From 1534c7178f192e09b4c452dc785c08a54a7eb02f Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:39:27 -0400 Subject: [PATCH 1/6] feat(sota): SIDECAR staging streams + the papers relevance router (living-sota slice 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec-20260905-103000 D3 / S3 (living-sota slice 2): per-campaign append-only transition streams beside the session ledger (.sota- staging.jsonl under sessions/, the hopper under the reserved stem 'hopper'), event-id idempotent, PIPE_BUF+O_APPEND line-atomic; stage/drop/accept are all appended transition lines — entries never mutated, state always derived by replay. The acceptance stamp is the sole sanctioned non-job append (PI-instructed, instruction provenance required). Expiry drops with the recorded expired-without-review line; O2 compaction pinned at 28 days (recorded by an appended compact line, accepted/pending never removed). The papers router enumerates active campaigns via the session-ledger parse target (session-*.md titles + verdict tables, per-file named degradation) and stages matches with review-by/expiry stamps; below-threshold and no-campaign matches go to the hopper. The campaign ledger itself is never touched (the nine-section grammar holds unamended). --- packages/amico-run/src/sota_router.ts | 298 ++++++++++ packages/amico-run/src/sota_staging.ts | 582 +++++++++++++++++++ packages/amico-run/test/sota_router.test.ts | 219 +++++++ packages/amico-run/test/sota_staging.test.ts | 410 +++++++++++++ 4 files changed, 1509 insertions(+) create mode 100644 packages/amico-run/src/sota_router.ts create mode 100644 packages/amico-run/src/sota_staging.ts create mode 100644 packages/amico-run/test/sota_router.test.ts create mode 100644 packages/amico-run/test/sota_staging.test.ts diff --git a/packages/amico-run/src/sota_router.ts b/packages/amico-run/src/sota_router.ts new file mode 100644 index 00000000..6cf7d857 --- /dev/null +++ b/packages/amico-run/src/sota_router.ts @@ -0,0 +1,298 @@ +// sota_router.ts — the relevance router (#living-sota slice 2, spec +// spec-20260905-103000 D3 / S3): new papers (and, via sota_watcher.ts, watched +// -repo events) match against ACTIVE campaigns — matched items append `stage` +// lines to the matching campaign's SIDECAR staging stream; below-threshold +// matches and no-campaign matches go to the HOPPER stream. A match NEVER +// touches the campaign ledger itself (the nine-section grammar holds +// unamended — the stage lands in the sidecar BESIDE it, per the pinned +// append-target convention). +// +// ── The campaign enumeration: the session-ledger parse target ───────────── +// +// ACTIVE campaigns are the `session-*.md` ledgers under the personal vault's +// sessions/ dir whose §2 VERDICT TABLE carries an OPEN row — a status cell +// with no terminal marker (MERGED / APPROVED / CLOSED / SAVED / SHIPPED / +// DONE / DROPPED / WONTFIX / REJECTED / SUPERSEDED). The matching corpus is +// the ledger's own identity: its H1 title + §1's objective line + the OPEN +// rows' item cells — an active campaign is matchable by what it IS working +// on, not by a hand-curated keyword list. A malformed or verdict-table-less +// ledger degrades per-file with a NAMED reason (the sweep proceeds — one +// broken ledger never blinds the router); an unreadable file is skipped. +// +// ── The score (explainable, word-boundary, reproducible) ────────────────── +// +// A paper matches a campaign iff ≥ RELEVANCE_THRESHOLD distinct campaign +// terms hit the paper's title+abstract (word-boundary, the digest's +// countMatches discipline — "control" never matches "controller" mid-word). +// THRESHOLD = 2: one shared word is noise ("gate", "system"); two distinct +// salient terms from a campaign's own identity is the minimum explainable +// match, and the stage line RECORDS which terms matched. Best match wins; +// ties break by campaign id ascending — the router is deterministic. +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + appendStageLine, + stagingStreamPath, + HOPPER_CAMPAIGN, + type StagingProvenance, +} from "./sota_staging.js"; + +export { HOPPER_CAMPAIGN }; + +export const RELEVANCE_THRESHOLD = 2; + +/** The hopper's named reasons (the stage line records WHY it went there). */ +export const REASON_BELOW_THRESHOLD = "below-threshold"; +export const REASON_NO_CAMPAIGN_MATCH = "no-campaign-match"; + +// ── the campaign enumeration (the session-ledger parse target) ───────────── + +export interface CampaignLedgerInfo { + /** The ledger file stem (`session-20260901-rydberg-cz`) — the sidecar's campaign. */ + id: string; + /** The H1 title after `# Session ledger — `. */ + title: string; + /** §1's first non-empty line (the one-line objective, the digest convention). */ + objectiveLine: string; + /** The OPEN verdict-table item cells (the campaign's live work). */ + open: string[]; + /** The salient-term corpus (title + objective + open items, stopworded). */ + terms: string[]; +} + +export interface CampaignEnumeration { + campaigns: CampaignLedgerInfo[]; + /** Per-file named degradation — a malformed ledger never fails the sweep. */ + skipped: { file: string; reason: string }[]; +} + +/** A verdict row is CLOSED when its status cell carries a terminal marker. */ +const TERMINAL_MARKERS = ["MERGED", "APPROVED", "CLOSED", "SAVED", "SHIPPED", "DONE", "DROPPED", "WONTFIX", "REJECTED", "SUPERSEDED"]; + +/** The section grammar: `## N.` numbered headers (the campaign_ledger.ts + * grammar, mirrored lean — amico-run cannot import the extension). */ +const SECTION_HEADER = /^##\s+§?\s*(\d+)\s*[.):\s]/; +const TABLE_LINE = /^\s*\|.*\|\s*$/; +const SEPARATOR_ROW = /^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$/; + +function tableRows(section: string): string[][] { + return section + .split("\n") + .filter((l) => TABLE_LINE.test(l) && !SEPARATOR_ROW.test(l)) + .map((l) => + l + .trim() + .replace(/^\|/, "") + .replace(/\|$/, "") + .split("|") + .map((c) => c.trim()), + ); +} + +/** §1's first non-empty line — the one-line objective (bullets stripped, + * capped like the digest's render). */ +function objectiveLine(section1: string): string { + const first = section1.split("\n").find((l) => l.trim() !== "") ?? ""; + return first.trim().replace(/^[-*]\s+/, "").slice(0, 240); +} + +// Short common words add nothing to relevance and drown the score — a compact +// stoplist (the term floor is 2 chars, so gate names like "cz" survive). +const STOPWORDS = new Set([ + "the", "and", "for", "with", "this", "that", "from", "into", "over", "under", + "are", "was", "were", "has", "had", "have", "will", "shall", "should", "would", + "been", "being", "its", "his", "her", "our", "your", "their", "them", "they", + "than", "then", "when", "what", "which", "while", "where", "these", "those", + "any", "all", "both", "each", "few", "more", "most", "other", "some", "such", + "only", "own", "same", "very", "just", "also", "next", "past", "per", "pre", + "not", "but", "out", "off", "via", "one", "two", "six", "ten", "onto", "upon", + "across", "against", "between", "through", "during", "before", "after", + "above", "below", "up", "down", "in", "on", "at", "by", "of", "to", "as", + "is", "it", "an", "or", "we", "no", "so", "if", "do", "did", "done", "run", + "runs", "ran", "row", "rows", "per", "use", "used", "uses", "make", "made", + "item", "items", "spec", "specs", "status", "review", "verdict", "table", + "objective", "campaign", "session", "ledger", "directives", "standing", +]); + +function salientTerms(corpus: string): string[] { + const tokens = corpus + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length >= 2 && !STOPWORDS.has(t) && !/^\d+$/.test(t)); + return [...new Set(tokens)]; +} + +/** Enumerate the ACTIVE campaigns under the personal vault's sessions/ dir — + * the session-ledger parse target (titles + verdict tables). Degrades + * per-file with named reasons; never throws on the sweep's account. */ +export function enumerateCampaigns(sessionsDir: string): CampaignEnumeration { + const campaigns: CampaignLedgerInfo[] = []; + const skipped: { file: string; reason: string }[] = []; + if (!existsSync(sessionsDir)) return { campaigns, skipped }; + const files = readdirSync(sessionsDir).filter((n) => n.startsWith("session-") && n.endsWith(".md")).sort(); + for (const name of files) { + let text: string; + try { + text = readFileSync(join(sessionsDir, name), "utf8"); + } catch { + skipped.push({ file: name, reason: "unreadable — skipped" }); + continue; + } + try { + // strip frontmatter if present (the splitFrontmatter rule: an + // unterminated block is body, not frontmatter) + let body = text; + if (text.startsWith("---")) { + const close = text.split("\n").slice(1, 65).findIndex((l) => l.trim() === "---"); + if (close >= 0) body = text.split("\n").slice(close + 2).join("\n"); + } + const title = body.match(/^#\s+Session ledger\s*[—-]\s*(.+)$/m)?.[1]?.trim() ?? ""; + // split on the numbered headers; §1 and §2 are the identity + const sections = new Map(); + let current: number | null = null; + for (const line of body.split("\n")) { + const m = line.match(SECTION_HEADER); + if (m) { + current = Number(m[1]); + continue; + } + if (current !== null) sections.set(current, `${sections.get(current) ?? ""}\n${line}`); + } + const s2 = sections.get(2) ?? ""; + const rows = tableRows(s2); + if (rows.length === 0) { + skipped.push({ file: name, reason: "no verdict table — not a matchable campaign ledger" }); + continue; + } + const open: string[] = []; + for (const cells of rows.slice(1)) { + // header row first (|item|status|); a row whose STATUS cell carries no + // terminal marker is OPEN — the campaign's live work + const status = (cells[1] ?? "").replace(/[*`]/g, "").toUpperCase(); + if (status === "") continue; + if (TERMINAL_MARKERS.some((m) => status.includes(m))) continue; + open.push((cells[0] ?? "").replace(/[*`]/g, "").trim()); + } + if (open.length === 0) continue; // all-terminal: a wrapped campaign, not matchable + const obj = objectiveLine(sections.get(1) ?? ""); + campaigns.push({ + id: name.slice(0, -3), + title, + objectiveLine: obj, + open, + terms: salientTerms(`${title}\n${obj}\n${open.join("\n")}`), + }); + } catch { + skipped.push({ file: name, reason: "unparseable ledger — skipped with a named reason, the sweep proceeds" }); + } + } + return { campaigns, skipped }; +} + +// ── the score (word-boundary, explainable, deterministic) ────────────────── + +const escapeRe = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +function termHits(text: string, terms: string[]): string[] { + const lower = text.toLowerCase(); + const hits: string[] = []; + for (const t of terms) { + if (new RegExp(`(^|[^a-z0-9-])${escapeRe(t)}([^a-z0-9-]|$)`).test(lower)) hits.push(t); + } + return hits; +} + +export interface RouteItem { + title: string; + detail: string; +} + +export type RouteResult = + | { target: "campaign"; campaign: CampaignLedgerInfo; score: number; matched: string[] } + | { target: "hopper"; reason: "below-threshold" | "no-campaign-match"; score: number; matched: string[] }; + +/** Route one survey item (a paper, a watcher event) against the active + * campaigns: best distinct-term score wins (ties → campaign id asc); below + * threshold → the hopper; no campaigns → the hopper. */ +export function routeItem(item: RouteItem, campaigns: CampaignLedgerInfo[]): RouteResult { + if (campaigns.length === 0) { + return { target: "hopper", reason: REASON_NO_CAMPAIGN_MATCH, score: 0, matched: [] }; + } + let best: { campaign: CampaignLedgerInfo; score: number; matched: string[] } | null = null; + for (const c of [...campaigns].sort((a, b) => (a.id < b.id ? -1 : 1))) { + const matched = termHits(`${item.title}\n${item.detail}`, c.terms); + if (best === null || matched.length > best.score) best = { campaign: c, score: matched.length, matched }; + } + if (best !== null && best.score >= RELEVANCE_THRESHOLD) { + return { target: "campaign", campaign: best.campaign, score: best.score, matched: best.matched }; + } + return { + target: "hopper", + reason: REASON_BELOW_THRESHOLD, + score: best?.score ?? 0, + matched: best?.matched ?? [], + }; +} + +// ── the digest's routing pass (ONE APPENDER: the digest job) ──────────────── + +export interface PapersRoutingItem { + arxiv: string; + title: string; + abstract: string; +} + +export interface RoutePapersOpts { + sessionsDir: string; + provenance: StagingProvenance; + nowMs?: () => number; +} + +export interface RoutePapersResult { + staged: { event_id: string; campaign: string; matched: string[] }[]; + hopper: { event_id: string; campaign: string; reason: string }[]; + deduped: string[]; +} + +/** Route a batch of papers (the digest's picks) into the staging streams: + * matched → the campaign sidecar; below-threshold / no-match → the hopper + * stream. Idempotent by event id — the daily job re-running on the same + * corpus dedupes centrally (double-delivery impossible). The campaign LEDGER + * is never touched. */ +export function routePapersToStaging(opts: RoutePapersOpts & { items: PapersRoutingItem[] }): RoutePapersResult { + const { campaigns } = enumerateCampaigns(opts.sessionsDir); + const result: RoutePapersResult = { staged: [], hopper: [], deduped: [] }; + for (const p of opts.items) { + const route = routeItem({ title: p.title, detail: p.abstract }, campaigns); + const event_id = `arxiv:${p.arxiv}`; + const entry = + route.target === "campaign" + ? { + event_id, + campaign: route.campaign.id, + kind: "paper" as const, + title: p.title, + url: `https://arxiv.org/abs/${p.arxiv}`, + provenance: opts.provenance, + matched: route.matched, + } + : { + event_id, + campaign: HOPPER_CAMPAIGN, + kind: "paper" as const, + title: p.title, + url: `https://arxiv.org/abs/${p.arxiv}`, + provenance: opts.provenance, + matched: route.matched, + reason: route.reason, + }; + const r = appendStageLine(stagingStreamPath(opts.sessionsDir, entry.campaign), entry, { nowMs: opts.nowMs }); + if (r.appended) { + if (route.target === "campaign") result.staged.push({ event_id, campaign: route.campaign.id, matched: route.matched }); + else result.hopper.push({ event_id, campaign: HOPPER_CAMPAIGN, reason: route.reason }); + } else { + result.deduped.push(event_id); + } + } + return result; +} diff --git a/packages/amico-run/src/sota_staging.ts b/packages/amico-run/src/sota_staging.ts new file mode 100644 index 00000000..4ee03a40 --- /dev/null +++ b/packages/amico-run/src/sota_staging.ts @@ -0,0 +1,582 @@ +// sota_staging.ts — the per-campaign SIDECAR staging streams (#living-sota +// slice 2, spec spec-20260905-103000 D3 / S3): append-only transition streams +// BESIDE the session ledger — `.sota-staging.jsonl` under the +// personal vault's sessions/ dir (the nine-section ledger grammar holds +// UNAMENDED; the sidecar is a SEPARATE file, one writer class, no collision +// with the live campaign session's own ledger writes). The hopper fallback is +// the same machinery under the reserved stem `hopper` — the flywheel's drain +// extends to it (the coordination note's feed). +// +// ── The transition grammar (all appended lines; entries never mutated; +// staging state always DERIVED by replay) ──────────────────────────────── +// +// stage {ev, seq, ts, event_id, campaign, kind, title, url, provenance, +// matched?, reason?, review_by, expires_at} +// accept {ev, seq, ts, event_id, campaign, kind, title, url, +// instructed_by: "PI", instruction: {channel, note, received_at}} +// drop {ev, seq, ts, event_id, campaign, reason, recorded} +// compact{ev, seq, ts, removed, window_days, high_water_seq, compacted_at} +// +// Event ids are the EXTERNAL identity (arxiv: for papers, github: for +// watcher events) — idempotency keys, so double-delivery is impossible: the +// appender dedupes centrally by event id (ONE APPENDER — the digest, the +// watcher, and the weekly synthesis are the ONLY stage/drop writers; the +// read-then-append here is safe ONLY because that writer class is single, +// which is exactly what the one-appender invariant buys). +// +// The accept stamp is the SOLE sanctioned non-job append: an agent writing on +// the PI's EXPLICIT instruction records the decision — `instructed_by: "PI"` +// + the instruction provenance (channel, note, received_at). Its schema rides +// the bridge fixtures (obligation O3: fixtures/bridge/2026-09-05-sota-staging, +// validated by scripts/validate_bridge_replay.mjs). +// +// ── Line-atomicity (the coordination-ledger discipline) ──────────────────── +// +// Every line is one JSON object ≤ PIPE_BUF (4096 B) written with O_APPEND: +// a concurrent reader never sees a torn line, and `kill -9` loses at most the +// line in flight. A LIVE reader (deriveStagingState) skips an incomplete +// trailing line and keeps the rest; a torn line MID-stream is corruption the +// writers refuse to create (throw, never a torn append). +// +// ── The stamps ───────────────────────────────────────────────────────────── +// +// REVIEW_BY_DAYS = 7: one full weekly-brief cycle — the awaiting-the-eye +// listing the PI reads weekly names the review-by date. +// EXPIRES_AFTER_DAYS = 14: a SECOND full weekly cycle of grace, then the +// job-appended drop with the recorded reason (expired-without-review) — +// missing one weekly is human, missing two is the chronic-non-review signal +// the health stamp exists to render ("reads sick, not clean"). +// COMPACTION_WINDOW_DAYS = 28 (O2, the pinned number): expired-and-dropped +// chains compact once their drop is 28 days old. Why 28: the trailing +// expired-without-review health stamp needs a month of history to show a +// month of neglect — four weekly digests, the human triage horizon — and +// beyond that the quiet-failure noise is archival, not actionable (the +// bounded-window discipline of sota_history: keep what the trailing window +// reads, nothing more). The compaction REWRITES the stream atomically and +// RECORDS itself as an appended compact line (removed, window, the +// pre-compaction high-water seq) — never a silent mutation. Accepted and +// pending entries are NEVER compacted: derivation reads accepted-only, so +// the accepted record stands forever. +import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { PIPE_BUF } from "./ledger.js"; + +export const SIDECAR_SUFFIX = ".sota-staging.jsonl"; + +/** The reserved stem for the fallback stream — the hopper feed (the flywheel's + * drain expectations extend to it, per the coordination note). */ +export const HOPPER_CAMPAIGN = "hopper"; + +/** The review-by stamp: one full weekly-brief cycle. */ +export const REVIEW_BY_DAYS = 7; + +/** The expiry stamp: a second full weekly cycle of grace, then the recorded drop. */ +export const EXPIRES_AFTER_DAYS = 14; + +/** O2: the compaction window — expired-and-dropped chains compact at 28 days + * (four weekly digests; the trailing health window stays computable). */ +export const COMPACTION_WINDOW_DAYS = 28; + +/** The named drop reason for a staged match past its expiry without review. */ +export const EXPIRED_WITHOUT_REVIEW = "expired-without-review"; + +// ── types ─────────────────────────────────────────────────────────────────── + +export type StagingKind = "paper" | "release" | "changelog" | "issue"; + +/** The provenance stamp every stage line carries (where the match CAME from — + * a cache read never launders as a live fetch). */ +export interface StagingProvenance { + job: string; // papers-digest | sota-watcher | … + via: string; // cache | fetched | feed + source: string; // the fetch surface, named + fetched_at: string; // ISO-8601 + [key: string]: unknown; // job-specific stamps (repo, surface, query, …) +} + +/** The PI-instruction provenance the accept stamp records (the schema is the + * human decision's record — an accept without it is refused). */ +export interface AcceptInstruction { + channel: string; // chat | review | issue — where the instruction arrived + note: string; // the PI's instruction, verbatim or near-verbatim + received_at?: string; // ISO-8601; defaults to the stamp's ts +} + +export interface StageEntryInput { + event_id: string; + campaign: string; + kind: StagingKind; + title: string; + url: string; + provenance: StagingProvenance; + /** The matched campaign terms (explainable: the line says WHY it staged here). */ + matched?: string[]; + /** For hopper entries: the named reason (below-threshold | no-campaign-match). */ + reason?: string; +} + +interface StageLine { + ev: "stage"; + seq: number; + ts: string; + event_id: string; + campaign: string; + kind: StagingKind; + title: string; + url: string; + provenance: StagingProvenance; + matched?: string[]; + reason?: string; + review_by: string; + expires_at: string; +} +interface AcceptLine { + ev: "accept"; + seq: number; + ts: string; + event_id: string; + campaign: string; + kind?: string; + title?: string; + url?: string; + instructed_by: "PI"; + instruction: { channel: string; note: string; received_at: string }; +} +interface DropLine { + ev: "drop"; + seq: number; + ts: string; + event_id: string; + campaign: string; + reason: string; + recorded: string; +} +interface CompactLine { + ev: "compact"; + seq: number; + ts: string; + removed: number; + window_days: number; + high_water_seq: number; + compacted_at: string; +} +/** Unknown-but-well-formed ev values are CARRIED (reader opacity — the bridge + * doctrine's forward-compat rule); `record` is the raw line. */ +export type StagingLine = (StageLine | AcceptLine | DropLine | CompactLine | { ev: string; seq: number; ts: string; [k: string]: unknown }) & { + seq: number; + ts: string; +}; + +export interface StagedEntryState { + event_id: string; + campaign: string; + kind?: string; + title?: string; + url?: string; + provenance?: StagingProvenance; + matched?: string[]; + reason?: string; + review_by?: string; + expires_at?: string; + state: "staged" | "accepted" | "dropped"; + /** Set when an accept/drop line has no stage line behind it — carried by the + * reader (never fatal); the WRITERS refuse to create it, the validator reds it. */ + orphan?: boolean; + accepted_at?: string; + instruction?: { channel: string; note: string; received_at: string }; + drop_reason?: string; + dropped_at?: string; +} + +export interface StagingState { + lines: StagingLine[]; + entries: Map; +} + +export interface StagingOpts { + nowMs?: () => number; +} + +export type AppendResult = + | { appended: true } + | { appended: false; reason: string }; + +// ── paths ─────────────────────────────────────────────────────────────────── + +/** The sidecar stream path for one campaign stem (a session-ledger file stem, + * or `hopper`), beside the ledger under sessions/. */ +export function stagingStreamPath(sessionsDir: string, campaign: string): string { + return join(sessionsDir, campaign + SIDECAR_SUFFIX); +} + +/** Every campaign stem with a sidecar stream under sessions/ (the sweep and + * the awaiting-the-eye render enumerate these; stems are file names minus + * the suffix — no directory reaches out of sessions/). */ +export function listStagingStreams(sessionsDir: string): string[] { + if (!existsSync(sessionsDir)) return []; + try { + return readdirSync(sessionsDir) + .filter((n) => n.endsWith(SIDECAR_SUFFIX)) + .map((n) => n.slice(0, -SIDECAR_SUFFIX.length)) + .sort(); + } catch { + return []; + } +} + +// ── the low-level append (ONE writer discipline: callers are the jobs, or +// the accept stamp on the PI's instruction) ─────────────────────────────── + +function writeLine(path: string, line: StagingLine): void { + const text = JSON.stringify(line) + "\n"; + const bytes = Buffer.byteLength(text, "utf8"); + if (bytes > PIPE_BUF) { + throw new Error( + `staging line exceeds PIPE_BUF: ${bytes} B > ${PIPE_BUF} B — O_APPEND atomicity holds only under the ceiling; truncate the payload, never the discipline`, + ); + } + mkdirSync(join(path, ".."), { recursive: true }); + appendFileSync(path, text, { flag: "a" }); // O_APPEND: atomic per line +} + +// ── reading + derivation (readers tolerate; writers are strict) ───────────── + +export interface ReadStagingResult { + lines: StagingLine[]; + /** The stream ends on a whole line (false = a torn in-flight tail was skipped). */ + tornTail: boolean; +} + +/** Read a stream as raw lines. A torn FINAL line (live in-flight write) is + * skipped; a torn line ELSEWHERE is corruption — the writers' atomic + * appends never create it, so the reader throws (the validator reds it). */ +export function readStagingStream(path: string): ReadStagingResult { + if (!existsSync(path)) return { lines: [], tornTail: false }; + const text = readFileSync(path, "utf8"); + if (text === "") return { lines: [], tornTail: false }; + const parts = text.split("\n"); + // a trailing "" is the final newline; a non-"" tail is a torn in-flight line + let tornTail = false; + let rawLines: string[]; + if (parts[parts.length - 1] === "") rawLines = parts.slice(0, -1); + else { + rawLines = parts.slice(0, -1); + tornTail = true; // the incomplete final chunk is skipped by a LIVE reader + } + const lines: StagingLine[] = []; + rawLines.forEach((l, i) => { + if (l.trim() === "") return; + let v: unknown; + try { + v = JSON.parse(l); + } catch { + throw new Error( + `staging stream corruption: ${path} line ${i + 1} is not a whole JSON object — the writers append atomically under PIPE_BUF, so a torn MID-stream line was never a live write`, + ); + } + if (v === null || typeof v !== "object" || Array.isArray(v)) { + throw new Error(`staging stream corruption: ${path} line ${i + 1} is not a JSON object`); + } + lines.push(v as StagingLine); + }); + return { lines, tornTail }; +} + +/** Derive staging state by replay — the ONLY way state exists. Per event_id: + * the first line must be a stage (accept/drop behind none is carried as + * `orphan`, never fatal to the reader); accept and drop are terminal; unknown + * ev values are carried through untouched (opacity). */ +export function deriveStagingState(path: string): StagingState { + const { lines } = readStagingStream(path); + const entries = new Map(); + for (const line of lines) { + const rec = line as Record; + const ev = typeof rec.ev === "string" ? rec.ev : ""; + if (ev !== "stage" && ev !== "accept" && ev !== "drop") continue; // carried (compact/unknown), no state change + const id = typeof rec.event_id === "string" ? rec.event_id : ""; + if (id === "") continue; + const existing = entries.get(id); + if (ev === "stage") { + if (existing === undefined) { + entries.set(id, { + event_id: id, + campaign: typeof rec.campaign === "string" ? rec.campaign : "", + kind: typeof rec.kind === "string" ? rec.kind : undefined, + title: typeof rec.title === "string" ? rec.title : undefined, + url: typeof rec.url === "string" ? rec.url : undefined, + provenance: (rec.provenance as StagingProvenance | undefined) ?? undefined, + matched: Array.isArray(rec.matched) ? (rec.matched as string[]) : undefined, + reason: typeof rec.reason === "string" ? rec.reason : undefined, + review_by: typeof rec.review_by === "string" ? rec.review_by : undefined, + expires_at: typeof rec.expires_at === "string" ? rec.expires_at : undefined, + state: "staged", + }); + } + continue; // a duplicate stage is deduped at write time; a reader carries it inertly + } + if (existing === undefined) { + entries.set(id, { + event_id: id, + campaign: typeof rec.campaign === "string" ? rec.campaign : "", + state: ev === "accept" ? "accepted" : "dropped", + orphan: true, // carried, never fatal — the writers refuse to create it + ...(ev === "accept" + ? { accepted_at: typeof rec.ts === "string" ? rec.ts : undefined } + : { drop_reason: typeof rec.reason === "string" ? rec.reason : undefined, dropped_at: typeof rec.recorded === "string" ? rec.recorded : undefined }), + }); + continue; + } + if (existing.state !== "staged") continue; // terminal stays terminal (idempotent replay) + if (ev === "accept") { + existing.state = "accepted"; + existing.accepted_at = typeof rec.ts === "string" ? rec.ts : undefined; + if (rec.instruction && typeof rec.instruction === "object") { + existing.instruction = rec.instruction as { channel: string; note: string; received_at: string }; + } + } else { + existing.state = "dropped"; + existing.drop_reason = typeof rec.reason === "string" ? rec.reason : undefined; + existing.dropped_at = typeof rec.recorded === "string" ? rec.recorded : undefined; + } + } + return { lines, entries }; +} + +// ── the writers (STRICT: they validate their own input; readers tolerate) ──── + +function assertStageInput(entry: StageEntryInput): void { + if (entry.event_id.trim() === "") throw new Error("stage line: event_id is required (the idempotency key)"); + if (entry.campaign.trim() === "") throw new Error("stage line: campaign is required"); + if (!/^(paper|release|changelog|issue)$/.test(entry.kind)) throw new Error(`stage line: unknown kind "${entry.kind}"`); + if (typeof entry.url !== "string" || entry.url === "") throw new Error("stage line: url is required (every match is cited)"); + const p = entry.provenance; + if (typeof p.job !== "string" || p.job === "" || typeof p.via !== "string" || p.via === "" || typeof p.source !== "string" || p.source === "" || typeof p.fetched_at !== "string" || p.fetched_at === "") { + throw new Error("stage line: provenance must carry {job, via, source, fetched_at} — a match never lands unprovenance-stamped"); + } +} + +/** Append a `stage` transition — the digest's and the watcher's writer + * (ONE APPENDER). Idempotent by event id: a delivery whose event is already + * in the stream (staged, accepted, or dropped) is a no-op with the named + * reason — double-delivery is impossible. */ +export function appendStageLine(path: string, entry: StageEntryInput, opts: StagingOpts = {}): AppendResult { + assertStageInput(entry); + const nowMs = opts.nowMs ?? Date.now; + const { lines } = readStagingStream(path); + if (lines.some((l) => (l as { event_id?: unknown }).event_id === entry.event_id)) { + return { appended: false, reason: `duplicate-delivery: ${entry.event_id} is already in the stream (deduped centrally by event id)` }; + } + const ts = new Date(nowMs()).toISOString(); + const line: StageLine = { + ev: "stage", + seq: lines.length + 1, // seq IS the line count at write time + ts, + event_id: entry.event_id, + campaign: entry.campaign, + kind: entry.kind, + title: entry.title.slice(0, 240), + url: entry.url, + provenance: entry.provenance, + ...(entry.matched !== undefined ? { matched: entry.matched } : {}), + ...(entry.reason !== undefined ? { reason: entry.reason } : {}), + review_by: new Date(nowMs() + REVIEW_BY_DAYS * 86_400_000).toISOString(), + expires_at: new Date(nowMs() + EXPIRES_AFTER_DAYS * 86_400_000).toISOString(), + }; + writeLine(path, line); + return { appended: true }; +} + +/** The PI-instructed acceptance stamp — the SOLE sanctioned non-job append: an + * agent acting on the PI's EXPLICIT instruction records the human decision. + * Idempotent by (event, accept); refused without a staged match, refused on + * a dropped match (terminal), refused without the instruction provenance. */ +export function appendAcceptStamp(path: string, event_id: string, instruction: AcceptInstruction, opts: StagingOpts = {}): AppendResult { + const nowMs = opts.nowMs ?? Date.now; + if (typeof instruction.note !== "string" || instruction.note.trim() === "") { + return { appended: false, reason: "refused: the acceptance stamp records the PI's explicit instruction — the instruction note is required (an unstamped acceptance is a laundered one)" }; + } + const { lines, entries } = deriveStagingState(path); + const entry = entries.get(event_id); + if (entry === undefined || entry.orphan === true) { + return { appended: false, reason: `refused: no staged match for ${event_id} — the stamp is the record of a human decision ON a staged match, never free-floating` }; + } + if (entry.state === "accepted") { + return { appended: false, reason: `already-accepted: ${event_id} (idempotent — the stamp records once)` }; + } + if (entry.state === "dropped") { + return { appended: false, reason: `refused: ${event_id} was dropped (${entry.drop_reason ?? "unknown reason"}) — a drop is terminal; re-stage the match if the PI wants it` }; + } + const ts = new Date(nowMs()).toISOString(); + const line: AcceptLine = { + ev: "accept", + seq: lines.length + 1, + ts, + event_id, + campaign: entry.campaign, + ...(entry.kind !== undefined ? { kind: entry.kind } : {}), + ...(entry.title !== undefined ? { title: entry.title } : {}), + ...(entry.url !== undefined ? { url: entry.url } : {}), + instructed_by: "PI", + instruction: { + channel: instruction.channel, + note: instruction.note.slice(0, 240), + received_at: instruction.received_at ?? ts, + }, + }; + writeLine(path, line); + return { appended: true }; +} + +/** Append a `drop` transition — the expiry writer inside the digest/synthesis + * job (ONE APPENDER). Refused on an accepted match (acceptance is terminal) + * and without a stage; idempotent on an already-dropped match. */ +export function appendDropLine(path: string, event_id: string, reason: string, opts: StagingOpts = {}): AppendResult { + const nowMs = opts.nowMs ?? Date.now; + const { lines, entries } = deriveStagingState(path); + const entry = entries.get(event_id); + if (entry === undefined || entry.orphan === true) { + return { appended: false, reason: `refused: no staged match for ${event_id} — a drop line records the fate of a staged match` }; + } + if (entry.state === "accepted") { + return { appended: false, reason: `refused: ${event_id} is ACCEPTED — acceptance is terminal, an accepted match never drops` }; + } + if (entry.state === "dropped") { + return { appended: false, reason: `already-dropped: ${event_id} (idempotent)` }; + } + const ts = new Date(nowMs()).toISOString(); + const line: DropLine = { + ev: "drop", + seq: lines.length + 1, + ts, + event_id, + campaign: entry.campaign, + reason, + recorded: ts, // the recorded line + }; + writeLine(path, line); + return { appended: true }; +} + +/** The expiry sweep (the weekly job's drop writer): every staged match past + * its `expires_at` gets a job-appended drop line with the recorded reason. + * A pipeline that stages and drops without ever being reviewed reads SICK + * via the expired-without-review count, not clean. */ +export function sweepExpiry(path: string, opts: StagingOpts = {}): { dropped: string[] } { + const nowMs = opts.nowMs ?? Date.now; + const now = nowMs(); + const { entries } = deriveStagingState(path); + const dropped: string[] = []; + for (const entry of entries.values()) { + if (entry.state !== "staged" || entry.expires_at === undefined) continue; + if (Date.parse(entry.expires_at) > now) continue; // not yet expired — stage-before-count holds + const r = appendDropLine(path, entry.event_id, EXPIRED_WITHOUT_REVIEW, opts); + if (r.appended) dropped.push(entry.event_id); + } + dropped.sort(); // deterministic across map order + return { dropped }; +} + +/** O2 — the compaction pass: expired-and-dropped chains whose drop is older + * than the 28-day window are removed (the whole chain: stage + drop), the + * rewrite is ATOMIC (tmp+rename), and the stream RECORDS its own compaction + * as an appended compact line — never a silent mutation. Accepted and + * pending entries are never removed; seq is renumbered so seq = line count + * holds for the rewritten stream (the compact line carries the + * pre-compaction high-water seq — nothing is lost). */ +export function compactStagingStream(path: string, opts: StagingOpts = {}): { removed: number } { + const nowMs = opts.nowMs ?? Date.now; + const now = nowMs(); + const { lines, entries } = deriveStagingState(path); + const eligible = new Set(); + for (const entry of entries.values()) { + if (entry.state !== "dropped" || entry.drop_reason !== EXPIRED_WITHOUT_REVIEW || entry.dropped_at === undefined) continue; + if (now - Date.parse(entry.dropped_at) <= COMPACTION_WINDOW_DAYS * 86_400_000) continue; + eligible.add(entry.event_id); + } + if (eligible.size === 0) return { removed: 0 }; // idle compaction is not an event — no noise lines + const kept = lines.filter((l) => { + const rec = l as { ev?: unknown; event_id?: unknown }; + if (typeof rec.event_id === "string" && eligible.has(rec.event_id)) return false; // the whole chain goes + return true; + }); + const highWater = lines.length; + const ts = new Date(now).toISOString(); + const compactLine: CompactLine = { + ev: "compact", + seq: kept.length + 1, + ts, + removed: highWater - kept.length, // LINES removed (each chain is its stage+transition lines) + window_days: COMPACTION_WINDOW_DAYS, + high_water_seq: highWater, + compacted_at: ts, + }; + const out = [...kept, compactLine].map((l, i) => ({ ...(l as Record), seq: i + 1 })); + const text = out.map((l) => JSON.stringify(l)).join("\n") + "\n"; + const tmp = `${path}.tmp-${process.pid}`; + writeFileSync(tmp, text); + renameSync(tmp, path); // atomic: a reader sees the whole stream or the old one + return { removed: highWater - kept.length }; +} + +// ── the awaiting-the-eye render (derived state ONLY; never currency) ──────── + +/** The trailing-window EXPIRED-WITHOUT-REVIEW count — the brief's health + * stamp input (a pipeline that stages and drops without review reads sick). */ +export function expiredWithoutReviewCount(path: string, opts: StagingOpts = {}): number { + const nowMs = opts.nowMs ?? Date.now; + const now = nowMs(); + const { entries } = deriveStagingState(path); + let n = 0; + for (const e of entries.values()) { + if (e.state !== "dropped" || e.drop_reason !== EXPIRED_WITHOUT_REVIEW || e.dropped_at === undefined) continue; + if (now - Date.parse(e.dropped_at) <= COMPACTION_WINDOW_DAYS * 86_400_000) n++; + } + return n; +} + +/** The awaiting-the-eye listing: everything PENDING (staged, un-reviewed, + * provenance-stamped) + the expired-without-review counts — rendered from + * DERIVED state. Accepted material is currency and appears NOWHERE here; a + * false-positive match can never launder into strategy — the human eye sits + * between the match and the composition. */ +export function renderAwaitingTheEye(paths: string[], opts: StagingOpts = {}): string { + const nowMs = opts.nowMs ?? Date.now; + const now = nowMs(); + const groups: { stem: string; pending: StagedEntryState[]; expired: number }[] = []; + let pendingTotal = 0; + let expiredTotal = 0; + for (const p of paths) { + const stem = p.split("/").pop()?.replace(/\.sota-staging\.jsonl$/, "") ?? "(stream)"; + const { entries } = deriveStagingState(p); + const pending = [...entries.values()].filter((e) => e.state === "staged").sort((a, b) => a.event_id.localeCompare(b.event_id)); + const expired = expiredWithoutReviewCount(p, { nowMs: () => now }); + if (pending.length === 0 && expired === 0) continue; + groups.push({ stem, pending, expired }); + pendingTotal += pending.length; + expiredTotal += expired; + } + const lines: string[] = [ + "## Awaiting the eye — staged matches pending review (never rendered as currency)", + `pending staged matches: ${pendingTotal} · expired-without-review (trailing ${COMPACTION_WINDOW_DAYS} days): ${expiredTotal}`, + "", + ]; + if (groups.length === 0) { + lines.push("_nothing pending — the staged streams are empty._"); + return lines.join("\n"); + } + for (const g of groups) { + lines.push(`### ${g.stem} (${g.pending.length} pending${g.expired > 0 ? `, ${g.expired} expired without review` : ""})`); + for (const e of g.pending) { + lines.push(`- **${e.title ?? "(untitled)"}** (${e.event_id})${e.url ? ` — ${e.url}` : ""}`); + const prov = e.provenance; + const provText = prov ? `${prov.job} via ${prov.via} (${prov.source})` : "provenance absent"; + const matched = e.matched && e.matched.length > 0 ? ` · matched: ${e.matched.join(", ")}` : ""; + const reason = e.reason ? ` · ${e.reason}` : ""; + const reviewBy = e.review_by ? ` · review by ${e.review_by.slice(0, 10)}` : ""; + lines.push(` _staged ${e.expires_at ? `expires ${e.expires_at.slice(0, 10)}` : ""}${reviewBy} · provenance: ${provText}${matched}${reason}_`); + } + lines.push(""); + } + lines.push("_derivation reads accepted-only: nothing above counts until the PI's accept stamp lands._"); + return lines.join("\n"); +} diff --git a/packages/amico-run/test/sota_router.test.ts b/packages/amico-run/test/sota_router.test.ts new file mode 100644 index 00000000..b4e58ef5 --- /dev/null +++ b/packages/amico-run/test/sota_router.test.ts @@ -0,0 +1,219 @@ +// sota_router.test.ts — the relevance router (#living-sota slice 2, spec +// spec-20260905-103000 D3 / S3): new papers match against ACTIVE campaigns — +// the campaign enumeration is the session-ledger parse target (sessions/ +// session-*.md verdict tables + titles); matched items append `stage` lines to +// the matching campaign's SIDECAR, or the HOPPER when no campaign matches; +// below-threshold → the hopper. A matched paper never enters a campaign +// LEDGER (the nine-section grammar holds unamended — the stage lands in the +// sidecar BESIDE it); a malformed ledger degrades per-file with a named +// reason, never failing the sweep. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + enumerateCampaigns, + routeItem, + routePapersToStaging, + RELEVANCE_THRESHOLD, + HOPPER_CAMPAIGN, + type CampaignLedgerInfo, +} from "../src/sota_router.js"; +import { stagingStreamPath, deriveStagingState, SIDECAR_SUFFIX, EXPIRED_WITHOUT_REVIEW } from "../src/sota_staging.js"; + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "sota-router-")); + mkdirSync(join(dir, "sessions"), { recursive: true }); +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +// ── fixture campaign ledgers (the session-ledger grammar, verbatim shape) ──── + +const ACTIVE_LEDGER = `--- +type: session-ledger +campaign: Rydberg blockade scaling +--- + +# Session ledger — Rydberg blockade scaling + +Campaign: push the Rydberg CZ gate fidelity past 0.9999 on the 8-atom register. + +## 1. Objective & standing directives + +- Objective: push the Rydberg CZ gate fidelity past 0.9999 on the 8-atom register, blockade regime. + +## 2. Verdict table + +| item | status | +|---|---| +| H1 blockade radius calibration | **FAILED (3 runs) — retry queued** | +| H2 shaped pulse vs square pulse CZ | pending | +| spec-20260901-rydberg-cz | **approved** | + +## 3. Active work + +- none in flight +`; + +const DONE_LEDGER = `# Session ledger — Wrapped old campaign + +## 2. Verdict table + +| item | status | +|---|---| +| S1 readme truth | **MERGED (PR #1)** | +`; + +const NO_TABLE_LEDGER = `# Session ledger — A bare note + +Some prose, no verdict table at all. +`; + +function writeLedger(name: string, text: string): string { + writeFileSync(join(dir, "sessions", name), text); + return name.replace(/\.md$/, ""); +} + +const ACTIVE = "session-20260901-rydberg-cz"; +const DONE = "session-20260801-wrapped-campaign"; + +function seedLedgers(): void { + writeLedger(ACTIVE + ".md", ACTIVE_LEDGER); + writeLedger(DONE + ".md", DONE_LEDGER); + writeLedger("session-20260802-bare-note.md", NO_TABLE_LEDGER); + writeLedger("CHECKOUTS.md", "not a ledger"); // never matches the session-*.md glob +} + +describe("enumerateCampaigns — the session-ledger parse target (titles + verdict tables)", () => { + it("enumerates ACTIVE campaigns: open verdict rows keep a campaign matchable; terminal-only ones do not", () => { + seedLedgers(); + const { campaigns, skipped } = enumerateCampaigns(join(dir, "sessions")); + expect(campaigns.map((c) => c.id)).toEqual([ACTIVE]); // the all-terminal ledger is not matchable + const active = campaigns[0] as CampaignLedgerInfo; + expect(active.title).toBe("Rydberg blockade scaling"); + expect(active.open).toEqual([ + "H1 blockade radius calibration", + "H2 shaped pulse vs square pulse CZ", + ]); + // the skipped carry NAMED reasons (degrade per-file, never fail the sweep) + expect(skipped).toEqual([{ file: "session-20260802-bare-note.md", reason: expect.stringMatching(/no verdict table/i) }]); + }); + + it("an unreadable ledger file is skipped with a named reason — the sweep proceeds", () => { + // a directory with a ledger-shaped name: readFileSync throws EISDIR — the + // per-file degradation path, never a sweep failure + mkdirSync(join(dir, "sessions", "session-20260701-garbage.md")); + const { campaigns, skipped } = enumerateCampaigns(join(dir, "sessions")); + expect(campaigns).toEqual([]); + expect(skipped).toEqual([{ file: "session-20260701-garbage.md", reason: expect.stringMatching(/unreadable/i) }]); + }); + + it("a missing/empty sessions dir enumerates to nothing (the router falls back to the hopper)", () => { + expect(enumerateCampaigns(join(dir, "nope")).campaigns).toEqual([]); + }); +}); + +describe("routeItem — relevance against the campaign corpora (explainable, word-boundary)", () => { + it("a paper sharing >= threshold distinct campaign terms routes to the campaign, with the matched terms recorded", () => { + seedLedgers(); + const { campaigns } = enumerateCampaigns(join(dir, "sessions")); + const r = routeItem( + { title: "Fast Rydberg CZ gates via optimal control", detail: "We shape pulses in the blockade regime; the CZ gate reaches 0.9999." }, + campaigns, + ); + expect(r.target).toBe("campaign"); + expect(r.campaign?.id).toBe(ACTIVE); + expect(r.matched.length).toBeGreaterThanOrEqual(RELEVANCE_THRESHOLD); + expect(r.matched).toContain("rydberg"); + expect(r.matched).toContain("cz"); + }); + + it("a paper below the threshold routes to the HOPPER (sub-threshold material is awaiting-the-eye, never currency)", () => { + seedLedgers(); + const { campaigns } = enumerateCampaigns(join(dir, "sessions")); + const r = routeItem({ title: "Protein folding via deep learning", detail: "AlphaFold-style pipelines for structure prediction." }, campaigns); + expect(r.target).toBe("hopper"); + expect(r.reason).toBe("below-threshold"); + }); + + it("no campaigns at all → the hopper with the named no-campaign-match reason", () => { + const r = routeItem({ title: "Anything at all", detail: "whatever" }, []); + expect(r.target).toBe("hopper"); + expect(r.reason).toBe("no-campaign-match"); + }); + + it("ties break deterministically (campaign id asc) — the router is reproducible", () => { + const a: CampaignLedgerInfo = { id: "session-b", title: "Zeta rydberg cz", objectiveLine: "", open: [], terms: ["rydberg", "cz"] }; + const b: CampaignLedgerInfo = { id: "session-a", title: "Alpha rydberg cz", objectiveLine: "", open: [], terms: ["rydberg", "cz"] }; + const r = routeItem({ title: "rydberg cz", detail: "" }, [a, b]); + expect(r.campaign?.id).toBe("session-a"); + }); +}); + +describe("routePapersToStaging — the digest's routing pass (stage lines, idempotent)", () => { + const PROV = { + job: "papers-digest", + via: "fetched", + source: "arXiv export API over HTTPS", + fetched_at: "2026-09-05T10:00:00.000Z", + }; + + it("a matched paper lands in the matching campaign's SIDECAR with stamps; below-threshold to the hopper; the LEDGER is untouched", () => { + seedLedgers(); + const r = routePapersToStaging({ + items: [ + { arxiv: "2606.05060", title: "Fast Rydberg CZ gates via optimal control", abstract: "We shape pulses in the blockade regime; the CZ gate reaches 0.9999." }, + { arxiv: "2606.99999", title: "Protein folding via deep learning", abstract: "AlphaFold-style pipelines." }, + ], + sessionsDir: join(dir, "sessions"), + provenance: PROV, + nowMs: () => 1_000_000_000_000, + }); + expect(r.staged).toHaveLength(1); + expect(r.staged[0]).toMatchObject({ event_id: "arxiv:2606.05060", campaign: ACTIVE }); + expect(r.hopper).toHaveLength(1); + expect(r.hopper[0]).toMatchObject({ event_id: "arxiv:2606.99999", campaign: HOPPER_CAMPAIGN, reason: "below-threshold" }); + // the stage line lives in the SIDECAR beside the ledger, never in the ledger + const sidecar = stagingStreamPath(join(dir, "sessions"), ACTIVE); + const st = deriveStagingState(sidecar); + expect(st.entries.get("arxiv:2606.05060")?.state).toBe("staged"); + expect(st.entries.get("arxiv:2606.05060")?.kind).toBe("paper"); + expect(readFileSync(join(dir, "sessions", ACTIVE + ".md"), "utf8")).toBe(ACTIVE_LEDGER); // the grammar holds unamended + // the hopper stream carries the below-threshold item with its reason + const hop = deriveStagingState(stagingStreamPath(join(dir, "sessions"), HOPPER_CAMPAIGN)); + expect(hop.entries.get("arxiv:2606.99999")?.state).toBe("staged"); + expect((readFileSync(stagingStreamPath(join(dir, "sessions"), HOPPER_CAMPAIGN), "utf8").match(/below-threshold/g) ?? []).length).toBe(1); + }); + + it("re-running the digest on the same papers dedupes by event id — one stage line, centrally, no double delivery", () => { + seedLedgers(); + const items = [{ arxiv: "2606.05060", title: "Fast Rydberg CZ gates via optimal control", abstract: "blockade regime CZ" }]; + const once = routePapersToStaging({ items, sessionsDir: join(dir, "sessions"), provenance: PROV, nowMs: () => 1 }); + const twice = routePapersToStaging({ items, sessionsDir: join(dir, "sessions"), provenance: PROV, nowMs: () => 2 }); + expect(once.staged).toHaveLength(1); + expect(twice.staged).toHaveLength(0); + expect(twice.deduped).toEqual(["arxiv:2606.05060"]); + const raw = readFileSync(stagingStreamPath(join(dir, "sessions"), ACTIVE), "utf8").trim().split("\n"); + expect(raw).toHaveLength(1); + }); + + it("the stage lines carry the digest's provenance stamp and the expiry stamps (S3's fixture shape)", () => { + seedLedgers(); + routePapersToStaging({ + items: [{ arxiv: "2606.05060", title: "Fast Rydberg CZ gates via optimal control", abstract: "blockade regime CZ" }], + sessionsDir: join(dir, "sessions"), + provenance: PROV, + nowMs: () => 1_000_000_000_000, + }); + const line = JSON.parse(readFileSync(stagingStreamPath(join(dir, "sessions"), ACTIVE), "utf8")) as Record; + expect(line.provenance).toEqual(PROV); + expect(line.event_id).toBe("arxiv:2606.05060"); + expect(line.url).toBe("https://arxiv.org/abs/2606.05060"); + expect(line.kind).toBe("paper"); + expect(typeof line.expires_at).toBe("string"); + expect(line.reason ?? "matched").not.toBe(EXPIRED_WITHOUT_REVIEW); // a stage line is a stage, never a drop + }); +}); diff --git a/packages/amico-run/test/sota_staging.test.ts b/packages/amico-run/test/sota_staging.test.ts new file mode 100644 index 00000000..b4985f7d --- /dev/null +++ b/packages/amico-run/test/sota_staging.test.ts @@ -0,0 +1,410 @@ +// sota_staging.test.ts — the SIDECAR staging streams (#living-sota slice 2, +// spec-20260905-103000 D3 / S3): per-campaign append-only transition streams +// beside the session ledger, event-id idempotent, PIPE_BUF + O_APPEND +// line-atomic; stage/drop/accept are ALL appended transition lines — entries +// never mutated, staging state always DERIVED from the stream; the acceptance +// stamp is the SOLE sanctioned non-job append (PI-instructed, its schema +// provenance-stamped); expiry drops with the recorded drop line; compaction +// (O2) removes expired-and-dropped chains past the window, recorded by an +// appended compact line, never touching accepted or pending entries. +import { describe, it, expect } from "vitest"; +import { mkdtempSync, readFileSync, writeFileSync, existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + SIDECAR_SUFFIX, + HOPPER_CAMPAIGN, + REVIEW_BY_DAYS, + EXPIRES_AFTER_DAYS, + COMPACTION_WINDOW_DAYS, + EXPIRED_WITHOUT_REVIEW, + stagingStreamPath, + appendStageLine, + appendAcceptStamp, + appendDropLine, + sweepExpiry, + compactStagingStream, + deriveStagingState, + readStagingStream, + renderAwaitingTheEye, + expiredWithoutReviewCount, + type StageEntryInput, + type StagingProvenance, +} from "../src/sota_staging.js"; +import { PIPE_BUF } from "../src/ledger.js"; + +function sessions(): string { + return mkdtempSync(join(tmpdir(), "sota-staging-")); +} + +function vclock(start = 1_000_000_000_000): { nowMs: () => number; jump: (ms: number) => void } { + let t = start; + return { nowMs: () => t, jump: (ms: number) => (t += ms) }; +} + +const PROV: StagingProvenance = { + job: "papers-digest", + via: "fetched", + source: "arXiv export API over HTTPS", + fetched_at: "2026-09-05T10:00:00.000Z", +}; + +function paperEntry(event_id = "arxiv:2606.05060", campaign = "session-20260901-rydberg-cz"): StageEntryInput { + return { + event_id, + campaign, + kind: "paper", + title: "Fast Rydberg CZ gates via shaped pulses", + url: "https://arxiv.org/abs/2606.05060", + provenance: PROV, + matched: ["rydberg", "cz"], + }; +} + +describe("the SIDECAR staging stream — stage with stamps (S3 cell 1)", () => { + it("a stage line lands beside the ledger with review-by/expiry stamps, provenance, and a monotonic seq", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + expect(path).toBe(join(dir, "session-20260901-rydberg-cz" + SIDECAR_SUFFIX)); + const r = appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + expect(r.appended).toBe(true); + const raw = readFileSync(path, "utf8"); + expect(raw.endsWith("\n")).toBe(true); // whole, flushed line — O_APPEND atomicity + const line = JSON.parse(raw.trim()) as Record; + expect(line.ev).toBe("stage"); + expect(line.seq).toBe(1); // seq IS the line count at write time + expect(line.event_id).toBe("arxiv:2606.05060"); + expect(line.campaign).toBe("session-20260901-rydberg-cz"); + expect(line.kind).toBe("paper"); + expect(line.provenance).toEqual(PROV); // provenance-stamped + expect(line.matched).toEqual(["rydberg", "cz"]); + const ts = Date.parse(String(line.ts)); + expect(Date.parse(String(line.review_by))).toBe(ts + REVIEW_BY_DAYS * 86_400_000); // the review-by stamp + expect(Date.parse(String(line.expires_at))).toBe(ts + EXPIRES_AFTER_DAYS * 86_400_000); // the expiry stamp + }); + + it("seq is monotonic across appends (the stream replays in write order)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry("arxiv:1", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + appendStageLine(path, paperEntry("arxiv:2", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + const { lines } = readStagingStream(path); + expect(lines.map((l) => (l as { seq: number }).seq)).toEqual([1, 2]); + }); + + it("a line that would exceed PIPE_BUF is refused — O_APPEND atomicity holds only under the ceiling", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + const big = paperEntry(); + // the title is capped at 240 chars by the writer — the honest way past the + // ceiling is a provenance extra (a job stamp gone wrong), never a torn append + big.provenance = { ...PROV, note: "x".repeat(PIPE_BUF) }; + expect(() => appendStageLine(path, big)).toThrow(/PIPE_BUF/); + expect(existsSync(path)).toBe(false); // nothing torn landed + }); +}); + +describe("idempotency — double-delivery is impossible (S3 cell)", () => { + it("the same event_id staged twice produces ONE line (idempotent by event id)", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + const a = appendStageLine(path, paperEntry()); + const b = appendStageLine(path, paperEntry()); + expect(a.appended).toBe(true); + expect(b.appended).toBe(false); + expect(b.reason).toMatch(/already staged|duplicate/); + const { lines } = readStagingStream(path); + expect(lines).toHaveLength(1); + }); + + it("the digest re-running tomorrow on the same paper dedupes centrally (state derived, one stage line)", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry()); + appendStageLine(path, paperEntry()); + appendStageLine(path, paperEntry()); + expect(readStagingStream(path).lines).toHaveLength(1); + expect(deriveStagingState(path).entries.get("arxiv:2606.05060")?.state).toBe("staged"); + }); +}); + +describe("the acceptance stamp — the PI-instructed promote (S3 cell 2; O3's schema)", () => { + it("an accepted match promotes by an appended accept line carrying the PI-instruction provenance", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + const r = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "the blockade number is load-bearing — accept" }, { nowMs: c.nowMs }); + expect(r.appended).toBe(true); + const { lines } = readStagingStream(path); + const accept = lines[1] as Record; + expect(accept.ev).toBe("accept"); + expect(accept.event_id).toBe("arxiv:2606.05060"); + expect(accept.instructed_by).toBe("PI"); + expect(accept.instruction).toEqual({ + channel: "chat", + note: "the blockade number is load-bearing — accept", + received_at: new Date(c.nowMs()).toISOString(), + }); + expect(deriveStagingState(path).entries.get("arxiv:2606.05060")?.state).toBe("accepted"); + }); + + it("a second accept stamp is an idempotent no-op (event-id-idempotent stamp)", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry()); + appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "first" }); + const again = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "repeated instruction" }); + expect(again.appended).toBe(false); + expect(again.reason).toMatch(/already accepted|idempotent/i); + expect(readStagingStream(path).lines).toHaveLength(2); + }); + + it("an accept without a staged match is REFUSED — the stamp is a decision's record, never free-floating", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + const r = appendAcceptStamp(path, "arxiv:never-staged", { channel: "chat", note: "x" }); + expect(r.appended).toBe(false); + expect(r.reason).toMatch(/no staged match/i); + expect(existsSync(path)).toBe(false); + }); + + it("an accept for an already-dropped match is REFUSED (a drop is terminal)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(path, { nowMs: c.nowMs }); + const r = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "late" }, { nowMs: c.nowMs }); + expect(r.appended).toBe(false); + expect(r.reason).toMatch(/dropped|terminal/i); + }); + + it("the instruction note is REQUIRED — an unstamped acceptance is refused (the schema records the instruction)", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry()); + const r = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "" }); + expect(r.appended).toBe(false); + expect(r.reason).toMatch(/instruction|note/i); + }); +}); + +describe("expiry — a match past its review window drops with the recorded line (S3 cell 3)", () => { + it("the sweep appends a drop line with the recorded reason; the state derives dropped", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); // past the expiry stamp + const { dropped } = sweepExpiry(path, { nowMs: c.nowMs }); + expect(dropped).toEqual(["arxiv:2606.05060"]); + const { lines } = readStagingStream(path); + const drop = lines[1] as Record; + expect(drop.ev).toBe("drop"); + expect(drop.reason).toBe(EXPIRED_WITHOUT_REVIEW); + expect(typeof drop.recorded).toBe("string"); // the recorded line + expect(deriveStagingState(path).entries.get("arxiv:2606.05060")?.state).toBe("dropped"); + }); + + it("a NOT-yet-expired staged match is NOT dropped (stage-before-count, no premature laundering)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + c.jump((EXPIRES_AFTER_DAYS - 1) * 86_400_000); + const { dropped } = sweepExpiry(path, { nowMs: c.nowMs }); + expect(dropped).toEqual([]); + expect(deriveStagingState(path).entries.get("arxiv:2606.05060")?.state).toBe("staged"); + }); + + it("the sweep is idempotent (one recorded drop, never two)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry(), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(path, { nowMs: c.nowMs }); + const second = sweepExpiry(path, { nowMs: c.nowMs }); + expect(second.dropped).toEqual([]); + expect(readStagingStream(path).lines).toHaveLength(2); + }); + + it("a drop on an ACCEPTED match is refused — acceptance is terminal", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry()); + appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "accept" }); + const r = appendDropLine(path, "arxiv:2606.05060", EXPIRED_WITHOUT_REVIEW); + expect(r.appended).toBe(false); + expect(r.reason).toMatch(/accepted|terminal/i); + }); +}); + +describe("the hopper fallback stream (S3 cell 4)", () => { + it("below-threshold / unmatched material lands in the hopper stream under sessions/, same shape", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, HOPPER_CAMPAIGN); + expect(path).toBe(join(dir, "hopper" + SIDECAR_SUFFIX)); + const r = appendStageLine(path, { + event_id: "arxiv:2606.99999", + campaign: HOPPER_CAMPAIGN, + kind: "paper", + title: "Protein folding via deep learning", + url: "https://arxiv.org/abs/2606.99999", + provenance: PROV, + matched: [], + reason: "below-threshold", + }); + expect(r.appended).toBe(true); + const line = readStagingStream(path).lines[0] as Record; + expect(line.campaign).toBe("hopper"); + expect(line.reason).toBe("below-threshold"); + }); +}); + +describe("derivation — staging state is always DERIVED from the stream (the invariant)", () => { + it("readers tolerate unknown ev values (opacity: carried, never fatal) and skip a torn in-flight tail", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry()); + // a future writer class appends an unknown transition kind — readers carry it + writeFileSync(path, '{"ev":"triage-tag","seq":2,"ts":"2026-09-05T10:05:00.000Z","event_id":"arxiv:2606.05060","campaign":"session-20260901-rydberg-cz","tag":"flywheel-drain"}\n', { flag: "a" }); + const st = deriveStagingState(path); + expect(st.entries.get("arxiv:2606.05060")?.state).toBe("staged"); + expect(st.lines).toHaveLength(2); + // a LIVE reader skips an incomplete trailing line — the rest stands + writeFileSync(path, '{"ev":"stage","seq":3,', { flag: "a" }); + const st2 = deriveStagingState(path); + expect(st2.entries.get("arxiv:2606.05060")?.state).toBe("staged"); + }); + + it("an orphan accept/drop (no stage) is carried as an orphan — readers never fail; the WRITERS refuse", () => { + const dir = sessions(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + writeFileSync(path, '{"ev":"accept","seq":1,"ts":"2026-09-05T10:00:00.000Z","event_id":"arxiv:ghost","campaign":"session-20260901-rydberg-cz","instructed_by":"PI"}\n'); + const st = deriveStagingState(path); + const ghost = st.entries.get("arxiv:ghost"); + expect(ghost?.orphan).toBe(true); // carried, never a crash — the validator reds it + }); + + it("a missing stream derives to empty (the fresh-campaign state), never a throw", () => { + const dir = sessions(); + const st = deriveStagingState(stagingStreamPath(dir, "session-2099-01-01-never")); + expect(st.entries.size).toBe(0); + }); +}); + +describe("compaction — O2: expired-and-dropped chains compact past the window, recorded, never silent", () => { + it("a dropped chain older than the compaction window is removed, the compact line records it, seqs renumber", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry("arxiv:old-dropped", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(path, { nowMs: c.nowMs }); // drops old-dropped (recorded) + // pending + accepted staged AFTER the sweep — fresh stamps, not yet expiring + appendStageLine(path, paperEntry("arxiv:pending", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + appendStageLine(path, paperEntry("arxiv:accepted", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + appendAcceptStamp(path, "arxiv:accepted", { channel: "chat", note: "accept" }, { nowMs: c.nowMs }); + c.jump(COMPACTION_WINDOW_DAYS * 86_400_000 + 1); // the drop is now past the window + const r = compactStagingStream(path, { nowMs: c.nowMs }); + expect(r.removed).toBe(2); // the stage + drop chain of the expired-and-dropped match + const { lines } = readStagingStream(path); + const compact = lines[lines.length - 1] as Record; + expect(compact.ev).toBe("compact"); + expect(compact.removed).toBe(2); + expect(compact.window_days).toBe(COMPACTION_WINDOW_DAYS); + expect(compact.high_water_seq).toBe(5); // the pre-compaction line count — nothing lost + expect(lines.map((l) => (l as { seq: number }).seq)).toEqual([1, 2, 3, 4]); // renumbered: seq = line count again + // the pending and accepted entries survive; the dropped one is gone + const st = deriveStagingState(path); + expect(st.entries.get("arxiv:pending")?.state).toBe("staged"); + expect(st.entries.get("arxiv:accepted")?.state).toBe("accepted"); + expect(st.entries.has("arxiv:old-dropped")).toBe(false); + }); + + it("accepted and pending entries are NEVER compacted (derivation reads accepted-only — the record stands)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry("arxiv:accepted", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + appendAcceptStamp(path, "arxiv:accepted", { channel: "chat", note: "accept" }, { nowMs: c.nowMs }); + c.jump(10 * COMPACTION_WINDOW_DAYS * 86_400_000); // any age — accepted survives + const r = compactStagingStream(path, { nowMs: c.nowMs }); + expect(r.removed).toBe(0); + expect(readStagingStream(path).lines).toHaveLength(2); // untouched, no idle compact noise + }); + + it("a recent drop is NOT compacted — the health stamp's trailing window stays computable", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry("arxiv:fresh-drop", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(path, { nowMs: c.nowMs }); + c.jump((COMPACTION_WINDOW_DAYS - 1) * 86_400_000); // dropped, but inside the window + expect(compactStagingStream(path, { nowMs: c.nowMs }).removed).toBe(0); + }); +}); + +describe("the awaiting-the-eye listing — rendered from DERIVED state (S3 cell)", () => { + it("lists pending staged matches with citations + provenance + review-by, NEVER as currency; counts the expired", () => { + const dir = sessions(); + const c = vclock(); + const campaignPath = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(campaignPath, paperEntry("arxiv:2606.07777", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + appendAcceptStamp(campaignPath, "arxiv:2606.07777", { channel: "chat", note: "accept" }, { nowMs: c.nowMs }); + appendStageLine(campaignPath, paperEntry("arxiv:2606.08888", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(campaignPath, { nowMs: c.nowMs }); // 08888 drops expired (chronic non-review) + // 05060 stages FRESH after the sweep — the current pending item + appendStageLine(campaignPath, paperEntry("arxiv:2606.05060", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + const hopperPath = stagingStreamPath(dir, HOPPER_CAMPAIGN); + appendStageLine( + hopperPath, + { + event_id: "arxiv:2606.99999", + campaign: HOPPER_CAMPAIGN, + kind: "paper", + title: "Protein folding via deep learning", + url: "https://arxiv.org/abs/2606.99999", + provenance: PROV, + matched: [], + reason: "below-threshold", + }, + { nowMs: c.nowMs }, + ); + const text = renderAwaitingTheEye([campaignPath, hopperPath], { nowMs: c.nowMs }); + expect(text).toMatch(/awaiting the eye/i); + expect(text).toContain("session-20260901-rydberg-cz"); + expect(text).toContain("Fast Rydberg CZ gates via shaped pulses"); + expect(text).toContain("https://arxiv.org/abs/2606.05060"); // CITED + expect(text).toContain("review by"); // the review-by stamp renders + expect(text).toContain("papers-digest"); // provenance renders + expect(text).toContain("hopper"); + expect(text).toContain("Protein folding via deep learning"); // sub-threshold material — in the awaiting listing + expect(text).toMatch(/expired without review.*1|1.*expired without review/i); // the count line + expect(text).toContain("never rendered as currency"); // the register line, verbatim + // the ACCEPTED item is currency, not awaiting — it does not appear as pending + expect(text).not.toContain("https://arxiv.org/abs/2606.07777"); + }); + + it("expiredWithoutReviewCount — the trailing-window health stamp input (chronic non-review masks itself)", () => { + const dir = sessions(); + const c = vclock(); + const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); + appendStageLine(path, paperEntry("arxiv:a", "session-20260901-rydberg-cz"), { nowMs: c.nowMs }); + c.jump(EXPIRES_AFTER_DAYS * 86_400_000 + 1); + sweepExpiry(path, { nowMs: c.nowMs }); + const now = c.nowMs(); + expect(expiredWithoutReviewCount(path, { nowMs: () => now })).toBe(1); + c.jump(COMPACTION_WINDOW_DAYS * 86_400_000 + 1); + expect(expiredWithoutReviewCount(path, { nowMs: c.nowMs })).toBe(0); // out of the trailing window + }); +}); + +void rmSync; From 9606314d95f14756a84abe0ad324efcff92db823 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:44:20 -0400 Subject: [PATCH 2/6] feat(sota): the SOTA watcher + the staged-routing verb surface (living-sota slice 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec-20260905-103000 D3 / S3: release/changelog/issue events from the watched-repo registry ride the IDENTICAL staged path as papers — the slice-1 codebase lens provides the fetch (canonical repos, one-fetcher seam), the watcher adds the routing (campaign enumeration -> stage lines / hopper fallback, github: event ids, repo+surface provenance). Verb surface: amico sota watcher (the job driver), accept (the PI-instructed stamp, the sole sanctioned non-job append, --note required, idempotent, refusals named), awaiting-the-eye (derived-state pending listing + expired-without- review counts), sweep (the weekly job's expiry drops + O2 compaction pass). Fetch-failed surfaces are named failures; the survey never blocks. --- packages/amico-run/src/sota_verb.ts | 124 ++++++++++-- packages/amico-run/src/sota_watcher.ts | 96 +++++++++ packages/amico-run/src/verbs.ts | 13 +- packages/amico-run/test/sota_router.test.ts | 11 +- packages/amico-run/test/sota_staging.test.ts | 19 +- packages/amico-run/test/sota_verb.test.ts | 194 +++++++++++++++++++ packages/amico-run/test/sota_watcher.test.ts | 147 ++++++++++++++ 7 files changed, 576 insertions(+), 28 deletions(-) create mode 100644 packages/amico-run/src/sota_watcher.ts create mode 100644 packages/amico-run/test/sota_watcher.test.ts diff --git a/packages/amico-run/src/sota_verb.ts b/packages/amico-run/src/sota_verb.ts index f51861a7..558338de 100644 --- a/packages/amico-run/src/sota_verb.ts +++ b/packages/amico-run/src/sota_verb.ts @@ -1,16 +1,28 @@ -// sota_verb.ts — `amico sota papers|codebase` (#820, spec-20260905-103000 -// living-sota D1): the SOTA survey surface agents drive — one on-demand -// arXiv query through the fleet-wide serialized queue (papers lens), or one -// watched-repo fetch round via the GitHub API (codebase lens). Both are the -// one-fetcher seam's clients: cache → queue → fetch, never a direct -// transport. The survey never blocks: queue-timeout and fetch failures are -// NAMED outcomes with the disclosed alternative (read the cache, or record -// the waiver) — exit 1 with the named reason, never a hang, never a silent -// empty. +// sota_verb.ts — `amico sota …` (#820 + living-sota slice 2, spec +// spec-20260905-103000 D1/D3): the SOTA survey surface agents drive. +// +// papers | codebase — the slice-1 read-only lenses (fetch + report) +// watcher — the SOTA watcher: one codebase-lens round ROUTED +// through the staged path (stage lines; hopper fallback) +// accept — the PI-instructed acceptance stamp: the SOLE +// sanctioned non-job append (an agent records the human +// decision on the explicit instruction; --note required) +// awaiting-the-eye — the pending listing, rendered from DERIVED state +// (never currency; expired-without-review counts ride) +// sweep — the weekly job's expiry + compaction driver (O2) +// +// The survey never blocks: every failure is a NAMED outcome with the +// disclosed alternative, never a hang, never a silent empty. ONE APPENDER: +// watcher/sweep are the job drivers (the only stage/drop writers); accept is +// the sanctioned human-instructed exception. import type { VerbResult } from "./verbs.js"; import { runPapersLens } from "./sota_papers.js"; import { runCodebaseLens } from "./sota_codebase.js"; import { sotaRoot } from "./sota_fetch.js"; +import { runSotaWatcher } from "./sota_watcher.js"; +import { appendAcceptStamp, renderAwaitingTheEye, sweepExpiry, compactStagingStream, listStagingStreams, stagingStreamPath, expiredWithoutReviewCount, deriveStagingState } from "./sota_staging.js"; +import { resolveMountStack, personalMount } from "./mounts.js"; +import { join } from "node:path"; function flagValue(argv: string[], name: string): string | undefined { const i = argv.indexOf(name); @@ -23,15 +35,39 @@ function flagValues(argv: string[], name: string): string[] { return out; } -const USAGE = `amico sota — the SOTA survey surface (read-only toward the world) +/** The sessions dir the staged routing enumerates/writes: $AMICO_SOTA_SESSIONS + * wins (hermetic escape); production is the personal vault mount's sessions/ + * — the session-ledger parse target. */ +export function sotaSessionsDir(): string { + const env = process.env.AMICO_SOTA_SESSIONS; + if (env && env.trim() !== "") return env; + const stack = resolveMountStack(); + const personal = personalMount(stack); + return join(personal?.path ?? "", "sessions"); +} + +/** A campaign stem names its sidecar under sessions/ — a file name, never a + * path (the traversal guard doubles as the shape check). */ +const CAMPAIGN_STEM_OK = /^(session-[A-Za-z0-9_-]+|hopper)$/; + +const USAGE = `amico sota — the SOTA survey surface (read-only toward the world; staged routing for the loop) amico sota papers --query "" [--top N] [--root ] one on-demand arXiv query through the fleet-wide serialized queue amico sota codebase [--repo owner/name]... [--root ] - one watched-repo fetch round via the GitHub API against canonical repos`; + one watched-repo fetch round via the GitHub API against canonical repos + amico sota watcher [--repo owner/name]... [--root ] [--sessions ] + the SOTA watcher: the codebase round routed through the staged streams + amico sota accept --campaign --event --note "" [--sessions ] + the PI-instructed acceptance stamp (the sole sanctioned non-job append) + amico sota awaiting-the-eye [--sessions ] + the pending staged matches + expired-without-review counts (derived state) + amico sota sweep [--sessions ] + the weekly job's expiry drops + compaction pass over every staging stream`; export async function sotaVerb(argv: string[]): Promise { const head = argv[0] ?? ""; const root = flagValue(argv, "--root") ?? sotaRoot(); + const sessions = flagValue(argv, "--sessions") ?? sotaSessionsDir(); if (head === "papers") { const query = flagValue(argv, "--query"); @@ -82,5 +118,71 @@ export async function sotaVerb(argv: string[]): Promise { }; } + if (head === "watcher") { + const repos = flagValues(argv, "--repo"); + for (const r of repos) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(r)) { + return { json: { ok: false, error: `watcher: --repo must be a canonical owner/name (got "${r}") — never a local checkout path` }, code: 64 }; + } + } + const res = await runSotaWatcher({ root, sessionsDir: sessions, repos: repos.length > 0 ? repos : undefined }); + return { + json: { + ok: true, + staged: res.staged, + hopper: res.hopper, + deduped: res.deduped, + lens_ok: res.lens.ok, + brief: res.lens.brief, + }, + code: 0, + }; + } + + if (head === "accept") { + const campaign = flagValue(argv, "--campaign"); + const event = flagValue(argv, "--event"); + const note = flagValue(argv, "--note"); + const channel = flagValue(argv, "--channel") ?? "chat"; + if (!campaign || !event || !note || note.trim() === "") { + return { json: { ok: false, error: "accept: --campaign, --event, and --note are all required (the stamp records the PI's explicit instruction — an unstamped acceptance is refused)", usage: USAGE }, code: 64 }; + } + if (!CAMPAIGN_STEM_OK.test(campaign)) { + return { json: { ok: false, error: `accept: --campaign must be a session-ledger stem (or "hopper") naming its sidecar — got "${campaign}"` }, code: 64 }; + } + const res = appendAcceptStamp(stagingStreamPath(sessions, campaign), event, { channel, note }); + if (!res.appended) { + // already-accepted is the idempotent success; every other refusal is the named failure + if (/already-accepted/i.test(res.reason)) return { json: { ok: true, idempotent: true, detail: res.reason }, code: 0 }; + return { json: { ok: false, error: res.reason }, code: 1 }; + } + return { json: { ok: true, accepted: event, campaign }, code: 0 }; + } + + if (head === "awaiting-the-eye") { + const stems = listStagingStreams(sessions); + const paths = stems.map((s) => stagingStreamPath(sessions, s)); + const text = renderAwaitingTheEye(paths); + let pending = 0; + let expired = 0; + for (const p of paths) { + const { entries } = deriveStagingState(p); + pending += [...entries.values()].filter((e) => e.state === "staged").length; + expired += expiredWithoutReviewCount(p); + } + return { json: { ok: true, text, pending, expired_without_review: expired, streams: stems }, code: 0 }; + } + + if (head === "sweep") { + const stems = listStagingStreams(sessions); + const streams = stems.map((campaign) => { + const path = stagingStreamPath(sessions, campaign); + const { dropped } = sweepExpiry(path); + const { removed } = compactStagingStream(path); + return { campaign, dropped, compacted: removed }; + }); + return { json: { ok: true, streams }, code: 0 }; + } + return { json: { ok: false, error: `sota: unknown lens "${head}"`, usage: USAGE }, code: 64 }; } diff --git a/packages/amico-run/src/sota_watcher.ts b/packages/amico-run/src/sota_watcher.ts new file mode 100644 index 00000000..323ae501 --- /dev/null +++ b/packages/amico-run/src/sota_watcher.ts @@ -0,0 +1,96 @@ +// sota_watcher.ts — the SOTA watcher (#living-sota slice 2, spec +// spec-20260905-103000 D3 / S3): release/changelog/issue events from the +// watched-repo registry ride the IDENTICAL staged path as papers. The slice-1 +// codebase lens (sota_codebase.ts) provides the FETCH — the GitHub API against +// canonical repos, through the one-fetcher seam, never a local fork checkout; +// the watcher adds the ROUTING: the lens's keyword-matched events enumerate +// the active campaigns (sota_router.ts) and append stage lines through the +// same sota_staging machinery — same hopper fallback, same review-by/expiry +// stamps, same event-id idempotency (double delivery impossible). +// +// ONE APPENDER (D4): the watcher is one of the ONLY three stage/drop writers +// (digest, watcher, weekly synthesis); concurrent agents signal matches, the +// job dedupes centrally by event id. A fetch-failed surface is a NAMED failure +// in the round — the watcher never silently skips, and stages nothing for it +// (the entry accrues toward retire-or-confirm in the registry, slice 1). +import { runCodebaseLens, type CodebaseEvent, type CodebaseLensOpts, type CodebaseLensResult } from "./sota_codebase.js"; +import type { FetchSurface } from "@amicode/schema"; +import { enumerateCampaigns, routeItem, REASON_BELOW_THRESHOLD, REASON_NO_CAMPAIGN_MATCH, type CampaignLedgerInfo } from "./sota_router.js"; +import { appendStageLine, stagingStreamPath, HOPPER_CAMPAIGN, type StagingProvenance } from "./sota_staging.js"; + +/** The watcher's opts: the codebase lens round (root, fetchFn, clock, queue + * overrides) + the sessions dir the routing enumerates. */ +export interface SotaWatchOpts extends CodebaseLensOpts { + sessionsDir: string; +} + +export interface WatcherEvent { + event_id: string; + campaign: string; + matched: string[]; +} + +export interface WatcherResult { + lens: CodebaseLensResult; + staged: WatcherEvent[]; + hopper: { event_id: string; reason: string }[]; + deduped: string[]; +} + +const KIND_BY_SURFACE: Record = { + releases: "release", + changelog: "changelog", + issues: "issue", +}; + +/** One watcher round: the codebase lens's fetch + the identical routing pass. + * Every matched event — whatever the campaign enumeration matches — stages + * through sota_staging (idempotent by event id); below-threshold and + * no-campaign events go to the hopper stream. */ +export async function runSotaWatcher(opts: SotaWatchOpts): Promise { + const nowMs = opts.nowMs ?? Date.now; + const lens = await runCodebaseLens(opts); + const { campaigns } = enumerateCampaigns(opts.sessionsDir); + const stampIso = lens.stamp.iso; + const staged: WatcherEvent[] = []; + const hopper: { event_id: string; reason: string }[] = []; + const deduped: string[] = []; + for (const repo of lens.repos) { + for (const surface of repo.surfaces) { + for (const e of surface.events) { + const route = routeItem({ title: e.title, detail: e.detail }, campaigns); + const campaign = route.target === "campaign" ? route.campaign.id : HOPPER_CAMPAIGN; + const event_id = `github:${e.id}`; + const provenance: StagingProvenance = { + job: "sota-watcher", + via: surface.via, + source: "GitHub API against canonical repos", + fetched_at: stampIso, + repo: repo.repo, + surface: surface.surface, + }; + const r = appendStageLine( + stagingStreamPath(opts.sessionsDir, campaign), + { + event_id, + campaign, + kind: KIND_BY_SURFACE[surface.surface], + title: e.title, + url: e.url, + provenance, + matched: route.matched, + ...(route.target === "hopper" ? { reason: route.reason } : {}), + }, + { nowMs }, + ); + if (!r.appended) { + deduped.push(event_id); + continue; + } + if (route.target === "hopper") hopper.push({ event_id, reason: route.reason }); + else staged.push({ event_id, campaign, matched: route.matched }); + } + } + } + return { lens, staged, hopper, deduped }; +} diff --git a/packages/amico-run/src/verbs.ts b/packages/amico-run/src/verbs.ts index c120e12e..c98a99bd 100644 --- a/packages/amico-run/src/verbs.ts +++ b/packages/amico-run/src/verbs.ts @@ -228,17 +228,18 @@ const project: Verb = { run: projectVerb, }; -// sota — the living-SOTA survey surface (#820, spec-20260905-103000 D1): +// sota — the living-SOTA survey surface (#820, spec-20260905-103000 D1+D3): // the dual-lens read-only survey (papers via the arXiv export API through // the fleet-wide serialized queue; codebases via the GitHub API against the -// watched-repo registry's canonical repos). This slice fetches and reports — -// staged routing/receipts are later slices; a survey that cannot run is a -// NAMED outcome, never a block. +// watched-repo registry's canonical repos) + the staged routing (the watcher, +// the PI-instructed accept stamp, the awaiting-the-eye listing, the expiry/ +// compaction sweep). A survey that cannot run is a NAMED outcome, never a +// block; a match never counts until the PI's accept stamp lands. const sota: Verb = { name: "sota", - summary: "the SOTA survey: papers (arXiv export API via the serialized queue) / codebase (GitHub API over the watched-repo registry)", + summary: "the SOTA survey + staged routing: papers/codebase lenses, the watcher, the accept stamp, the awaiting-the-eye listing, the sweep", generalizes: "the loops' external-currency survey step (the sota-review skill's driving surface)", - slice: "living-sota (spec-20260905-103000 D1)", + slice: "living-sota (spec-20260905-103000 D1+D3)", run: sotaVerb, }; diff --git a/packages/amico-run/test/sota_router.test.ts b/packages/amico-run/test/sota_router.test.ts index b4e58ef5..53f89d47 100644 --- a/packages/amico-run/test/sota_router.test.ts +++ b/packages/amico-run/test/sota_router.test.ts @@ -124,8 +124,8 @@ describe("routeItem — relevance against the campaign corpora (explainable, wor { title: "Fast Rydberg CZ gates via optimal control", detail: "We shape pulses in the blockade regime; the CZ gate reaches 0.9999." }, campaigns, ); - expect(r.target).toBe("campaign"); - expect(r.campaign?.id).toBe(ACTIVE); + if (r.target !== "campaign") throw new Error("expected a campaign route"); + expect(r.campaign.id).toBe(ACTIVE); expect(r.matched.length).toBeGreaterThanOrEqual(RELEVANCE_THRESHOLD); expect(r.matched).toContain("rydberg"); expect(r.matched).toContain("cz"); @@ -135,13 +135,13 @@ describe("routeItem — relevance against the campaign corpora (explainable, wor seedLedgers(); const { campaigns } = enumerateCampaigns(join(dir, "sessions")); const r = routeItem({ title: "Protein folding via deep learning", detail: "AlphaFold-style pipelines for structure prediction." }, campaigns); - expect(r.target).toBe("hopper"); + if (r.target !== "hopper") throw new Error("expected a hopper route"); expect(r.reason).toBe("below-threshold"); }); it("no campaigns at all → the hopper with the named no-campaign-match reason", () => { const r = routeItem({ title: "Anything at all", detail: "whatever" }, []); - expect(r.target).toBe("hopper"); + if (r.target !== "hopper") throw new Error("expected a hopper route"); expect(r.reason).toBe("no-campaign-match"); }); @@ -149,7 +149,8 @@ describe("routeItem — relevance against the campaign corpora (explainable, wor const a: CampaignLedgerInfo = { id: "session-b", title: "Zeta rydberg cz", objectiveLine: "", open: [], terms: ["rydberg", "cz"] }; const b: CampaignLedgerInfo = { id: "session-a", title: "Alpha rydberg cz", objectiveLine: "", open: [], terms: ["rydberg", "cz"] }; const r = routeItem({ title: "rydberg cz", detail: "" }, [a, b]); - expect(r.campaign?.id).toBe("session-a"); + if (r.target !== "campaign") throw new Error("expected a campaign route"); + expect(r.campaign.id).toBe("session-a"); }); }); diff --git a/packages/amico-run/test/sota_staging.test.ts b/packages/amico-run/test/sota_staging.test.ts index b4985f7d..bc0e6099 100644 --- a/packages/amico-run/test/sota_staging.test.ts +++ b/packages/amico-run/test/sota_staging.test.ts @@ -49,6 +49,13 @@ const PROV: StagingProvenance = { fetched_at: "2026-09-05T10:00:00.000Z", }; + +/** Narrow an AppendResult to its refusal reason (the appended:true arm has none). */ +function refusal(r: import("../src/sota_staging.js").AppendResult): string { + if (r.appended) throw new Error("expected a refusal, got an append"); + return r.reason; +} + function paperEntry(event_id = "arxiv:2606.05060", campaign = "session-20260901-rydberg-cz"): StageEntryInput { return { event_id, @@ -114,7 +121,7 @@ describe("idempotency — double-delivery is impossible (S3 cell)", () => { const b = appendStageLine(path, paperEntry()); expect(a.appended).toBe(true); expect(b.appended).toBe(false); - expect(b.reason).toMatch(/already staged|duplicate/); + expect(refusal(b)).toMatch(/already staged|duplicate/); const { lines } = readStagingStream(path); expect(lines).toHaveLength(1); }); @@ -158,7 +165,7 @@ describe("the acceptance stamp — the PI-instructed promote (S3 cell 2; O3's sc appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "first" }); const again = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "repeated instruction" }); expect(again.appended).toBe(false); - expect(again.reason).toMatch(/already accepted|idempotent/i); + expect(refusal(again)).toMatch(/already accepted|idempotent/i); expect(readStagingStream(path).lines).toHaveLength(2); }); @@ -167,7 +174,7 @@ describe("the acceptance stamp — the PI-instructed promote (S3 cell 2; O3's sc const path = stagingStreamPath(dir, "session-20260901-rydberg-cz"); const r = appendAcceptStamp(path, "arxiv:never-staged", { channel: "chat", note: "x" }); expect(r.appended).toBe(false); - expect(r.reason).toMatch(/no staged match/i); + expect(refusal(r)).toMatch(/no staged match/i); expect(existsSync(path)).toBe(false); }); @@ -180,7 +187,7 @@ describe("the acceptance stamp — the PI-instructed promote (S3 cell 2; O3's sc sweepExpiry(path, { nowMs: c.nowMs }); const r = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "late" }, { nowMs: c.nowMs }); expect(r.appended).toBe(false); - expect(r.reason).toMatch(/dropped|terminal/i); + expect(refusal(r)).toMatch(/dropped|terminal/i); }); it("the instruction note is REQUIRED — an unstamped acceptance is refused (the schema records the instruction)", () => { @@ -189,7 +196,7 @@ describe("the acceptance stamp — the PI-instructed promote (S3 cell 2; O3's sc appendStageLine(path, paperEntry()); const r = appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "" }); expect(r.appended).toBe(false); - expect(r.reason).toMatch(/instruction|note/i); + expect(refusal(r)).toMatch(/instruction|note/i); }); }); @@ -240,7 +247,7 @@ describe("expiry — a match past its review window drops with the recorded line appendAcceptStamp(path, "arxiv:2606.05060", { channel: "chat", note: "accept" }); const r = appendDropLine(path, "arxiv:2606.05060", EXPIRED_WITHOUT_REVIEW); expect(r.appended).toBe(false); - expect(r.reason).toMatch(/accepted|terminal/i); + expect(refusal(r)).toMatch(/accepted|terminal/i); }); }); diff --git a/packages/amico-run/test/sota_verb.test.ts b/packages/amico-run/test/sota_verb.test.ts index f2ea82d1..ee6feac0 100644 --- a/packages/amico-run/test/sota_verb.test.ts +++ b/packages/amico-run/test/sota_verb.test.ts @@ -125,3 +125,197 @@ match_keywords = ["trajectory"] expect((res.json as { error: string }).error).toMatch(/unknown lens/); }); }); + +// ── living-sota slice 2: the staged-routing verb surface (#living-sota D3) ──── + +import { mkdtempSync as mkdtemp2, mkdirSync as mkdir2, writeFileSync as wf2, readFileSync as rf2 } from "node:fs"; +import { + appendStageLine, + stagingStreamPath, + deriveStagingState, + EXPIRES_AFTER_DAYS, + HOPPER_CAMPAIGN, +} from "../src/sota_staging.js"; + +const ACTIVE_LEDGER = `# Session ledger — Rydberg trajectory integrator + +## 2. Verdict table + +| item | status | +|---|---| +| H1 integrator drift audit | pending | +`; + +function verbSessions(): { dir: string; sessions: string } { + const dir = mkdtemp2(join(tmpdir(), "sota-verb2-")); + const sessions = join(dir, "sessions"); + mkdir2(sessions, { recursive: true }); + wf2(join(sessions, "session-20260901-rydberg-integrator.md"), ACTIVE_LEDGER); + return { dir, sessions }; +} + +describe("`amico sota watcher` — one watched-repo round routed through staging", () => { + it("rides the codebase lens and stages matched events into the campaign sidecar (hermetic: seeded cache)", async () => { + const r = root(); + writeFileSync(registryPath(r), ` +schema_version = "1" +failure_threshold = 7 + +[[repos]] +repo = "example/piccolo-adjacent" +why_watched = "API shifts adjacent to our authoring map" +domains = ["julia-optimal-control"] +fetch_surface = ["releases"] +match_keywords = ["trajectory", "rydberg"] +`); + mkdirSync(join(r, "fetch-cache"), { recursive: true }); + writeFileSync(cachePath(r, githubApiUrl("example/piccolo-adjacent", "releases")), JSON.stringify({ url: "x", fetched_at: new Date().toISOString(), body: RELEASES_JSON, count: 1 }) + "\n"); + const { sessions } = verbSessions(); + const res = await sotaVerb(["watcher", "--root", r, "--sessions", sessions]); + expect(res.code).toBe(0); + const j = res.json as { ok: boolean; staged: { event_id: string; campaign: string }[]; deduped: string[] }; + expect(j.ok).toBe(true); + expect(j.staged).toHaveLength(1); + expect(j.staged[0]).toMatchObject({ event_id: "github:example/piccolo-adjacent@v0.9.0", campaign: "session-20260901-rydberg-integrator" }); + expect(deriveStagingState(stagingStreamPath(sessions, "session-20260901-rydberg-integrator")).entries.get("github:example/piccolo-adjacent@v0.9.0")?.state).toBe("staged"); + }); +}); + +describe("`amico sota accept` — the PI-instructed acceptance stamp (the sole sanctioned non-job append)", () => { + it("promotes a staged match on the PI's explicit instruction, recording the instruction provenance", async () => { + const { sessions } = verbSessions(); + // stage one item through the machinery (the digest's writer) + appendStageLine(stagingStreamPath(sessions, "session-20260901-rydberg-integrator"), { + event_id: "arxiv:2606.05060", + campaign: "session-20260901-rydberg-integrator", + kind: "paper", + title: "Fast Rydberg trajectory gates", + url: "https://arxiv.org/abs/2606.05060", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + matched: ["rydberg", "trajectory"], + }); + const res = await sotaVerb([ + "accept", + "--campaign", "session-20260901-rydberg-integrator", + "--event", "arxiv:2606.05060", + "--note", "the integrator drift is load-bearing — accept", + "--sessions", sessions, + ]); + expect(res.code).toBe(0); + expect(deriveStagingState(stagingStreamPath(sessions, "session-20260901-rydberg-integrator")).entries.get("arxiv:2606.05060")?.state).toBe("accepted"); + }); + + it("an accept without a staged match is a NAMED refusal (exit 1) — never a free-floating stamp", async () => { + const { sessions } = verbSessions(); + const res = await sotaVerb(["accept", "--campaign", "session-20260901-rydberg-integrator", "--event", "arxiv:ghost", "--note", "x", "--sessions", sessions]); + expect(res.code).toBe(1); + expect((res.json as { error: string }).error).toMatch(/no staged match/i); + }); + + it("a second accept is an idempotent no-op (exit 0, still one accept line)", async () => { + const { sessions } = verbSessions(); + const path = stagingStreamPath(sessions, "session-20260901-rydberg-integrator"); + appendStageLine(path, { + event_id: "arxiv:2606.05060", + campaign: "session-20260901-rydberg-integrator", + kind: "paper", + title: "t", + url: "https://arxiv.org/abs/2606.05060", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + }); + await sotaVerb(["accept", "--campaign", "session-20260901-rydberg-integrator", "--event", "arxiv:2606.05060", "--note", "first", "--sessions", sessions]); + const again = await sotaVerb(["accept", "--campaign", "session-20260901-rydberg-integrator", "--event", "arxiv:2606.05060", "--note", "again", "--sessions", sessions]); + expect(again.code).toBe(0); + const lines = rf2(path, "utf8").trim().split("\n"); + expect(lines).toHaveLength(2); // stage + accept — the stamp records once + }); + + it("a missing --note is a usage error — the schema records the instruction, an unstamped acceptance is refused", async () => { + const { sessions } = verbSessions(); + const res = await sotaVerb(["accept", "--campaign", "session-20260901-rydberg-integrator", "--event", "x", "--sessions", sessions]); + expect(res.code).toBe(64); + }); + + it("a campaign stem that could traverse is a usage error (the sidecar path is a file name, never a path)", async () => { + const { sessions } = verbSessions(); + const res = await sotaVerb(["accept", "--campaign", "../../etc", "--event", "x", "--note", "x", "--sessions", sessions]); + expect(res.code).toBe(64); + }); +}); + +describe("`amico sota awaiting-the-eye` — the pending listing from derived state", () => { + it("renders pending staged matches + expired-without-review counts, never accepted material", async () => { + const { sessions } = verbSessions(); + const path = stagingStreamPath(sessions, "session-20260901-rydberg-integrator"); + appendStageLine(path, { + event_id: "arxiv:2606.05060", + campaign: "session-20260901-rydberg-integrator", + kind: "paper", + title: "Fast Rydberg trajectory gates", + url: "https://arxiv.org/abs/2606.05060", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + matched: ["rydberg", "trajectory"], + }); + appendStageLine(stagingStreamPath(sessions, HOPPER_CAMPAIGN), { + event_id: "arxiv:2606.99999", + campaign: HOPPER_CAMPAIGN, + kind: "paper", + title: "Protein folding via deep learning", + url: "https://arxiv.org/abs/2606.99999", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + matched: [], + reason: "below-threshold", + }); + const res = await sotaVerb(["awaiting-the-eye", "--sessions", sessions]); + expect(res.code).toBe(0); + const j = res.json as { ok: boolean; text: string; pending: number; expired_without_review: number }; + expect(j.ok).toBe(true); + expect(j.pending).toBe(2); + expect(j.expired_without_review).toBe(0); + expect(j.text).toContain("Fast Rydberg trajectory gates"); + expect(j.text).toContain("Protein folding via deep learning"); + expect(j.text).toMatch(/never rendered as currency/i); + }); + + it("a swept-and-expired campaign reads in the expired count (chronic non-review masks itself)", async () => { + const { sessions } = verbSessions(); + const path = stagingStreamPath(sessions, "session-20260901-rydberg-integrator"); + const now = Date.now(); + appendStageLine(path, { + event_id: "arxiv:2606.05060", + campaign: "session-20260901-rydberg-integrator", + kind: "paper", + title: "old", + url: "https://arxiv.org/abs/2606.05060", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + }, { nowMs: () => now - (EXPIRES_AFTER_DAYS + 1) * 86_400_000 }); + const sweep = await sotaVerb(["sweep", "--sessions", sessions]); + expect(sweep.code).toBe(0); + const res = await sotaVerb(["awaiting-the-eye", "--sessions", sessions]); + const j = res.json as { pending: number; expired_without_review: number }; + expect(j.pending).toBe(0); + expect(j.expired_without_review).toBe(1); + }); +}); + +describe("`amico sota sweep` — the job's expiry + compaction pass (O2's cadence driver)", () => { + it("drops expired staged matches with the recorded line and compacts nothing inside the window", async () => { + const { sessions } = verbSessions(); + const path = stagingStreamPath(sessions, "session-20260901-rydberg-integrator"); + const now = Date.now(); + appendStageLine(path, { + event_id: "arxiv:2606.05060", + campaign: "session-20260901-rydberg-integrator", + kind: "paper", + title: "old", + url: "https://arxiv.org/abs/2606.05060", + provenance: { job: "papers-digest", via: "fetched", source: "arXiv export API over HTTPS", fetched_at: new Date().toISOString() }, + }, { nowMs: () => now - (EXPIRES_AFTER_DAYS + 1) * 86_400_000 }); + const res = await sotaVerb(["sweep", "--sessions", sessions]); + expect(res.code).toBe(0); + const j = res.json as { ok: boolean; streams: { campaign: string; dropped: string[]; compacted: number }[] }; + expect(j.streams).toHaveLength(1); + expect(j.streams[0]).toMatchObject({ campaign: "session-20260901-rydberg-integrator", dropped: ["arxiv:2606.05060"], compacted: 0 }); + expect(deriveStagingState(path).entries.get("arxiv:2606.05060")?.state).toBe("dropped"); + }); +}); diff --git a/packages/amico-run/test/sota_watcher.test.ts b/packages/amico-run/test/sota_watcher.test.ts new file mode 100644 index 00000000..7aeadb30 --- /dev/null +++ b/packages/amico-run/test/sota_watcher.test.ts @@ -0,0 +1,147 @@ +// sota_watcher.test.ts — the SOTA watcher (#living-sota slice 2, spec +// spec-20260905-103000 D3 / S3): release/changelog/issue events from the +// watched-repo registry ride the IDENTICAL staged path as papers — the slice-1 +// codebase lens provides the fetch (GitHub API against canonical repos, never +// a local checkout); the watcher adds the ROUTING: matched events enumerate +// the active campaigns and append stage lines through the same sota_staging +// machinery, idempotent by event id, provenance-stamped with the repo+surface. +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runSotaWatcher } from "../src/sota_watcher.js"; +import { githubApiUrl, registryPath } from "../src/sota_codebase.js"; +import { cachePath } from "../src/sota_fetch.js"; +import { stagingStreamPath, deriveStagingState } from "../src/sota_staging.js"; + +let root: string; +let sessionsDir: string; +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "sota-watcher-")); + sessionsDir = join(root, "sessions"); + mkdirSync(sessionsDir, { recursive: true }); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +const REGISTRY_TOML = ` +schema_version = "1" +failure_threshold = 7 + +[[repos]] +repo = "example/piccolo-adjacent" +why_watched = "API shifts adjacent to our authoring map" +domains = ["julia-optimal-control"] +fetch_surface = ["releases", "issues"] +match_keywords = ["trajectory", "rydberg"] +`; + +const RELEASES_JSON = JSON.stringify([ + { + tag_name: "v0.9.0", + name: "v0.9.0 — trajectory rework", + published_at: "2026-08-20T00:00:00Z", + html_url: "https://github.com/example/piccolo-adjacent/releases/tag/v0.9.0", + body: "breaking: the Rydberg trajectory API now requires integrator selection", + }, +]); + +const ISSUES_JSON = JSON.stringify([ + { + number: 42, + title: "Rydberg subsystem levels regression", + html_url: "https://github.com/example/piccolo-adjacent/issues/42", + updated_at: "2026-08-22T00:00:00Z", + body: "the trajectory integrator regressed for rydberg subsystem_levels", + }, +]); + +const ACTIVE_LEDGER = `# Session ledger — Rydberg trajectory integrator + +## 1. Objective & standing directives + +- Objective: keep the rydberg integrator selection honest across the trajectory rework. + +## 2. Verdict table + +| item | status | +|---|---| +| H1 integrator drift audit | pending | +`; + +const CLOCK_MS = 1_000_000_000_000; + +function seed(): void { + writeFileSync(registryPath(root), REGISTRY_TOML); + mkdirSync(join(root, "fetch-cache"), { recursive: true }); + // the cache seed stamps AT the frozen fake clock — the TTL freshness check + // compares fetched_at against the same nowMs the watcher runs on + const stamp = new Date(CLOCK_MS).toISOString(); + writeFileSync(cachePath(root, githubApiUrl("example/piccolo-adjacent", "releases")), JSON.stringify({ url: "x", fetched_at: stamp, body: RELEASES_JSON, count: 1 }) + "\n"); + writeFileSync(cachePath(root, githubApiUrl("example/piccolo-adjacent", "issues")), JSON.stringify({ url: "x", fetched_at: stamp, body: ISSUES_JSON, count: 1 }) + "\n"); + writeFileSync(join(sessionsDir, "session-20260901-rydberg-integrator.md"), ACTIVE_LEDGER); +} + +describe("runSotaWatcher — the watcher rides the IDENTICAL staged path (S3, the watcher criterion)", () => { + it("release + issue events route into the matching campaign's sidecar with github event ids and repo/surface provenance", async () => { + seed(); + const res = await runSotaWatcher({ root, sessionsDir, nowMs: () => CLOCK_MS }); + expect(res.lens.ok).toBe(true); + expect(res.staged).toHaveLength(2); + expect(res.staged.map((s) => s.event_id).sort()).toEqual(["github:example/piccolo-adjacent#42", "github:example/piccolo-adjacent@v0.9.0"]); + // the identical staged path: the campaign sidecar, kinds release/issue, stamps present + const path = stagingStreamPath(sessionsDir, "session-20260901-rydberg-integrator"); + const st = deriveStagingState(path); + const release = st.entries.get("github:example/piccolo-adjacent@v0.9.0"); + expect(release?.state).toBe("staged"); + expect(release?.kind).toBe("release"); + expect(release?.url).toBe("https://github.com/example/piccolo-adjacent/releases/tag/v0.9.0"); // CITED + expect(release?.provenance?.job).toBe("sota-watcher"); + expect((release?.provenance as Record).repo).toBe("example/piccolo-adjacent"); + expect((release?.provenance as Record).surface).toBe("releases"); + expect(typeof release?.review_by).toBe("string"); // the review-by stamp rides + const issue = st.entries.get("github:example/piccolo-adjacent#42"); + expect(issue?.kind).toBe("issue"); + expect(issue?.matched).toContain("rydberg"); + }); + + it("a second watcher round on the same payloads dedupes by event id — no double delivery, one stage line per event", async () => { + seed(); + await runSotaWatcher({ root, sessionsDir, nowMs: () => CLOCK_MS }); + const twice = await runSotaWatcher({ root, sessionsDir, nowMs: () => CLOCK_MS }); + expect(twice.staged).toHaveLength(0); + expect(twice.deduped.sort()).toEqual(["github:example/piccolo-adjacent#42", "github:example/piccolo-adjacent@v0.9.0"]); + const raw = readFileSync(stagingStreamPath(sessionsDir, "session-20260901-rydberg-integrator"), "utf8").trim().split("\n"); + expect(raw).toHaveLength(2); + }); + + it("no campaign match → the hopper stream, same shape (the identical fixtures, hopper fallback)", async () => { + seed(); + rmSync(join(sessionsDir, "session-20260901-rydberg-integrator.md")); // no active campaigns + const res = await runSotaWatcher({ root, sessionsDir, nowMs: () => CLOCK_MS }); + expect(res.staged).toHaveLength(0); + expect(res.hopper).toHaveLength(2); + const st = deriveStagingState(stagingStreamPath(sessionsDir, "hopper")); + expect(st.entries.get("github:example/piccolo-adjacent@v0.9.0")?.state).toBe("staged"); + expect(st.entries.get("github:example/piccolo-adjacent@v0.9.0")?.reason).toBe("no-campaign-match"); + }); + + it("a fetch-failed surface is a NAMED failure in the round summary — the watcher never silently skips it", async () => { + seed(); + // the releases cache stays (that surface reads the cache and stages); the + // issues surface has NO cache and the injected transport 403s — a NAMED + // failure in the round, while the release still routes + rmSync(cachePath(root, githubApiUrl("example/piccolo-adjacent", "issues"))); + const res = await runSotaWatcher({ + root, + sessionsDir, + nowMs: () => CLOCK_MS, + fetchFn: (async () => ({ ok: false as const, status: 403, error: "HTTP 403 (rate-limited unauthenticated GitHub)" })) as never, + }); + expect(res.staged).toHaveLength(1); // the cached release routed + const issues = res.lens.repos[0].surfaces.find((s) => s.surface === "issues"); + expect(issues?.ok).toBe(false); + expect(issues?.error).toMatch(/403/); // the named failure — never a silent skip + }); +}); From 27bd56e679e176cb51857cabd8fe83a4cd8305cd Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:46:25 -0400 Subject: [PATCH 3/6] feat(sota): the papers digest gains the staged relevance router (living-sota slice 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec-20260905-103000 D3 / S3: 'amico papers digest --route' routes the lab-corpus picks against the active campaigns — matches stage into the campaign sidecar, below-threshold/no-match into the hopper, idempotent by event id. Routing is explicit (never a side effect of a dry run) and never fatal (a routing failure is a named routed.errors entry; the survey never blocks). --feed-xml is the deterministic fixture seam (zero transports in tests); --sessions overrides the personal-vault sessions dir. --- packages/amico-run/src/papers_digest_verb.ts | 67 +++++++++++++- packages/amico-run/src/papers_verb.ts | 2 +- packages/amico-run/test/papers_verb.test.ts | 96 ++++++++++++++++++++ 3 files changed, 159 insertions(+), 6 deletions(-) diff --git a/packages/amico-run/src/papers_digest_verb.ts b/packages/amico-run/src/papers_digest_verb.ts index 6ca14948..b59eeca9 100644 --- a/packages/amico-run/src/papers_digest_verb.ts +++ b/packages/amico-run/src/papers_digest_verb.ts @@ -2,6 +2,16 @@ // (#412): fetch → rank against the lab corpus → dedup → print (dry-run default) // or post to Slack as the Amico bot. The posted-state file makes reruns // idempotent. Runs on the server (the Slack token is server-only by posture). +// +// living-sota slice 2 (spec-20260905-103000 D3): `--route` adds the digest's +// RELEVANCE ROUTER — the lab-corpus picks route against the active campaigns +// and stage into the per-campaign SIDECAR streams (below-threshold / no-match +// → the hopper stream), idempotent by event id. Routing is explicit (the +// daily job's act, never a side effect of a dry run) and NEVER fatal: a +// routing failure is a named `routed.errors` entry and the digest proceeds — +// the survey never blocks. `--feed-xml ` is the deterministic fixture +// seam (read the feed from a file instead of the wire — the same zero-dep +// escape as AMICO_PAPERS_VAULTS). import { parseArxivRss, buildProfile, @@ -20,6 +30,8 @@ import { execFileSync } from "node:child_process"; import { homedir } from "node:os"; import { join } from "node:path"; import type { VerbResult } from "./verbs.js"; +import { routePapersToStaging, type RoutePapersResult } from "./sota_router.js"; +import { sotaSessionsDir } from "./sota_verb.js"; function flagValue(argv: string[], name: string): string | undefined { const i = argv.indexOf(name); @@ -56,16 +68,27 @@ export async function papersDigestVerb(argv: string[]): Promise { const top = Number(flagValue(argv, "--top") ?? 5); const post = flagValue(argv, "--post"); const dryRun = argv.includes("--dry-run") || post === undefined; + const route = argv.includes("--route"); + const feedXml = flagValue(argv, "--feed-xml"); + const sessionsDir = flagValue(argv, "--sessions") ?? sotaSessionsDir(); // hermetic escapes (tests) → studio ladder (production) const vaults = process.env.AMICO_PAPERS_VAULTS ?? studioPathsOrLegacy().vaultsRoot; const library = process.env.AMICO_PAPERS_LIBRARY ?? join(homedir(), ".amico", "library"); let xml: string; - try { - xml = fetchFeed(feedUrl(feed)); - } catch (e) { - return { json: { ok: false, error: `feed fetch failed: ${e}` }, code: 1 }; + if (feedXml !== undefined) { + try { + xml = readFileSync(feedXml, "utf8"); // the deterministic fixture seam — no transport + } catch (e) { + return { json: { ok: false, error: `--feed-xml unreadable: ${e}` }, code: 64 }; + } + } else { + try { + xml = fetchFeed(feedUrl(feed)); + } catch (e) { + return { json: { ok: false, error: `feed fetch failed: ${e}` }, code: 1 }; + } } const items = parseArxivRss(xml); if (items.length === 0) { @@ -85,6 +108,31 @@ export async function papersDigestVerb(argv: string[]): Promise { }; } + // the relevance router (living-sota D3): the picks stage into the campaign + // SIDECARs / the hopper. Explicit (--route), NEVER fatal: a routing failure + // is a named error and the digest proceeds — the survey never blocks. + let routed: RoutePapersResult | undefined; + const routingErrors: string[] = []; + const routedField = () => (routed !== undefined || routingErrors.length > 0 + ? { routed: { staged: [], hopper: [], deduped: [], ...routed, errors: routingErrors } } + : {}); + if (route) { + try { + routed = routePapersToStaging({ + items: r.picks.map((p) => ({ arxiv: p.item.arxiv, title: p.item.title, abstract: p.item.abstract })), + sessionsDir, + provenance: { + job: "papers-digest", + via: "feed", + source: `arXiv RSS ${feed}`, + fetched_at: new Date().toISOString(), + }, + }); + } catch (e) { + routingErrors.push(`routing failed: ${(e as Error).message} — the digest proceeds; the router re-runs on the next pass`); + } + } + const text = formatDigest({ feedName: feed, total: items.length, picks: r.picks, skipped: r.skipped }); if (dryRun) { @@ -95,6 +143,7 @@ export async function papersDigestVerb(argv: string[]): Promise { text, fingerprint: digestFingerprint(text), counts: { total: items.length, picks: r.picks.length, skipped_corpus: r.skipped.corpus.length, dropped: r.dropped.length }, + ...routedField(), }, code: 0, }; @@ -104,7 +153,15 @@ export async function papersDigestVerb(argv: string[]): Promise { if (!res.ok) return { json: { ok: false, error: `slack post failed: ${res.error}` }, code: 1 }; writePostedIds(r.picks.map((p) => p.item.arxiv)); return { - json: { ok: true, posted: true, channel: post, ts: res.ts, fingerprint: digestFingerprint(text), counts: { total: items.length, picks: r.picks.length } }, + json: { + ok: true, + posted: true, + channel: post, + ts: res.ts, + fingerprint: digestFingerprint(text), + counts: { total: items.length, picks: r.picks.length }, + ...routedField(), + }, code: 0, }; } diff --git a/packages/amico-run/src/papers_verb.ts b/packages/amico-run/src/papers_verb.ts index 2f6ae114..a29f05c8 100644 --- a/packages/amico-run/src/papers_verb.ts +++ b/packages/amico-run/src/papers_verb.ts @@ -18,7 +18,7 @@ export async function papersVerb(argv: string[]): Promise { if (sub === "list") return papersList(rest); if (sub === "digest") return papersDigestVerb(rest); return { - json: { ok: false, error: `papers: unknown subcommand '${sub ?? ""}' — usage: amico papers list […] | amico papers digest [--feed ] [--top ] [--dry-run|--post ]` }, + json: { ok: false, error: `papers: unknown subcommand '${sub ?? ""}' — usage: amico papers list […] | amico papers digest [--feed | --feed-xml ] [--top ] [--route [--sessions ]] [--dry-run|--post ]` }, code: 64, }; } diff --git a/packages/amico-run/test/papers_verb.test.ts b/packages/amico-run/test/papers_verb.test.ts index dee053d0..28f95fe0 100644 --- a/packages/amico-run/test/papers_verb.test.ts +++ b/packages/amico-run/test/papers_verb.test.ts @@ -84,3 +84,99 @@ describe("papersVerb", () => { expect(r.code).toBe(64); }); }); + +// ── living-sota slice 2: the digest's relevance router (spec D3, S3) ────────── +// the papers digest gains the staged router: its LAB-CORPUS picks route +// against the active campaigns; matches stage into the campaign sidecar, +// below-threshold/no-match into the hopper. Hermetic: --feed-xml seeds the +// feed from a fixture file (the deterministic seam) — zero transports. + +import { mkdtempSync as mkd2, writeFileSync as wf2, readFileSync as rf2, mkdirSync as mkd2d } from "node:fs"; +import { deriveStagingState, stagingStreamPath, HOPPER_CAMPAIGN } from "../src/sota_staging.js"; + +const FEED_XML = ` + +Fast Rydberg CZ gates via optimal controlhttp://arxiv.org/abs/2606.05060We shape pulses in the blockade regime; the CZ gate reaches 0.9999 on an eight-atom register. +Protein folding via deep learninghttp://arxiv.org/abs/2606.99999AlphaFold-style pipelines for structure prediction. +Unrelated condensed matter notehttp://arxiv.org/abs/2606.11111A note about something else entirely. +`; + +const ACTIVE_LEDGER_2 = `# Session ledger — Rydberg blockade scaling + +## 2. Verdict table + +| item | status | +|---|---| +| H1 blockade radius calibration | pending | +`; + +function corpusNote(terms: string[]): void { + // the lab corpus gives the profile the taste that picks the digest entries + // (a VALID library-paper record — the corpus fold skips invalid notes) + wf2(join(vaults, "mine", "papers", `note-${Math.random().toString(36).slice(2)}.md`), `---\ntype: paper\ntitle: "T"\nauthors: [Someone]\narxiv: "0000.00000"\nsystems: [${terms.join(", ")}]\ntags: [${terms.join(", ")}]\n---\n\n# t\n`); +} + +describe("`amico papers digest --route` — matched picks stage into the campaign sidecar (the digest's router)", () => { + it("routes the digest picks: matched -> campaign sidecar; sub-corpus-threshold/unmatched -> hopper; idempotent re-run", async () => { + corpusNote(["rydberg", "cz", "blockade", "protein"]); + const sessions = mkd2(join(tmpdir(), "papers-sessions-")); + wf2(join(sessions, "session-20260901-rydberg-blockade.md"), ACTIVE_LEDGER_2); + const feedFile = join(mkd2(join(tmpdir(), "papers-feed-")), "feed.xml"); + wf2(feedFile, FEED_XML); + const res = await papersVerb(["digest", "--feed-xml", feedFile, "--route", "--sessions", sessions]); + expect(res.code).toBe(0); + const j = res.json as { ok: boolean; routed: { staged: { event_id: string; campaign: string }[]; hopper: { event_id: string; reason: string }[]; deduped: string[] } }; + expect(j.ok).toBe(true); + // the rydberg paper matched the active campaign; protein + the unrelated note did not + expect(j.routed.staged).toHaveLength(1); + expect(j.routed.staged[0]).toMatchObject({ event_id: "arxiv:2606.05060", campaign: "session-20260901-rydberg-blockade" }); + expect(j.routed.hopper).toHaveLength(1); + expect(j.routed.hopper[0]).toMatchObject({ event_id: "arxiv:2606.99999" }); + const sidecar = deriveStagingState(stagingStreamPath(sessions, "session-20260901-rydberg-blockade")); + expect(sidecar.entries.get("arxiv:2606.05060")?.state).toBe("staged"); + expect(sidecar.entries.get("arxiv:2606.05060")?.provenance?.job).toBe("papers-digest"); + expect(deriveStagingState(stagingStreamPath(sessions, HOPPER_CAMPAIGN)).entries.get("arxiv:2606.99999")?.state).toBe("staged"); + // re-running the digest on the same feed dedupes by event id — one stage line each + const twice = await papersVerb(["digest", "--feed-xml", feedFile, "--route", "--sessions", sessions]); + const j2 = (twice.json as { routed: { deduped: string[] } }).routed; + expect(j2.deduped.sort()).toEqual(["arxiv:2606.05060", "arxiv:2606.99999"]); + expect(rf2(stagingStreamPath(sessions, "session-20260901-rydberg-blockade"), "utf8").trim().split("\n")).toHaveLength(1); + }); + + it("without --route the digest does NOT stage (routing is the job's explicit act, never a side effect of a dry run)", async () => { + corpusNote(["rydberg", "cz"]); + const sessions = mkd2(join(tmpdir(), "papers-sessions-")); + wf2(join(sessions, "session-20260901-rydberg-blockade.md"), ACTIVE_LEDGER_2); + const feedFile = join(mkd2(join(tmpdir(), "papers-feed-")), "feed.xml"); + wf2(feedFile, FEED_XML); + const res = await papersVerb(["digest", "--feed-xml", feedFile, "--sessions", sessions]); + expect(res.code).toBe(0); + expect((res.json as { routed?: unknown }).routed).toBeUndefined(); + expect(mkd2d).toBeTruthy(); // (import sanity for the fs helpers) + expect(!existsSyncSync(stagingStreamPath(sessions, "session-20260901-rydberg-blockade"))).toBe(true); + }); + + it("a routing failure is a NAMED, non-fatal outcome (the digest still reports; the survey never blocks)", async () => { + corpusNote(["rydberg", "cz"]); + // sessions dir points into a FILE — enumerate/read fails -> named routing error, digest exit 0 + const sessionsFile = join(mkd2(join(tmpdir(), "papers-badsessions-")), "sessions"); + wf2(sessionsFile, "not a dir"); + const feedFile = join(mkd2(join(tmpdir(), "papers-feed-")), "feed.xml"); + wf2(feedFile, FEED_XML); + const res = await papersVerb(["digest", "--feed-xml", feedFile, "--route", "--sessions", sessionsFile]); + expect(res.code).toBe(0); + const j = res.json as { ok: boolean; routed: { staged: unknown[]; errors: string[] } }; + expect(j.ok).toBe(true); + expect(j.routed.staged).toHaveLength(0); + expect(j.routed.errors.length).toBeGreaterThan(0); + }); +}); + +function existsSyncSync(p: string): boolean { + try { + rf2(p); + return true; + } catch { + return false; + } +} From 54da830720f03bb64aa98f04cef0161dcc1b635e Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:49:36 -0400 Subject: [PATCH 4/6] =?UTF-8?q?feat(bridge):=20the=20SOTA=20staging=20side?= =?UTF-8?q?car=20record=20kind=20=E2=80=94=20the=20acceptance-stamp=20sche?= =?UTF-8?q?ma=20rides=20the=20bridge=20fixtures=20(living-sota=20slice=202?= =?UTF-8?q?,=20O3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec-20260905-103000 D3 / S3 (extends the SEAM 4 bridge fixtures): the canonical sota-staging record dir (fixtures/bridge/2026-09-05-sota-staging) carries the per-campaign SIDECAR transition stream — stage lines with provenance + review-by/expiry stamps, the watcher release riding the identical shape, the PI-instructed accept stamp (instructed_by PI + the recorded instruction — the O3 schema), an expired-without-review drop, one pending stage — plus the hopper stream with a below-threshold stage and an unknown-ev opacity probe. validate_bridge_replay.mjs gains the sota-staging kind: stage unique by event id (double-delivery reds), transitions must reference their stage, accept XOR drop terminal, accept carries the PI instruction record, stamps and provenance required, unknown ev carried. docs/ledger-bridge-contract.md gains the (d) record-kind section. --- docs/ledger-bridge-contract.md | 41 ++++++- .../bridge/2026-09-05-sota-staging/README.md | 28 +++++ .../hopper.sota-staging.jsonl | 2 + ...20260831-bridge-fixture.sota-staging.jsonl | 6 + .../2026-09-05-sota-staging/staging.toml | 6 + packages/amico-run/fixtures/bridge/README.md | 21 +++- .../scripts/validate_bridge_replay.d.mts | 2 +- .../scripts/validate_bridge_replay.mjs | 114 +++++++++++++++++- packages/amico-run/test/bridge_replay.test.ts | 110 ++++++++++++++++- 9 files changed, 315 insertions(+), 15 deletions(-) create mode 100644 packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/README.md create mode 100644 packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/hopper.sota-staging.jsonl create mode 100644 packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/session-20260831-bridge-fixture.sota-staging.jsonl create mode 100644 packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/staging.toml diff --git a/docs/ledger-bridge-contract.md b/docs/ledger-bridge-contract.md index 0904eca0..aaf3b80d 100644 --- a/docs/ledger-bridge-contract.md +++ b/docs/ledger-bridge-contract.md @@ -1,4 +1,4 @@ -# The ledger bridge contract — one doctrine, three record kinds +# The ledger bridge contract — one doctrine, four record kinds > SEAM 4 of the outside-lab codesign (amicode's half; issue #704, part of #679; > design-of-record: `spec-20260831-120000-amicode-outside-lab-codesign.md`, SEAM 4). @@ -99,6 +99,28 @@ completes as this note + the fixtures, and amicode's reports stay readable through its own surfaces (the Run Inspector, the problem-workspace spine, the `amico-run` launch/verify path) exactly as they do today. +### (d) the SOTA staging sidecar (living-sota D3 — slice 2 of spec-20260905-103000) + +The per-campaign SIDECAR staging stream a session ledger carries BESIDE +itself — the nine-section ledger grammar holds unamended; the sidecar is a +SEPARATE file (`.sota-staging.jsonl` under the personal vault's +`sessions/`, one per campaign, plus the reserved `hopper` stream as the +no-match/below-threshold fallback). The doctrine's five clauses apply +verbatim: append-only transition lines (`stage` / `accept` / `drop` / +`compact`) at or under PIPE_BUF with O_APPEND line-atomicity; staging state +DERIVED by replay, never stored; entries never mutated (the windowed +compaction rewrite is the one sanctioned exception, and it RECORDS itself as +an appended `compact` line); unknown `ev` values carried by readers; and +single writer — the digest, the SOTA watcher, and the weekly synthesis are +the ONLY stage/drop writers, with ONE sanctioned non-job append: the +**acceptance stamp**, an agent appending an `accept` line on the PI's explicit +instruction, carrying `instructed_by: "PI"` plus the recorded instruction +(channel, note, received_at). Every `stage` line carries provenance +(job/via/source/fetched_at) and the review-by/expiry stamps; every transition +is keyed by its external `event_id` (idempotent — double delivery is +impossible); accept and drop are both terminal and exactly one lands. The +acceptance-stamp schema rides the bridge fixture below (obligation O3). + ## The replay fixtures + validator (amicode's half, shipped here) - `packages/amico-run/fixtures/bridge/amicode-run/` — one canonical amicode run @@ -110,11 +132,20 @@ through its own surfaces (the Run Inspector, the problem-workspace spine, the `task.toml` (`kind = "experiment"`), `progress.jsonl` with the known event kinds **plus one unknown `ev` on purpose** (the opacity rule, exercised, not asserted), `result.toml`, and the artifact the `artifact` event names. +- `packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/` — one + canonical SOTA staging sidecar record (living-sota slice 2): `staging.toml` + (the manifest), the campaign sidecar with a staged paper, a staged watcher + release, the PI-instructed accept stamp, an expired-without-review drop, + and a pending stage, plus the `hopper` stream carrying a below-threshold + stage and one unknown `ev` on purpose (the opacity probe, same as the + strumento fixture's). - `packages/amico-run/scripts/validate_bridge_replay.mjs` — the replay - validator: exit 0 on both fixtures, non-zero on doctrine violations (a torn - terminal marker, a mutated content hash, a missing terminal marker, a torn - append-only stream, a non-contiguous `seq`, an escaping/void artifact path, a - broken stdout-contract line). No Julia, no Python — the fixtures are + validator: exit 0 on all three fixtures, non-zero on doctrine violations (a + torn terminal marker, a mutated content hash, a missing terminal marker, a + torn append-only stream, a non-contiguous `seq`, an escaping/void artifact + path, a broken stdout-contract line, a laundered staging acceptance — an + accept without its stage behind it or without its PI-instruction record, a + double stage for one event id). No Julia, no Python — the fixtures are committed data. `node packages/amico-run/scripts/validate_bridge_replay.mjs` with no arguments validates both; pass a record dir to validate one. diff --git a/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/README.md b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/README.md new file mode 100644 index 00000000..14832b20 --- /dev/null +++ b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/README.md @@ -0,0 +1,28 @@ +# The SOTA staging sidecar record — the canonical stream shape (living-sota D3) + +One canonical SIDECAR staging record dir (the `sota-staging` record kind the +ledger-bridge validator replays): the per-campaign append-only transition +stream a campaign ledger carries BESIDE itself, plus the hopper fallback +stream. Synthetic values, real shapes — every line is a whole, flushed JSON +object ≤ PIPE_BUF appended with O_APPEND, `seq` is the line count at write +time, and state is derived by replay, never stored. + +- `staging.toml` — the record's manifest: the record kind, the campaign the + sidecar belongs to, and the stamped constants (review-by / expiry / + compaction windows). +- `session-20260831-bridge-fixture.sota-staging.jsonl` — the campaign sidecar: + a staged paper (provenance-stamped, review-by/expiry stamps), a staged + watcher release event (the watcher rides the IDENTICAL shape), the + PI-instructed accept stamp (instruction provenance recorded), a match that + expired without review (the recorded drop line), and one still-pending + stage. +- `hopper.sota-staging.jsonl` — the fallback stream: a below-threshold stage + line **plus one unknown `ev` on purpose** — the reader-opacity rule, + exercised by the fixture the same way the strumento fixture carries its + unknown `ev` probe. + +The writer discipline (ONE APPENDER — the digest, the watcher, and the +weekly synthesis are the only stage/drop writers; the accept stamp is the +sole sanctioned non-job append, an agent recording the PI's explicit +instruction) lives in `packages/amico-run/src/sota_staging.ts`; the replay +grammar is enforced by `packages/amico-run/scripts/validate_bridge_replay.mjs`. diff --git a/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/hopper.sota-staging.jsonl b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/hopper.sota-staging.jsonl new file mode 100644 index 00000000..41e4c9e6 --- /dev/null +++ b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/hopper.sota-staging.jsonl @@ -0,0 +1,2 @@ +{"ev":"stage","seq":1,"ts":"2026-09-05T09:00:00.000Z","event_id":"arxiv:2606.99999","campaign":"hopper","kind":"paper","title":"Protein folding via deep learning","url":"https://arxiv.org/abs/2606.99999","provenance":{"job":"papers-digest","via":"fetched","source":"arXiv export API over HTTPS","fetched_at":"2026-09-05T08:55:00.000Z","feed":"quant-ph"},"matched":[],"reason":"below-threshold","review_by":"2026-09-12T09:00:00.000Z","expires_at":"2026-09-19T09:00:00.000Z"} +{"ev":"triage-tag","seq":2,"ts":"2026-09-05T12:00:00.000Z","event_id":"arxiv:2606.99999","tag":"flywheel-drain"} diff --git a/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/session-20260831-bridge-fixture.sota-staging.jsonl b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/session-20260831-bridge-fixture.sota-staging.jsonl new file mode 100644 index 00000000..3b765742 --- /dev/null +++ b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/session-20260831-bridge-fixture.sota-staging.jsonl @@ -0,0 +1,6 @@ +{"ev":"stage","seq":1,"ts":"2026-09-05T09:00:00.000Z","event_id":"arxiv:2606.05060","campaign":"session-20260831-bridge-fixture","kind":"paper","title":"Fast Rydberg CZ gates via shaped pulses","url":"https://arxiv.org/abs/2606.05060","provenance":{"job":"papers-digest","via":"fetched","source":"arXiv export API over HTTPS","fetched_at":"2026-09-05T08:55:00.000Z","feed":"quant-ph"},"matched":["rydberg","cz"],"review_by":"2026-09-12T09:00:00.000Z","expires_at":"2026-09-19T09:00:00.000Z"} +{"ev":"stage","seq":2,"ts":"2026-09-05T09:05:00.000Z","event_id":"github:example/piccolo-adjacent@v0.9.0","campaign":"session-20260831-bridge-fixture","kind":"release","title":"v0.9.0 — trajectory rework","url":"https://github.com/example/piccolo-adjacent/releases/tag/v0.9.0","provenance":{"job":"sota-watcher","via":"cache","source":"GitHub API against canonical repos","fetched_at":"2026-09-05T09:04:00.000Z","repo":"example/piccolo-adjacent","surface":"releases"},"matched":["trajectory","rydberg"],"review_by":"2026-09-12T09:05:00.000Z","expires_at":"2026-09-19T09:05:00.000Z"} +{"ev":"accept","seq":3,"ts":"2026-09-06T10:00:00.000Z","event_id":"arxiv:2606.05060","campaign":"session-20260831-bridge-fixture","kind":"paper","title":"Fast Rydberg CZ gates via shaped pulses","url":"https://arxiv.org/abs/2606.05060","instructed_by":"PI","instruction":{"channel":"chat","note":"the blockade number is load-bearing — accept","received_at":"2026-09-06T09:58:00.000Z"}} +{"ev":"stage","seq":4,"ts":"2026-09-05T09:10:00.000Z","event_id":"arxiv:2606.11111","campaign":"session-20260831-bridge-fixture","kind":"paper","title":"Marginal drift note","url":"https://arxiv.org/abs/2606.11111","provenance":{"job":"papers-digest","via":"fetched","source":"arXiv export API over HTTPS","fetched_at":"2026-09-05T08:55:00.000Z","feed":"quant-ph"},"matched":["blockade"],"review_by":"2026-09-12T09:10:00.000Z","expires_at":"2026-09-19T09:10:00.000Z"} +{"ev":"drop","seq":5,"ts":"2026-09-19T09:00:01.000Z","event_id":"arxiv:2606.11111","campaign":"session-20260831-bridge-fixture","reason":"expired-without-review","recorded":"2026-09-19T09:00:01.000Z"} +{"ev":"stage","seq":6,"ts":"2026-09-20T08:02:00.000Z","event_id":"arxiv:2606.07777","campaign":"session-20260831-bridge-fixture","kind":"paper","title":"Rydberg register scaling revisited","url":"https://arxiv.org/abs/2606.07777","provenance":{"job":"papers-digest","via":"cache","source":"arXiv export API over HTTPS","fetched_at":"2026-09-20T08:01:00.000Z","feed":"quant-ph"},"matched":["rydberg","register"],"review_by":"2026-09-27T08:02:00.000Z","expires_at":"2026-10-04T08:02:00.000Z"} diff --git a/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/staging.toml b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/staging.toml new file mode 100644 index 00000000..177c2d67 --- /dev/null +++ b/packages/amico-run/fixtures/bridge/2026-09-05-sota-staging/staging.toml @@ -0,0 +1,6 @@ +kind = "sota-staging" +schema_version = "1" +campaign = "session-20260831-bridge-fixture" +review_by_days = 7 +expires_after_days = 14 +compaction_window_days = 28 diff --git a/packages/amico-run/fixtures/bridge/README.md b/packages/amico-run/fixtures/bridge/README.md index d9ffb7c6..001baebb 100644 --- a/packages/amico-run/fixtures/bridge/README.md +++ b/packages/amico-run/fixtures/bridge/README.md @@ -1,10 +1,12 @@ # SEAM 4 bridge fixtures — the canonical replay records -Two committed, synthetic, stable record dirs the Telaio fold must replay — -issued #704, amicode's half of the ledger bridge. The doctrine they carry is +Three committed, synthetic, stable record dirs the Telaio fold must replay — +issued #704, amicode's half of the ledger bridge (the sota-staging record is +the living-sota campaign's slice-2 extension, spec-20260905-103000 D3). +The doctrine they carry is [`docs/ledger-bridge-contract.md`](../../../../docs/ledger-bridge-contract.md); the validator is -`packages/amico-run/scripts/validate_bridge_replay.mjs` (exit 0 on both by +`packages/amico-run/scripts/validate_bridge_replay.mjs` (exit 0 on all by default; pass a dir to check one; the corruption directions are pinned by `packages/amico-run/test/bridge_replay.test.ts`). @@ -54,3 +56,16 @@ Named by its id — the TaskRecord contract binds `id == directory basename`. - `artifacts/fit_002.json` — the file the `artifact` event names; the contract requires a recorded artifact path to resolve to a real file inside the task dir, so it does. + +## `2026-09-05-sota-staging/` — one canonical SOTA staging sidecar record + +The per-campaign SIDECAR staging stream (living-sota slice 2, D3): the +append-only transition stream a campaign ledger carries BESIDE itself (the +nine-section ledger grammar unamended), plus the hopper fallback stream. +`staging.toml` is the manifest (record kind + campaign + the stamped +windows); `session-20260831-bridge-fixture.sota-staging.jsonl` carries a +staged paper, a staged watcher release, the PI-instructed accept stamp (the +acceptance-stamp schema — obligation O3 — lives HERE), an +expired-without-review drop, and a pending stage; `hopper.sota-staging.jsonl` +carries a below-threshold stage plus one unknown `ev` on purpose (the +reader-opacity probe, same as the strumento fixture's unknown `ev`). diff --git a/packages/amico-run/scripts/validate_bridge_replay.d.mts b/packages/amico-run/scripts/validate_bridge_replay.d.mts index 605ec9f6..65f9f667 100644 --- a/packages/amico-run/scripts/validate_bridge_replay.d.mts +++ b/packages/amico-run/scripts/validate_bridge_replay.d.mts @@ -2,7 +2,7 @@ // amico-run tsconfig includes test/, unlike the extension package's — hence // this declaration rather than an untyped import). -export type BridgeRecordKind = "amicode-run" | "strumento-task"; +export type BridgeRecordKind = "amicode-run" | "strumento-task" | "sota-staging"; export interface BridgeValidation { ok: boolean; diff --git a/packages/amico-run/scripts/validate_bridge_replay.mjs b/packages/amico-run/scripts/validate_bridge_replay.mjs index a611a576..0f82ce77 100644 --- a/packages/amico-run/scripts/validate_bridge_replay.mjs +++ b/packages/amico-run/scripts/validate_bridge_replay.mjs @@ -25,7 +25,7 @@ // belongs to the .d.mts surface next to it, the assert_built_bundles.mjs // pattern). import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { parse as parseToml } from "smol-toml"; @@ -63,14 +63,16 @@ export function defaultFixtureDirs() { return [ { kind: "amicode-run", dir: join(PKG_ROOT, "fixtures", "bridge", "amicode-run") }, { kind: "strumento-task", dir: join(PKG_ROOT, "fixtures", "bridge", "2026-08-31-strumento-task-b3a7") }, + { kind: "sota-staging", dir: join(PKG_ROOT, "fixtures", "bridge", "2026-09-05-sota-staging") }, ]; } /** Infer the record kind from the directory's own manifest (the record is the - * truth; neither kind ever contains the other's manifest). */ + * truth; no kind ever contains another's manifest). */ export function inferRecordKind(dir) { if (existsSync(join(dir, "run.toml"))) return "amicode-run"; if (existsSync(join(dir, "task.toml"))) return "strumento-task"; + if (existsSync(join(dir, "staging.toml"))) return "sota-staging"; return undefined; } @@ -326,6 +328,109 @@ function validateStrumentoTask(dir, errors) { } } +// ─── the SOTA staging sidecar record (living-sota D3, slice 2) ─────────────── +// +// The per-campaign SIDECAR staging stream: append-only transition lines +// (stage/accept/drop/compact) beside the session ledger; state derived by +// replay. The grammar this validator enforces IS the acceptance-stamp schema +// (obligation O3): an accept line carries instructed_by: "PI" + the recorded +// instruction; transitions reference their stage; a stage is unique by event +// id; accept and drop are mutually terminal; unknown `ev` values are carried +// (reader opacity — the fixture ships one on purpose). + +const STAGING_KINDS = new Set(["paper", "release", "changelog", "issue"]); + +function validateSotaStaging(dir, errors) { + const manifest = readTomlFile("staging.toml", dir, errors); + let campaign; + if (manifest !== undefined) { + if (manifest.kind !== "sota-staging") { + errors.push("staging.toml: kind is not sota-staging — the manifest names the record kind"); + } + if (typeof manifest.campaign !== "string" || manifest.campaign === "") { + errors.push("staging.toml: campaign missing — the manifest names the sidecar's campaign"); + } else { + campaign = manifest.campaign; + } + } + // the campaign's own sidecar must be present + if (campaign !== undefined && !existsSync(join(dir, `${campaign}.sota-staging.jsonl`))) { + errors.push(`${campaign}.sota-staging.jsonl: missing — the manifest's campaign must have its sidecar`); + } + + let streams = []; + try { + streams = readdirSync(dir).filter((n) => n.endsWith(".sota-staging.jsonl")); + } catch { + /* none */ + } + if (streams.length === 0) { + errors.push("no *.sota-staging.jsonl stream in the record — a staging record is its streams"); + } + for (const name of streams) { + validateStagingStream(dir, name, errors); + } +} + +function validateStagingStream(dir, name, errors) { + const stem = name.replace(/\.sota-staging\.jsonl$/, ""); + const events = readJsonlFile(name, dir, errors); + if (events === undefined) return; + const seenStage = new Set(); + const terminal = new Map(); // event_id → ev ("accept" | "drop") + events.forEach((e, i) => { + const at = `${name}: line ${i + 1}`; + if (e.seq !== i + 1) errors.push(`${at}: seq is ${e.seq} — seq is the line count at write time (monotonic from 1)`); + if (typeof e.ts !== "string" || !ISO_RE.test(e.ts)) errors.push(`${at}: ts missing or not ISO-8601`); + const ev = typeof e.ev === "string" ? e.ev : ""; + if (ev === "" || typeof e.event_id !== "string" || e.event_id === "") { + if (ev !== "") errors.push(`${at}: event_id missing — every transition is keyed by its external identity`); + else if (typeof e.ev !== "string") errors.push(`${at}: ev missing — unknown ev VALUES skip, but ev itself is required`); + return; + } + if (ev === "stage") { + if (typeof e.campaign !== "string" || e.campaign !== stem) { + errors.push(`${at}: campaign "${e.campaign}" ≠ the stream's stem "${stem}" — a sidecar carries its own campaign only`); + } + if (!STAGING_KINDS.has(e.kind)) errors.push(`${at}: kind "${e.kind}" outside paper|release|changelog|issue`); + if (typeof e.title !== "string" || e.title === "") errors.push(`${at}: title missing — the listing renders it`); + if (typeof e.url !== "string" || e.url === "") errors.push(`${at}: url missing — every staged match is cited`); + if (e.provenance === null || typeof e.provenance !== "object" || Array.isArray(e.provenance)) { + errors.push(`${at}: provenance missing — a match never lands unprovenance-stamped`); + } else { + for (const k of ["job", "via", "source", "fetched_at"]) { + if (typeof e.provenance[k] !== "string" || e.provenance[k] === "") { + errors.push(`${at}: provenance.${k} missing — the stamp carries {job, via, source, fetched_at}`); + } + } + } + if (typeof e.review_by !== "string" || !ISO_RE.test(e.review_by)) errors.push(`${at}: review_by missing or not ISO-8601 — the review-by stamp is the shape`); + if (typeof e.expires_at !== "string" || !ISO_RE.test(e.expires_at)) errors.push(`${at}: expires_at missing or not ISO-8601 — the expiry stamp is the shape`); + if (seenStage.has(e.event_id)) errors.push(`${at}: a second stage for ${e.event_id} — double-delivery is impossible (idempotent by event id)`); + seenStage.add(e.event_id); + return; + } + if (ev === "accept") { + // O3: the acceptance-stamp schema — the PI-instructed record + if (e.instructed_by !== "PI") errors.push(`${at}: instructed_by is not "PI" — the stamp records the human decision, not a job append`); + if (e.instruction === null || typeof e.instruction !== "object" || Array.isArray(e.instruction)) { + errors.push(`${at}: instruction missing — an unstamped acceptance is a laundered one`); + } else { + if (typeof e.instruction.channel !== "string" || e.instruction.channel === "") errors.push(`${at}: instruction.channel missing`); + if (typeof e.instruction.note !== "string" || e.instruction.note.trim() === "") errors.push(`${at}: instruction.note missing — the PI's explicit instruction is required`); + if (typeof e.instruction.received_at !== "string" || !ISO_RE.test(e.instruction.received_at)) errors.push(`${at}: instruction.received_at missing or not ISO-8601`); + } + } else if (ev === "drop") { + if (typeof e.reason !== "string" || e.reason === "") errors.push(`${at}: reason missing — a drop is a recorded line, never a silent skip`); + if (typeof e.recorded !== "string" || !ISO_RE.test(e.recorded)) errors.push(`${at}: recorded missing or not ISO-8601`); + } + // transition grammar: references its stage; accept XOR drop per event id + if (!seenStage.has(e.event_id)) errors.push(`${at}: ${ev} for ${e.event_id} with no stage behind it — a transition records the fate of a STAGED match`); + else if (terminal.has(e.event_id)) errors.push(`${at}: ${ev} after ${terminal.get(e.event_id)} for ${e.event_id} — accept and drop are both terminal, exactly one lands`); + else terminal.set(e.event_id, ev); + }); +} + /** Validate one record directory against the bridge doctrine. Pure: reads the * dir, returns every violation it finds (never throws). */ export function validateBridgeRecord(dir, kind) { @@ -336,12 +441,13 @@ export function validateBridgeRecord(dir, kind) { ok: false, kind: "amicode-run", errors: [ - `${dir}: no record manifest found (run.toml for an amicode run dir, task.toml for a strumento task dir)`, + `${dir}: no record manifest found (run.toml for an amicode run dir, task.toml for a strumento task dir, staging.toml for a SOTA staging record)`, ], }; } if (k === "amicode-run") validateAmicodeRun(dir, errors); - else validateStrumentoTask(dir, errors); + else if (k === "strumento-task") validateStrumentoTask(dir, errors); + else if (k === "sota-staging") validateSotaStaging(dir, errors); return { ok: errors.length === 0, kind: k, errors }; } diff --git a/packages/amico-run/test/bridge_replay.test.ts b/packages/amico-run/test/bridge_replay.test.ts index 33413f00..6a4b135e 100644 --- a/packages/amico-run/test/bridge_replay.test.ts +++ b/packages/amico-run/test/bridge_replay.test.ts @@ -28,12 +28,16 @@ const AMICODE_FIXTURE = join(BRIDGE_FIXTURES, "amicode-run"); // The strumento record dir is named BY ITS ID (the contract: "the id is always // the directory's basename") — the fixture honors the shape it replays. const STRUMENTO_FIXTURE = join(BRIDGE_FIXTURES, "2026-08-31-strumento-task-b3a7"); +// living-sota slice 2: the SOTA staging sidecar record (the acceptance-stamp +// schema rides THIS fixture — obligation O3). +const SOTA_STAGING_FIXTURE = join(BRIDGE_FIXTURES, "2026-09-05-sota-staging"); +const SIDECAR = "session-20260831-bridge-fixture.sota-staging.jsonl"; /** Copy a committed fixture to a tmp dir and hand the copy to `mutate` — * corruption tests never touch the committed bytes. The copy keeps the * fixture's basename (the strumento id contract binds id == basename). */ -function mutatedFixture(kind: "amicode-run" | "strumento-task", mutate: (dir: string) => void): string { - const src = kind === "amicode-run" ? AMICODE_FIXTURE : STRUMENTO_FIXTURE; +function mutatedFixture(kind: "amicode-run" | "strumento-task" | "sota-staging", mutate: (dir: string) => void): string { + const src = kind === "amicode-run" ? AMICODE_FIXTURE : kind === "strumento-task" ? STRUMENTO_FIXTURE : SOTA_STAGING_FIXTURE; const dir = join(mkdtempSync(join(tmpdir(), "bridge-replay-")), basename(src)); cpSync(src, dir, { recursive: true }); mutate(dir); @@ -309,3 +313,105 @@ describe("SEAM 4 replay fixtures — the canonical records validate", () => { expect(r.ok).toBe(true); }); }); + +// ─── the SOTA staging sidecar record (living-sota slice 2, O3) ────────────── +// The acceptance-stamp schema rides THIS fixture: stage/accept/drop/compact +// transition lines, event-id idempotent, provenance-stamped, the PI-instructed +// accept carrying its instruction record. The corruption directions exercise +// the transition grammar — a laundered accept, a double stage, an orphan +// transition, a silent drop — each reds by name. + +describe("living-sota staging record — the committed fixture validates; the transition grammar reds on corruption", () => { + it("the committed sota-staging fixture is present and validates against the grammar", () => { + expect(existsSync(join(SOTA_STAGING_FIXTURE, "staging.toml"))).toBe(true); + const r = validateBridgeRecord(SOTA_STAGING_FIXTURE, "sota-staging"); + expect(r.errors).toEqual([]); + expect(r.ok).toBe(true); + }); + + it("the fixture's own unknown-ev probe (the hopper stream's triage-tag) validates by design — readers carry, never fail", () => { + const raw = readFileSync(join(SOTA_STAGING_FIXTURE, "hopper.sota-staging.jsonl"), "utf8"); + expect(raw).toContain('"ev":"triage-tag"'); + const r = validateBridgeRecord(SOTA_STAGING_FIXTURE, "sota-staging"); + expect(r.ok).toBe(true); + }); + + it("a duplicate stage line (double delivery) reds — idempotency is the shape, not the writer's mood", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => + s + + '{"ev":"stage","seq":7,"ts":"2026-09-21T08:00:00.000Z","event_id":"arxiv:2606.05060","campaign":"session-20260831-bridge-fixture","kind":"paper","title":"dup","url":"https://arxiv.org/abs/2606.05060","provenance":{"job":"papers-digest","via":"cache","source":"arXiv export API over HTTPS","fetched_at":"2026-09-21T07:00:00.000Z"},"review_by":"2026-09-28T08:00:00.000Z","expires_at":"2026-10-05T08:00:00.000Z"}\n', + ); + }); + expectNotOk(dir, "sota-staging", /second stage|double-delivery/); + }); + + it("an accept with no stage behind it reds (a free-floating stamp is a laundered acceptance)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => + s + + '{"ev":"accept","seq":7,"ts":"2026-09-21T08:00:00.000Z","event_id":"arxiv:9999.99999","campaign":"session-20260831-bridge-fixture","instructed_by":"PI","instruction":{"channel":"chat","note":"ghost","received_at":"2026-09-21T08:00:00.000Z"}}\n', + ); + }); + expectNotOk(dir, "sota-staging", /no stage behind it/); + }); + + it("a drop after an accept reds — accept and drop are both terminal, exactly one lands", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => + s + + '{"ev":"drop","seq":7,"ts":"2026-09-21T08:00:00.000Z","event_id":"arxiv:2606.05060","campaign":"session-20260831-bridge-fixture","reason":"expired-without-review","recorded":"2026-09-21T08:00:00.000Z"}\n', + ); + }); + expectNotOk(dir, "sota-staging", /terminal/); + }); + + it("an accept without the PI instruction record reds — the schema IS the human decision's record (O3)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => s.replace(/"instructed_by":"PI","instruction":\{[^}]*\}/, '"instructed_by":"job"')); + }); + expectNotOk(dir, "sota-staging", /instructed_by is not "PI"/); + }); + + it("a stage line without provenance reds — a match never lands unprovenance-stamped", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => s.replace(/"provenance":\{"job":"papers-digest"[^}]*\}/g, '"provenance":{}')); + }); + expectNotOk(dir, "sota-staging", /provenance/); + }); + + it("a stage line missing its review-by/expiry stamps reds (the stamps are the shape)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + // remove the fields cleanly (keeping the line whole JSON) — the corruption + // under test is the MISSING STAMP, not a torn line + edit(d, SIDECAR, (s) => + s.replace( + ',"review_by":"2026-09-12T09:10:00.000Z","expires_at":"2026-09-19T09:10:00.000Z"', + "", + ), + ); + }); + expectNotOk(dir, "sota-staging", /review_by/); + }); + + it("a stage line whose campaign disagrees with the stream's stem reds (a sidecar carries its own campaign)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => s.replace('"campaign":"session-20260831-bridge-fixture","kind":"paper","title":"Marginal drift note"', '"campaign":"some-other-campaign","kind":"paper","title":"Marginal drift note"')); + }); + expectNotOk(dir, "sota-staging", /≠ the stream's stem/); + }); + + it("a torn mid-stream line reds (the writers append whole flushed lines under PIPE_BUF)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => s.slice(0, s.length - 60)); + }); + expectNotOk(dir, "sota-staging", /torn/); + }); + + it("a non-contiguous seq reds (an append-only violation: a dropped or replayed line)", () => { + const dir = mutatedFixture("sota-staging", (d) => { + edit(d, SIDECAR, (s) => s.replace('"seq":5', '"seq":9')); + }); + expectNotOk(dir, "sota-staging", /seq/); + }); +}); From 727005d4c57d507b2d566e78b59e2503bf7a2264 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:50:27 -0400 Subject: [PATCH 5/6] =?UTF-8?q?docs(sota-review=20skill):=20the=20staged-r?= =?UTF-8?q?outing=20layer=20is=20landed=20=E2=80=94=20the=20lenses=20fetch?= =?UTF-8?q?=20and=20report,=20they=20never=20append=20(living-sota=20slice?= =?UTF-8?q?=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec-20260905-103000 D3: the skill's honest-scope section now names the staged routing as its own layer with enumerated writers (digest --route, the watcher, the PI-instructed accept stamp, the awaiting-the-eye listing); the on-demand lenses stay read-only. The skill lint strengthens with the slice-2 surfaces: the new modules join the scraping-pattern and sanctioned- host greps, the stage-before-count rule and the enumerated writer commands are pinned in the text. --- .../extension/skills/sota-review/SKILL.md | 21 +++++++++++----- .../extension/test/sota_review_skill.test.ts | 24 +++++++++++++++---- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/packages/extension/skills/sota-review/SKILL.md b/packages/extension/skills/sota-review/SKILL.md index 4fdf0458..b3b48d91 100644 --- a/packages/extension/skills/sota-review/SKILL.md +++ b/packages/extension/skills/sota-review/SKILL.md @@ -12,9 +12,11 @@ revision: 1 One survey, two lenses, one discipline: **read-only toward the world, append-only toward the vault**. The survey seeds hypotheses and design decisions; it never blocks the loop (a survey that cannot run records a -NAMED outcome and the loop proceeds), and its matches route through staging -for a human eye before they count as currency — this skill fetches and -reports; the staged routing is a later layer. +NAMED outcome and the loop proceeds), and its matches **stage before they +count**: the daily digest and the SOTA watcher append `stage` lines to +per-campaign SIDECAR staging streams beside the session ledgers, and nothing +counts as currency until the PI's accept stamp lands. This skill's lenses +fetch and report — they never append; the staged routing is its own layer. ## Usage @@ -128,6 +130,13 @@ Run through this list before publishing any brief: - **Network unavailable**: the survey records the named outcome and the loop proceeds (currency is a seed, not a stall). The brief renders the failure and its provenance honestly. -- **This slice fetches and reports**: it does not stage matches into a - campaign ledger, render strategy compositions, or gate anything — those - layers exist later; do not simulate their effects. +- **The lenses fetch and report; they never append.** Staged routing is its + own layer, and its writers are enumerated: the papers digest + (`amico papers digest --route`) and the SOTA watcher + (`amico sota watcher`) are the match writers; the **accept stamp** + (`amico sota accept`, recording the PI's explicit instruction) is the only + sanctioned other writer. The **awaiting-the-eye listing** + (`amico sota awaiting-the-eye`) renders everything pending — staged + matches are never rendered as currency until accepted. No strategy + composition renders here either — that layer lives elsewhere; do not + simulate its effects. diff --git a/packages/extension/test/sota_review_skill.test.ts b/packages/extension/test/sota_review_skill.test.ts index 89cf4309..7984dfe8 100644 --- a/packages/extension/test/sota_review_skill.test.ts +++ b/packages/extension/test/sota_review_skill.test.ts @@ -94,9 +94,19 @@ describe("the sota-review skill carries both lenses with the recipe verbatim (#8 expect(skill).toMatch(/amico sota codebase/); }); - it("honest scope — this slice fetches and reports (staging is a named later layer)", () => { - expect(flat).toMatch(/fetches and reports/i); - expect(flat).toMatch(/later layer|later slices/i); + it("honest scope — the lenses fetch and report; the staged routing is its OWN layer with enumerated writers (living-sota slice 2)", () => { + expect(flat).toMatch(/fetches and reports|fetch and report/i); + // the staged-routing layer is LANDED, not "later": the skill names its writers + expect(flat).toMatch(/staged routing is its\s*own layer/i); + expect(skill).toMatch(/amico papers digest --route/); + expect(skill).toMatch(/amico sota watcher/); + expect(skill).toMatch(/amico sota accept/); + expect(skill).toMatch(/amico sota awaiting-the-eye/); + // stage-before-count: nothing counts until the PI's accept stamp lands + expect(flat).toMatch(/never rendered as currency/i); + expect(flat).toMatch(/PI's accept stamp|accept stamp lands/i); + // the lenses NEVER append — the on-demand survey commands are read-only + expect(flat).toMatch(/they never append/i); }); }); @@ -117,6 +127,9 @@ describe("scraping patterns are banned by grep (#820 S1 — the recipe's rules, "the codebase lens (sota_codebase.ts)": readFileSync(join(AMICO_RUN, "src", "sota_codebase.ts"), "utf8"), "the fetch seam (sota_fetch.ts)": readFileSync(join(AMICO_RUN, "src", "sota_fetch.ts"), "utf8"), "the verb (sota_verb.ts)": readFileSync(join(AMICO_RUN, "src", "sota_verb.ts"), "utf8"), + "the staging streams (sota_staging.ts)": readFileSync(join(AMICO_RUN, "src", "sota_staging.ts"), "utf8"), + "the router (sota_router.ts)": readFileSync(join(AMICO_RUN, "src", "sota_router.ts"), "utf8"), + "the watcher (sota_watcher.ts)": readFileSync(join(AMICO_RUN, "src", "sota_watcher.ts"), "utf8"), }; for (const [name, text] of Object.entries(SURFACES)) { @@ -127,10 +140,13 @@ describe("scraping patterns are banned by grep (#820 S1 — the recipe's rules, } } - it("the only network endpoints in the lens code are the sanctioned APIs (arXiv export over https; GitHub API)", () => { + it("the only network endpoints in the lens + staging code are the sanctioned APIs (arXiv export over https; GitHub API; cited abs/release/issue URLs)", () => { const lensCode = [ readFileSync(join(AMICO_RUN, "src", "sota_papers.ts"), "utf8"), readFileSync(join(AMICO_RUN, "src", "sota_codebase.ts"), "utf8"), + readFileSync(join(AMICO_RUN, "src", "sota_router.ts"), "utf8"), + readFileSync(join(AMICO_RUN, "src", "sota_watcher.ts"), "utf8"), + readFileSync(join(AMICO_RUN, "src", "sota_staging.ts"), "utf8"), ].join("\n"); const httpsHosts = [...lensCode.matchAll(/https:\/\/([a-z0-9.-]+)/gi)].map((m) => m[1]); const sanctioned = new Set(["export.arxiv.org", "arxiv.org", "api.github.com", "github.com"]); From 871d62acd506317f83cc4a0c49160ccb1e3722c9 Mon Sep 17 00:00:00 2001 From: Aaron Trowbridge Date: Sat, 5 Sep 2026 20:53:15 -0400 Subject: [PATCH 6/6] =?UTF-8?q?test(sota):=20the=20changelog-surface=20wat?= =?UTF-8?q?cher=20cell=20=E2=80=94=20release=20notes=20ride=20the=20identi?= =?UTF-8?q?cal=20staged=20path=20(living-sota=20slice=202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the S3 cell list: release AND changelog AND issue events all ride the identical staged fixtures (kind mapping changelog; the releases endpoint serves the changelog surface on GitHub-shaped data — slice 1's githubApiUrl). --- packages/amico-run/test/sota_watcher.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/amico-run/test/sota_watcher.test.ts b/packages/amico-run/test/sota_watcher.test.ts index 7aeadb30..eabc3c56 100644 --- a/packages/amico-run/test/sota_watcher.test.ts +++ b/packages/amico-run/test/sota_watcher.test.ts @@ -116,6 +116,23 @@ describe("runSotaWatcher — the watcher rides the IDENTICAL staged path (S3, th expect(raw).toHaveLength(2); }); + it("a changelog-surface event rides the identical path (kind: changelog — release notes ARE the changelog on GitHub-shaped data)", async () => { + seed(); + // swap the registry's surfaces to changelog-only; the endpoint is the + // releases URL (slice 1's githubApiUrl), so the seeded cache serves it + writeFileSync( + registryPath(root), + REGISTRY_TOML.replace('fetch_surface = ["releases", "issues"]', 'fetch_surface = ["changelog"]'), + ); + const res = await runSotaWatcher({ root, sessionsDir, nowMs: () => CLOCK_MS }); + expect(res.staged).toHaveLength(1); + const st = deriveStagingState(stagingStreamPath(sessionsDir, "session-20260901-rydberg-integrator")); + const changelog = st.entries.get("github:example/piccolo-adjacent@v0.9.0"); + expect(changelog?.kind).toBe("changelog"); // the kind maps, the shape is identical + expect(changelog?.state).toBe("staged"); + expect((changelog?.provenance as Record).surface).toBe("changelog"); + }); + it("no campaign match → the hopper stream, same shape (the identical fixtures, hopper fallback)", async () => { seed(); rmSync(join(sessionsDir, "session-20260901-rydberg-integrator.md")); // no active campaigns