From b89091a2425e91c0df90d39cea38412565f6e944 Mon Sep 17 00:00:00 2001 From: Joob1n Date: Wed, 16 Sep 2026 23:14:45 +0800 Subject: [PATCH] feat(runtime): recall the files a message carried MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A screenshot pasted under "have a look" left no trace in the transcript text, so the message was unreachable by any term a person would think to search. Recall now matches a message on the names of the files it carried and returns those files as metadata beside the passage. Materials are metadata only — name, kind, media type, size. A material carries the address `Read` accepts exactly when it is reachable from the Session asking: an attachment read resolves against the calling Session and refuses one stored elsewhere, so offering the address across a Session boundary would invite a call that can only fail. Elsewhere the file is named but has no address, which is the honest answer. A file name is user-authored text, so it leaves through the same redaction the passage body does, and a credential-shaped name can no more be matched than it can be returned. An attachment that does not match the current shape is skipped rather than projected as a material named `undefined`. A message whose whole content was a pasted file has no text of its own and used to be dropped from every passage, which would have made the file unreachable in exactly the case this layer exists for; such a message now projects with empty text and its materials. No new tool: `Read` already accepts `maka://runtime/attachments/`. The candidate scan needs no change either — a file name is a JSON string value of the event that carried it, so a payload scan still offers every Session that could match. Claude-Session: https://claude.ai/code/session_014ajaRxC4jydavY9nYUFj5J --- packages/core/src/__tests__/recall.test.ts | 180 ++++++++++++++++++ packages/core/src/recall.ts | 141 ++++++++++++-- .../__tests__/recall-ledger-corpus.test.ts | 71 +++++++ packages/runtime/src/recall-tools.ts | 17 ++ 4 files changed, 395 insertions(+), 14 deletions(-) diff --git a/packages/core/src/__tests__/recall.test.ts b/packages/core/src/__tests__/recall.test.ts index f838696cd6..da55e9e736 100644 --- a/packages/core/src/__tests__/recall.test.ts +++ b/packages/core/src/__tests__/recall.test.ts @@ -53,6 +53,30 @@ function userMessage(id: string, turnId: string, text: string): StoredMessage { return { type: 'user', id, turnId, ts: (nextTs += 1000), text } as StoredMessage; } +function attachment(name: string, overrides: Record = {}) { + return { + kind: 'image', + name, + mimeType: 'image/png', + bytes: 2048, + ref: { + kind: 'session_file', + sessionId: 's-shot', + relativePath: 'art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + ...overrides, + }; +} + +function userMessageWithFiles( + id: string, + turnId: string, + text: string, + attachments: readonly ReturnType[], +): StoredMessage { + return { type: 'user', id, turnId, ts: (nextTs += 1000), text, attachments } as StoredMessage; +} + function assistantMessage(id: string, turnId: string, text: string): StoredMessage { return { type: 'assistant', id, turnId, ts: (nextTs += 1000), text } as StoredMessage; } @@ -868,6 +892,162 @@ test('the passage budget keeps the nearest context and says when it cut the rest ); }); +/** + * A screenshot pasted under "have a look" leaves no trace in the text, so the + * message is unreachable by any term a person would think of. Matching the + * file name is what makes the file findable at all. + */ +test('a message is reachable by the name of the file it carried', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '你看看', [attachment('pipeline-failure.png')]), + assistantMessage('a1', 'ts', '这是 CI 挂了'), + ], + }, + ]); + for (const deps of [scanDeps(data), candidateDeps(data)]) { + const result = await runRecall({ terms: ['pipeline-failure'] }, deps, { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual(anchorIds(result.passages), ['u1']); + const anchor = result.passages[0]?.messages.find((message) => message.isAnchor); + // The name is matched, but it comes back as a material rather than as text + // the user never typed. + assert.equal(anchor?.text, '你看看'); + assert.deepEqual(anchor?.materials, [ + { + name: 'pipeline-failure.png', + kind: 'image', + mimeType: 'image/png', + bytes: 2048, + resource: 'maka://runtime/attachments/art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + ]); + } +}); + +test('a message whose whole content was a file is still a passage', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '', [attachment('bundle-size.png')]), + assistantMessage('a1', 'ts', '包体积涨了'), + ], + }, + ]); + const result = await runRecall({ terms: ['bundle-size'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual(anchorIds(result.passages), ['u1']); + assert.equal(result.passages[0]?.messages[0]?.text, ''); +}); + +/** + * An attachment read resolves against the calling Session and refuses one + * stored elsewhere, so offering the address across a Session boundary would + * invite a call that can only fail. + */ +test('a material outside the asking Session is named without an address', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [userMessageWithFiles('u1', 'ts', '看这个', [attachment('trace.png')])], + }, + ]); + const elsewhere = await runRecall({ terms: ['trace'] }, scanDeps(data), { + activeSessionId: 's-other', + }); + assert.ok(elsewhere.ok); + const material = elsewhere.passages[0]?.messages[0]?.materials?.[0]; + assert.equal(material?.name, 'trace.png'); + assert.equal(material?.resource, undefined); + assert.equal('resource' in (material ?? {}), false); + + const here = await runRecall({ terms: ['trace'] }, scanDeps(data), { activeSessionId: 's-shot' }); + assert.ok(here.ok); + assert.ok(here.passages[0]?.messages[0]?.materials?.[0]?.resource); +}); + +test('a material whose ref has no readable address is named without one', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '本地文件', [ + attachment('notes.md', { + kind: 'doc', + mimeType: 'text/markdown', + ref: { kind: 'external_file', absolutePath: '/tmp/notes.md' }, + }), + ]), + ], + }, + ]); + const result = await runRecall({ terms: ['notes.md'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.equal(result.passages[0]?.messages[0]?.materials?.[0]?.resource, undefined); +}); + +test('a malformed attachment is skipped rather than named `undefined`', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + { + type: 'user', + id: 'u1', + turnId: 'ts', + ts: 1, + text: '看这个', + attachments: [{ kind: 'image' }, null, attachment('real.png')], + } as unknown as StoredMessage, + ], + }, + ]); + const result = await runRecall({ terms: ['看这个'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual( + result.passages[0]?.messages[0]?.materials?.map((material) => material.name), + ['real.png'], + ); +}); + +test('a credential-shaped file name is redacted on the way out', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '配置截图', [ + attachment('ghp_0123456789abcdefghij-console.png'), + ]), + ], + }, + ]); + const result = await runRecall({ terms: ['配置截图'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + const name = result.passages[0]?.messages[0]?.materials?.[0]?.name ?? ''; + assert.doesNotMatch(name, /ghp_0123456789abcdefghij/u); + assert.match(name, /\[redacted\]/u); + + // And the same term cannot be used to probe for it. + const probe = await runRecall({ terms: ['0123456789abcdefghij'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(probe.ok); + assert.equal(probe.passages.length, 0); +}); + test('a Session the source names but recall did not ask about is not read', async () => { const data = mixedCorpus(); const read: string[] = []; diff --git a/packages/core/src/recall.ts b/packages/core/src/recall.ts index 0714666052..f350356b6a 100644 --- a/packages/core/src/recall.ts +++ b/packages/core/src/recall.ts @@ -54,6 +54,8 @@ * density reflects machine output rather than relevance. */ +import { formatAttachmentResourceRef, MAX_ATTACHMENT_COUNT } from './attachments.js'; +import type { AttachmentRef } from './events.js'; import { validateWorkspacePrivacyContext } from './incognito.js'; import { redactSecrets } from './redaction.js'; import { SEARCH_QUERY_MAX_CHARS } from './search.js'; @@ -195,6 +197,76 @@ function stripSyntheticText(text: string, patterns: readonly RegExp[] | undefine return stripped; } +/** + * One file a message carried. Metadata only: recall never returns bytes, and a + * material is worth returning precisely because its bytes are expensive. + */ +export interface RecallMaterial { + readonly name: string; + readonly kind: AttachmentRef['kind']; + readonly mimeType: string; + readonly bytes: number; + /** + * Address `Read` accepts, present only when the material is reachable from + * the Session asking. Attachment reads resolve against the calling Session + * and refuse anything stored elsewhere, so offering the address across a + * Session boundary would invite a call that can only fail. + */ + readonly resource?: string; +} + +/** + * The files a message carried. A screenshot pasted under "have a look" leaves + * no trace in the text, so without this the message is unreachable by any + * term the user would think to search. + */ +export function recallMaterials(message: StoredMessage): readonly RecallMaterial[] { + const attachments = (message as { attachments?: unknown }).attachments; + if (!Array.isArray(attachments) || attachments.length === 0) return []; + const materials: RecallMaterial[] = []; + // A record older or stranger than the current shape still has to project + // something usable or nothing at all, never a material named `undefined`. + for (const candidate of attachments.slice(0, MAX_ATTACHMENT_COUNT)) { + if (!isAttachmentRef(candidate)) continue; + const resource = formatAttachmentResourceRef(candidate.ref); + materials.push({ + name: candidate.name, + kind: candidate.kind, + mimeType: candidate.mimeType, + bytes: candidate.bytes, + ...(resource ? { resource } : {}), + }); + } + return materials; +} + +function isAttachmentRef(value: unknown): value is AttachmentRef { + if (value === null || typeof value !== 'object') return false; + const candidate = value as Partial; + return ( + typeof candidate.name === 'string' && + candidate.name.length > 0 && + typeof candidate.kind === 'string' && + typeof candidate.mimeType === 'string' && + typeof candidate.bytes === 'number' && + candidate.ref !== null && + typeof candidate.ref === 'object' + ); +} + +/** + * What the predicate runs on: a message's prose plus the names of the files it + * carried, so a material is reachable by the name a person would remember. + * Names are matched but not returned as text — they come back as materials. + */ +function recallMatchableText(message: StoredMessage): string | undefined { + const prose = recallSearchableText(message); + const materials = recallMaterials(message); + if (materials.length === 0) return prose; + const names = materials.map((material) => material.name).join('\n'); + return prose === undefined || prose.length === 0 ? names : `${prose}\n${names}`; +} + /** Visits every string value in a JSON-like value, in document order, until the visitor declines. */ function collectStringLeaves(value: unknown, visit: (leaf: string) => boolean): boolean { if (typeof value === 'string') return visit(value); @@ -250,6 +322,8 @@ export interface RecallPassageMessage { readonly text: string; readonly timestamp: number; readonly isAnchor: boolean; + /** Files this message carried; omitted when it carried none. */ + readonly materials?: readonly RecallMaterial[]; } export interface RecallPassage { @@ -464,7 +538,12 @@ export async function runRecall( const sessionById = new Map(sessions.map((session) => [session.id, session])); const anchors = applySessionQuota(collected.hits, limit); - const passages = assemblePassages(collected.transcripts, anchors, sessionById); + const passages = assemblePassages( + collected.transcripts, + anchors, + sessionById, + options.activeSessionId, + ); return { ok: true, @@ -557,8 +636,7 @@ export async function expandRecallPassage( } const message = transcript.find( - (candidate) => - candidate.id === anchorMessageId && recallSearchableText(candidate) !== undefined, + (candidate) => candidate.id === anchorMessageId && isPassageMessage(candidate), ); if (!message) { return { ok: false, reason: 'not_found', message: 'That passage anchor was not found.' }; @@ -587,6 +665,7 @@ export async function expandRecallPassage( transcript, session, RECALL_PASSAGE_MAX_BYTES, + options.activeSessionId, { before, after }, ); if (!built) { @@ -866,7 +945,7 @@ function verify( return undefined; } - const raw = recallSearchableText(message); + const raw = recallMatchableText(message); if (raw === undefined) return undefined; // A hit is a term that occurs in the text as it was stored *and* still // occurs once secrets are redacted. Redaction alone is the security @@ -1006,6 +1085,7 @@ function assemblePassages( transcripts: ReadonlyMap, anchors: readonly VerifiedHit[], sessionById: ReadonlyMap, + activeSessionId: string | undefined, ): RecallPassage[] { if (anchors.length === 0) return []; const passages: RecallPassage[] = []; @@ -1014,7 +1094,13 @@ function assemblePassages( if (remaining <= 0) break; const transcript = transcripts.get(anchor.sessionId); if (!transcript) continue; - const built = buildPassage(anchor, transcript, sessionById.get(anchor.sessionId), remaining); + const built = buildPassage( + anchor, + transcript, + sessionById.get(anchor.sessionId), + remaining, + activeSessionId, + ); if (!built) continue; remaining -= built.bytes; passages.push(built.passage); @@ -1027,6 +1113,7 @@ function buildPassage( transcript: readonly StoredMessage[], session: SessionSummary | undefined, budget: number, + activeSessionId: string | undefined, span: { readonly before: number; readonly after: number } = { before: RECALL_PASSAGE_NEIGHBOURS, after: RECALL_PASSAGE_NEIGHBOURS, @@ -1051,7 +1138,7 @@ function buildPassage( const render = (message: StoredMessage, isAnchor: boolean): void => { if (rendered.has(message.id)) return; - const projected = projectPassageMessage(message, isAnchor); + const projected = projectPassageMessage(message, isAnchor, activeSessionId, anchor.sessionId); if (!projected) return; const overhead = Buffer.byteLength(JSON.stringify({ ...projected, text: '' }), 'utf8'); if (remaining <= overhead) { @@ -1090,11 +1177,7 @@ function buildPassage( // same way one beyond the span does. Reporting otherwise would tell a caller // there is nothing more in a direction recall just cut short. const omitted = (entries: readonly { message: StoredMessage }[]): boolean => - entries.some( - (entry) => - !rendered.has(entry.message.id) && - projectPassageMessage(entry.message, false) !== undefined, - ); + entries.some((entry) => !rendered.has(entry.message.id) && isPassageMessage(entry.message)); const passage: RecallPassage = { sessionId: anchor.sessionId, @@ -1167,14 +1250,37 @@ function isPassageNeighbour(message: StoredMessage): boolean { return message.type === 'user' || message.type === 'assistant' || message.type === 'tool_call'; } +/** + * Whether a message can appear in a passage at all. Redaction rewrites text + * but never empties it, so this decides the same set `projectPassageMessage` + * does without paying for redaction on every message a lookup walks past. + */ +function isPassageMessage(message: StoredMessage): boolean { + const raw = recallSearchableText(message); + if (raw !== undefined && raw.trim().length > 0) return true; + return recallMaterials(message).length > 0; +} + function projectPassageMessage( message: StoredMessage, isAnchor: boolean, + activeSessionId?: string, + sessionId?: string, ): RecallPassageMessage | undefined { const raw = recallSearchableText(message); - if (raw === undefined) return undefined; - const text = redactSecrets(raw).trim(); - if (text.length === 0) return undefined; + const text = raw === undefined ? '' : redactSecrets(raw).trim(); + // A message whose whole content was a pasted file has no text of its own. + // Dropping it would make the file unreachable in exactly the case this + // layer exists for. + // A file name is user-authored text like any other, so it leaves through the + // same redaction the passage body does; the address beside it is a runtime + // identifier and carries nothing to redact. + const materials = recallMaterials(message).map((material) => ({ + ...material, + name: redactSecrets(material.name), + })); + if (text.length === 0 && materials.length === 0) return undefined; + const reachable = sessionId !== undefined && sessionId === activeSessionId; return { messageId: message.id, role: passageRole(message), @@ -1182,6 +1288,13 @@ function projectPassageMessage( text, timestamp: message.ts, isAnchor, + ...(materials.length > 0 + ? { + materials: reachable + ? materials + : materials.map(({ resource: _resource, ...rest }) => rest), + } + : {}), }; } diff --git a/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts b/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts index d372d00fd9..db0fff9d51 100644 --- a/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts +++ b/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts @@ -232,6 +232,77 @@ describe('recall over the corpus a live workspace actually has', () => { }); }); + /** + * The file name has to survive into the event payload for a payload scan to + * offer the Session, and into the projection for the predicate to match it. + * A double could satisfy either half alone. + */ + test('a pasted file is reachable by name through the real stores', async () => { + await withWorkspace(async (workspace) => { + const session = await workspace.sessions.create(makeInput('screenshots')); + const runId = 'run-shot'; + const turnId = `turn-${runId}`; + const identity = { sessionId: session.id, invocationId: runId, runId, turnId }; + await workspace.runtime.appendRuntimeEvent( + session.id, + runId, + testInvocationOpenedEvent({ sessionId: session.id, runId, turnId, openedAt: 100 }), + ); + await workspace.runtime.appendRuntimeEvent(session.id, runId, { + ...identity, + id: `${runId}-user`, + ts: 101, + partial: false, + role: 'user', + author: 'user', + content: { + kind: 'text', + text: '你看看', + attachments: [ + { + kind: 'image', + name: 'pipeline-failure.png', + mimeType: 'image/png', + bytes: 2048, + ref: { + kind: 'session_file', + sessionId: session.id, + relativePath: 'art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + }, + ], + }, + }); + await workspace.runtime.appendRuntimeEvent(session.id, runId, { + ...identity, + id: `${runId}-end`, + ts: 103, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }); + + const options = { includeArchived: true, activeSessionId: session.id }; + for (const [name, deps] of [ + ['full', scanDeps(workspace)], + ['narrowed', narrowedDeps(workspace)], + ] as const) { + const result = await runRecall({ terms: ['pipeline-failure'] }, deps, options); + assert.ok(result.ok, name); + assert.deepEqual(anchors(result), [`${runId}-user`], name); + const material = result.passages[0]?.messages[0]?.materials?.[0]; + assert.equal(material?.name, 'pipeline-failure.png', name); + assert.equal( + material?.resource, + 'maka://runtime/attachments/art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + name, + ); + } + }); + }); + test('narrowing reads only the Sessions that can match', async () => { await withWorkspace(async (workspace) => { const hit = await workspace.sessions.create(makeInput('hit')); diff --git a/packages/runtime/src/recall-tools.ts b/packages/runtime/src/recall-tools.ts index fe4dfd34d4..1c5a0a384a 100644 --- a/packages/runtime/src/recall-tools.ts +++ b/packages/runtime/src/recall-tools.ts @@ -58,6 +58,8 @@ export function buildRecallTool(deps: RecallToolDeps): MakaTool { 'Supply a few distinct literal terms rather than a sentence: matching is case-insensitive substring, OR-combined, ' + 'and results rank higher when they contain more of the terms. Returns distilled facts, ranked transcript passages ' + 'that already carry the surrounding exchange, and a note on what the search did not reach. ' + + 'A message that carried files lists them under materials, matched by file name; a material carrying a resource ' + + 'address can be opened with Read, and one without it lives in another Session and cannot be opened from here. ' + 'One Recall call usually suffices; use RecallMore only when a passage is cut short.', parameters: z .object({ @@ -232,6 +234,21 @@ function projectPassage(passage: RecallPassage, activeSessionId: string) { timestamp: message.timestamp, ...(message.isAnchor ? { is_anchor: true } : {}), text: message.text, + // Metadata only. `resource` is present exactly when the file is + // readable from the Session asking; elsewhere the material is named + // but has no address, because an attachment read resolves against the + // calling Session and would refuse one stored in another. + ...(message.materials + ? { + materials: message.materials.map((material) => ({ + name: material.name, + kind: material.kind, + mime_type: material.mimeType, + bytes: material.bytes, + ...(material.resource ? { resource: material.resource } : {}), + })), + } + : {}), })), }; }