diff --git a/packages/core/src/__tests__/recall.test.ts b/packages/core/src/__tests__/recall.test.ts index f838696cd6..da55e9e736 100644 --- a/packages/core/src/__tests__/recall.test.ts +++ b/packages/core/src/__tests__/recall.test.ts @@ -53,6 +53,30 @@ function userMessage(id: string, turnId: string, text: string): StoredMessage { return { type: 'user', id, turnId, ts: (nextTs += 1000), text } as StoredMessage; } +function attachment(name: string, overrides: Record = {}) { + return { + kind: 'image', + name, + mimeType: 'image/png', + bytes: 2048, + ref: { + kind: 'session_file', + sessionId: 's-shot', + relativePath: 'art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + ...overrides, + }; +} + +function userMessageWithFiles( + id: string, + turnId: string, + text: string, + attachments: readonly ReturnType[], +): StoredMessage { + return { type: 'user', id, turnId, ts: (nextTs += 1000), text, attachments } as StoredMessage; +} + function assistantMessage(id: string, turnId: string, text: string): StoredMessage { return { type: 'assistant', id, turnId, ts: (nextTs += 1000), text } as StoredMessage; } @@ -868,6 +892,162 @@ test('the passage budget keeps the nearest context and says when it cut the rest ); }); +/** + * A screenshot pasted under "have a look" leaves no trace in the text, so the + * message is unreachable by any term a person would think of. Matching the + * file name is what makes the file findable at all. + */ +test('a message is reachable by the name of the file it carried', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '你看看', [attachment('pipeline-failure.png')]), + assistantMessage('a1', 'ts', '这是 CI 挂了'), + ], + }, + ]); + for (const deps of [scanDeps(data), candidateDeps(data)]) { + const result = await runRecall({ terms: ['pipeline-failure'] }, deps, { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual(anchorIds(result.passages), ['u1']); + const anchor = result.passages[0]?.messages.find((message) => message.isAnchor); + // The name is matched, but it comes back as a material rather than as text + // the user never typed. + assert.equal(anchor?.text, '你看看'); + assert.deepEqual(anchor?.materials, [ + { + name: 'pipeline-failure.png', + kind: 'image', + mimeType: 'image/png', + bytes: 2048, + resource: 'maka://runtime/attachments/art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + ]); + } +}); + +test('a message whose whole content was a file is still a passage', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '', [attachment('bundle-size.png')]), + assistantMessage('a1', 'ts', '包体积涨了'), + ], + }, + ]); + const result = await runRecall({ terms: ['bundle-size'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual(anchorIds(result.passages), ['u1']); + assert.equal(result.passages[0]?.messages[0]?.text, ''); +}); + +/** + * An attachment read resolves against the calling Session and refuses one + * stored elsewhere, so offering the address across a Session boundary would + * invite a call that can only fail. + */ +test('a material outside the asking Session is named without an address', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [userMessageWithFiles('u1', 'ts', '看这个', [attachment('trace.png')])], + }, + ]); + const elsewhere = await runRecall({ terms: ['trace'] }, scanDeps(data), { + activeSessionId: 's-other', + }); + assert.ok(elsewhere.ok); + const material = elsewhere.passages[0]?.messages[0]?.materials?.[0]; + assert.equal(material?.name, 'trace.png'); + assert.equal(material?.resource, undefined); + assert.equal('resource' in (material ?? {}), false); + + const here = await runRecall({ terms: ['trace'] }, scanDeps(data), { activeSessionId: 's-shot' }); + assert.ok(here.ok); + assert.ok(here.passages[0]?.messages[0]?.materials?.[0]?.resource); +}); + +test('a material whose ref has no readable address is named without one', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '本地文件', [ + attachment('notes.md', { + kind: 'doc', + mimeType: 'text/markdown', + ref: { kind: 'external_file', absolutePath: '/tmp/notes.md' }, + }), + ]), + ], + }, + ]); + const result = await runRecall({ terms: ['notes.md'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.equal(result.passages[0]?.messages[0]?.materials?.[0]?.resource, undefined); +}); + +test('a malformed attachment is skipped rather than named `undefined`', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + { + type: 'user', + id: 'u1', + turnId: 'ts', + ts: 1, + text: '看这个', + attachments: [{ kind: 'image' }, null, attachment('real.png')], + } as unknown as StoredMessage, + ], + }, + ]); + const result = await runRecall({ terms: ['看这个'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + assert.deepEqual( + result.passages[0]?.messages[0]?.materials?.map((material) => material.name), + ['real.png'], + ); +}); + +test('a credential-shaped file name is redacted on the way out', async () => { + const data = corpus([ + { + session: session('s-shot', 'screenshots'), + messages: [ + userMessageWithFiles('u1', 'ts', '配置截图', [ + attachment('ghp_0123456789abcdefghij-console.png'), + ]), + ], + }, + ]); + const result = await runRecall({ terms: ['配置截图'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(result.ok); + const name = result.passages[0]?.messages[0]?.materials?.[0]?.name ?? ''; + assert.doesNotMatch(name, /ghp_0123456789abcdefghij/u); + assert.match(name, /\[redacted\]/u); + + // And the same term cannot be used to probe for it. + const probe = await runRecall({ terms: ['0123456789abcdefghij'] }, scanDeps(data), { + activeSessionId: 's-shot', + }); + assert.ok(probe.ok); + assert.equal(probe.passages.length, 0); +}); + test('a Session the source names but recall did not ask about is not read', async () => { const data = mixedCorpus(); const read: string[] = []; diff --git a/packages/core/src/recall.ts b/packages/core/src/recall.ts index 0714666052..f350356b6a 100644 --- a/packages/core/src/recall.ts +++ b/packages/core/src/recall.ts @@ -54,6 +54,8 @@ * density reflects machine output rather than relevance. */ +import { formatAttachmentResourceRef, MAX_ATTACHMENT_COUNT } from './attachments.js'; +import type { AttachmentRef } from './events.js'; import { validateWorkspacePrivacyContext } from './incognito.js'; import { redactSecrets } from './redaction.js'; import { SEARCH_QUERY_MAX_CHARS } from './search.js'; @@ -195,6 +197,76 @@ function stripSyntheticText(text: string, patterns: readonly RegExp[] | undefine return stripped; } +/** + * One file a message carried. Metadata only: recall never returns bytes, and a + * material is worth returning precisely because its bytes are expensive. + */ +export interface RecallMaterial { + readonly name: string; + readonly kind: AttachmentRef['kind']; + readonly mimeType: string; + readonly bytes: number; + /** + * Address `Read` accepts, present only when the material is reachable from + * the Session asking. Attachment reads resolve against the calling Session + * and refuse anything stored elsewhere, so offering the address across a + * Session boundary would invite a call that can only fail. + */ + readonly resource?: string; +} + +/** + * The files a message carried. A screenshot pasted under "have a look" leaves + * no trace in the text, so without this the message is unreachable by any + * term the user would think to search. + */ +export function recallMaterials(message: StoredMessage): readonly RecallMaterial[] { + const attachments = (message as { attachments?: unknown }).attachments; + if (!Array.isArray(attachments) || attachments.length === 0) return []; + const materials: RecallMaterial[] = []; + // A record older or stranger than the current shape still has to project + // something usable or nothing at all, never a material named `undefined`. + for (const candidate of attachments.slice(0, MAX_ATTACHMENT_COUNT)) { + if (!isAttachmentRef(candidate)) continue; + const resource = formatAttachmentResourceRef(candidate.ref); + materials.push({ + name: candidate.name, + kind: candidate.kind, + mimeType: candidate.mimeType, + bytes: candidate.bytes, + ...(resource ? { resource } : {}), + }); + } + return materials; +} + +function isAttachmentRef(value: unknown): value is AttachmentRef { + if (value === null || typeof value !== 'object') return false; + const candidate = value as Partial; + return ( + typeof candidate.name === 'string' && + candidate.name.length > 0 && + typeof candidate.kind === 'string' && + typeof candidate.mimeType === 'string' && + typeof candidate.bytes === 'number' && + candidate.ref !== null && + typeof candidate.ref === 'object' + ); +} + +/** + * What the predicate runs on: a message's prose plus the names of the files it + * carried, so a material is reachable by the name a person would remember. + * Names are matched but not returned as text — they come back as materials. + */ +function recallMatchableText(message: StoredMessage): string | undefined { + const prose = recallSearchableText(message); + const materials = recallMaterials(message); + if (materials.length === 0) return prose; + const names = materials.map((material) => material.name).join('\n'); + return prose === undefined || prose.length === 0 ? names : `${prose}\n${names}`; +} + /** Visits every string value in a JSON-like value, in document order, until the visitor declines. */ function collectStringLeaves(value: unknown, visit: (leaf: string) => boolean): boolean { if (typeof value === 'string') return visit(value); @@ -250,6 +322,8 @@ export interface RecallPassageMessage { readonly text: string; readonly timestamp: number; readonly isAnchor: boolean; + /** Files this message carried; omitted when it carried none. */ + readonly materials?: readonly RecallMaterial[]; } export interface RecallPassage { @@ -464,7 +538,12 @@ export async function runRecall( const sessionById = new Map(sessions.map((session) => [session.id, session])); const anchors = applySessionQuota(collected.hits, limit); - const passages = assemblePassages(collected.transcripts, anchors, sessionById); + const passages = assemblePassages( + collected.transcripts, + anchors, + sessionById, + options.activeSessionId, + ); return { ok: true, @@ -557,8 +636,7 @@ export async function expandRecallPassage( } const message = transcript.find( - (candidate) => - candidate.id === anchorMessageId && recallSearchableText(candidate) !== undefined, + (candidate) => candidate.id === anchorMessageId && isPassageMessage(candidate), ); if (!message) { return { ok: false, reason: 'not_found', message: 'That passage anchor was not found.' }; @@ -587,6 +665,7 @@ export async function expandRecallPassage( transcript, session, RECALL_PASSAGE_MAX_BYTES, + options.activeSessionId, { before, after }, ); if (!built) { @@ -866,7 +945,7 @@ function verify( return undefined; } - const raw = recallSearchableText(message); + const raw = recallMatchableText(message); if (raw === undefined) return undefined; // A hit is a term that occurs in the text as it was stored *and* still // occurs once secrets are redacted. Redaction alone is the security @@ -1006,6 +1085,7 @@ function assemblePassages( transcripts: ReadonlyMap, anchors: readonly VerifiedHit[], sessionById: ReadonlyMap, + activeSessionId: string | undefined, ): RecallPassage[] { if (anchors.length === 0) return []; const passages: RecallPassage[] = []; @@ -1014,7 +1094,13 @@ function assemblePassages( if (remaining <= 0) break; const transcript = transcripts.get(anchor.sessionId); if (!transcript) continue; - const built = buildPassage(anchor, transcript, sessionById.get(anchor.sessionId), remaining); + const built = buildPassage( + anchor, + transcript, + sessionById.get(anchor.sessionId), + remaining, + activeSessionId, + ); if (!built) continue; remaining -= built.bytes; passages.push(built.passage); @@ -1027,6 +1113,7 @@ function buildPassage( transcript: readonly StoredMessage[], session: SessionSummary | undefined, budget: number, + activeSessionId: string | undefined, span: { readonly before: number; readonly after: number } = { before: RECALL_PASSAGE_NEIGHBOURS, after: RECALL_PASSAGE_NEIGHBOURS, @@ -1051,7 +1138,7 @@ function buildPassage( const render = (message: StoredMessage, isAnchor: boolean): void => { if (rendered.has(message.id)) return; - const projected = projectPassageMessage(message, isAnchor); + const projected = projectPassageMessage(message, isAnchor, activeSessionId, anchor.sessionId); if (!projected) return; const overhead = Buffer.byteLength(JSON.stringify({ ...projected, text: '' }), 'utf8'); if (remaining <= overhead) { @@ -1090,11 +1177,7 @@ function buildPassage( // same way one beyond the span does. Reporting otherwise would tell a caller // there is nothing more in a direction recall just cut short. const omitted = (entries: readonly { message: StoredMessage }[]): boolean => - entries.some( - (entry) => - !rendered.has(entry.message.id) && - projectPassageMessage(entry.message, false) !== undefined, - ); + entries.some((entry) => !rendered.has(entry.message.id) && isPassageMessage(entry.message)); const passage: RecallPassage = { sessionId: anchor.sessionId, @@ -1167,14 +1250,37 @@ function isPassageNeighbour(message: StoredMessage): boolean { return message.type === 'user' || message.type === 'assistant' || message.type === 'tool_call'; } +/** + * Whether a message can appear in a passage at all. Redaction rewrites text + * but never empties it, so this decides the same set `projectPassageMessage` + * does without paying for redaction on every message a lookup walks past. + */ +function isPassageMessage(message: StoredMessage): boolean { + const raw = recallSearchableText(message); + if (raw !== undefined && raw.trim().length > 0) return true; + return recallMaterials(message).length > 0; +} + function projectPassageMessage( message: StoredMessage, isAnchor: boolean, + activeSessionId?: string, + sessionId?: string, ): RecallPassageMessage | undefined { const raw = recallSearchableText(message); - if (raw === undefined) return undefined; - const text = redactSecrets(raw).trim(); - if (text.length === 0) return undefined; + const text = raw === undefined ? '' : redactSecrets(raw).trim(); + // A message whose whole content was a pasted file has no text of its own. + // Dropping it would make the file unreachable in exactly the case this + // layer exists for. + // A file name is user-authored text like any other, so it leaves through the + // same redaction the passage body does; the address beside it is a runtime + // identifier and carries nothing to redact. + const materials = recallMaterials(message).map((material) => ({ + ...material, + name: redactSecrets(material.name), + })); + if (text.length === 0 && materials.length === 0) return undefined; + const reachable = sessionId !== undefined && sessionId === activeSessionId; return { messageId: message.id, role: passageRole(message), @@ -1182,6 +1288,13 @@ function projectPassageMessage( text, timestamp: message.ts, isAnchor, + ...(materials.length > 0 + ? { + materials: reachable + ? materials + : materials.map(({ resource: _resource, ...rest }) => rest), + } + : {}), }; } diff --git a/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts b/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts index d372d00fd9..db0fff9d51 100644 --- a/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts +++ b/packages/runtime/src/__tests__/recall-ledger-corpus.test.ts @@ -232,6 +232,77 @@ describe('recall over the corpus a live workspace actually has', () => { }); }); + /** + * The file name has to survive into the event payload for a payload scan to + * offer the Session, and into the projection for the predicate to match it. + * A double could satisfy either half alone. + */ + test('a pasted file is reachable by name through the real stores', async () => { + await withWorkspace(async (workspace) => { + const session = await workspace.sessions.create(makeInput('screenshots')); + const runId = 'run-shot'; + const turnId = `turn-${runId}`; + const identity = { sessionId: session.id, invocationId: runId, runId, turnId }; + await workspace.runtime.appendRuntimeEvent( + session.id, + runId, + testInvocationOpenedEvent({ sessionId: session.id, runId, turnId, openedAt: 100 }), + ); + await workspace.runtime.appendRuntimeEvent(session.id, runId, { + ...identity, + id: `${runId}-user`, + ts: 101, + partial: false, + role: 'user', + author: 'user', + content: { + kind: 'text', + text: '你看看', + attachments: [ + { + kind: 'image', + name: 'pipeline-failure.png', + mimeType: 'image/png', + bytes: 2048, + ref: { + kind: 'session_file', + sessionId: session.id, + relativePath: 'art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + }, + }, + ], + }, + }); + await workspace.runtime.appendRuntimeEvent(session.id, runId, { + ...identity, + id: `${runId}-end`, + ts: 103, + partial: false, + role: 'system', + author: 'system', + status: 'completed', + actions: { endInvocation: true }, + }); + + const options = { includeArchived: true, activeSessionId: session.id }; + for (const [name, deps] of [ + ['full', scanDeps(workspace)], + ['narrowed', narrowedDeps(workspace)], + ] as const) { + const result = await runRecall({ terms: ['pipeline-failure'] }, deps, options); + assert.ok(result.ok, name); + assert.deepEqual(anchors(result), [`${runId}-user`], name); + const material = result.passages[0]?.messages[0]?.materials?.[0]; + assert.equal(material?.name, 'pipeline-failure.png', name); + assert.equal( + material?.resource, + 'maka://runtime/attachments/art_01HQ8Z3K4M5N6P7Q8R9S0T1V2W', + name, + ); + } + }); + }); + test('narrowing reads only the Sessions that can match', async () => { await withWorkspace(async (workspace) => { const hit = await workspace.sessions.create(makeInput('hit')); diff --git a/packages/runtime/src/recall-tools.ts b/packages/runtime/src/recall-tools.ts index fe4dfd34d4..1c5a0a384a 100644 --- a/packages/runtime/src/recall-tools.ts +++ b/packages/runtime/src/recall-tools.ts @@ -58,6 +58,8 @@ export function buildRecallTool(deps: RecallToolDeps): MakaTool { 'Supply a few distinct literal terms rather than a sentence: matching is case-insensitive substring, OR-combined, ' + 'and results rank higher when they contain more of the terms. Returns distilled facts, ranked transcript passages ' + 'that already carry the surrounding exchange, and a note on what the search did not reach. ' + + 'A message that carried files lists them under materials, matched by file name; a material carrying a resource ' + + 'address can be opened with Read, and one without it lives in another Session and cannot be opened from here. ' + 'One Recall call usually suffices; use RecallMore only when a passage is cut short.', parameters: z .object({ @@ -232,6 +234,21 @@ function projectPassage(passage: RecallPassage, activeSessionId: string) { timestamp: message.timestamp, ...(message.isAnchor ? { is_anchor: true } : {}), text: message.text, + // Metadata only. `resource` is present exactly when the file is + // readable from the Session asking; elsewhere the material is named + // but has no address, because an attachment read resolves against the + // calling Session and would refuse one stored in another. + ...(message.materials + ? { + materials: message.materials.map((material) => ({ + name: material.name, + kind: material.kind, + mime_type: material.mimeType, + bytes: material.bytes, + ...(material.resource ? { resource: material.resource } : {}), + })), + } + : {}), })), }; }