From d6a670d9a871afa5f68a8b82590dcf11327f90c4 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:31:36 +0800 Subject: [PATCH 1/8] fix(storage): bound Codex filesystem cursors Generated-by: Codex --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../__tests__/codex-session-adapter.test.ts | 39 +++++++++++++++++++ packages/storage/src/codex-session-adapter.ts | 35 +++++++++-------- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 6e4bbf63a2..0c5ce7caa7 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -106,7 +106,7 @@ Host requests at most `page size + 1` source entries. Each returned entry carrie Codex keeps no server-side catalog snapshot, SQLite transaction, TTL, or LRU across requests. Cursors are bound to the current query. State DB pages order by `(sort_key DESC, id DESC)`, where `sort_key` is computed once by the query, selected alongside the row, and read straight back off that row to build the cursor — so the position a cursor names is by construction the position the query ordered by. Seconds and milliseconds are normalized into that one numeric key before ordering. The first page reads the newest `state_N.sqlite`; if that generation cannot be read, the page is served by the filesystem fallback rather than by an older generation, because a lower generation is the snapshot frozen at the last bump and is missing everything created since. Continuation names the generation it started on, uses a SQL keyset condition, and stays strict — a missing original generation invalidates the cursor, and a transient read failure remains a persistence failure. Connections close after each request. -The filesystem fallback orders by `(mtime DESC, relative path ASC)` across active and optionally archived roots. One traversal has a `maxCatalogCandidates` file bound; exceeding it returns a typed source limit. Stat-known keys reject candidates that cannot enter the current page before reading their bounded heads. The page retains at most `limit + 1` matching summaries instead of materializing the corpus or rescanning it repeatedly for deep pages. +The filesystem fallback orders by `(mtime DESC, fixed-size path identity ASC)` across active and optionally archived roots. The identity is derived once from the relative rollout path, so deeply nested paths cannot enlarge the cursor past the Host wire bound. One traversal has a `maxCatalogCandidates` file bound; exceeding it returns a typed source limit. Stat-known keys reject candidates that cannot enter the current page before reading their bounded heads. The page retains at most `limit + 1` matching summaries instead of materializing the corpus or rescanning it repeatedly for deep pages. Keysets resume strictly after the last delivered record. A live source updated between pages may move ahead of the cursor and be absent from that traversal; a fresh catalog query sees the new order. The design does not claim a stable snapshot of mutable external data. diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index acc5a3b253..1ff65d3587 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -171,7 +171,7 @@ Codex 不保存跨请求的 SQLite 事务、catalog snapshot、TTL 或 LRU 状 两条来源路径分别使用自己的稳定排序键: 1. **state DB**:`(sort_key DESC, id DESC)`。`sort_key` 由查询算一次、随行一起选出,cursor 直接读回该行上的这个值 —— 这样 cursor 指向的位置必然就是查询排序的位置。秒级与毫秒级时间戳先统一成这一个数值键再排序。首页只读最新的 `state_N.sqlite`;若该 generation 读不了,本页改由 filesystem fallback 回答,而**不是**退到更旧的 generation:更旧那本是上一次跃迁时冻结的快照,跃迁之后新建的会话都不在里面。cursor 记录起始 generation,续页仍只读打开同一个文件,通过 `WHERE` keyset 条件继续,并保持严格 —— 原 generation 已删除或不可读时 cursor 明确失效,读失败仍是 persistence failure。连接用完立即关闭。 -2. **filesystem fallback**:`(mtime DESC, relative path ASC)`。一次遍历 active 和可选 archived roots,以 `maxCatalogCandidates` 限制遍历的文件数;超过上限返回 typed limit error。候选先用 stat 已知排序键与当前页尾比较,只有可能进入当前页的候选才读取有界 head 并完成 query/path 校验;内存最多保留 `limit + 1` 个匹配摘要,不物化整个 catalog,也不为深分页重复扫描多轮。 +2. **filesystem fallback**:`(mtime DESC, fixed-size path identity ASC)`。identity 由相对 rollout path 一次派生,因此深层路径不会使 cursor 超过 Host wire 上限。一次遍历 active 和可选 archived roots,以 `maxCatalogCandidates` 限制遍历的文件数;超过上限返回 typed limit error。候选先用 stat 已知排序键与当前页尾比较,只有可能进入当前页的候选才读取有界 head 并完成 query/path 校验;内存最多保留 `limit + 1` 个匹配摘要,不物化整个 catalog,也不为深分页重复扫描多轮。 keyset 的语义是“继续读取严格排在最后交付项之后的记录”。如果一个尚未读取的 live Session 在两页之间更新并移动到 cursor 之前,本次遍历可能看不到它,但不会因此重复已经交付的行;重新打开或刷新 catalog 会看到当前最新顺序。这是实时可变来源下不持有 snapshot 的明确边界。 diff --git a/packages/storage/src/__tests__/codex-session-adapter.test.ts b/packages/storage/src/__tests__/codex-session-adapter.test.ts index a3a5a01538..1586de4680 100644 --- a/packages/storage/src/__tests__/codex-session-adapter.test.ts +++ b/packages/storage/src/__tests__/codex-session-adapter.test.ts @@ -698,6 +698,45 @@ describe('CodexSessionAdapter', () => { }); }); + test('filesystem cursor stays wire-bounded for deeply nested rollout paths', async () => { + await withCodexHome(async (codexHome) => { + const nestedDirectory = join( + codexHome, + 'sessions', + ...Array.from({ length: 12 }, (_, index) => `${index}-${'nested'.repeat(8)}`), + ); + await mkdir(nestedDirectory, { recursive: true }); + const nestedId = 'codex-deep-cursor'; + const nestedPath = join(nestedDirectory, `rollout-${nestedId}.jsonl`); + await writeFile(nestedPath, minimalRollout(nestedId, '/workspace/root', 'deep')); + const shallowPath = await seedMinimalRollout( + codexHome, + 'codex-shallow-cursor', + false, + '/workspace/root', + 'shallow', + ); + const tied = new Date('2026-08-08T00:00:00Z'); + await utimes(nestedPath, tied, tied); + await utimes(shallowPath, tied, tied); + const adapter = new CodexSessionAdapter({ codexHome }); + + const first = await adapter.listSessionPage({ limit: 1 }); + assert.equal(first.hasMore, true); + assert.ok(Buffer.byteLength(first.items[0]!.nextCursor, 'utf8') <= 512); + const second = await adapter.listSessionPage({ + cursor: first.items[0]!.nextCursor, + limit: 1, + }); + + assert.equal(second.hasMore, false); + assert.deepEqual( + new Set([...first.items, ...second.items].map(({ summary }) => summary.id)), + new Set([nestedId, 'codex-shallow-cursor']), + ); + }); + }); + test('database keyset paging stays on the state generation that issued the cursor', async () => { await withCodexHome(async (codexHome) => { const oldRows: StateRow[] = []; diff --git a/packages/storage/src/codex-session-adapter.ts b/packages/storage/src/codex-session-adapter.ts index 6efc91ce72..4023d28793 100644 --- a/packages/storage/src/codex-session-adapter.ts +++ b/packages/storage/src/codex-session-adapter.ts @@ -17,6 +17,7 @@ * under the License. */ +import { createHash } from 'node:crypto'; import { open, opendir, readdir, realpath, stat } from 'node:fs/promises'; import { homedir } from 'node:os'; import { basename, join, resolve, sep } from 'node:path'; @@ -103,7 +104,7 @@ type CodexCatalogKeyset = readonly sortTimestamp: number; readonly id: string; } - | { readonly kind: 'filesystem'; readonly mtimeMs: number; readonly pathKey: string }; + | { readonly kind: 'filesystem'; readonly mtimeMs: number; readonly pathIdentity: string }; /** * Read-only adapter for Codex rollout JSONL. @@ -335,7 +336,7 @@ export class CodexSessionAdapter implements ExternalSessionAdapter { interface RolloutCandidate { path: string; - catalogKey: string; + catalogIdentity: string; mtimeMs: number; archived: boolean; } @@ -1007,9 +1008,10 @@ async function* iterateRolloutFiles( entry.name.endsWith('.jsonl') ) { try { + const catalogKey = `${archived ? 'a' : 's'}/${relativePath}`; yield { path, - catalogKey: `${archived ? 'a' : 's'}/${relativePath}`, + catalogIdentity: createHash('sha256').update(catalogKey).digest('base64url'), mtimeMs: (await stat(path)).mtimeMs, archived, }; @@ -1076,10 +1078,10 @@ async function nextRolloutCatalogBatch( } function compareRolloutCandidates( - left: Pick, - right: Pick, + left: Pick, + right: Pick, ): number { - return right.mtimeMs - left.mtimeMs || left.catalogKey.localeCompare(right.catalogKey); + return right.mtimeMs - left.mtimeMs || left.catalogIdentity.localeCompare(right.catalogIdentity); } function rolloutCandidateIsAfter( @@ -1089,7 +1091,7 @@ function rolloutCandidateIsAfter( return ( compareRolloutCandidates(candidate, { mtimeMs: keyset.mtimeMs, - catalogKey: keyset.pathKey, + catalogIdentity: keyset.pathIdentity, }) > 0 ); } @@ -1097,7 +1099,11 @@ function rolloutCandidateIsAfter( function candidateKeyset( candidate: RolloutCandidate, ): Extract { - return { kind: 'filesystem', mtimeMs: candidate.mtimeMs, pathKey: candidate.catalogKey }; + return { + kind: 'filesystem', + mtimeMs: candidate.mtimeMs, + pathIdentity: candidate.catalogIdentity, + }; } function encodeCatalogKeyset( @@ -1108,7 +1114,7 @@ function encodeCatalogKeyset( if (keyset.kind === 'database') { return `d:${queryHash}:${Buffer.from(keyset.stateDatabase).toString('base64url')}:${encodeCursorNumber(keyset.sortTimestamp)}:${Buffer.from(keyset.id).toString('base64url')}`; } - return `f:${queryHash}:${encodeCursorNumber(keyset.mtimeMs)}:${Buffer.from(keyset.pathKey).toString('base64url')}`; + return `f:${queryHash}:${encodeCursorNumber(keyset.mtimeMs)}:${keyset.pathIdentity}`; } function decodeCatalogKeyset( @@ -1141,16 +1147,11 @@ function decodeCatalogKeyset( if (parts[0] === 'f') { if (parts.length !== 4) throw new ExternalSessionCatalogCursorError(); const mtimeMs = decodeCursorNumber(parts[2]!); - const encodedPathKey = parts[3]!; - const pathKey = Buffer.from(encodedPathKey, 'base64url').toString('utf8'); - if ( - Buffer.byteLength(pathKey, 'utf8') > 320 || - Buffer.from(pathKey).toString('base64url') !== encodedPathKey || - !/^[as]\/[^\u0000-\u001f\u007f]+$/.test(pathKey) - ) { + const pathIdentity = parts[3]!; + if (!/^[A-Za-z0-9_-]{43}$/.test(pathIdentity)) { throw new ExternalSessionCatalogCursorError(); } - return { kind: 'filesystem', mtimeMs, pathKey }; + return { kind: 'filesystem', mtimeMs, pathIdentity }; } throw new ExternalSessionCatalogCursorError(); } From 1b13b30b189906467252f265cba48112a9616db8 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:32:35 +0800 Subject: [PATCH 2/8] fix(core): match external Session source ids Generated-by: Codex --- docs/architecture/external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../core/src/__tests__/external-session-query.test.ts | 10 ++++++++++ packages/core/src/external-session.ts | 5 +++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 0c5ce7caa7..30cfb0751b 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -49,7 +49,7 @@ The design does not track source updates or incrementally synchronize an importe | Obligation | Single authority | Public seam | | --- | --- | --- | | Source format, discovery, filtering, paging, decoding, and conversion | Corresponding Storage adapter | `listSessionPage(query)`, `readSession(id)` | -| Shared query, sanitize, and limit contracts | Core external-session | Contracts consumed by adapters and Host | +| Shared query, source Session ID/title/cwd matching, sanitize, and limit contracts | Core external-session | Contracts consumed by adapters and Host | | Workspace resolution, import concurrency, result classification, staging, publication, and recovery | Runtime Host external-session coordinator | `external-session.catalog.query`, `external-session.import` | | Current published import count and recent Maka Session IDs | Storage Session authority | `lookupExternalSessionImports(adapterId, sourceSessionIds, limit)`, projected by Host as `importState` | | Provider admission for stored history | Runtime replay planner | `buildRuntimeEventModelReplayPlan`; continuation has separate admission | diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 1ff65d3587..93b3f0f17a 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -79,7 +79,7 @@ TUI / Desktop 选择来源与外部 Session ## 1 · 模块边界 -- `packages/core/external-session` 只定义跨来源 contract、query、sanitize 和 limit 语义,不理解任何来源的文件格式。 +- `packages/core/external-session` 定义跨来源 contract、query、来源 Session ID/标题/cwd 匹配、sanitize 和 limit 语义,不理解任何来源的文件格式。 - storage adapter 各自拥有 Claude、Codex、OpenCode 的发现、筛选、分页、解码和消息转换规则。 - Runtime Host 拥有 workspace 解析、wire 边界、导入并发、错误分类、暂存和发布。 - TUI 与 Desktop 只展示 catalog、提交选择、按稳定结果码更新交互,不解析来源数据或错误字符串。 diff --git a/packages/core/src/__tests__/external-session-query.test.ts b/packages/core/src/__tests__/external-session-query.test.ts index dad026aebf..435db97500 100644 --- a/packages/core/src/__tests__/external-session-query.test.ts +++ b/packages/core/src/__tests__/external-session-query.test.ts @@ -50,6 +50,16 @@ describe('externalSessionMatchesQuery', () => { assert.equal(externalSessionMatchesQuery(summary(), { text: '/Users/z' }), true); }); + test('a term matches the source Session id', () => { + assert.equal(externalSessionMatchesQuery(summary(), { text: 'SESS-1' }), true); + assert.equal( + externalSessionMatchesQuery(summary({ id: '01JY7W3FKJZKQ2G7A2P9V4M8NE' }), { + text: 'q2g7a2p9', + }), + true, + ); + }); + test('a term that matches neither excludes the row', () => { assert.equal(externalSessionMatchesQuery(summary(), { text: 'kubernetes' }), false); }); diff --git a/packages/core/src/external-session.ts b/packages/core/src/external-session.ts index b47658c01f..077dafec52 100644 --- a/packages/core/src/external-session.ts +++ b/packages/core/src/external-session.ts @@ -128,8 +128,8 @@ export function externalSessionMatchesQuery( if (query.cwd !== undefined && !sameExternalSessionPath(summary.cwd, query.cwd)) return false; const text = normalizeExternalSessionQueryText(query.text); if (text === undefined) return true; - // Title and path, because those are the two things a user remembers about a - // conversation they are looking for. Both already sit on the summary, so + // Source id, title, and path are the bounded summary fields a user can paste + // or remember about a conversation. All already sit on the summary, so // matching costs no extra reads. Message content is deliberately excluded: // it would mean opening every transcript on every keystroke. // @@ -143,6 +143,7 @@ export function externalSessionMatchesQuery( // `\\n` from finding the very title it names. The path pair has no such // ambiguity: a separator there is a separator. return ( + normalizeExternalSessionMatchText(summary.id).includes(text) || normalizeExternalSessionMatchText(summary.name).includes(text) || foldExternalSessionPathSeparators(normalizeExternalSessionMatchText(summary.cwd)).includes( foldExternalSessionPathSeparators(text), From 47aac7448f5fd75061ebe76d4b90cc84a1b2d840 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:35:36 +0800 Subject: [PATCH 3/8] fix(storage): project imports before commit starts Generated-by: Codex --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../execution-provider-conformance.test.ts | 38 +++++++++++++++++++ packages/storage/src/session-store.ts | 7 +--- .../src/test-only/memory-execution-session.ts | 32 ++++++++++------ 5 files changed, 63 insertions(+), 18 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 30cfb0751b..10dff7bf04 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -118,7 +118,7 @@ Storage counts extant published Sessions whose immutable `externalOrigin` matche ## 10 · Staging, publication, and recovery -Importer validates canonical input before the durable commit attempt. It creates a `transcriptLedgerVersion: 0` staged Session, materializes its Ledger, then publishes it as a usable Session. A pre-materialization failure deletes staging; Host startup `recover()` processes remaining version-0 Sessions. Only published copies count in the catalog. Host coalesces concurrent imports of the same `(adapterId, sourceSessionId)` onto one in-flight Promise. An explicit import after completion creates an independent copy. +Importer validates canonical input and completes deterministic catalog projection before announcing the durable commit attempt. It creates a `transcriptLedgerVersion: 0` staged Session, materializes its Ledger, then publishes it as a usable Session. A pre-materialization failure deletes staging; Host startup `recover()` processes remaining version-0 Sessions. Only published copies count in the catalog. Host coalesces concurrent imports of the same `(adapterId, sourceSessionId)` onto one in-flight Promise. An explicit import after completion creates an independent copy. ## 11 · Unknown outcome and client interaction diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 93b3f0f17a..442ad144e1 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -185,7 +185,7 @@ Host 同时限制每页项目数和编码后的 JSON 字节数。若下一项使 ## 10 · 暂存、发布与恢复 -导入先创建 `transcriptLedgerVersion: 0` 的暂存 Session。Ledger 物化完成后才升级为已发布状态,并出现在任务列表和 catalog 的副本统计中。 +导入在宣布持久化提交开始前,先完成 canonical 输入校验和确定性 catalog 投影。然后创建 `transcriptLedgerVersion: 0` 的暂存 Session。Ledger 物化完成后才升级为已发布状态,并出现在任务列表和 catalog 的副本统计中。 - 物化前失败:删除暂存 Session。 - Host 重启:`recover()` 继续处理版本 0 的 Session。 diff --git a/packages/storage/src/__tests__/execution-provider-conformance.test.ts b/packages/storage/src/__tests__/execution-provider-conformance.test.ts index 598c731d90..8965e757d7 100644 --- a/packages/storage/src/__tests__/execution-provider-conformance.test.ts +++ b/packages/storage/src/__tests__/execution-provider-conformance.test.ts @@ -621,6 +621,44 @@ for (const backend of ['Local', 'Memory'] as const) { }); }, ); + test(backend + ': imported message projection finishes before commit starts', async () => { + await withProvider(make(), async ({ sessionStore: s }, root) => { + let commitStarted = false; + const replaceDescriptor = Object.getOwnPropertyDescriptor(String.prototype, 'replace')!; + Object.defineProperty(String.prototype, 'replace', { + ...replaceDescriptor, + value(this: string, ...args: unknown[]) { + if (String(this) === 'force projection failure') { + throw new Error('forced projection failure'); + } + return Reflect.apply(replaceDescriptor.value as String['replace'], this, args) as string; + }, + }); + try { + await assert.rejects( + s.createImportedSession( + sessionInput(root), + [ + { + type: 'user', + id: 'imported-user', + turnId: 'imported-turn', + ts: 1, + text: 'force projection failure', + }, + ], + { adapterId: 'fake', sourceSessionId: 'source' }, + { onCommitStarted: () => (commitStarted = true) }, + ), + /forced projection failure/, + ); + assert.equal(commitStarted, false); + assert.deepEqual(await s.listHeaders(), []); + } finally { + Object.defineProperty(String.prototype, 'replace', replaceDescriptor); + } + }); + }); test( backend + ': catalog pagination visits mixed-case tied IDs exactly once in Local order', async () => { diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 186e32c6f0..e3ff652139 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -213,12 +213,9 @@ class SqliteSessionStore implements SessionAuthorityStore { externalOrigin, transcriptLedgerVersion: 0, }; + const catalogProjection = projectSessionCatalogMessages(canonicalMessages); options.onCommitStarted?.(); - const outcome = await this.metadata.importSession( - header, - canonicalMessages, - projectSessionCatalogMessages(canonicalMessages), - ); + const outcome = await this.metadata.importSession(header, canonicalMessages, catalogProjection); if (outcome !== 'imported') { throw new Error(`Generated Session id already exists: ${header.id}`); } diff --git a/packages/storage/src/test-only/memory-execution-session.ts b/packages/storage/src/test-only/memory-execution-session.ts index a9c8edda70..12fc145913 100644 --- a/packages/storage/src/test-only/memory-execution-session.ts +++ b/packages/storage/src/test-only/memory-execution-session.ts @@ -72,10 +72,7 @@ import { normalizeProvenSteeringMessageHandoff, type PendingMessageAdmission, } from '../message-admission-store.js'; -import { - projectSessionCatalogMessages, - lastMessagePreviewForMessages, -} from '../session-message-projection.js'; +import { projectSessionCatalogMessages } from '../session-message-projection.js'; import { type MemoryExecutionAuthority, copy, @@ -245,10 +242,14 @@ function catalog(s: MemoryState, id: string): SessionCatalogRecord { }, }; } -function project(s: MemoryState, id: string, values: readonly StoredMessage[]): void { +function project( + s: MemoryState, + id: string, + values: readonly StoredMessage[], + projection = projectSessionCatalogMessages(values), +): void { const current = requireHeader(s, id); - const projection = projectSessionCatalogMessages(values); - const preview = lastMessagePreviewForMessages(values); + const preview = projection.lastMessagePreview; if (preview !== undefined) rows(s, 'previews').set(id, preview); const visible = values.some((m) => m.type === 'user' || m.type === 'assistant'); if (visible || projection.lastMessageAt !== undefined) { @@ -261,10 +262,18 @@ function project(s: MemoryState, id: string, values: readonly StoredMessage[]): } } function append(s: MemoryState, id: string, inputs: readonly StoredMessage[]): void { + const canonicalValues = inputs.map((input) => decodeCanonicalMessage(copy(input))); + appendCanonical(s, id, canonicalValues); +} +function appendCanonical( + s: MemoryState, + id: string, + canonicalValues: readonly StoredMessage[], + projection = projectSessionCatalogMessages(canonicalValues), +): void { requireHeader(s, id); const list = messages(s).get(id)!; - for (const input of inputs) { - const message = decodeCanonicalMessage(copy(input)); + for (const message of canonicalValues) { const previous = list.find((m) => m.id === message.id); if (previous) { if (!equal(previous, message)) conflict('Message identity changed'); @@ -272,7 +281,7 @@ function append(s: MemoryState, id: string, inputs: readonly StoredMessage[]): v } list.push(message); } - project(s, id, inputs); + project(s, id, canonicalValues, projection); } function probe(s: MemoryState, id: string, fingerprint: string) { assertSafeSessionId(id); @@ -423,10 +432,11 @@ export function createMemorySessionStore( externalOrigin: copy(origin), transcriptLedgerVersion: 0 as const, }; + const catalogProjection = projectSessionCatalogMessages(canonicalValues); options?.onCommitStarted?.(); return write('session.import', (s) => { insert(s, h); - append(s, h.id, canonicalValues); + appendCanonical(s, h.id, canonicalValues, catalogProjection); return requireHeader(s, h.id).header; }); }, From 94e8329a3e342efd18a596260bad8b77104cc5f4 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:36:20 +0800 Subject: [PATCH 4/8] fix(storage): hide staged memory imports Generated-by: Codex --- .../execution-provider-conformance.test.ts | 28 +++++++++++++++++++ .../src/test-only/memory-execution-session.ts | 23 +++++++++------ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/storage/src/__tests__/execution-provider-conformance.test.ts b/packages/storage/src/__tests__/execution-provider-conformance.test.ts index 8965e757d7..481ab57e9c 100644 --- a/packages/storage/src/__tests__/execution-provider-conformance.test.ts +++ b/packages/storage/src/__tests__/execution-provider-conformance.test.ts @@ -659,6 +659,34 @@ for (const backend of ['Local', 'Memory'] as const) { } }); }); + test(backend + ': external import lookup excludes staged Sessions', async () => { + await withProvider(make(), async ({ sessionStore: s }, root) => { + const createImport = (sourceSessionId: string) => + s.createImportedSession(sessionInput(root), [], { + adapterId: 'fake', + sourceSessionId, + }); + const published = await createImport('shared-source'); + await createImport('shared-source'); + await createImport('staged-only'); + await s.updateHeader(published.id, { transcriptLedgerVersion: 1 }); + + assert.deepEqual( + await s.lookupExternalSessionImports( + 'fake', + ['shared-source', 'staged-only', 'missing'], + 8, + ), + [ + { + sourceSessionId: 'shared-source', + livePublishedImportCount: 1, + recentSessionIds: [published.id], + }, + ], + ); + }); + }); test( backend + ': catalog pagination visits mixed-case tied IDs exactly once in Local order', async () => { diff --git a/packages/storage/src/test-only/memory-execution-session.ts b/packages/storage/src/test-only/memory-execution-session.ts index 12fc145913..f59db462a1 100644 --- a/packages/storage/src/test-only/memory-execution-session.ts +++ b/packages/storage/src/test-only/memory-execution-session.ts @@ -442,20 +442,25 @@ export function createMemorySessionStore( }, lookupExternalSessionImports: async (adapterId, sourceIds, limit) => read((s) => - sourceIds.map((sourceSessionId) => { + sourceIds.flatMap((sourceSessionId) => { const matches = [...headers(s).values()].filter( (h) => + h.header.transcriptLedgerVersion === 1 && h.header.externalOrigin?.adapterId === adapterId && h.header.externalOrigin.sourceSessionId === sourceSessionId, ); - return { - sourceSessionId, - livePublishedImportCount: matches.length, - recentSessionIds: matches - .sort((x, y) => y.header.createdAt - x.header.createdAt) - .slice(0, limit) - .map((h) => h.header.id), - }; + return matches.length === 0 + ? [] + : [ + { + sourceSessionId, + livePublishedImportCount: matches.length, + recentSessionIds: matches + .sort((x, y) => y.header.createdAt - x.header.createdAt) + .slice(0, limit) + .map((h) => h.header.id), + }, + ]; }), ), createSubagent: async (input, initial) => From e363e9814ee6c1eff313994527ad3c07ef551d1f Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:38:14 +0800 Subject: [PATCH 5/8] fix(storage): unify Codex timestamp ordering Generated-by: Codex --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../__tests__/codex-session-adapter.test.ts | 10 +++--- packages/storage/src/codex-session-adapter.ts | 32 +++++++++---------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 10dff7bf04..9e833f3279 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -104,7 +104,7 @@ Host requests at most `page size + 1` source entries. Each returned entry carrie ## 8 · Codex keyset catalog -Codex keeps no server-side catalog snapshot, SQLite transaction, TTL, or LRU across requests. Cursors are bound to the current query. State DB pages order by `(sort_key DESC, id DESC)`, where `sort_key` is computed once by the query, selected alongside the row, and read straight back off that row to build the cursor — so the position a cursor names is by construction the position the query ordered by. Seconds and milliseconds are normalized into that one numeric key before ordering. The first page reads the newest `state_N.sqlite`; if that generation cannot be read, the page is served by the filesystem fallback rather than by an older generation, because a lower generation is the snapshot frozen at the last bump and is missing everything created since. Continuation names the generation it started on, uses a SQL keyset condition, and stays strict — a missing original generation invalidates the cursor, and a transient read failure remains a persistence failure. Connections close after each request. +Codex keeps no server-side catalog snapshot, SQLite transaction, TTL, or LRU across requests. Cursors are bound to the current query. State DB pages order by `(sort_key DESC, id DESC)`, where `sort_key` is computed once by the query, selected alongside the row, and read straight back off that row to build the cursor — so the position a cursor names is by construction the position the query ordered by. One adapter normalizer accepts finite numeric or numeric-string epoch seconds/milliseconds and parseable date-time strings such as ISO 8601; SQLite ordering, cursor position, and displayed summary timestamps all call that same rule. The first page reads the newest `state_N.sqlite`; if that generation cannot be read, the page is served by the filesystem fallback rather than by an older generation, because a lower generation is the snapshot frozen at the last bump and is missing everything created since. Continuation names the generation it started on, uses a SQL keyset condition, and stays strict — a missing original generation invalidates the cursor, and a transient read failure remains a persistence failure. Connections close after each request. The filesystem fallback orders by `(mtime DESC, fixed-size path identity ASC)` across active and optionally archived roots. The identity is derived once from the relative rollout path, so deeply nested paths cannot enlarge the cursor past the Host wire bound. One traversal has a `maxCatalogCandidates` file bound; exceeding it returns a typed source limit. Stat-known keys reject candidates that cannot enter the current page before reading their bounded heads. The page retains at most `limit + 1` matching summaries instead of materializing the corpus or rescanning it repeatedly for deep pages. diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 442ad144e1..3c2aaae791 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -170,7 +170,7 @@ Codex 不保存跨请求的 SQLite 事务、catalog snapshot、TTL 或 LRU 状 两条来源路径分别使用自己的稳定排序键: -1. **state DB**:`(sort_key DESC, id DESC)`。`sort_key` 由查询算一次、随行一起选出,cursor 直接读回该行上的这个值 —— 这样 cursor 指向的位置必然就是查询排序的位置。秒级与毫秒级时间戳先统一成这一个数值键再排序。首页只读最新的 `state_N.sqlite`;若该 generation 读不了,本页改由 filesystem fallback 回答,而**不是**退到更旧的 generation:更旧那本是上一次跃迁时冻结的快照,跃迁之后新建的会话都不在里面。cursor 记录起始 generation,续页仍只读打开同一个文件,通过 `WHERE` keyset 条件继续,并保持严格 —— 原 generation 已删除或不可读时 cursor 明确失效,读失败仍是 persistence failure。连接用完立即关闭。 +1. **state DB**:`(sort_key DESC, id DESC)`。`sort_key` 由查询算一次、随行一起选出,cursor 直接读回该行上的这个值 —— 这样 cursor 指向的位置必然就是查询排序的位置。adapter 中的唯一 normalizer 接受有限数值或数字字符串形式的 epoch 秒/毫秒,以及 ISO 8601 等可解析 date-time 字符串;SQLite 排序、cursor 位置和展示的摘要时间都调用同一条规则。首页只读最新的 `state_N.sqlite`;若该 generation 读不了,本页改由 filesystem fallback 回答,而**不是**退到更旧的 generation:更旧那本是上一次跃迁时冻结的快照,跃迁之后新建的会话都不在里面。cursor 记录起始 generation,续页仍只读打开同一个文件,通过 `WHERE` keyset 条件继续,并保持严格 —— 原 generation 已删除或不可读时 cursor 明确失效,读失败仍是 persistence failure。连接用完立即关闭。 2. **filesystem fallback**:`(mtime DESC, fixed-size path identity ASC)`。identity 由相对 rollout path 一次派生,因此深层路径不会使 cursor 超过 Host wire 上限。一次遍历 active 和可选 archived roots,以 `maxCatalogCandidates` 限制遍历的文件数;超过上限返回 typed limit error。候选先用 stat 已知排序键与当前页尾比较,只有可能进入当前页的候选才读取有界 head 并完成 query/path 校验;内存最多保留 `limit + 1` 个匹配摘要,不物化整个 catalog,也不为深分页重复扫描多轮。 keyset 的语义是“继续读取严格排在最后交付项之后的记录”。如果一个尚未读取的 live Session 在两页之间更新并移动到 cursor 之前,本次遍历可能看不到它,但不会因此重复已经交付的行;重新打开或刷新 catalog 会看到当前最新顺序。这是实时可变来源下不持有 snapshot 的明确边界。 diff --git a/packages/storage/src/__tests__/codex-session-adapter.test.ts b/packages/storage/src/__tests__/codex-session-adapter.test.ts index 1586de4680..a9f6cb8bce 100644 --- a/packages/storage/src/__tests__/codex-session-adapter.test.ts +++ b/packages/storage/src/__tests__/codex-session-adapter.test.ts @@ -952,12 +952,10 @@ describe('CodexSessionAdapter', () => { cursor = result.items.at(-1)?.nextCursor; assert.ok(cursor, 'a page that reports hasMore must carry a next cursor'); } - // The cursor has to name the position the query ordered by. Recomputing - // the key in JS put `codex-text` at 1000 while the SQL key it was ordered - // by is text, so the next page asked for `key < 1000`, matched nothing, - // and reported the catalog exhausted — the other two Conversations were - // dropped without an error. - assert.deepEqual(seen.sort(), ['codex-a', 'codex-b', 'codex-text']); + // The same normalization must determine display time, SQL order, and + // cursor position. The ISO value is in 2026, so it precedes the small + // numeric fixtures instead of being cast to the number 2026. + assert.deepEqual(seen, ['codex-text', 'codex-a', 'codex-b']); }); }); diff --git a/packages/storage/src/codex-session-adapter.ts b/packages/storage/src/codex-session-adapter.ts index 4023d28793..9d30a92ebe 100644 --- a/packages/storage/src/codex-session-adapter.ts +++ b/packages/storage/src/codex-session-adapter.ts @@ -49,6 +49,7 @@ const CODEX_ROLLOUT_READ_BYTES = 64 * 1024; const CODEX_ROLLOUT_MAX_RECORD_BYTES = 64 * 1024 * 1024; const CODEX_ROLLOUT_MAX_CONVERTED_BYTES = 256 * 1024 * 1024; const CODEX_ROLLOUT_MAX_MESSAGES = 250_000; +const CODEX_EPOCH_MS_SQL_FUNCTION = 'maka_codex_epoch_ms'; const CODEX_SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; const CODEX_SUPPORTED_THREAD_SOURCES = ['cli', 'exec', 'vscode', 'atlas', 'chatgpt'] as const; const CODEX_UNSAFE_PATH_CHARS = @@ -273,6 +274,7 @@ export class CodexSessionAdapter implements ExternalSessionAdapter { try { const sqlite = await import('node:sqlite'); db = new sqlite.DatabaseSync(dbPath, { readOnly: true }); + registerCodexEpochNormalization(db); const spec = codexThreadQuery(db, query, undefined, keyset); if (!spec) return undefined; const items: ExternalSessionCatalogPage['items'][number][] = []; @@ -870,6 +872,7 @@ async function readCodexThreadRows( const sqlite = await import('node:sqlite'); const db = new sqlite.DatabaseSync(dbPath, { readOnly: true }); try { + registerCodexEpochNormalization(db); const spec = codexThreadQuery(db, query, exactId); if (!spec) return undefined; return db.prepare(spec.sql).all(...spec.params) as CodexThreadRow[]; @@ -922,22 +925,11 @@ function codexThreadQuery( const orderColumns = ['updated_at_ms', 'updated_at', 'created_at_ms', 'created_at'].filter( (column) => columns.has(column), ); - // One authority for the ordering key. It is computed here, selected as - // `sort_key`, and read straight back off the row to build the cursor, so the - // position a cursor names is by construction the position the query ordered - // by. Recomputing it in JS let the two drift on a stored TEXT value: SQL - // keeps the text as the key and orders it above every number, while the JS - // fallback chain skips that column and lands on a different one. A TEXT key - // never satisfies a numeric comparison, so the disagreement did not surface - // as a duplicate — it ended the traversal at that page and dropped every - // Conversation after it without an error. Casting first also keeps the key - // numeric, which is what the cursor's 8-byte encoding requires. - const orderValues = orderColumns.map((column) => { - const numeric = `CAST(${column} AS REAL)`; - return column.endsWith('_ms') - ? numeric - : `(CASE WHEN ${numeric} >= 1000000000000 THEN ${numeric} ELSE ${numeric} * 1000 END)`; - }); + // The same normalizer owns displayed timestamps and this SQL ordering key. + // Selecting the key with the row then makes the cursor name exactly the + // position the query used, including ISO text stored in an INTEGER-affinity + // column. + const orderValues = orderColumns.map((column) => `${CODEX_EPOCH_MS_SQL_FUNCTION}(${column})`); const orderExpression = orderValues.length > 0 ? `coalesce(${orderValues.join(', ')}, 0)` : '0'; if (keyset) { where.push(`(${orderExpression} < ? OR (${orderExpression} = ? AND id < ?))`); @@ -968,6 +960,14 @@ function finiteNumber(value: unknown): number | undefined { return undefined; } +function registerCodexEpochNormalization(db: DatabaseSync): void { + db.function( + CODEX_EPOCH_MS_SQL_FUNCTION, + { deterministic: true, directOnly: true }, + (value) => normalizeEpochMs(value) ?? null, + ); +} + async function codexStateDbsNewestFirst(codexHome: string): Promise { try { const root = await realpath(codexHome); From e0b1d8ba9087c70cb7a89c4917e41857bf78ac13 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Wed, 16 Sep 2026 23:42:51 +0800 Subject: [PATCH 6/8] fix(cli): coalesce external catalog search Generated-by: Codex --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../cli/src/__tests__/pi-tui-runner.test.ts | 70 +++++++++++++++++++ packages/cli/src/pi-tui-runner.ts | 43 ++++++++++-- 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 9e833f3279..2d8bfcbc05 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -94,7 +94,7 @@ The Ledger preserves a source transcript that starts with assistant content. `bu ## 6 · Workspace scope -The Runtime Host TUI external-session surface decides scope. With a current workspace target, it offers current workspace and all, defaulting to current; without a target, it offers only all. If the target disappears before a scoped query, the surface rejects that request instead of silently broadening it. TUI runner forwards this choice rather than deriving scope from the Session driver. External-source scope is independent of the Maka task list's Current/All filter. +The Runtime Host TUI external-session surface decides scope. With a current workspace target, it offers current workspace and all, defaulting to current; without a target, it offers only all. If the target disappears before a scoped query, the surface rejects that request instead of silently broadening it. TUI runner forwards this choice rather than deriving scope from the Session driver. External-source scope is independent of the Maka task list's Current/All filter. The TUI runner briefly coalesces consecutive search edits before querying the Host. Every edit advances the same request revision immediately, so an older in-flight response cannot repaint the catalog while the newer query waits for its debounce. ## 7 · Adapter-owned paging diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 3c2aaae791..92a2c2489b 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -152,7 +152,7 @@ Runtime Host TUI 的 external-session surface 是 workspace scope 的唯一决 - 没有 workspace target:surface 只提供“全部”,界面不显示无效的 workspace 切换。 - 如果发起当前 workspace 查询时 target 已失效,surface 明确拒绝,不会省略 workspace 参数后静默查询全部。 -TUI runner 只展示 surface 给出的选项并转发用户选择,不再读取 Session driver 自行推导 scope。这个 scope 仍与 Maka Session 列表的 Current/All 标签无关;后者只控制原生 Session 列表的展示范围。 +TUI runner 只展示 surface 给出的选项并转发用户选择,不再读取 Session driver 自行推导 scope。这个 scope 仍与 Maka Session 列表的 Current/All 标签无关;后者只控制原生 Session 列表的展示范围。TUI runner 会在查询 Host 前短暂合并连续的搜索输入;每次输入仍会立即推进同一个 request revision,因此新查询等待 debounce 时,旧的在途响应也不能回写 catalog。 ## 7 · 分页由 adapter 拥有 diff --git a/packages/cli/src/__tests__/pi-tui-runner.test.ts b/packages/cli/src/__tests__/pi-tui-runner.test.ts index 386d2055b4..471cf46681 100644 --- a/packages/cli/src/__tests__/pi-tui-runner.test.ts +++ b/packages/cli/src/__tests__/pi-tui-runner.test.ts @@ -42,6 +42,7 @@ import type { InteractionFormResponse } from '@maka/core/interaction'; import type { SkillInvocationResult } from '@maka/core/skill-invocation'; import type { AgentGraphClientSnapshot, + ExternalSessionCatalogItem, TurnMessageSubmitResult, } from '@maka/runtime-host/protocol'; import { SessionActivityRegistry } from '@maka/runtime/goal-turn-lifecycle'; @@ -6506,6 +6507,75 @@ Slug openai-work await run; }); + test('coalesces external catalog search while retiring stale responses immediately', async () => { + const terminal = new FakeTerminal(); + const driver = new SlashCommandDriver([]); + const queries: Array = []; + let resolveStale!: (page: { sessions: ExternalSessionCatalogItem[]; nextCursor: null }) => void; + const externalSessions = { + listScopes: () => ['all'] as const, + listSources: async () => ['codex'], + listSessions: async ({ text }: { text?: string }) => { + queries.push(text); + if (text === 'code') { + return new Promise<{ sessions: ExternalSessionCatalogItem[]; nextCursor: null }>( + (resolve) => { + resolveStale = resolve; + }, + ); + } + return { sessions: [], nextCursor: null }; + }, + importSession: async () => { + throw new Error('unused'); + }, + }; + const run = runMakaPiTui({ + title: 'Maka', + driver, + cwd: '/repo', + model: 'claude-sonnet-4-5', + connectionSlug: 'claude-subscription', + permissionMode: 'ask', + terminal, + externalSessions, + }); + + terminal.input('/session'); + terminal.input('\r'); + await waitFor(() => plainTerminalOutput(terminal.output()).includes('Import external session')); + terminal.input('\r'); + await waitFor(() => queries.length === 1); + + terminal.input('c'); + terminal.input('o'); + terminal.input('d'); + terminal.input('e'); + assert.deepEqual(queries, [undefined]); + await waitFor(() => queries.length === 2); + + terminal.input('x'); + resolveStale({ + sessions: [ + { + id: 'stale', + name: 'Stale code result', + hostCwd: '/repo', + importState: { importedCount: 0, importedSessionIds: [], isImporting: false }, + }, + ], + nextCursor: null, + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.doesNotMatch(plainTerminalOutput(terminal.screenOutput()), /Stale code result/); + await waitFor(() => queries.length === 3); + assert.deepEqual(queries, [undefined, 'code', 'codex']); + + terminal.input('\x1b'); + exitMaka(terminal); + await run; + }); + test('reports the durable Session id when import succeeds but opening fails', async () => { const terminal = new FakeTerminal(); const driver = new FailingSwitchSessionDriver([]); diff --git a/packages/cli/src/pi-tui-runner.ts b/packages/cli/src/pi-tui-runner.ts index 4560744a59..01bb2a7859 100644 --- a/packages/cli/src/pi-tui-runner.ts +++ b/packages/cli/src/pi-tui-runner.ts @@ -186,6 +186,8 @@ import { getTuiPrimaryGuidance } from './tui-primary-guidance.js'; import { TUI_COPY_RESOURCES } from './tui-copy-catalog.js'; import type { GoalControlAction, GoalProjection } from '@maka/runtime-host/protocol'; +const EXTERNAL_SESSION_SEARCH_DEBOUNCE_MS = 120; + export interface MakaPiTuiInput { /** Launcher command used in resume and recovery instructions. */ cliCommand?: string; @@ -3110,16 +3112,30 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { let sessions: readonly ExternalSessionCatalogItem[] = []; let nextCursor: string | null = null; let revision = 0; + let cancelScheduledSearch: (() => void) | undefined; + let pageClosed = false; let overlay: OverlayHandle | undefined; let search: SessionSearchOverlay | undefined; let byValue = new Map(); - const closeOverlay = () => overlay?.hide(); + const dropScheduledSearch = (): boolean => { + const hadScheduledSearch = cancelScheduledSearch !== undefined; + cancelScheduledSearch?.(); + cancelScheduledSearch = undefined; + return hadScheduledSearch; + }; + const closeOverlay = (): void => { + pageClosed = true; + dropScheduledSearch(); + revision += 1; + overlay?.hide(); + }; const toggleScope = (): void => { const alternate = input.externalSessions ?.listScopes() .find((candidate) => candidate !== scope); if (!alternate) return; + dropScheduledSearch(); scope = alternate; void load(false); }; @@ -3194,10 +3210,23 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { notice, onQuery: (text) => { query = text; - void load(false); + dropScheduledSearch(); + // Retire an older in-flight response immediately. Waiting until the + // debounce fires would let it repaint results for the previous query. + const requestRevision = ++revision; + const handle = setTimeout(() => { + cancelScheduledSearch = undefined; + void load(false, undefined, requestRevision); + }, EXTERNAL_SESSION_SEARCH_DEBOUNCE_MS); + handle.unref(); + cancelScheduledSearch = () => clearTimeout(handle); }, onSelect: (item) => { if (item.value === 'external:load-more' && nextCursor) { + if (dropScheduledSearch()) { + void load(false); + return; + } void load(true, nextCursor); return; } @@ -3267,8 +3296,12 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { overlay = showBottomPicker(search); }; - const load = async (append: boolean, cursor?: string): Promise => { - const requestRevision = ++revision; + const load = async ( + append: boolean, + cursor?: string, + scheduledRevision?: number, + ): Promise => { + const requestRevision = scheduledRevision ?? ++revision; try { const page = await input.externalSessions!.listSessions({ adapterId, @@ -3276,7 +3309,7 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { ...(cursor ? { cursor } : {}), ...(query ? { text: query } : {}), }); - if (requestRevision !== revision || closed || turnRunning) return; + if (requestRevision !== revision || pageClosed || closed || turnRunning) return; sessions = append ? [...sessions, ...page.sessions] : page.sessions; nextCursor = page.nextCursor; render(); From 2884233db5a45dc2a2d9c2801f49bd40cd1453ec Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Thu, 17 Sep 2026 01:17:22 +0800 Subject: [PATCH 7/8] fix(import): close external session boundary gaps --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../cli/src/__tests__/pi-tui-runner.test.ts | 58 ++++++++++++++++++- packages/cli/src/pi-tui-runner.ts | 21 ++++++- .../execution-provider-conformance.test.ts | 15 ++++- .../src/test-only/memory-execution-session.ts | 2 + 6 files changed, 92 insertions(+), 8 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 2d8bfcbc05..8378dd9ad3 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -94,7 +94,7 @@ The Ledger preserves a source transcript that starts with assistant content. `bu ## 6 · Workspace scope -The Runtime Host TUI external-session surface decides scope. With a current workspace target, it offers current workspace and all, defaulting to current; without a target, it offers only all. If the target disappears before a scoped query, the surface rejects that request instead of silently broadening it. TUI runner forwards this choice rather than deriving scope from the Session driver. External-source scope is independent of the Maka task list's Current/All filter. The TUI runner briefly coalesces consecutive search edits before querying the Host. Every edit advances the same request revision immediately, so an older in-flight response cannot repaint the catalog while the newer query waits for its debounce. +The Runtime Host TUI external-session surface decides scope. With a current workspace target, it offers current workspace and all, defaulting to current; without a target, it offers only all. If the target disappears before a scoped query, the surface rejects that request instead of silently broadening it. TUI runner forwards this choice rather than deriving scope from the Session driver. External-source scope is independent of the Maka task list's Current/All filter. The TUI runner briefly coalesces consecutive search edits before querying the Host. Every edit advances the same request revision immediately and retires the displayed rows and cursor, so an older in-flight response cannot repaint or paginate the catalog while the newer query waits for its debounce. ## 7 · Adapter-owned paging diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 92a2c2489b..8b94f81717 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -152,7 +152,7 @@ Runtime Host TUI 的 external-session surface 是 workspace scope 的唯一决 - 没有 workspace target:surface 只提供“全部”,界面不显示无效的 workspace 切换。 - 如果发起当前 workspace 查询时 target 已失效,surface 明确拒绝,不会省略 workspace 参数后静默查询全部。 -TUI runner 只展示 surface 给出的选项并转发用户选择,不再读取 Session driver 自行推导 scope。这个 scope 仍与 Maka Session 列表的 Current/All 标签无关;后者只控制原生 Session 列表的展示范围。TUI runner 会在查询 Host 前短暂合并连续的搜索输入;每次输入仍会立即推进同一个 request revision,因此新查询等待 debounce 时,旧的在途响应也不能回写 catalog。 +TUI runner 只展示 surface 给出的选项并转发用户选择,不再读取 Session driver 自行推导 scope。这个 scope 仍与 Maka Session 列表的 Current/All 标签无关;后者只控制原生 Session 列表的展示范围。TUI runner 会在查询 Host 前短暂合并连续的搜索输入;每次输入都会立即推进同一个 request revision,并清除当前显示的 rows 与 cursor,因此新查询等待 debounce 时,旧的在途响应既不能回写 catalog,也不能继续旧分页。 ## 7 · 分页由 adapter 拥有 diff --git a/packages/cli/src/__tests__/pi-tui-runner.test.ts b/packages/cli/src/__tests__/pi-tui-runner.test.ts index 471cf46681..3a37c56117 100644 --- a/packages/cli/src/__tests__/pi-tui-runner.test.ts +++ b/packages/cli/src/__tests__/pi-tui-runner.test.ts @@ -6511,12 +6511,14 @@ Slug openai-work const terminal = new FakeTerminal(); const driver = new SlashCommandDriver([]); const queries: Array = []; + const requests: Array<{ text?: string; cursor?: string }> = []; let resolveStale!: (page: { sessions: ExternalSessionCatalogItem[]; nextCursor: null }) => void; const externalSessions = { listScopes: () => ['all'] as const, listSources: async () => ['codex'], - listSessions: async ({ text }: { text?: string }) => { + listSessions: async ({ text, cursor }: { text?: string; cursor?: string }) => { queries.push(text); + requests.push({ ...(text === undefined ? {} : { text }), ...(cursor ? { cursor } : {}) }); if (text === 'code') { return new Promise<{ sessions: ExternalSessionCatalogItem[]; nextCursor: null }>( (resolve) => { @@ -6524,7 +6526,19 @@ Slug openai-work }, ); } - return { sessions: [], nextCursor: null }; + return text === undefined + ? { + sessions: [ + { + id: 'old', + name: 'Old empty-query result', + hostCwd: '/repo', + importState: { importedCount: 0, importedSessionIds: [], isImporting: false }, + }, + ], + nextCursor: 'old-next', + } + : { sessions: [], nextCursor: null }; }, importSession: async () => { throw new Error('unused'); @@ -6553,6 +6567,8 @@ Slug openai-work terminal.input('e'); assert.deepEqual(queries, [undefined]); await waitFor(() => queries.length === 2); + assert.equal(requests[1]?.cursor, undefined); + assert.doesNotMatch(plainTerminalOutput(terminal.screenOutput()), /Old empty-query result/); terminal.input('x'); resolveStale({ @@ -6576,6 +6592,44 @@ Slug openai-work await run; }); + test('cancels external catalog search timers during runner shutdown', async () => { + const terminal = new FakeTerminal(); + const driver = new SlashCommandDriver([]); + let listCalls = 0; + const externalSessions = { + listScopes: () => ['all'] as const, + listSources: async () => ['codex'], + listSessions: async () => { + listCalls += 1; + return { sessions: [], nextCursor: null }; + }, + importSession: async () => { + throw new Error('unused'); + }, + }; + const run = runMakaPiTui({ + title: 'Maka', + driver, + cwd: '/repo', + model: 'claude-sonnet-4-5', + connectionSlug: 'claude-subscription', + permissionMode: 'ask', + terminal, + externalSessions, + }); + + terminal.input('/session'); + terminal.input('\r'); + await waitFor(() => plainTerminalOutput(terminal.output()).includes('Import external session')); + terminal.input('\r'); + await waitFor(() => listCalls === 1); + terminal.input('x'); + exitMaka(terminal); + await run; + await delay(160); + assert.equal(listCalls, 1); + }); + test('reports the durable Session id when import succeeds but opening fails', async () => { const terminal = new FakeTerminal(); const driver = new FailingSwitchSessionDriver([]); diff --git a/packages/cli/src/pi-tui-runner.ts b/packages/cli/src/pi-tui-runner.ts index 01bb2a7859..7e4ffa1932 100644 --- a/packages/cli/src/pi-tui-runner.ts +++ b/packages/cli/src/pi-tui-runner.ts @@ -569,6 +569,7 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { let connectionIdentityNotice: string | undefined; let busy = false; let closed = false; + const cancelScheduledExternalSearches = new Set<() => void>(); let currentActivityCompletion: Promise | undefined; let permissionResponseInFlightRequestId: string | null = null; // Session recap (issue #1055): an in-flight lock shared by manual and @@ -1147,6 +1148,8 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { attention.reset(); // Stop asking the terminal for focus reports before handing it back. terminal.write(DISABLE_FOCUS_REPORTING); + for (const cancel of cancelScheduledExternalSearches) cancel(); + cancelScheduledExternalSearches.clear(); tui.stop(); }; @@ -3120,10 +3123,17 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { const dropScheduledSearch = (): boolean => { const hadScheduledSearch = cancelScheduledSearch !== undefined; + if (cancelScheduledSearch) cancelScheduledExternalSearches.delete(cancelScheduledSearch); cancelScheduledSearch?.(); cancelScheduledSearch = undefined; return hadScheduledSearch; }; + const resetCatalogPage = (): void => { + sessions = []; + nextCursor = null; + byValue = new Map(); + render(); + }; const closeOverlay = (): void => { pageClosed = true; dropScheduledSearch(); @@ -3137,6 +3147,7 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { if (!alternate) return; dropScheduledSearch(); scope = alternate; + resetCatalogPage(); void load(false); }; const render = (): void => { @@ -3211,15 +3222,21 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { onQuery: (text) => { query = text; dropScheduledSearch(); + resetCatalogPage(); // Retire an older in-flight response immediately. Waiting until the // debounce fires would let it repaint results for the previous query. const requestRevision = ++revision; + let cancel: (() => void) | undefined; const handle = setTimeout(() => { + if (cancel) cancelScheduledExternalSearches.delete(cancel); cancelScheduledSearch = undefined; + if (closed || pageClosed) return; void load(false, undefined, requestRevision); }, EXTERNAL_SESSION_SEARCH_DEBOUNCE_MS); handle.unref(); - cancelScheduledSearch = () => clearTimeout(handle); + cancel = () => clearTimeout(handle); + cancelScheduledSearch = cancel; + cancelScheduledExternalSearches.add(cancel); }, onSelect: (item) => { if (item.value === 'external:load-more' && nextCursor) { @@ -3314,7 +3331,7 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { nextCursor = page.nextCursor; render(); } catch { - if (requestRevision !== revision) return; + if (requestRevision !== revision || pageClosed || closed) return; state.entries.push({ kind: 'notice', level: 'error', text: copy.externalCatalogFailed }); requestRender(); } diff --git a/packages/storage/src/__tests__/execution-provider-conformance.test.ts b/packages/storage/src/__tests__/execution-provider-conformance.test.ts index 481ab57e9c..0f94b204c6 100644 --- a/packages/storage/src/__tests__/execution-provider-conformance.test.ts +++ b/packages/storage/src/__tests__/execution-provider-conformance.test.ts @@ -667,8 +667,8 @@ for (const backend of ['Local', 'Memory'] as const) { sourceSessionId, }); const published = await createImport('shared-source'); - await createImport('shared-source'); - await createImport('staged-only'); + const stagedShared = await createImport('shared-source'); + const stagedOnly = await createImport('staged-only'); await s.updateHeader(published.id, { transcriptLedgerVersion: 1 }); assert.deepEqual( @@ -685,6 +685,17 @@ for (const backend of ['Local', 'Memory'] as const) { }, ], ); + + const page = await s.listCatalogPage(undefined, undefined, 8); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') throw new Error('Expected a catalog page'); + assert.deepEqual( + page.records.map((record) => record.header.id), + [published.id], + ); + await assert.rejects(s.readCatalogRecord(stagedShared.id), SessionNotFoundError); + await assert.rejects(s.readCatalogRecord(stagedOnly.id), SessionNotFoundError); + assert.equal((await s.readCatalogRecord(published.id)).header.id, published.id); }); }); test( diff --git a/packages/storage/src/test-only/memory-execution-session.ts b/packages/storage/src/test-only/memory-execution-session.ts index f59db462a1..7f13cf0098 100644 --- a/packages/storage/src/test-only/memory-execution-session.ts +++ b/packages/storage/src/test-only/memory-execution-session.ts @@ -510,6 +510,7 @@ export function createMemorySessionStore( const ordinary = id !== HUB && header.role === undefined; const coordination = id === HUB && header.role === WORKHUB_COORDINATION_SESSION_ROLE; if ( + header.transcriptLedgerVersion === 0 || header.conversationCopy?.state === 'preparing' || (!ordinary && !(roleScope === 'recoverable' && coordination)) ) @@ -1247,6 +1248,7 @@ function selectCatalog(s: MemoryState, filter: Parameters r.header.role !== WORKHUB_COORDINATION_SESSION_ROLE && + r.header.transcriptLedgerVersion !== 0 && r.header.conversationCopy?.state !== 'preparing' && (filter?.subagentParentSessionId === undefined || r.header.subagentParent?.parentSessionId === filter.subagentParentSessionId), From e8590d4fef810e75cbcf8475803bc6bca838e861 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:27:08 +0800 Subject: [PATCH 8/8] fix(import): close remaining external-session review gaps --- .../external-session-import-design.md | 2 +- .../external-session-import-design.zh-CN.md | 2 +- .../cli/src/__tests__/pi-tui-runner.test.ts | 10 ++- packages/cli/src/pi-tui-runner.ts | 13 +-- packages/core/src/external-session.ts | 2 +- .../external-session-coordinator.test.ts | 27 +++++- .../src/server/execution-composition.ts | 9 +- .../__tests__/codex-session-adapter.test.ts | 28 +++++-- .../execution-provider-conformance.test.ts | 82 +++++++++++-------- packages/storage/src/codex-session-adapter.ts | 4 +- .../storage/src/session-store-contract.ts | 7 +- packages/storage/src/session-store.ts | 8 +- .../src/test-only/memory-execution-session.ts | 2 +- 13 files changed, 131 insertions(+), 65 deletions(-) diff --git a/docs/architecture/external-session-import-design.md b/docs/architecture/external-session-import-design.md index 8378dd9ad3..e80bc5fcb3 100644 --- a/docs/architecture/external-session-import-design.md +++ b/docs/architecture/external-session-import-design.md @@ -106,7 +106,7 @@ Host requests at most `page size + 1` source entries. Each returned entry carrie Codex keeps no server-side catalog snapshot, SQLite transaction, TTL, or LRU across requests. Cursors are bound to the current query. State DB pages order by `(sort_key DESC, id DESC)`, where `sort_key` is computed once by the query, selected alongside the row, and read straight back off that row to build the cursor — so the position a cursor names is by construction the position the query ordered by. One adapter normalizer accepts finite numeric or numeric-string epoch seconds/milliseconds and parseable date-time strings such as ISO 8601; SQLite ordering, cursor position, and displayed summary timestamps all call that same rule. The first page reads the newest `state_N.sqlite`; if that generation cannot be read, the page is served by the filesystem fallback rather than by an older generation, because a lower generation is the snapshot frozen at the last bump and is missing everything created since. Continuation names the generation it started on, uses a SQL keyset condition, and stays strict — a missing original generation invalidates the cursor, and a transient read failure remains a persistence failure. Connections close after each request. -The filesystem fallback orders by `(mtime DESC, fixed-size path identity ASC)` across active and optionally archived roots. The identity is derived once from the relative rollout path, so deeply nested paths cannot enlarge the cursor past the Host wire bound. One traversal has a `maxCatalogCandidates` file bound; exceeding it returns a typed source limit. Stat-known keys reject candidates that cannot enter the current page before reading their bounded heads. The page retains at most `limit + 1` matching summaries instead of materializing the corpus or rescanning it repeatedly for deep pages. +The filesystem fallback orders by `(mtime DESC, fixed-size path identity ASC)` across active and optionally archived roots. Its opaque cursor uses the versioned `f2` filesystem tag; the identity is derived once from the relative rollout path, so deeply nested paths cannot enlarge the cursor past the Host wire bound. One traversal has a `maxCatalogCandidates` file bound; exceeding it returns a typed source limit. Stat-known keys reject candidates that cannot enter the current page before reading their bounded heads. The page retains at most `limit + 1` matching summaries instead of materializing the corpus or rescanning it repeatedly for deep pages. Keysets resume strictly after the last delivered record. A live source updated between pages may move ahead of the cursor and be absent from that traversal; a fresh catalog query sees the new order. The design does not claim a stable snapshot of mutable external data. diff --git a/docs/architecture/external-session-import-design.zh-CN.md b/docs/architecture/external-session-import-design.zh-CN.md index 8b94f81717..175e499387 100644 --- a/docs/architecture/external-session-import-design.zh-CN.md +++ b/docs/architecture/external-session-import-design.zh-CN.md @@ -171,7 +171,7 @@ Codex 不保存跨请求的 SQLite 事务、catalog snapshot、TTL 或 LRU 状 两条来源路径分别使用自己的稳定排序键: 1. **state DB**:`(sort_key DESC, id DESC)`。`sort_key` 由查询算一次、随行一起选出,cursor 直接读回该行上的这个值 —— 这样 cursor 指向的位置必然就是查询排序的位置。adapter 中的唯一 normalizer 接受有限数值或数字字符串形式的 epoch 秒/毫秒,以及 ISO 8601 等可解析 date-time 字符串;SQLite 排序、cursor 位置和展示的摘要时间都调用同一条规则。首页只读最新的 `state_N.sqlite`;若该 generation 读不了,本页改由 filesystem fallback 回答,而**不是**退到更旧的 generation:更旧那本是上一次跃迁时冻结的快照,跃迁之后新建的会话都不在里面。cursor 记录起始 generation,续页仍只读打开同一个文件,通过 `WHERE` keyset 条件继续,并保持严格 —— 原 generation 已删除或不可读时 cursor 明确失效,读失败仍是 persistence failure。连接用完立即关闭。 -2. **filesystem fallback**:`(mtime DESC, fixed-size path identity ASC)`。identity 由相对 rollout path 一次派生,因此深层路径不会使 cursor 超过 Host wire 上限。一次遍历 active 和可选 archived roots,以 `maxCatalogCandidates` 限制遍历的文件数;超过上限返回 typed limit error。候选先用 stat 已知排序键与当前页尾比较,只有可能进入当前页的候选才读取有界 head 并完成 query/path 校验;内存最多保留 `limit + 1` 个匹配摘要,不物化整个 catalog,也不为深分页重复扫描多轮。 +2. **filesystem fallback**:`(mtime DESC, fixed-size path identity ASC)`。opaque cursor 使用版本化的 `f2` filesystem tag;identity 由相对 rollout path 一次派生,因此深层路径不会使 cursor 超过 Host wire 上限。一次遍历 active 和可选 archived roots,以 `maxCatalogCandidates` 限制遍历的文件数;超过上限返回 typed limit error。候选先用 stat 已知排序键与当前页尾比较,只有可能进入当前页的候选才读取有界 head 并完成 query/path 校验;内存最多保留 `limit + 1` 个匹配摘要,不物化整个 catalog,也不为深分页重复扫描多轮。 keyset 的语义是“继续读取严格排在最后交付项之后的记录”。如果一个尚未读取的 live Session 在两页之间更新并移动到 cursor 之前,本次遍历可能看不到它,但不会因此重复已经交付的行;重新打开或刷新 catalog 会看到当前最新顺序。这是实时可变来源下不持有 snapshot 的明确边界。 diff --git a/packages/cli/src/__tests__/pi-tui-runner.test.ts b/packages/cli/src/__tests__/pi-tui-runner.test.ts index 3a37c56117..5affa9143a 100644 --- a/packages/cli/src/__tests__/pi-tui-runner.test.ts +++ b/packages/cli/src/__tests__/pi-tui-runner.test.ts @@ -6507,7 +6507,7 @@ Slug openai-work await run; }); - test('coalesces external catalog search while retiring stale responses immediately', async () => { + test('coalesces external catalog search while retiring stale responses immediately', async (t) => { const terminal = new FakeTerminal(); const driver = new SlashCommandDriver([]); const queries: Array = []; @@ -6570,6 +6570,14 @@ Slug openai-work assert.equal(requests[1]?.cursor, undefined); assert.doesNotMatch(plainTerminalOutput(terminal.screenOutput()), /Old empty-query result/); + t.mock.timers.enable({ apis: ['setTimeout'] }); + terminal.input(' '); + t.mock.timers.tick(121); + await new Promise((resolve) => setImmediate(resolve)); + t.mock.timers.reset(); + assert.deepEqual(queries, [undefined, 'code']); + terminal.input('\x7f'); + terminal.input('x'); resolveStale({ sessions: [ diff --git a/packages/cli/src/pi-tui-runner.ts b/packages/cli/src/pi-tui-runner.ts index 7e4ffa1932..477345dd6d 100644 --- a/packages/cli/src/pi-tui-runner.ts +++ b/packages/cli/src/pi-tui-runner.ts @@ -32,7 +32,10 @@ import { type Terminal, } from '@earendil-works/pi-tui'; import type { PermissionMode } from '@maka/core/permission'; -import type { ExternalSessionLimit } from '@maka/core/external-session'; +import { + normalizeExternalSessionQueryText, + type ExternalSessionLimit, +} from '@maka/core/external-session'; import { CurrentTodoStore, TodoOverlay, renderTodoIndicator } from './pi-tui-todo.js'; import { isThinkingLevel, type ThinkingLevel } from '@maka/core/model-thinking'; import { deriveConnectionSlug, type ProviderType } from '@maka/core/llm-connections'; @@ -3115,6 +3118,7 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { let sessions: readonly ExternalSessionCatalogItem[] = []; let nextCursor: string | null = null; let revision = 0; + let normalizedQuery = normalizeExternalSessionQueryText(query); let cancelScheduledSearch: (() => void) | undefined; let pageClosed = false; let overlay: OverlayHandle | undefined; @@ -3220,7 +3224,10 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { emptyText: sessions.length === 0 ? copy.externalEmpty : copy.externalUnavailable, notice, onQuery: (text) => { + const nextNormalizedQuery = normalizeExternalSessionQueryText(text); query = text; + if (nextNormalizedQuery === normalizedQuery) return; + normalizedQuery = nextNormalizedQuery; dropScheduledSearch(); resetCatalogPage(); // Retire an older in-flight response immediately. Waiting until the @@ -3240,10 +3247,6 @@ export async function runMakaPiTui(input: MakaPiTuiInput): Promise { }, onSelect: (item) => { if (item.value === 'external:load-more' && nextCursor) { - if (dropScheduledSearch()) { - void load(false); - return; - } void load(true, nextCursor); return; } diff --git a/packages/core/src/external-session.ts b/packages/core/src/external-session.ts index 077dafec52..528647efbb 100644 --- a/packages/core/src/external-session.ts +++ b/packages/core/src/external-session.ts @@ -32,7 +32,7 @@ export interface ExternalSessionQuery { cwd?: string; includeArchived?: boolean; /** - * Free text matched against a summary's title and cwd. + * Free text matched against a summary's source id, title, and cwd. * * Applied by the adapter, before paging. Filtering an assembled page would * search only the rows already fetched, which on a 1128-session source is diff --git a/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts b/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts index b58573171a..986006c474 100644 --- a/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/external-session-coordinator.test.ts @@ -740,7 +740,8 @@ test('reports conversion errors before persistence and store uncertainty after e assert.equal(canonicalizationFailure.drainRequests(), 0); const persistenceFailure = coordinatorFixture([adapterFixture()], { - createImportedSession: async () => { + createImportedSession: async (_input, _messages, _externalOrigin, options) => { + options?.onCommitStarted?.(); throw new Error('commit acknowledgement lost'); }, }); @@ -759,6 +760,26 @@ test('reports conversion errors before persistence and store uncertainty after e }, ); assert.equal(persistenceFailure.drainRequests(), 1); + + const projectionFailure = coordinatorFixture([adapterFixture()], { + createImportedSession: async () => { + throw new Error('forced projection failure'); + }, + }); + assert.deepEqual( + await projectionFailure.coordinator.handlers['external-session.import']( + { adapterId: 'codex', sourceSessionId: 'source-0' }, + context, + ), + { + ok: false, + error: { + code: 'source_unreadable', + message: 'External Session could not be read or converted', + }, + }, + ); + assert.equal(projectionFailure.drainRequests(), 0); }); test('classifies source absence only through the adapter error authority', async () => { @@ -882,7 +903,8 @@ test('does not classify untyped source errors or errors after persistence as sou }, ); const committed = coordinatorFixture([adapterFixture()], { - createImportedSession: async () => { + createImportedSession: async (_input, _messages, _externalOrigin, options) => { + options?.onCommitStarted?.(); throw new ExternalSessionLimitError('record_bytes', 100, 'private persistence details'); }, }); @@ -1091,7 +1113,6 @@ function coordinatorFixture( if (!storeOverrides.createImportedSession) { return defaultCreate(input, messages, externalOrigin, options); } - options?.onCommitStarted?.(); return storeOverrides.createImportedSession(input, messages, externalOrigin, options); }, lookupExternalSessionImports: async (adapterId, sourceSessionIds, recentSessionIdLimit) => { diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index af992b24af..17a589e17f 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -29,6 +29,7 @@ import type { RuntimeExecutionConnection } from '@maka/core/llm-connections'; import { generalizedErrorMessage } from '@maka/core/redaction'; import { emptyPlanSessionState } from '@maka/core/plan'; import { readLogicalRuntimeExecutionForRun } from '@maka/core/runtime-logical-execution'; +import { foldForMatch } from '@maka/core/thread-search'; import type { PermissionMode } from '@maka/core/permission'; import { runtimeInvocationOutcome, @@ -1709,19 +1710,17 @@ export async function createExecutionRuntimeHostComposition( }); }, search: async (request, caller) => { - const query = request.query.toLocaleLowerCase(); + const query = foldForMatch(request.query); const sessions = await visibleAgentSessions(sessionQueryInitiator(caller)); const matches: typeof sessions = []; for (const session of sessions) { - const headerText = `${session.name}\n${session.cwd ?? ''}`.toLocaleLowerCase(); + const headerText = foldForMatch(`${session.name}\n${session.cwd ?? ''}`); if (headerText.includes(query)) { matches.push(session); continue; } const messages = await requireSessionManager(manager).getMessages(session.id); - if ( - messages.some((message) => JSON.stringify(message).toLocaleLowerCase().includes(query)) - ) { + if (messages.some((message) => foldForMatch(JSON.stringify(message)).includes(query))) { matches.push(session); } } diff --git a/packages/storage/src/__tests__/codex-session-adapter.test.ts b/packages/storage/src/__tests__/codex-session-adapter.test.ts index a9f6cb8bce..b3ed4e2263 100644 --- a/packages/storage/src/__tests__/codex-session-adapter.test.ts +++ b/packages/storage/src/__tests__/codex-session-adapter.test.ts @@ -658,7 +658,7 @@ describe('CodexSessionAdapter', () => { }); }); - test('filesystem keyset paging uses one path order across equal-mtime pages', async () => { + test('filesystem keyset paging uses one digest order across equal-mtime pages', async () => { await withCodexHome(async (codexHome) => { const underscore = await seedMinimalRollout( codexHome, @@ -680,19 +680,31 @@ describe('CodexSessionAdapter', () => { const adapter = new CodexSessionAdapter({ codexHome }); const first = await adapter.listSessionPage!({ limit: 1 }); - assert.deepEqual( - first.items.map(({ summary }) => summary.id), - ['codex_a'], - ); + assert.equal(first.items.length, 1); assert.equal(first.hasMore, true); + const cursor = first.items[0]!.nextCursor; + const [tag, queryHash, timestamp, identity] = cursor.split(':'); + if (!queryHash || !timestamp || !identity) throw new Error('Expected a filesystem cursor'); + assert.equal(tag, 'f2'); + for (const invalidCursor of [ + `f:${queryHash}:${timestamp}:${identity}`, + `f2:${queryHash}:${timestamp}:${identity.slice(0, -1)}`, + `f2:${queryHash}:${timestamp}:${identity}A`, + `f2:${queryHash}:${timestamp}:${identity.slice(0, -1)}+`, + ]) { + await assert.rejects( + adapter.listSessionPage!({ cursor: invalidCursor, limit: 1 }), + ExternalSessionCatalogCursorError, + ); + } const second = await adapter.listSessionPage!({ - cursor: first.items[0]!.nextCursor, + cursor, limit: 1, }); assert.deepEqual( - second.items.map(({ summary }) => summary.id), - ['codex-a'], + new Set([...first.items, ...second.items].map(({ summary }) => summary.id)), + new Set(['codex_a', 'codex-a']), ); assert.equal(second.hasMore, false); }); diff --git a/packages/storage/src/__tests__/execution-provider-conformance.test.ts b/packages/storage/src/__tests__/execution-provider-conformance.test.ts index 0f94b204c6..7bf55771b5 100644 --- a/packages/storage/src/__tests__/execution-provider-conformance.test.ts +++ b/packages/storage/src/__tests__/execution-provider-conformance.test.ts @@ -621,51 +621,60 @@ for (const backend of ['Local', 'Memory'] as const) { }); }, ); - test(backend + ': imported message projection finishes before commit starts', async () => { + test(backend + ': imported message projection finishes before commit starts', async (t) => { await withProvider(make(), async ({ sessionStore: s }, root) => { let commitStarted = false; - const replaceDescriptor = Object.getOwnPropertyDescriptor(String.prototype, 'replace')!; - Object.defineProperty(String.prototype, 'replace', { - ...replaceDescriptor, - value(this: string, ...args: unknown[]) { + const originalReplace = String.prototype.replace; + t.mock.method( + String.prototype, + 'replace', + function (this: string, ...args: Parameters) { if (String(this) === 'force projection failure') { throw new Error('forced projection failure'); } - return Reflect.apply(replaceDescriptor.value as String['replace'], this, args) as string; + return Reflect.apply(originalReplace, this, args) as string; }, - }); - try { - await assert.rejects( - s.createImportedSession( - sessionInput(root), - [ - { - type: 'user', - id: 'imported-user', - turnId: 'imported-turn', - ts: 1, - text: 'force projection failure', - }, - ], - { adapterId: 'fake', sourceSessionId: 'source' }, - { onCommitStarted: () => (commitStarted = true) }, - ), - /forced projection failure/, - ); - assert.equal(commitStarted, false); - assert.deepEqual(await s.listHeaders(), []); - } finally { - Object.defineProperty(String.prototype, 'replace', replaceDescriptor); - } + ); + await assert.rejects( + s.createImportedSession( + sessionInput(root), + [ + { + type: 'user', + id: 'imported-user', + turnId: 'imported-turn', + ts: 1, + text: 'force projection failure', + }, + ], + { adapterId: 'fake', sourceSessionId: 'source' }, + { onCommitStarted: () => (commitStarted = true) }, + ), + /forced projection failure/, + ); + assert.equal(commitStarted, false); + assert.deepEqual(await s.listHeaders(), []); }); }); test(backend + ': external import lookup excludes staged Sessions', async () => { await withProvider(make(), async ({ sessionStore: s }, root) => { const createImport = (sourceSessionId: string) => - s.createImportedSession(sessionInput(root), [], { - adapterId: 'fake', - sourceSessionId, - }); + s.createImportedSession( + sessionInput(root), + [ + { + type: 'user', + id: `imported-${sourceSessionId}`, + turnId: `turn-${sourceSessionId}`, + ts: 1, + text: `imported ${sourceSessionId}`, + }, + ], + { + adapterId: 'fake', + sourceSessionId, + }, + ); const published = await createImport('shared-source'); const stagedShared = await createImport('shared-source'); const stagedOnly = await createImport('staged-only'); @@ -686,6 +695,11 @@ for (const backend of ['Local', 'Memory'] as const) { ], ); + assert.deepEqual( + (await s.list()).map((session) => session.id), + [published.id], + ); + const page = await s.listCatalogPage(undefined, undefined, 8); assert.equal(page.kind, 'page'); if (page.kind !== 'page') throw new Error('Expected a catalog page'); diff --git a/packages/storage/src/codex-session-adapter.ts b/packages/storage/src/codex-session-adapter.ts index 9d30a92ebe..7b375d6127 100644 --- a/packages/storage/src/codex-session-adapter.ts +++ b/packages/storage/src/codex-session-adapter.ts @@ -1114,7 +1114,7 @@ function encodeCatalogKeyset( if (keyset.kind === 'database') { return `d:${queryHash}:${Buffer.from(keyset.stateDatabase).toString('base64url')}:${encodeCursorNumber(keyset.sortTimestamp)}:${Buffer.from(keyset.id).toString('base64url')}`; } - return `f:${queryHash}:${encodeCursorNumber(keyset.mtimeMs)}:${keyset.pathIdentity}`; + return `f2:${queryHash}:${encodeCursorNumber(keyset.mtimeMs)}:${keyset.pathIdentity}`; } function decodeCatalogKeyset( @@ -1144,7 +1144,7 @@ function decodeCatalogKeyset( } return { kind: 'database', stateDatabase, sortTimestamp, id }; } - if (parts[0] === 'f') { + if (parts[0] === 'f2') { if (parts.length !== 4) throw new ExternalSessionCatalogCursorError(); const mtimeMs = decodeCursorNumber(parts[2]!); const pathIdentity = parts[3]!; diff --git a/packages/storage/src/session-store-contract.ts b/packages/storage/src/session-store-contract.ts index ead6f923ab..20622f3b59 100644 --- a/packages/storage/src/session-store-contract.ts +++ b/packages/storage/src/session-store-contract.ts @@ -418,7 +418,12 @@ export interface SessionAuthorityStore extends SessionStore, MessageAdmissionSto subscribeTranscriptChanges(listener: (sessionId: string) => void): () => void; /** Wait until the durable authority is ready for cross-domain transactions. */ ready(): Promise; - /** Atomically create a Session from already-converted Maka raw messages. */ + /** + * Atomically create a Session from already-converted Maka raw messages. + * Implementations must finish canonical validation and deterministic catalog + * projection before invoking `onCommitStarted`; failures before that callback + * have not started durable commit and must not leave a staged Session. + */ createImportedSession( input: CreateSessionInput, messages: readonly StoredMessage[], diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index e3ff652139..8b19bce890 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -570,6 +570,7 @@ class SqliteSessionStore implements SessionAuthorityStore { async list(filter?: SessionListFilter): Promise { await this.ensureReady(); return (await this.metadata.list(filter, 'ordinary')) + .filter((record) => record.header.transcriptLedgerVersion !== 0) .filter((record) => record.header.conversationCopy?.state !== 'preparing') .map((record) => toCatalogSummary(record.header, record.lastMessagePreview)); } @@ -723,10 +724,13 @@ class SqliteSessionStore implements SessionAuthorityStore { async appendMessages(sessionId: string, messages: StoredMessage[]): Promise { if (messages.length === 0) return; await this.ensureReady(); + const canonicalMessages = messages.map((message) => + decodeCanonicalMessage(JSON.parse(JSON.stringify(message)) as unknown), + ); await this.metadata.appendMessages( sessionId, - messages, - projectSessionCatalogMessages(messages), + canonicalMessages, + projectSessionCatalogMessages(canonicalMessages), ); for (const listener of this.transcriptChangeListeners) listener(sessionId); } diff --git a/packages/storage/src/test-only/memory-execution-session.ts b/packages/storage/src/test-only/memory-execution-session.ts index 7f13cf0098..84e690217b 100644 --- a/packages/storage/src/test-only/memory-execution-session.ts +++ b/packages/storage/src/test-only/memory-execution-session.ts @@ -445,7 +445,7 @@ export function createMemorySessionStore( sourceIds.flatMap((sourceSessionId) => { const matches = [...headers(s).values()].filter( (h) => - h.header.transcriptLedgerVersion === 1 && + h.header.transcriptLedgerVersion !== 0 && h.header.externalOrigin?.adapterId === adapterId && h.header.externalOrigin.sourceSessionId === sourceSessionId, );