diff --git a/CHANGELOG.md b/CHANGELOG.md index ea38d7a..9d86faa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 17.1.0 + +- Preserve document identity through reciprocal-rank fusion; same-id pages from different origins can no longer replace matching bytes. +- Build run-scoped citation handles and graph edges through the existing citation resolver. Unique ids remain unchanged; ambiguous origins are explicit and every returned handle resolves to the page actually ranked. +- Expose existing invalidation, tag and kind filters to `knowledge_search` without changing its defaults, write intake, or access scope. +- Share one typed reciprocal-rank fusion implementation between document ranking and the unchanged public string-key helper. + + ## 17.0.3 — 2026-09-20 Allow consumers to install Eval 0.182 or 0.183 alongside Knowledge. diff --git a/README.md b/README.md index 18e317c..36d0d39 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent- ## Install ```bash -pnpm add @tangle-network/agent-knowledge@17.0.3 @tangle-network/agent-eval@0.183.0 @tangle-network/agent-interface@2.10.0 +pnpm add @tangle-network/agent-knowledge@17.1.0 @tangle-network/agent-eval@0.183.0 @tangle-network/agent-interface@2.10.0 ``` Requires Node.js 20.19 or later. diff --git a/docs/run-scoped-citations.md b/docs/run-scoped-citations.md index 2d29fae..05df394 100644 --- a/docs/run-scoped-citations.md +++ b/docs/run-scoped-citations.md @@ -31,6 +31,36 @@ cites: Use `parseKnowledgeCitationReference()` and `formatKnowledgeCitationReference()` rather than assembling qualified strings in application code. +Names containing the delimiter are supported, not banned. When the legacy spelling would +lose identity, the formatter emits `knowledge-ref:v1:` followed by a URI-encoded JSON +`[origin, pageId]` tuple; a null origin means unqualified. For example, origin +`inherited:a::b` with page `c` must not alias origin `inherited:a` with page `b::c`. +Ordinary handles retain their previous bytes, and percent sequences in legacy handles +remain literal. The new prefix is reserved for encoded references; use the formatter +(or the structured reference API) for a literal page id beginning with that prefix. +Malformed encoded handles fail explicitly. Historical ambiguous handles are not guessed +or silently rewritten; retain the original evidence and qualify a new reference from +its known origin. + +## Search and read use the same identity + +`buildKnowledgeBrief` and `knowledge_search` rank distinct visible documents without +merging pages that share a bare id. Their returned `citationIds` and rendered links +use the resolver's qualified form whenever the full visible chain is ambiguous, +even if only one of those pages matches the query or survives a filter. Unique +ordinary ids retain their previous short form. Returned pages and retrieval +receipts preserve the original bytes and origins, not rewritten page objects. + +Use each returned handle directly with `knowledge_read`, `knowledge_resolve`, or +`cites`; a query for a local page cannot be substituted with a same-id inherited +page. Ambiguous or malformed outgoing links are not graph-ranking evidence; +citation audit still sees the unchanged source page and can diagnose them. + +`knowledge_search` accepts optional `excludeInvalidated`, `tags`, and `kinds` using +the existing brief semantics. For historical research, `excludeInvalidated: false` +includes refuted approaches; this does not make their claims valid or change the +host defaults for subsequent calls. Receipt identities capture the selected filters. + ## Resolution ```ts diff --git a/package.json b/package.json index ebe2117..6cf7c0b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "17.0.3", + "version": "17.1.0", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { diff --git a/src/citation-encoding.test.ts b/src/citation-encoding.test.ts new file mode 100644 index 0000000..2867a99 --- /dev/null +++ b/src/citation-encoding.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest' +import { + formatKnowledgeCitationReference, + parseKnowledgeCitationReference, + resolveKnowledgeCitation, +} from './citation-resolution' +import { buildKnowledgeBrief } from './knowledge-brief' +import type { PageOrigin } from './run-scoped' +import type { KnowledgePage } from './types' + +const page = (id: string, text: string): KnowledgePage => ({ + id, + text, + title: text, + path: `${id}.md`, + outLinks: [], + tags: [], + sourceIds: [], + frontmatter: { id }, +}) + +describe('citation serialization is injective, including legacy delimiter collisions', () => { + it('round trips every supported origin/page pair without banning existing names', () => { + const origins: (PageOrigin | undefined)[] = [ + undefined, + 'here', + 'shared', + 'inherited:a', + 'inherited:a::b', + 'inherited:a:', + 'inherited:a%3A%3Ab', + 'inherited:∑:研究', + 'inherited:%/::x', + ] + const ids = [ + 'c', + 'b::c', + 'here::literal', + 'shared::', + 'knowledge-ref:v1:literal', + '%3A%3A', + 'quantum-∑', + 'a::b::c', + ] + const handles = new Set() + for (const origin of origins) { + for (const pageId of ids) { + const reference = { pageId, ...(origin === undefined ? {} : { origin }) } + const handle = formatKnowledgeCitationReference(reference) + expect(parseKnowledgeCitationReference(handle)).toEqual(reference) + expect(handles.has(handle)).toBe(false) + handles.add(handle) + } + } + }) + + it('leaves normal handles and legacy literal percent sequences unchanged', () => { + expect(formatKnowledgeCitationReference({ origin: 'inherited:a', pageId: 'c' })).toBe( + 'inherited:a::c', + ) + expect(parseKnowledgeCitationReference('inherited:a%3A%3Ab::c')).toEqual({ + origin: 'inherited:a%3A%3Ab', + pageId: 'c', + }) + expect(formatKnowledgeCitationReference({ pageId: 'plain' })).toBe('plain') + }) + + it('keeps the two former colliding origins distinct through search and resolution', () => { + const first = page('c', 'Quantum first') + const second = page('b::c', 'Quantum second') + const visible = [ + { page: first, origin: 'inherited:a::b' as const }, + { page: second, origin: 'inherited:a' as const }, + { page: page('c', 'Banana local'), origin: 'here' as const }, + ] + const result = buildKnowledgeBrief(visible, 'quantum') + expect(result.results).toHaveLength(2) + expect(new Set(result.citationIds).size).toBe(2) + for (const [index, hit] of result.results.entries()) { + const resolved = resolveKnowledgeCitation( + visible, + parseKnowledgeCitationReference(result.citationIds[index]!), + ) + expect(resolved.resolved?.page).toBe(hit.page) + expect(resolved.resolved?.origin).toBe(hit.origin) + } + }) + + it('retrieves literal reserved-prefix page names through the same formatter', () => { + const literal = page('knowledge-ref:v1:literal', 'Quantum reserved name') + const visible = [{ page: literal, origin: 'here' as const }] + const brief = buildKnowledgeBrief(visible, 'quantum') + expect(brief.results).toHaveLength(1) + expect( + resolveKnowledgeCitation(visible, parseKnowledgeCitationReference(brief.citationIds[0]!)) + .resolved?.page, + ).toBe(literal) + }) + + it('rejects malformed encoded references instead of silently changing their identity', () => { + for (const value of ['%', '%5B%5D', '%5Bnull%5D', '%5Btrue%2C%22p%22%5D']) { + expect(() => parseKnowledgeCitationReference(`knowledge-ref:v1:${value}`)).toThrow( + /invalid encoded knowledge citation/, + ) + } + }) +}) diff --git a/src/citation-resolution.ts b/src/citation-resolution.ts index 66c45c0..f3097d3 100644 --- a/src/citation-resolution.ts +++ b/src/citation-resolution.ts @@ -91,17 +91,40 @@ export class KnowledgeCitationAuditError extends Error { } } +// Legacy origin::page handles stay readable and byte-compatible. Use an explicit, +// versioned tuple only when delimiters or a reserved prefix would lose identity. +const ENCODED_REFERENCE_PREFIX = 'knowledge-ref:v1:' + /** * Parse the persisted citation form. * * `page-id` is unqualified. `here::page-id`, `shared::page-id`, and * `inherited:::page-id` bind an intentional duplicate to one origin. + * Ambiguous names use `knowledge-ref:v1:` plus a URI-encoded JSON [origin, pageId] + * tuple (null origin means unqualified). Always use the formatter to mint handles. + * Legacy percent sequences are literal; old records are not silently reinterpreted. */ export function parseKnowledgeCitationReference(value: string): KnowledgeCitationReference { if (typeof value !== 'string' || value.trim().length === 0) { throw new TypeError('persisted knowledge citation must be a non-empty string') } const normalized = value.trim() + if (normalized.startsWith(ENCODED_REFERENCE_PREFIX)) { + try { + const tuple: unknown = JSON.parse( + decodeURIComponent(normalized.slice(ENCODED_REFERENCE_PREFIX.length)), + ) + if (!Array.isArray(tuple) || tuple.length !== 2) { + throw new TypeError('encoded citation must contain an origin/page tuple') + } + return normalizeReference({ + pageId: tuple[1], + ...(tuple[0] === null ? {} : { origin: tuple[0] }), + }) + } catch (cause) { + throw new TypeError('invalid encoded knowledge citation', { cause }) + } + } const separator = normalized.indexOf('::') if (separator < 0) return Object.freeze({ pageId: normalized }) const possibleOrigin = normalized.slice(0, separator) @@ -115,9 +138,20 @@ export function parseKnowledgeCitationReference(value: string): KnowledgeCitatio /** Serialize one reference into the canonical frontmatter representation. */ export function formatKnowledgeCitationReference(reference: KnowledgeCitationReference): string { const normalized = normalizeReference(reference) - return normalized.origin === undefined - ? normalized.pageId - : `${normalized.origin}::${normalized.pageId}` + const legacy = + normalized.origin === undefined + ? normalized.pageId + : `${normalized.origin}::${normalized.pageId}` + try { + const parsed = parseKnowledgeCitationReference(legacy) + if (parsed.pageId === normalized.pageId && parsed.origin === normalized.origin) return legacy + } catch (error) { + // A literal page id can itself look like a malformed reserved handle. + if (!(error instanceof TypeError)) throw error + } + return `${ENCODED_REFERENCE_PREFIX}${encodeURIComponent( + JSON.stringify([normalized.origin ?? null, normalized.pageId]), + )}` } /** Resolve one reference against an already materialized visibility chain. */ diff --git a/src/knowledge-brief.ts b/src/knowledge-brief.ts index 228b627..5310f88 100644 --- a/src/knowledge-brief.ts +++ b/src/knowledge-brief.ts @@ -10,14 +10,20 @@ * Pure: no clock, no filesystem, no network. */ import { canonicalCandidateDigest, type Sha256Digest } from '@tangle-network/agent-interface' +import { + assertKnowledgeCitationsResolved, + formatKnowledgeCitationReference, + parseKnowledgeCitationReference, + resolveKnowledgeCitation, +} from './citation-resolution' import type { OriginatedKnowledgeSearchResult } from './knowledge-use-receipts' -import type { OriginatedPage, PageOrigin } from './run-scoped' +import type { OriginatedPage } from './run-scoped' import { KNOWLEDGE_SEARCH_RETRIEVER_ID, type KnowledgeSearchHit, searchKnowledgePages, } from './search' -import type { KnowledgeId } from './types' +import type { KnowledgeId, KnowledgePage } from './types' /** Pages in a brief when the caller names no limit. */ export const DEFAULT_KNOWLEDGE_BRIEF_LIMIT = 5 @@ -87,21 +93,64 @@ export function buildKnowledgeBrief( throw new Error(`knowledge brief maxChars must be a non-negative integer, got ${maxChars}`) } - const originByPage = new Map() - for (const entry of visiblePages) originByPage.set(entry.page, entry.origin) - - const ranked = searchKnowledgePages( - visiblePages.map((entry) => entry.page), - question, - { - limit, - excludeInvalidated, - ...(options.tags === undefined ? {} : { tags: options.tags }), - ...(options.kinds === undefined ? {} : { kinds: options.kinds }), - }, - ) + // Resolve citation addresses with the same owner used by read/write validation. + // Keep a small candidate list per id so resolving links does not rescan the corpus. + const byId = new Map() + for (const entry of visiblePages) { + const matches = byId.get(entry.page.id) ?? [] + matches.push(entry) + byId.set(entry.page.id, matches) + } + const qualified = (entry: OriginatedPage) => + formatKnowledgeCitationReference({ pageId: entry.page.id, origin: entry.origin }) + const originals = new Map() + const pages = visiblePages.map((entry) => { + const page: KnowledgePage = { + ...entry.page, + id: qualified(entry), + outLinks: entry.page.outLinks.flatMap((link: string) => { + // A malformed link is not a ranking edge; retain the page unchanged so + // citation audit can still diagnose it without making retrieval unavailable. + try { + const reference = parseKnowledgeCitationReference(link) + const resolution = resolveKnowledgeCitation(byId.get(reference.pageId) ?? [], reference) + return resolution.resolved ? [qualified(resolution.resolved)] : [] + } catch (error) { + if (error instanceof TypeError) return [] + throw error + } + }), + } + originals.set(page, entry) + return page + }) + const ranked = searchKnowledgePages(pages, question, { + limit, + excludeInvalidated, + ...(options.tags === undefined ? {} : { tags: options.tags }), + ...(options.kinds === undefined ? {} : { kinds: options.kinds }), + }).map((hit) => { + const entry = originals.get(hit.page)! + const candidates = byId.get(entry.page.id)! + const reference = { + pageId: entry.page.id, + ...(candidates.length === 1 && + formatKnowledgeCitationReference({ pageId: entry.page.id }) === entry.page.id + ? {} + : { origin: entry.origin }), + } + // The whole visible chain determines ambiguity, not just this query's filtered hits. + // Reusing one id within the SAME origin cannot be repaired with a qualifier. + assertKnowledgeCitationsResolved(candidates, [reference]) + return { + ...hit, + page: entry.page, + origin: entry.origin, + citationId: formatKnowledgeCitationReference(reference), + } + }) - const hits: KnowledgeSearchHit[] = [] + const hits: (KnowledgeSearchHit & OriginatedKnowledgeSearchResult)[] = [] const lines: string[] = [] let length = 0 for (const hit of ranked) { @@ -125,9 +174,7 @@ export function buildKnowledgeBrief( }), hits: Object.freeze(hits), citationIds: Object.freeze(hits.map((hit) => hit.citationId)), - results: Object.freeze( - hits.map((hit) => Object.freeze({ ...hit, origin: originOf(originByPage, hit) })), - ), + results: Object.freeze(hits.map((hit) => Object.freeze(hit))), text: lines.join('\n'), }) } @@ -138,14 +185,3 @@ function briefLine(hit: KnowledgeSearchHit): string { ? `- [${hit.citationId}] ${hit.page.title}` : `- [${hit.citationId}] ${hit.page.title} — ${snippet}` } - -function originOf( - originByPage: ReadonlyMap, - hit: KnowledgeSearchHit, -): PageOrigin { - const origin = originByPage.get(hit.page) - if (origin === undefined) { - throw new Error(`knowledge brief ranked a page outside the visible chain: ${hit.page.path}`) - } - return origin -} diff --git a/src/knowledge-tools.test.ts b/src/knowledge-tools.test.ts index 88e958e..8a668e9 100644 --- a/src/knowledge-tools.test.ts +++ b/src/knowledge-tools.test.ts @@ -220,3 +220,66 @@ describe('createKnowledgeRetrievalDisposition', () => { ) }) }) + +describe('search controls use the existing brief semantics', () => { + it('allows an agent to inspect invalidated history without changing host defaults', async () => { + const historical = { + id: 'refuted', + path: 'knowledge/refuted.md', + title: 'Refuted quantum approach', + text: 'Quantum decoding.', + frontmatter: { kind: 'finding' }, + sourceIds: [], + tags: ['history'], + outLinks: [], + invalidation: { + verdict: 'contradicted' as const, + observedAt: '2026-09-20T00:00:00Z', + reason: 'Counterexample.', + }, + } + const scopedStores = { + ...stores, + loadChain: async () => [{ page: historical, origin: 'here' as const }], + } + const search = createKnowledgeTools({ + stores: scopedStores, + runId: 'run-a', + retrieverVersion: 'test', + }).find((tool) => tool.name === 'knowledge_search')! + const invoke = async (input: unknown) => + (await search.handler(input, {})) as { citationIds: string[] } + expect((await invoke({ question: 'quantum' })).citationIds).toEqual([]) + expect( + ( + await invoke({ + question: 'quantum', + excludeInvalidated: false, + tags: ['history'], + kinds: ['finding'], + }) + ).citationIds, + ).toEqual(['refuted']) + expect( + (await invoke({ question: 'quantum', excludeInvalidated: false, tags: ['other'] })) + .citationIds, + ).toEqual([]) + expect((await invoke({ question: 'quantum' })).citationIds).toEqual([]) + }) + + it('search-to-read round trips origin-qualified handles through the actual tools and stores', async () => { + await writePage(stores.storePath('run-a'), 'state', 'Quantum decoding.') + await initKnowledgeBase(shared) + await writePage(shared, 'state', 'Banana schedules.') + const search = await call('knowledge_search', { question: 'quantum' }) + expect(search.citationIds).toEqual(['here::state']) + const found = await call('knowledge_read', { pageId: (search.citationIds as string[])[0] }) + expect(found.status).toBe('resolved') + expect(found.page).toMatchObject({ + origin: 'here', + pageId: 'state', + text: expect.stringContaining('Quantum'), + }) + expect(recorded[0]?.results[0]).toMatchObject({ origin: 'here', pageId: 'state' }) + }) +}) diff --git a/src/knowledge-tools.ts b/src/knowledge-tools.ts index 29708b7..60e7761 100644 --- a/src/knowledge-tools.ts +++ b/src/knowledge-tools.ts @@ -60,6 +60,12 @@ export interface CreateKnowledgeToolsOptions { const searchInput = z.object({ question: z.string().min(1), limit: z.int().min(1).max(50).optional(), + excludeInvalidated: z + .boolean() + .optional() + .describe('False includes refuted pages for historical research.'), + tags: z.array(z.string()).optional(), + kinds: z.array(z.string()).optional(), }) const readInput = z.object({ pageId: z.string().min(1) }) const recordInput = z.object({ @@ -94,13 +100,18 @@ export function createKnowledgeTools(options: CreateKnowledgeToolsOptions): Tool return [ tool( 'knowledge_search', - 'Search the knowledge this run can see and return a brief with the ids to cite.', + 'Search visible knowledge with unambiguous citation handles. Optionally include refuted history or filter tags and kinds.', searchInput, async (input) => { const chain = await stores.loadChain(runId) const brief = buildKnowledgeBrief(chain, input.question, { ...options.brief, ...(input.limit === undefined ? {} : { limit: input.limit }), + ...(input.excludeInvalidated === undefined + ? {} + : { excludeInvalidated: input.excludeInvalidated }), + ...(input.tags === undefined ? {} : { tags: input.tags }), + ...(input.kinds === undefined ? {} : { kinds: input.kinds }), }) const visibility = createKnowledgeVisibilitySnapshot(chain) const visibilityArtifact = options.recordRetrieval diff --git a/src/search-origins.test.ts b/src/search-origins.test.ts new file mode 100644 index 0000000..aa36eca --- /dev/null +++ b/src/search-origins.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from 'vitest' +import { parseKnowledgeCitationReference, resolveKnowledgeCitation } from './citation-resolution' +import { buildKnowledgeBrief } from './knowledge-brief' +import { + assertKnowledgeRetrievalMatchesVisibility, + createKnowledgeRetrievalReceipt, + createKnowledgeVisibilitySnapshot, +} from './knowledge-use-receipts' +import type { OriginatedPage } from './run-scoped' +import { searchKnowledgePages } from './search' +import type { KnowledgePage } from './types' + +const page = (id: string, title: string, text: string, outLinks: string[] = []): KnowledgePage => ({ + id, + title, + path: `knowledge/${id}.md`, + text, + outLinks, + tags: [], + sourceIds: [], + frontmatter: { id }, +}) +const quantum = page('state', 'Quantum error correction', 'Quantum syndrome decoding.') +const banana = page('state', 'Banana logistics', 'Banana shipment schedules.') +const visible: OriginatedPage[] = [ + { page: quantum, origin: 'here' }, + { page: banana, origin: 'inherited:prior' }, +] + +function assertRoundTrips(chain: OriginatedPage[], question: string) { + const brief = buildKnowledgeBrief(chain, question, { limit: 50 }) + for (const hit of brief.results) { + const reference = brief.citationIds[hit.rank - 1]! + const resolved = resolveKnowledgeCitation(chain, parseKnowledgeCitationReference(reference)) + expect(resolved.status).toBe('resolved') + expect(resolved.resolved?.page).toBe(hit.page) + expect(resolved.resolved?.origin).toBe(hit.origin) + } + const visibility = createKnowledgeVisibilitySnapshot(chain) + const receipt = createKnowledgeRetrievalReceipt({ + runId: 'origin-test', + query: brief.question, + retriever: { + id: brief.retrieverId, + version: 'test', + configDigest: brief.retrieverConfigDigest, + }, + visibility, + results: brief.results, + }) + assertKnowledgeRetrievalMatchesVisibility(receipt, visibility) + return brief +} + +describe('retrieval preserves document identity across origins', () => { + it('does not replace a matching document with an unrelated same-id document, in either order', () => { + for (const chain of [visible, [...visible].reverse()]) { + const raw = searchKnowledgePages( + chain.map((entry) => entry.page), + 'quantum', + ) + expect(raw).toHaveLength(1) + expect(raw[0]!.page).toBe(quantum) + const brief = assertRoundTrips(chain, 'quantum') + expect(brief.citationIds).toEqual(['here::state']) + expect(brief.text).toContain('Quantum error correction') + expect(brief.text).not.toContain('Banana') + } + }) + + it('keeps both independently matching same-id pages rather than fusing them into one', () => { + const a = page('state', 'Quantum A', 'quantum') + const b = page('state', 'Quantum B', 'quantum') + expect(searchKnowledgePages([a, b], 'quantum').map((hit) => hit.page)).toEqual([a, b]) + }) + + it('can distinguish the same page object exposed at several origins', () => { + const chain: OriginatedPage[] = ['here', 'shared', 'inherited:prior'].map((origin) => ({ + page: quantum, + origin: origin as OriginatedPage['origin'], + })) + const before = JSON.stringify(chain) + const result = assertRoundTrips(chain, 'quantum') + expect(new Set(result.citationIds).size).toBe(3) + expect(result.results.every((hit) => hit.page === quantum)).toBe(true) + expect(assertRoundTrips([...chain].reverse(), 'quantum').citationIds).toEqual( + result.citationIds, + ) + expect(JSON.stringify(chain)).toBe(before) + }) + + it('qualifies against all visible pages, even when only one duplicate survives the filters', () => { + const chain: OriginatedPage[] = [ + { origin: 'here', page: { ...quantum, tags: ['selected'] } }, + { origin: 'shared', page: { ...banana, tags: ['excluded'] } }, + ] + const brief = buildKnowledgeBrief(chain, 'quantum', { tags: ['selected'], limit: 1 }) + expect(brief.citationIds).toEqual(['here::state']) + expect( + resolveKnowledgeCitation(chain, parseKnowledgeCitationReference(brief.citationIds[0]!)) + .resolved?.page, + ).toBe(chain[0]!.page) + }) + + it('preserves unqualified handles for unique ordinary page ids', () => { + expect(assertRoundTrips([visible[0]!], 'quantum').citationIds).toEqual(['state']) + }) + + it('keeps text, hits and receipt results aligned after the rendering bound', () => { + const brief = buildKnowledgeBrief(visible, 'quantum', { maxChars: 1 }) + expect(brief.text).toBe('') + expect(brief.hits).toEqual([]) + expect(brief.results).toEqual([]) + expect(brief.citationIds).toEqual([]) + }) + + it('refuses a genuinely ambiguous same-origin citation rather than choosing a page', () => { + expect(() => + buildKnowledgeBrief( + [ + { origin: 'here', page: quantum }, + { origin: 'here', page: { ...banana, path: 'knowledge/other.md' } }, + ], + 'quantum', + ), + ).toThrow(/ambiguous/) + }) + + it('uses the existing citation resolver for graph edges, not bare id coincidence', () => { + const chain: OriginatedPage[] = [ + ...visible, + { + origin: 'here', + page: page('right-link', 'Correct neighbor', 'A useful neighbor.', ['here::state']), + }, + { + origin: 'here', + page: page('wrong-link', 'Unrelated neighbor', 'Another neighbor.', [ + 'inherited:prior::state', + ]), + }, + { + origin: 'here', + page: page('ambiguous-link', 'Ambiguous neighbor', 'Ambiguous.', ['state']), + }, + ] + const brief = assertRoundTrips(chain, 'quantum') + expect([...brief.citationIds].sort()).toEqual(['here::state', 'right-link']) + }) + + it('does not infer an unscoped graph edge from an ambiguous bare id', () => { + const linked = page('link', 'Neighbor', 'Neighbor.', ['state']) + expect( + searchKnowledgePages([quantum, banana, linked], 'quantum').map((hit) => hit.page), + ).toEqual([quantum]) + }) + + it('keeps searchable evidence with malformed outgoing links without inventing an edge', () => { + const malformed = { ...quantum, outLinks: ['inherited:::state'] } + const chain: OriginatedPage[] = [{ origin: 'here', page: malformed }] + const result = assertRoundTrips(chain, 'quantum') + expect(result.results[0]!.page).toBe(malformed) + expect(result.results[0]!.page.outLinks).toEqual(['inherited:::state']) + }) +}) diff --git a/src/search.ts b/src/search.ts index 22117a3..3411404 100644 --- a/src/search.ts +++ b/src/search.ts @@ -39,8 +39,8 @@ export interface SearchKnowledgeOptions { /** * A retrieval result with an explicit citation handle. * - * `citationId` is exactly `page.id`; later writes should persist this value - * when they cite the page. Keeping it at the result's top level prevents tool + * Unscoped search returns `page.id`; a run-scoped brief qualifies ambiguous ids + * through the citation resolver. Later writes should persist the returned value. Keeping it at the result's top level prevents tool * renderers from accidentally hiding the only stable handle a model can copy. */ export interface KnowledgeSearchHit extends KnowledgeSearchResult { @@ -90,17 +90,14 @@ export function searchKnowledgePages( ? assertLexicalIndexMatches(options.lexicalIndex, pages) : buildKnowledgeLexicalIndex(pages) const lexicalRanked = rankLexical(matched, trimmed, lexicalIndex) - const graphRanked = rankByGraph(matched, lexicalRanked) - const scores = reciprocalRankFusion([ - lexicalRanked.map((p) => p.id), - graphRanked.map((p) => p.id), - ]) - const byId = new Map(matched.map((page) => [page.id, page])) + const graphRanked = rankByGraph(matched, lexicalRanked, pages) + // Stable ids are citation addresses, not unique document identities across stores. + // Fuse the same page objects used by BM25, then return those exact pages. + const scores = fuseRanks([lexicalRanked, graphRanked]) const ranked = [...scores.entries()] - .map(([id, score]) => ({ page: byId.get(id), score })) - .filter((item): item is { page: KnowledgePage; score: number } => Boolean(item.page)) - .sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)) + .map(([page, score]) => ({ page, score })) + .sort((a, b) => b.score - a.score || comparePages(a.page, b.page)) .slice(0, limit) // Normalize against the top hit so callers can compare against natural @@ -122,7 +119,12 @@ export function searchKnowledgePages( } export function reciprocalRankFusion(rankLists: string[][], k = RRF_K): Map { - const scores = new Map() + return fuseRanks(rankLists, k) +} + +// The public string-key helper and document retrieval share one ranking implementation. +function fuseRanks(rankLists: readonly (readonly T[])[], k = RRF_K): Map { + const scores = new Map() for (const list of rankLists) { list.forEach((id, idx) => { scores.set(id, (scores.get(id) ?? 0) + 1 / (k + idx + 1)) @@ -190,7 +192,7 @@ function rankLexical( if (score === 0 && tier === 0) return [] return [{ page, tier, score }] }) - .sort((a, b) => b.tier - a.tier || b.score - a.score || a.page.path.localeCompare(b.page.path)) + .sort((a, b) => b.tier - a.tier || b.score - a.score || comparePages(a.page, b.page)) .map((item) => item.page) } @@ -202,9 +204,22 @@ function phraseTier(page: KnowledgePage, phrase: string): number { return 0 } -function rankByGraph(pages: KnowledgePage[], lexicalRanked: KnowledgePage[]): KnowledgePage[] { +function rankByGraph( + pages: KnowledgePage[], + lexicalRanked: KnowledgePage[], + visiblePages: readonly KnowledgePage[], +): KnowledgePage[] { if (lexicalRanked.length === 0) return [] - const seeds = new Set(lexicalRanked.slice(0, 5).map((page) => page.id)) + // A bare link to a duplicate id has no unambiguous target. Scoped callers + // qualify links before ranking; unscoped callers must not invent that edge. + const counts = new Map() + for (const page of visiblePages) counts.set(page.id, (counts.get(page.id) ?? 0) + 1) + const seeds = new Set( + lexicalRanked + .slice(0, 5) + .filter((page) => counts.get(page.id) === 1) + .map((page) => page.id), + ) return pages .map((page) => ({ page, @@ -215,10 +230,14 @@ function rankByGraph(pages: KnowledgePage[], lexicalRanked: KnowledgePage[]): Kn ).length, })) .filter((item) => item.score > 0) - .sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)) + .sort((a, b) => b.score - a.score || comparePages(a.page, b.page)) .map((item) => item.page) } +function comparePages(a: KnowledgePage, b: KnowledgePage): number { + return a.path.localeCompare(b.path) || a.id.localeCompare(b.id) +} + function buildSnippet(text: string, query: string): string { const compact = text.replace(/\s+/g, ' ').trim() const idx = compact.toLowerCase().indexOf(query.toLowerCase())