From ac3218fe18bc2f2e180eaa00c7dd7d43b6e86fd4 Mon Sep 17 00:00:00 2001 From: avivkeller Date: Fri, 7 Aug 2026 11:43:54 -0400 Subject: [PATCH 1/3] feat(caching): support incremental builds --- README.md | 26 +- docs/caching.md | 96 +++++ package-lock.json | 29 ++ packages/core/bin/commands/generate.mjs | 61 +-- packages/core/bin/commands/index.mjs | 3 +- packages/core/bin/commands/watch.mjs | 150 +++++++ packages/core/bin/utils.mjs | 82 ++++ packages/core/package.json | 1 + .../core/src/cache/__tests__/hash.test.mjs | 83 ++++ .../src/cache/__tests__/manifest.test.mjs | 66 +++ .../src/cache/__tests__/snapshot.test.mjs | 78 ++++ .../core/src/cache/__tests__/store.test.mjs | 109 +++++ packages/core/src/cache/hash.mjs | 80 ++++ packages/core/src/cache/index.mjs | 391 ++++++++++++++++++ packages/core/src/cache/manifest.mjs | 51 +++ packages/core/src/cache/salt.mjs | 187 +++++++++ packages/core/src/cache/snapshot.mjs | 43 ++ packages/core/src/cache/store.mjs | 187 +++++++++ packages/core/src/cache/types.d.ts | 80 ++++ packages/core/src/generators.mjs | 78 +++- .../core/src/generators/ast-js/generate.mjs | 8 +- packages/core/src/generators/ast/generate.mjs | 4 + .../src/generators/json-simple/generate.mjs | 14 +- .../core/src/generators/metadata/generate.mjs | 10 +- packages/core/src/threading/parallel.mjs | 50 ++- .../core/src/utils/configuration/index.mjs | 23 +- packages/core/src/utils/file.mjs | 52 ++- packages/core/src/utils/unist.mjs | 26 ++ .../legacy/src/legacy-html-all/generate.mjs | 3 +- packages/legacy/src/legacy-html/generate.mjs | 156 +++++-- .../legacy/src/legacy-json-all/generate.mjs | 2 +- packages/react/src/html/bundlers/vite.mjs | 38 +- packages/react/src/html/generate.mjs | 29 +- packages/react/src/html/index.mjs | 8 +- .../src/html/utils/__tests__/copying.test.mjs | 8 +- packages/react/src/html/utils/copying.mjs | 4 +- packages/react/src/html/utils/processing.mjs | 210 +++++++--- packages/react/src/jsx-ast/generate.mjs | 152 ++++++- packages/react/src/jsx-ast/index.mjs | 2 +- packages/react/src/orama-db/generate.mjs | 5 + packages/react/src/sitemap/generate.mjs | 9 +- packages/react/src/sitemap/types.d.ts | 6 + www/doc-kit.config.mjs | 3 + 43 files changed, 2469 insertions(+), 234 deletions(-) create mode 100644 docs/caching.md create mode 100644 packages/core/bin/commands/watch.mjs create mode 100644 packages/core/src/cache/__tests__/hash.test.mjs create mode 100644 packages/core/src/cache/__tests__/manifest.test.mjs create mode 100644 packages/core/src/cache/__tests__/snapshot.test.mjs create mode 100644 packages/core/src/cache/__tests__/store.test.mjs create mode 100644 packages/core/src/cache/hash.mjs create mode 100644 packages/core/src/cache/index.mjs create mode 100644 packages/core/src/cache/manifest.mjs create mode 100644 packages/core/src/cache/salt.mjs create mode 100644 packages/core/src/cache/snapshot.mjs create mode 100644 packages/core/src/cache/store.mjs create mode 100644 packages/core/src/cache/types.d.ts diff --git a/README.md b/README.md index b24f5c68..fb747407 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ Options: Commands: generate [options] Generate API docs + watch [options] Generate API docs, rebuilding whenever an input file + changes help [command] display help for command ``` @@ -78,8 +80,8 @@ Options: (json-simple, legacy-html, legacy-html-all, man-page, legacy-json, legacy-json-all, addon-verify, api-links, orama-db, llms-txt, - sitemap, web) or an import specifier for a custom - generator + sitemap, html) or an import specifier for a + custom generator --ignore Ignore file patterns (glob) -o, --output The output directory -p, --threads Number of threads to use (minimum: 1) @@ -91,9 +93,29 @@ Options: --index index.md URL or path --minify Minify? --type-map Type map URL or path + --no-cache Disable the on-disk build cache entirely + --force Ignore existing cache entries (the cache is still + written) + --cache-dir Build cache directory -h, --help display help for command ``` +### `watch` + +`watch` takes the same options as `generate`. It builds once, then rebuilds +whenever a file matching `--input` changes, until you interrupt it. + +```sh +npx doc-kit watch \ + --input "doc/api/*.md" \ + --target json-simple \ + --output out +``` + +Rebuilds go through the same [build cache](docs/caching.md) as `generate`, so a +rebuild only redoes the work the change actually affected. A document that +fails to parse is reported without ending the session. + ## Examples ### Legacy diff --git a/docs/caching.md b/docs/caching.md new file mode 100644 index 00000000..b92b0217 --- /dev/null +++ b/docs/caching.md @@ -0,0 +1,96 @@ +# Caching and Incremental Builds + +doc-kit keeps a durable on-disk cache between runs so that rebuilds only redo +the work a change actually affects. Caching is **on by default** and designed +to be invisible: every cache failure — corruption, version mismatch, deleted +files — silently degrades to a full, correct rebuild. If you ever get a wrong +output out of a cached build, that is a bug; please report it rather than +scripting around it. + +## What you get + +- **No-change rebuilds are skipped entirely.** When the input files, + configuration, and generator code are unchanged and every previously + written output still verifies on disk, the run finishes in well under a + second without loading any generator. +- **Partial rebuilds.** After editing one document, `legacy-html` re-renders + (and re-highlights, and re-minifies) only that document; the react `html` + generator rebuilds only the edited page's JSX and server-rendered HTML. + Pages whose bytes did not change are not rewritten, so their mtimes are + stable for downstream tooling. +- **Honest floors.** Markdown parsing and metadata extraction always re-run + (they are as fast as reading the cache would be), the synthetic `all` page + is rebuilt whenever anything changed (it folds every module by design), and + the client Vite build always runs over the full graph (partial input sets + change chunk hashing). For fast dev loops on large corpora, disable the all + page: + + ```js + export default { + 'jsx-ast': { generateAllPage: false }, + }; + ``` + +## How invalidation works + +Cache keys are content hashes — never timestamps. Every key is salted with: + +- each package's identity: the published version, or a content hash of its + `src/` tree when running from a workspace or `npm link` (so hacking on + doc-kit itself invalidates correctly); +- the resolved configuration (including the parsed changelog, index, and the + fetched `typeMap` bytes); +- the Node.js major version and a cache schema version. + +Anything the cache cannot fully account for (for example a theme `imports` +alias pointing at a directory) makes the affected entries uncacheable rather +than possibly stale. + +## Configuration + +```js +export default { + cache: { + enabled: true, // default + dir: 'node_modules/.cache/doc-kit', // default (falls back to .doc-kit-cache) + maxAgeDays: 7, // age-based object pruning + }, +}; +``` + +CLI flags and environment variables: + +| Surface | Effect | +| -------------------------- | --------------------------------------------- | +| `--no-cache` | Disable reads and writes for this run | +| `--force` | Ignore existing entries; still write new ones | +| `--cache-dir ` | Override the cache directory | +| `DOC_KIT_NO_CACHE=1` | Same as `--no-cache` | +| `DOC_KIT_CACHE_FORCE=1` | Same as `--force` | +| `DOC_KIT_CACHE_DIR` | Same as `--cache-dir` | +| `DOC_KIT_CACHE_STATS_FILE` | Write machine-readable run stats as JSON | + +## CI usage + +The cache is relocatable: keys contain no absolute paths, and outputs are +never inputs. Restoring the cache directory (for example with +`actions/cache`, keyed however you like — the cache self-invalidates by +content) turns unchanged-doc CI builds into sub-second no-ops: + +```yaml +- uses: actions/cache@v4 + with: + path: node_modules/.cache/doc-kit + key: doc-kit-${{ runner.os }}-${{ hashFiles('package-lock.json') }} +``` + +The output directory can always be deleted independently of the cache; a warm +run regenerates it byte-for-byte. + +## Guarantees and verification + +`scripts/cache-verify/index.mjs` asserts the invariants end to end on every +change: cold builds are byte-identical across runs and across +threading/chunking topologies; cached builds are byte-identical to +`--no-cache` builds; a wiped output directory or a corrupted cache silently +recovers; and one-file edits rebuild exactly the affected outputs. diff --git a/package-lock.json b/package-lock.json index b88fa56f..a7a84526 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3938,6 +3938,21 @@ "dev": true, "license": "MIT" }, + "node_modules/chokidar": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", + "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", + "license": "MIT", + "dependencies": { + "readdirp": "^5.0.0" + }, + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/classnames": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", @@ -9192,6 +9207,19 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/readdirp": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", + "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", + "license": "MIT", + "engines": { + "node": ">= 20.19.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/reading-time": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/reading-time/-/reading-time-1.5.0.tgz", @@ -11222,6 +11250,7 @@ "@swc/html-wasm": "^1.15.46", "@swc/wasm": "^1.15.46", "acorn": "^8.17.0", + "chokidar": "^5.0.0", "commander": "^15.0.0", "cosmiconfig": "^9.0.2", "dedent": "^1.7.2", diff --git a/packages/core/bin/commands/generate.mjs b/packages/core/bin/commands/generate.mjs index 69dc6f23..80aca46f 100644 --- a/packages/core/bin/commands/generate.mjs +++ b/packages/core/bin/commands/generate.mjs @@ -1,71 +1,16 @@ -import { Command, Option } from 'commander'; +import { Command } from 'commander'; -import { publicGenerators } from '../../src/generators/index.mjs'; import createGenerator from '../../src/generators.mjs'; import { assertRunnableOptions, setConfig, } from '../../src/utils/configuration/index.mjs'; -import { errorWrap } from '../utils.mjs'; +import { errorWrap, insertCommonOptions } from '../utils.mjs'; const { runGenerators } = createGenerator(); -/** - * @typedef {Object} CLIOptions - * @property {string} configFile - * @property {string[]} input - * @property {string[]} target - * @property {string[]} ignore - * @property {string} output - * @property {number} threads - * @property {number} chunkSize - * @property {string} version - * @property {string} changelog - * @property {string} gitRef - * @property {string} index - * @property {boolean} minify - * @property {string} typeMap - */ - -export default new Command('generate') +export default insertCommonOptions(new Command('generate')) .description('Generate API docs') - .addOption(new Option('--config-file ', 'Config file')) - - // Options that need to be converted into a configuration - .addOption( - new Option('-i, --input ', 'Input file patterns (glob)') - ) - .addOption( - new Option( - '-t, --target ', - 'Target generator(s): a built-in name ' + - `(${Object.keys(publicGenerators).join(', ')}) ` + - 'or an import specifier for a custom generator' - ) - ) - .addOption( - new Option('--ignore ', 'Ignore file patterns (glob)') - ) - .addOption(new Option('-o, --output ', 'The output directory')) - .addOption( - new Option( - '-p, --threads ', - 'Number of threads to use (minimum: 1)' - ) - ) - .addOption( - new Option( - '--chunk-size ', - 'Number of items to process per worker thread (minimum: 1)' - ) - ) - .addOption(new Option('-v, --version ', 'Target Node.js version')) - .addOption(new Option('-c, --changelog ', 'Changelog URL or path')) - .addOption(new Option('--git-ref ', 'Git ref')) - .addOption(new Option('--index ', 'index.md URL or path')) - .addOption(new Option('--minify', 'Minify?')) - .addOption(new Option('--type-map ', 'Type map URL or path')) - .action( errorWrap(async opts => { const config = await setConfig(opts); diff --git a/packages/core/bin/commands/index.mjs b/packages/core/bin/commands/index.mjs index 3e6d9d97..ce8b58ba 100644 --- a/packages/core/bin/commands/index.mjs +++ b/packages/core/bin/commands/index.mjs @@ -1,3 +1,4 @@ import generate from './generate.mjs'; +import watch from './watch.mjs'; -export default [generate]; +export default [generate, watch]; diff --git a/packages/core/bin/commands/watch.mjs b/packages/core/bin/commands/watch.mjs new file mode 100644 index 00000000..b598f3c1 --- /dev/null +++ b/packages/core/bin/commands/watch.mjs @@ -0,0 +1,150 @@ +import { matchesGlob, resolve } from 'node:path'; +import process from 'node:process'; + +import { watch } from 'chokidar'; +import { Command } from 'commander'; +import globParent from 'glob-parent'; + +import createGenerator, { SKIPPED } from '../../src/generators.mjs'; +import logger from '../../src/logger/index.mjs'; +import { + assertRunnableOptions, + setConfig, +} from '../../src/utils/configuration/index.mjs'; +import { errorWrap, insertCommonOptions } from '../utils.mjs'; + +const watchLogger = logger.child('watch'); + +const DEBOUNCE_MS = 100; + +/** + * Runs one generation pass. + * + * @param {import('../../src/utils/configuration/types').Configuration} config - The configuration + * @returns {Promise} + */ +const build = async config => { + const startedAt = Date.now(); + + try { + const { runGenerators } = createGenerator(); + + const results = await runGenerators(config); + const duration = Date.now() - startedAt; + + watchLogger.info( + results.every(result => result === SKIPPED) + ? `Already up to date (${duration}ms)` + : `Rebuilt in ${duration}ms` + ); + } catch (error) { + watchLogger.error(error); + } +}; + +/** + * Debounces rebuilds, and keeps them serialized + * + * @param {() => Promise} task - The work to run + * @returns {{schedule: () => void, cancel: () => void}} + */ +const createScheduler = task => { + let timer; + let running = false; + let queued = false; + + /** + * @returns {Promise} + */ + const run = async () => { + running = true; + + try { + do { + queued = false; + + await task(); + } while (queued); + } catch (error) { + watchLogger.error(error); + } finally { + running = false; + } + }; + + return { + /** + * @returns {void} + */ + schedule: () => { + if (running) { + queued = true; + + return; + } + + clearTimeout(timer); + + timer = setTimeout(run, DEBOUNCE_MS); + }, + + /** + * @returns {void} + */ + cancel: () => clearTimeout(timer), + }; +}; + +export default insertCommonOptions(new Command('watch')) + .description('Generate API docs, rebuilding whenever an input file changes') + .action( + errorWrap(async opts => { + const config = await setConfig(opts); + assertRunnableOptions(config); + + const patterns = config.global.input.map(pattern => resolve(pattern)); + + const ignored = (config.global.ignore ?? []).map(pattern => + resolve(pattern) + ); + + const scheduler = createScheduler(() => build(config)); + + const watcher = watch(patterns.map(globParent), { ignoreInitial: true }); + + watcher.on('all', (_event, path) => { + const target = resolve(path); + + if ( + patterns.some(pattern => matchesGlob(target, pattern)) && + !ignored.some(pattern => matchesGlob(target, pattern)) + ) { + scheduler.schedule(); + } + }); + + watcher.on('error', watchLogger.error); + + /** + * + */ + const stop = async () => { + watchLogger.info('Stopping'); + + scheduler.cancel(); + await watcher.close(); + + process.exit(0); + }; + + for (const signal of ['SIGINT', 'SIGTERM']) { + // TODO(@avivkeller): It's confusing for users when CTRL+C is pressed due to + // the blocking of the main thread + process.once(signal, stop); + } + + watchLogger.info(`Watching. Press Ctrl+C to stop`); + + scheduler.schedule(); + }) + ); diff --git a/packages/core/bin/utils.mjs b/packages/core/bin/utils.mjs index 561d9098..bf9c4bad 100644 --- a/packages/core/bin/utils.mjs +++ b/packages/core/bin/utils.mjs @@ -1,5 +1,30 @@ +import { Option } from 'commander'; + +import { publicGenerators } from '../src/generators/index.mjs'; import logger from '../src/logger/index.mjs'; +/** + * The options every runnable command accepts, as parsed by commander. + * + * @typedef {Object} CLIOptions + * @property {string} configFile + * @property {string[]} input + * @property {string[]} target + * @property {string[]} ignore + * @property {string} output + * @property {number} threads + * @property {number} chunkSize + * @property {string} version + * @property {string} changelog + * @property {string} gitRef + * @property {string} index + * @property {boolean} minify + * @property {string} typeMap + * @property {boolean} cache + * @property {boolean} force + * @property {string} cacheDir + */ + /** * Wraps a function to catch both synchronous and asynchronous errors. * @@ -16,3 +41,60 @@ export const errorWrap = process.exit(1); } }; + +/** + * Adds the options shared by every command that resolves a configuration and + * runs generators. + * + * @template {import('commander').Command} T + * @param {T} cmd - The command to extend + * @returns {T} The same command, for chaining + */ +export const insertCommonOptions = cmd => + cmd + .addOption(new Option('--config-file ', 'Config file')) + + // Options that need to be converted into a configuration + .addOption( + new Option('-i, --input ', 'Input file patterns (glob)') + ) + .addOption( + new Option( + '-t, --target ', + 'Target generator(s): a built-in name ' + + `(${Object.keys(publicGenerators).join(', ')}) ` + + 'or an import specifier for a custom generator' + ) + ) + .addOption( + new Option('--ignore ', 'Ignore file patterns (glob)') + ) + .addOption(new Option('-o, --output ', 'The output directory')) + .addOption( + new Option( + '-p, --threads ', + 'Number of threads to use (minimum: 1)' + ) + ) + .addOption( + new Option( + '--chunk-size ', + 'Number of items to process per worker thread (minimum: 1)' + ) + ) + .addOption(new Option('-v, --version ', 'Target Node.js version')) + .addOption(new Option('-c, --changelog ', 'Changelog URL or path')) + .addOption(new Option('--git-ref ', 'Git ref')) + .addOption(new Option('--index ', 'index.md URL or path')) + .addOption(new Option('--minify', 'Minify?')) + .addOption(new Option('--type-map ', 'Type map URL or path')) + .addOption( + new Option('--no-cache', 'Disable the on-disk build cache entirely') + ) + .addOption( + new Option( + '--force', + 'Ignore existing cache entries (the cache is still written)' + ) + ) + .addOption(new Option('--cache-dir ', 'Build cache directory')); diff --git a/packages/core/package.json b/packages/core/package.json index b1367d06..79f2aa05 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -46,6 +46,7 @@ "@swc/html-wasm": "^1.15.46", "@swc/wasm": "^1.15.46", "acorn": "^8.17.0", + "chokidar": "^5.0.0", "commander": "^15.0.0", "cosmiconfig": "^9.0.2", "dedent": "^1.7.2", diff --git a/packages/core/src/cache/__tests__/hash.test.mjs b/packages/core/src/cache/__tests__/hash.test.mjs new file mode 100644 index 00000000..e6ba410e --- /dev/null +++ b/packages/core/src/cache/__tests__/hash.test.mjs @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { canonicalJSON, combine, hashData, hashValue } from '../hash.mjs'; + +describe('hashData', () => { + it('returns a 32-char hex hash', () => { + assert.match(hashData('hello'), /^[0-9a-f]{32}$/); + }); + + it('differs for different data', () => { + assert.notEqual(hashData('a'), hashData('b')); + }); +}); + +describe('canonicalJSON', () => { + it('sorts object keys', () => { + assert.equal(canonicalJSON({ b: 1, a: 2 }), canonicalJSON({ a: 2, b: 1 })); + }); + + it('sorts keys recursively', () => { + assert.equal( + canonicalJSON({ x: { b: 1, a: 2 } }), + canonicalJSON({ x: { a: 2, b: 1 } }) + ); + }); + + it('omits undefined object values like JSON does', () => { + assert.equal( + canonicalJSON({ a: 1, b: undefined }), + canonicalJSON({ a: 1 }) + ); + }); + + it('serializes undefined array items as null like JSON does', () => { + assert.equal(canonicalJSON([undefined]), '[null]'); + }); + + it('serializes functions by source text', () => { + const fn = () => 42; + + assert.equal(canonicalJSON(fn), JSON.stringify(String(fn))); + }); + + it('marks cycles instead of throwing', () => { + const value = { a: 1 }; + value.self = value; + + assert.equal(canonicalJSON(value), '{"a":1,"self":"[circular]"}'); + }); + + it('allows shared (non-cyclic) references', () => { + const shared = { x: 1 }; + + assert.equal( + canonicalJSON({ a: shared, b: shared }), + '{"a":{"x":1},"b":{"x":1}}' + ); + }); +}); + +describe('hashValue', () => { + it('is key-order independent', () => { + assert.equal( + hashValue({ a: 1, b: [2, 3] }), + hashValue({ b: [2, 3], a: 1 }) + ); + }); + + it('differs for different values', () => { + assert.notEqual(hashValue({ a: 1 }), hashValue({ a: 2 })); + }); +}); + +describe('combine', () => { + it('is boundary-unambiguous', () => { + assert.notEqual(combine('ab', 'c'), combine('a', 'bc')); + }); + + it('is deterministic', () => { + assert.equal(combine('a', 'b'), combine('a', 'b')); + }); +}); diff --git a/packages/core/src/cache/__tests__/manifest.test.mjs b/packages/core/src/cache/__tests__/manifest.test.mjs new file mode 100644 index 00000000..8cc0c59c --- /dev/null +++ b/packages/core/src/cache/__tests__/manifest.test.mjs @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { loadManifest, saveManifest } from '../manifest.mjs'; +import { CACHE_SCHEMA } from '../salt.mjs'; + +const createTempDir = () => mkdtemp(join(tmpdir(), 'doc-kit-manifest-test-')); + +const PROFILE = { + completedAt: 1, + targets: { 'a-target': 'a'.repeat(32) }, + outputs: {}, + complete: true, +}; + +describe('manifest', () => { + it('returns a fresh manifest when none exists', async () => { + const manifest = await loadManifest(await createTempDir()); + + assert.deepEqual(manifest, { schema: CACHE_SCHEMA, profiles: {} }); + }); + + it('round-trips a profile', async () => { + const dir = await createTempDir(); + + await saveManifest(dir, { key: PROFILE }); + + assert.deepEqual((await loadManifest(dir)).profiles.key, PROFILE); + }); + + it('discards a corrupt manifest silently', async () => { + const dir = await createTempDir(); + + await writeFile(join(dir, 'manifest.json'), '{"schema": not json'); + + assert.deepEqual(await loadManifest(dir), { + schema: CACHE_SCHEMA, + profiles: {}, + }); + }); + + it('discards a manifest with a different schema', async () => { + const dir = await createTempDir(); + + await writeFile( + join(dir, 'manifest.json'), + JSON.stringify({ schema: -1, profiles: { stale: PROFILE } }) + ); + + assert.deepEqual((await loadManifest(dir)).profiles, {}); + }); + + it('merges with existing profiles on save', async () => { + const dir = await createTempDir(); + + await saveManifest(dir, { first: PROFILE }); + await saveManifest(dir, { second: { ...PROFILE, completedAt: 2 } }); + + const { profiles } = await loadManifest(dir); + + assert.deepEqual(Object.keys(profiles).sort(), ['first', 'second']); + }); +}); diff --git a/packages/core/src/cache/__tests__/snapshot.test.mjs b/packages/core/src/cache/__tests__/snapshot.test.mjs new file mode 100644 index 00000000..c52d6b94 --- /dev/null +++ b/packages/core/src/cache/__tests__/snapshot.test.mjs @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { createSnapshot } from '../snapshot.mjs'; + +const createFixtureDir = async files => { + const dir = await mkdtemp(join(tmpdir(), 'doc-kit-snapshot-test-')); + + for (const [name, content] of Object.entries(files)) { + await writeFile(join(dir, name), content); + } + + return dir; +}; + +describe('createSnapshot', () => { + it('lists files sorted with content hashes', async () => { + const dir = await createFixtureDir({ 'b.md': 'bee', 'a.md': 'ay' }); + + const snapshot = await createSnapshot([join(dir, '*.md')]); + + assert.deepEqual( + snapshot.files.map(file => file.rel), + ['a.md', 'b.md'] + ); + assert.ok(snapshot.files.every(file => /^[0-9a-f]{32}$/.test(file.hash))); + }); + + it('changes the digest when content changes', async () => { + const dir = await createFixtureDir({ 'a.md': 'one' }); + const before = await createSnapshot([join(dir, '*.md')]); + + await writeFile(join(dir, 'a.md'), 'two'); + const after = await createSnapshot([join(dir, '*.md')]); + + assert.notEqual(before.digest, after.digest); + }); + + it('changes the digest when a file is added', async () => { + const dir = await createFixtureDir({ 'a.md': 'one' }); + const before = await createSnapshot([join(dir, '*.md')]); + + await writeFile(join(dir, 'b.md'), 'two'); + const after = await createSnapshot([join(dir, '*.md')]); + + assert.notEqual(before.digest, after.digest); + }); + + it('is stable across identical content in different directories', async () => { + const dirA = await createFixtureDir({ 'a.md': 'same' }); + const dirB = await createFixtureDir({ 'a.md': 'same' }); + + const [snapA, snapB] = await Promise.all([ + createSnapshot([join(dirA, '*.md')]), + createSnapshot([join(dirB, '*.md')]), + ]); + + // Paths are glob-parent relative, so a relocated checkout hits. + assert.equal(snapA.digest, snapB.digest); + }); + + it('respects ignore patterns', async () => { + const dir = await createFixtureDir({ 'a.md': 'keep', 'skip.md': 'skip' }); + + const snapshot = await createSnapshot( + [join(dir, '*.md')], + [join(dir, 'skip.md')] + ); + + assert.deepEqual( + snapshot.files.map(file => file.rel), + ['a.md'] + ); + }); +}); diff --git a/packages/core/src/cache/__tests__/store.test.mjs b/packages/core/src/cache/__tests__/store.test.mjs new file mode 100644 index 00000000..1c98753d --- /dev/null +++ b/packages/core/src/cache/__tests__/store.test.mjs @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, utimes, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, it } from 'node:test'; + +import { createStore } from '../store.mjs'; + +const KEY_A = 'a'.repeat(32); +const KEY_B = 'b'.repeat(32); + +const createTempStore = async () => + createStore(await mkdtemp(join(tmpdir(), 'doc-kit-store-test-'))); + +describe('store', () => { + it('round-trips a value', async () => { + const store = await createTempStore(); + + store.put(KEY_A, 'value'); + await store.flush(); + + assert.equal(await store.get(KEY_A), 'value'); + }); + + it('misses on unknown keys', async () => { + const store = await createTempStore(); + + assert.equal(await store.get(KEY_A), null); + assert.equal(store.stats.misses, 1); + }); + + it('counts hits and writes', async () => { + const store = await createTempStore(); + + store.put(KEY_A, 'value'); + await store.flush(); + await store.get(KEY_A); + + assert.equal(store.stats.hits, 1); + assert.equal(store.stats.writes, 1); + }); + + it('memoizes: computes once, then serves from disk', async () => { + const store = await createTempStore(); + + let calls = 0; + const produce = () => { + calls++; + + return 'computed'; + }; + + assert.equal(await store.memo('ns', KEY_A, produce), 'computed'); + await store.flush(); + assert.equal(await store.memo('ns', KEY_A, produce), 'computed'); + assert.equal(calls, 1); + }); + + it('namespaces memo values', async () => { + const store = await createTempStore(); + + await store.memo('one', KEY_A, () => 'first'); + await store.flush(); + + assert.equal(await store.memo('two', KEY_A, () => 'second'), 'second'); + }); + + it('prunes objects older than the age threshold', async () => { + const dir = await mkdtemp(join(tmpdir(), 'doc-kit-store-test-')); + const store = createStore(dir); + + store.put(KEY_A, 'old'); + store.put(KEY_B, 'fresh'); + await store.flush(); + + const oldPath = join(dir, 'objects', KEY_A.slice(0, 2), KEY_A); + const past = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); + await utimes(oldPath, past, past); + + await store.prune(7); + + assert.equal(await store.get(KEY_A), null); + assert.equal(await store.get(KEY_B), 'fresh'); + }); + + it('writes atomically: no partial objects are left behind', async () => { + const dir = await mkdtemp(join(tmpdir(), 'doc-kit-store-test-')); + const store = createStore(dir); + + store.put(KEY_A, 'value'); + await store.flush(); + + const shard = await readdir(join(dir, 'objects', KEY_A.slice(0, 2))); + + assert.deepEqual(shard, [KEY_A]); + }); + + it('swallows write failures', async () => { + // A file where the objects dir should be makes every write fail. + const dir = await mkdtemp(join(tmpdir(), 'doc-kit-store-test-')); + await writeFile(join(dir, 'objects'), 'not a directory'); + + const store = createStore(dir); + + store.put(KEY_A, 'value'); + + await assert.doesNotReject(store.flush()); + }); +}); diff --git a/packages/core/src/cache/hash.mjs b/packages/core/src/cache/hash.mjs new file mode 100644 index 00000000..b05da938 --- /dev/null +++ b/packages/core/src/cache/hash.mjs @@ -0,0 +1,80 @@ +'use strict'; + +import { hash } from 'node:crypto'; + +/** + * Hashes raw data to the 128-bit hex key format used across the cache. + * Truncated SHA-256: collision-safe at this scale, half the key size. + * + * @param {string | Buffer} data - Data to hash + * @returns {string} 32-char hex hash + */ +export const hashData = data => hash('sha256', data, 'hex').slice(0, 32); + +/** + * Serializes a value into a canonical JSON string: object keys sorted, + * `undefined` treated as JSON does (omitted from objects, `null` in arrays), + * functions by source text, cycles marked. Two structurally equal values + * always produce the same string. + * + * @param {unknown} value - Value to serialize + * @param {Set} [seen] - Ancestry for cycle detection + * @returns {string} Canonical JSON + */ +export const canonicalJSON = (value, seen = new Set()) => { + if (value === undefined) { + return 'null'; + } + + if (typeof value === 'function') { + return JSON.stringify(String(value)); + } + + if (typeof value === 'bigint') { + return JSON.stringify(String(value)); + } + + if (value === null || typeof value !== 'object') { + return JSON.stringify(value); + } + + if (seen.has(value)) { + return '"[circular]"'; + } + + seen.add(value); + + try { + if (Array.isArray(value)) { + return `[${value.map(item => canonicalJSON(item, seen)).join(',')}]`; + } + + const entries = Object.entries(value) + .filter(([, val]) => val !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map( + ([key, val]) => `${JSON.stringify(key)}:${canonicalJSON(val, seen)}` + ); + + return `{${entries.join(',')}}`; + } finally { + seen.delete(value); + } +}; + +/** + * Hashes any JSON-serializable value canonically. + * + * @param {unknown} value - Value to hash + * @returns {string} 32-char hex hash + */ +export const hashValue = value => hashData(canonicalJSON(value)); + +/** + * Combines hash parts into one key. Parts are joined with an unambiguous + * separator so `('ab', 'c')` and `('a', 'bc')` differ. + * + * @param {...string} parts - Hashes or literals to combine + * @returns {string} 32-char hex hash + */ +export const combine = (...parts) => hashData(parts.join('\x00')); diff --git a/packages/core/src/cache/index.mjs b/packages/core/src/cache/index.mjs new file mode 100644 index 00000000..9a410527 --- /dev/null +++ b/packages/core/src/cache/index.mjs @@ -0,0 +1,391 @@ +'use strict'; + +import { existsSync } from 'node:fs'; +import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, resolve, sep } from 'node:path'; + +import { hashData, hashValue } from './hash.mjs'; +import { loadManifest, saveManifest } from './manifest.mjs'; +import { aggregateKey, chainSalt } from './salt.mjs'; +import { createSnapshot } from './snapshot.mjs'; +import { createStore } from './store.mjs'; +import { resolveGeneratorSpecifier } from '../generators/loader.mjs'; +import logger from '../logger/index.mjs'; +import { enforceArray } from '../utils/array.mjs'; +import { loadFromURL } from '../utils/loaders.mjs'; + +const cacheLogger = logger.child('cache'); + +/** @type {import('./types').BuildCache | null} */ +let context = null; + +/** + * The active build cache, or null when caching is disabled (or in a worker + * thread, where the cache is never set up). + * + * @returns {import('./types').BuildCache | null} + */ +export const getBuildCache = () => context; + +/** + * Resolves the cache directory: explicit configuration, environment override, + * then `node_modules/.cache/doc-kit` when a `node_modules` exists, falling + * back to `.doc-kit-cache`. + * + * @param {string} [explicit] - Configured directory, if any + * @returns {string} Absolute cache directory + */ +export const resolveCacheDir = explicit => { + const dir = + explicit || + process.env.DOC_KIT_CACHE_DIR || + (existsSync('node_modules') + ? join('node_modules', '.cache', 'doc-kit') + : '.doc-kit-cache'); + + return resolve(dir); +}; + +/** + * Checks that every output recorded by a previous run is still on disk with + * the recorded size and content hash. Any deviation disqualifies a skip — + * rerunning the generators is always the recovery path. + * + * @param {import('./types').Profile} profile - Prior run's profile + * @param {string} outputDir - The configured output directory + * @returns {Promise} Whether every output verifies + */ +const verifyOutputs = async (profile, outputDir) => { + const entries = Object.entries(profile.outputs); + + if (entries.length > 0 && !outputDir) { + return false; + } + + const checks = await Promise.all( + entries.map(async ([rel, record]) => { + try { + const path = join(outputDir, rel); + + if ((await stat(path)).size !== record.size) { + return false; + } + + return hashData(await readFile(path)) === record.hash; + } catch { + return false; + } + }) + ); + + return checks.every(Boolean); +}; + +/** + * Resolves a string `typeMap` configuration into its parsed content before + * salts are computed, so the salt covers the fetched bytes — a typeMap change + * must invalidate downstream even though no source file changed. + * + * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration + * @returns {Promise} + */ +const resolveTypeMap = async configuration => { + if (typeof configuration.metadata?.typeMap === 'string') { + configuration.metadata.typeMap = JSON.parse( + await loadFromURL(configuration.metadata.typeMap) + ); + } +}; + +/** + * Sets up the durable build cache for one run: hashes input snapshots, + * computes every target's aggregate key, and decides whether this invocation + * can skip entirely because its outputs are already on disk and verified. + * + * Any failure disables the cache for the run (with a debug log) — the build + * must never fail, or even get slower than a cold build, because of caching. + * + * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration + * @param {Map} generators - Loaded generators + * @param {string[]} targets - Resolved target specifiers + * @returns {Promise} + */ +export const setupBuildCache = async (configuration, generators, targets) => { + const settings = { enabled: true, maxAgeDays: 7, ...configuration.cache }; + + if (settings.enabled === false || process.env.DOC_KIT_NO_CACHE) { + return (context = null); + } + + const force = settings.force || Boolean(process.env.DOC_KIT_CACHE_FORCE); + + try { + const dir = resolveCacheDir(settings.dir); + const outputDir = configuration.global?.output; + const store = createStore(dir); + const manifest = await loadManifest(dir); + + await resolveTypeMap(configuration); + + /** @type {Map>} */ + const snapshots = new Map(); + + /** + * Memoized input snapshot for a root generator's configured globs. + * + * @param {string} specifier - Resolved root generator specifier + * @returns {Promise} + */ + const snapshotFor = specifier => { + if (!snapshots.has(specifier)) { + const { name } = generators.get(specifier); + const slice = configuration[name] ?? {}; + + snapshots.set( + specifier, + createSnapshot(enforceArray(slice.input ?? []), slice.ignore) + ); + } + + return snapshots.get(specifier); + }; + + /** + * Resolves a generator's `dependsOn` into a resolved specifier. + * + * @param {string} specifier - Resolved generator specifier + * @returns {string | undefined} + */ + const resolveDependency = specifier => { + const { dependsOn } = generators.get(specifier); + + return dependsOn && resolveGeneratorSpecifier(dependsOn); + }; + + /** @type {Record} */ + const targetKeys = {}; + + for (const target of targets) { + targetKeys[target] = await aggregateKey( + target, + generators, + resolveDependency, + configuration, + snapshotFor + ); + } + + const profileKey = hashValue({ + targets: [...targets].sort(), + input: configuration.global?.input, + output: outputDir, + }); + + // Skip is all-or-nothing per invocation shape: outputs cannot be + // attributed to individual concurrent generators, so a partial skip could + // not verify what it skips. Per-item leaf caches make partial misses + // cheap instead. + const profile = manifest.profiles[profileKey]; + + let skippedTargets = new Set(); + + if ( + !force && + profile?.complete && + targets.every(target => profile.targets[target] === targetKeys[target]) && + (await verifyOutputs(profile, outputDir)) + ) { + skippedTargets = new Set(targets); + } + + /** @type {Map>} */ + const chainSalts = new Map(); + + /** + * Memoized chain salt for a generator, for use in per-item leaf-cache + * keys: covers every code and configuration input in the generator's + * dependency chain, but not the input files (leaf keys add the specific + * file hashes they depend on). + * + * @param {string} specifier - Resolved generator specifier + * @returns {Promise} Chain salt + */ + const chainSaltFor = specifier => { + if (!chainSalts.has(specifier)) { + chainSalts.set( + specifier, + chainSalt( + specifier, + generators, + resolveDependency, + configuration + ).then(({ salt }) => salt) + ); + } + + return chainSalts.get(specifier); + }; + + /** @type {Map | null} */ + let sourceHashes = null; + + /** + * Content hash of the source file behind a module path (the extensionless + * `MetadataEntry.path`, e.g. `/fs`), or undefined when unknown — callers + * must treat unknown provenance as uncacheable. + * + * @param {string} modulePath - Extensionless module path with leading `/` + * @returns {Promise} + */ + const sourceHashFor = async modulePath => { + if (!sourceHashes) { + sourceHashes = new Map(); + + for (const snapshot of await Promise.all(snapshots.values())) { + for (const file of snapshot.files) { + const posixRel = file.rel.split(sep).join('/'); + + sourceHashes.set( + `/${posixRel.replace(/\.[0-9a-z]+$/i, '')}`, + file.hash + ); + } + } + } + + return sourceHashes.get(modulePath); + }; + + /** @type {Map} */ + const outputs = new Map(); + + /** + * Records an output file (hash + size, relative to the output directory) + * for whole-run verification on later invocations. + * + * @param {string} absPath - Absolute path of the written file + * @param {Buffer | string} content - The written content + * @returns {void} + */ + const recordOutput = (absPath, content) => { + if (!outputDir) { + return; + } + + const buffer = Buffer.isBuffer(content) ? content : Buffer.from(content); + + outputs.set(relative(resolve(outputDir), resolve(absPath)), { + hash: hashData(buffer), + size: buffer.byteLength, + }); + }; + + context = { + store, + dir, + skippedTargets, + recordOutput, + chainSalt: chainSaltFor, + sourceHash: sourceHashFor, + + /** + * Tracked file write: records the output and skips byte-identical + * writes so warm runs leave mtimes untouched. + * + * @param {string} file - Destination path + * @param {unknown} data - File contents + * @param {unknown[]} [rest] - Remaining fs.writeFile arguments + * @returns {Promise} + */ + writeTracked: async (file, data, rest = []) => { + if (typeof data !== 'string' && !Buffer.isBuffer(data)) { + // Untrackable payloads (streams, etc.) fall through untouched. + await mkdir(dirname(file), { recursive: true }); + + return writeFile(file, data, ...rest); + } + + const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data); + + recordOutput(file, buffer); + + // Skipping byte-identical writes keeps mtimes stable for downstream + // tooling and makes warm rebuilds mostly write-free. + const existing = await readFile(file).catch(() => null); + + if (existing?.equals(buffer)) { + return; + } + + await mkdir(dirname(file), { recursive: true }); + + return writeFile(file, buffer); + }, + + /** + * Flushes pending object writes, persists the run's profile (only on + * success, and only when the run actually executed), prunes stale + * objects, and reports stats. + * + * @param {boolean} success - Whether the run completed without error + * @returns {Promise} + */ + finalize: async success => { + await store.flush().catch(() => undefined); + + try { + if (success && skippedTargets.size === 0) { + await saveManifest(dir, { + [profileKey]: { + completedAt: Date.now(), + targets: targetKeys, + outputs: Object.fromEntries(outputs), + complete: true, + }, + }); + + await store.prune(settings.maxAgeDays); + } + } catch (error) { + cacheLogger.debug('Failed to persist cache manifest', { + error: error.message, + }); + } + + const stats = { + skippedTargets: [...skippedTargets], + targets: targetKeys, + outputsRecorded: outputs.size, + store: store.stats, + }; + + if (process.env.DOC_KIT_CACHE_STATS_FILE) { + await writeFile( + process.env.DOC_KIT_CACHE_STATS_FILE, + JSON.stringify(stats, null, 2) + ).catch(() => undefined); + } + + if (skippedTargets.size > 0) { + cacheLogger.info( + `Skipped ${skippedTargets.size} up-to-date target(s); outputs verified on disk` + ); + } else { + const { hits, misses, writes } = store.stats; + + cacheLogger.info( + `${hits} hits, ${misses} misses, ${writes} objects written, ${outputs.size} outputs recorded` + ); + } + + context = null; + }, + }; + + return context; + } catch (error) { + cacheLogger.debug(`Cache disabled for this run: ${error.message}`); + + return (context = null); + } +}; diff --git a/packages/core/src/cache/manifest.mjs b/packages/core/src/cache/manifest.mjs new file mode 100644 index 00000000..8ed3e5c4 --- /dev/null +++ b/packages/core/src/cache/manifest.mjs @@ -0,0 +1,51 @@ +'use strict'; + +import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { CACHE_SCHEMA } from './salt.mjs'; + +const MANIFEST = 'manifest.json'; + +/** + * Loads the cache manifest. Any parse or schema mismatch discards it + * wholesale — objects are self-keyed, so a lost manifest only costs the + * whole-skip records, never correctness. + * + * @param {string} dir - Cache directory + * @returns {Promise} + */ +export const loadManifest = async dir => { + try { + const manifest = JSON.parse(await readFile(join(dir, MANIFEST), 'utf-8')); + + if (manifest.schema === CACHE_SCHEMA) { + return manifest; + } + } catch { + // Fall through to a fresh manifest. + } + + return { schema: CACHE_SCHEMA, profiles: {} }; +}; + +/** + * Saves the manifest with a read-merge-write cycle: the on-disk manifest is + * re-read so a concurrent process's profiles survive, then the given profiles + * are merged in and the result written atomically (temp + rename). + * + * @param {string} dir - Cache directory + * @param {Record} profiles - Profiles to merge in + * @returns {Promise} + */ +export const saveManifest = async (dir, profiles) => { + const manifest = await loadManifest(dir); + + Object.assign(manifest.profiles, profiles); + + const tmp = join(dir, `${MANIFEST}.${process.pid}.tmp`); + + await mkdir(dir, { recursive: true }); + await writeFile(tmp, JSON.stringify(manifest)); + await rename(tmp, join(dir, MANIFEST)); +}; diff --git a/packages/core/src/cache/salt.mjs b/packages/core/src/cache/salt.mjs new file mode 100644 index 00000000..e1346288 --- /dev/null +++ b/packages/core/src/cache/salt.mjs @@ -0,0 +1,187 @@ +'use strict'; + +import { realpathSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { dirname, join, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { globSync } from 'tinyglobby'; + +import { combine, hashData, hashValue } from './hash.mjs'; + +/** + * Bump to invalidate every existing cache entry and manifest when the cache's + * own semantics change. + */ +export const CACHE_SCHEMA = 1; + +/** @type {Map>} */ +const packageSalts = new Map(); + +/** + * Finds the directory of the package that owns a resolved generator + * specifier, by resolving the module and walking up to its `package.json`. + * + * @param {string} specifier - Resolved generator import specifier + * @returns {Promise} Real (symlink-free) package directory + */ +const findPackageDir = async specifier => { + const modulePath = realpathSync( + fileURLToPath(import.meta.resolve(specifier)) + ); + + let dir = dirname(modulePath); + + while (true) { + try { + await readFile(join(dir, 'package.json')); + + return dir; + } catch { + const parent = dirname(dir); + + if (parent === dir) { + throw new Error(`No package.json found above ${modulePath}`); + } + + dir = parent; + } + } +}; + +/** + * Computes a salt identifying a package's code. Published installs (real path + * inside node_modules) are identified by name and version. Anything else — a + * workspace, an `npm link`, a raw checkout — is a development install whose + * version never changes while its code does, so the salt is a content hash of + * the package sources instead. + * + * @param {string} pkgDir - Real package directory + * @returns {Promise} Package salt + */ +const computePackageSalt = async pkgDir => { + const pkg = JSON.parse(await readFile(join(pkgDir, 'package.json'), 'utf-8')); + + if (pkgDir.split(sep).includes('node_modules')) { + return hashData(`${pkg.name}@${pkg.version}`); + } + + const sources = globSync('src/**/*', { + cwd: pkgDir, + onlyFiles: true, + ignore: ['**/__tests__/**', '**/*.test.mjs'], + }).sort(); + + const hashes = await Promise.all( + sources.map( + async rel => `${rel}:${hashData(await readFile(join(pkgDir, rel)))}` + ) + ); + + return hashData(`${pkg.name}@dev\n${hashes.join('\n')}`); +}; + +/** + * Memoized package salt for a resolved generator specifier. + * + * @param {string} specifier - Resolved generator import specifier + * @returns {Promise} Package salt + */ +export const packageSalt = specifier => { + if (!packageSalts.has(specifier)) { + packageSalts.set( + specifier, + findPackageDir(specifier).then(computePackageSalt) + ); + } + + return packageSalts.get(specifier); +}; + +/** + * Salt for one generator: its package's code identity plus its resolved + * configuration slice (which inherits the global configuration, so parsed + * changelog/index/version are covered). The output directory is excluded — + * where output lands does not change what it contains. + * + * @param {string} specifier - Resolved generator specifier + * @param {GeneratorMetadata} generator - Loaded generator + * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration + * @returns {Promise} Generator salt + */ +const generatorSalt = async (specifier, generator, configuration) => { + const slice = { ...(configuration[generator.name] ?? {}) }; + + delete slice.output; + + return combine(await packageSalt(specifier), hashValue(slice)); +}; + +/** + * Salt for a target's whole dependency chain: the cache schema, the Node.js + * major (worker/serialization behavior), and the salt of every generator from + * the target down to its root. Everything code- and configuration-shaped that + * can affect the target's output — but not the input files themselves. + * + * @param {string} target - Resolved target specifier + * @param {Map} generators - Loaded generators + * @param {(specifier: string) => string | undefined} resolveDependency - Maps a specifier to its resolved dependency + * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration + * @returns {Promise<{ salt: string, root: string }>} The chain salt and the chain's root specifier + */ +export const chainSalt = async ( + target, + generators, + resolveDependency, + configuration +) => { + const parts = [ + `schema:${CACHE_SCHEMA}`, + `node:${process.versions.node.split('.')[0]}`, + ]; + + let specifier = target; + let root = target; + + while (specifier) { + const generator = generators.get(specifier); + + parts.push(await generatorSalt(specifier, generator, configuration)); + + root = specifier; + specifier = resolveDependency(specifier); + } + + return { salt: combine(...parts), root }; +}; + +/** + * Aggregate key for a target: its chain salt plus the content digest of its + * root's input snapshot. Equality of this key means the target's outputs are + * already on disk and correct. + * + * @param {string} target - Resolved target specifier + * @param {Map} generators - Loaded generators + * @param {(specifier: string) => string | undefined} resolveDependency - Maps a specifier to its resolved dependency + * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration + * @param {(rootSpecifier: string) => Promise} snapshotFor - Snapshot accessor per root generator + * @returns {Promise} Aggregate key + */ +export const aggregateKey = async ( + target, + generators, + resolveDependency, + configuration, + snapshotFor +) => { + const { salt, root } = await chainSalt( + target, + generators, + resolveDependency, + configuration + ); + + const snapshot = await snapshotFor(root); + + return combine(salt, snapshot.digest); +}; diff --git a/packages/core/src/cache/snapshot.mjs b/packages/core/src/cache/snapshot.mjs new file mode 100644 index 00000000..fb9fd9f9 --- /dev/null +++ b/packages/core/src/cache/snapshot.mjs @@ -0,0 +1,43 @@ +'use strict'; + +import { readFile } from 'node:fs/promises'; +import { relative } from 'node:path'; + +import globParent from 'glob-parent'; +import { globSync } from 'tinyglobby'; + +import { hashData } from './hash.mjs'; + +/** + * Globs and content-hashes a root generator's input files. The digest covers + * the file set and every file's bytes, with paths relative to each pattern's + * glob parent so a relocated checkout produces the same digest. + * + * @param {string[]} patterns - Input glob patterns (as `ast`/`ast-js` consume them) + * @param {string[]} [ignore] - Ignore patterns + * @returns {Promise} + */ +export const createSnapshot = async (patterns, ignore) => { + const files = patterns.flatMap(pattern => { + const parent = globParent(pattern); + + return globSync(pattern, { ignore }).map(abs => ({ + abs, + rel: relative(parent, abs), + })); + }); + + files.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); + + await Promise.all( + files.map(async file => { + file.hash = hashData(await readFile(file.abs)); + }) + ); + + const digest = hashData( + files.map(file => `${file.rel}:${file.hash}`).join('\n') + ); + + return { files, digest }; +}; diff --git a/packages/core/src/cache/store.mjs b/packages/core/src/cache/store.mjs new file mode 100644 index 00000000..3a7e6bf0 --- /dev/null +++ b/packages/core/src/cache/store.mjs @@ -0,0 +1,187 @@ +'use strict'; + +import { + mkdir, + readFile, + readdir, + rename, + rm, + stat, + utimes, + writeFile, +} from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +import { combine } from './hash.mjs'; +import logger from '../logger/index.mjs'; + +const storeLogger = logger.child('cache'); + +/** + * Content-addressed object store. The filename IS the fully-salted input key, + * so objects are self-validating: lookup is existence, corruption is a miss. + * Writes are temp-file-then-rename — atomic, and idempotent under concurrent + * processes computing the same key. + * + * @param {string} dir - Cache directory + * @returns {import('./types').Store} + */ +export const createStore = dir => { + const objectsDir = join(dir, 'objects'); + const tmpDir = join(dir, 'tmp'); + + /** @type {Set>} */ + const pending = new Set(); + + const stats = { hits: 0, misses: 0, writes: 0, bytesWritten: 0 }; + + let tmpCounter = 0; + + /** + * @param {string} key - Object key + * @returns {string} Object path, sharded to keep directories small + */ + const pathFor = key => join(objectsDir, key.slice(0, 2), key); + + /** + * Reads an object; any failure is a miss. + * + * @param {string} key - Object key + * @returns {Promise} Stored value, or null + */ + const get = async key => { + try { + const value = await readFile(pathFor(key), 'utf-8'); + + stats.hits++; + + return value; + } catch { + stats.misses++; + + return null; + } + }; + + /** + * Persists an object write-behind: failures are logged and swallowed — + * the build must never fail because the cache did. + * + * @param {string} key - Object key + * @param {string} value - Value to store + * @returns {void} + */ + const put = (key, value) => { + const write = (async () => { + const tmp = join(tmpDir, `${process.pid}-${tmpCounter++}`); + + await mkdir(dirname(pathFor(key)), { recursive: true }); + await mkdir(tmpDir, { recursive: true }); + await writeFile(tmp, value); + await rename(tmp, pathFor(key)); + + stats.writes++; + stats.bytesWritten += Buffer.byteLength(value); + })().catch(error => + storeLogger.debug(`Cache write failed for ${key}`, { + error: error.message, + }) + ); + + pending.add(write); + write.finally(() => pending.delete(write)); + }; + + return { + get, + put, + stats, + + /** + * Checks that an object exists without reading it, refreshing its mtime + * so a concurrent age-based prune cannot delete an object another run + * just decided to rely on. + * + * @param {string} key - Object key + * @returns {Promise} Whether the object exists + */ + touch: async key => { + try { + const now = new Date(); + + await utimes(pathFor(key), now, now); + + return true; + } catch { + return false; + } + }, + + /** + * Durable memo: returns the cached value for `(namespace, key)` or runs + * `produce`, persists, and returns its result. + * + * @param {string} namespace - Distinguishes value kinds sharing key inputs + * @param {string} key - Input key + * @param {() => Promise | string} produce - Computes the value on miss + * @returns {Promise} The cached or computed value + */ + memo: async (namespace, key, produce) => { + const derived = combine(namespace, key); + const cached = await get(derived); + + if (cached !== null) { + return cached; + } + + const value = await produce(); + + put(derived, value); + + return value; + }, + + /** + * Awaits all in-flight writes. + * + * @returns {Promise} + */ + flush: () => Promise.all([...pending]).then(() => undefined), + + /** + * Deletes objects older than the given age. Deleting a live object only + * causes a future miss, so pruning needs no reference tracking. + * + * @param {number} maxAgeDays - Age threshold in days + * @returns {Promise} + */ + prune: async maxAgeDays => { + const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000; + + let shards; + + try { + shards = await readdir(objectsDir); + } catch { + return; + } + + for (const shard of shards) { + const shardDir = join(objectsDir, shard); + const objects = await readdir(shardDir).catch(() => []); + + for (const object of objects) { + const path = join(shardDir, object); + + try { + if ((await stat(path)).mtimeMs < cutoff) { + await rm(path, { force: true }); + } + } catch { + // A concurrent process may have pruned it already. + } + } + } + }, + }; +}; diff --git a/packages/core/src/cache/types.d.ts b/packages/core/src/cache/types.d.ts new file mode 100644 index 00000000..0a0eb28d --- /dev/null +++ b/packages/core/src/cache/types.d.ts @@ -0,0 +1,80 @@ +export interface SnapshotFile { + /** Absolute path on disk */ + abs: string; + /** Path relative to the pattern's glob parent (relocatable) */ + rel: string; + /** Content hash */ + hash: string; +} + +export interface Snapshot { + files: SnapshotFile[]; + /** Hash of the sorted rel:hash lines — covers file set and contents */ + digest: string; +} + +export interface StoreStats { + hits: number; + misses: number; + writes: number; + bytesWritten: number; +} + +export interface Store { + get(key: string): Promise; + put(key: string, value: string): void; + /** Existence check that refreshes the object's mtime (prune protection) */ + touch(key: string): Promise; + memo( + namespace: string, + key: string, + produce: () => Promise | string + ): Promise; + flush(): Promise; + prune(maxAgeDays: number): Promise; + stats: StoreStats; +} + +export interface OutputRecord { + hash: string; + size: number; +} + +export interface Profile { + completedAt: number; + /** Aggregate key per resolved target specifier */ + targets: Record; + /** Files the profile's last complete run wrote, relative to the output dir */ + outputs: Record; + complete: boolean; +} + +export interface Manifest { + schema: number; + profiles: Record; +} + +export interface CacheConfiguration { + enabled?: boolean; + /** No cache reads this run; results are still written (repopulate) */ + force?: boolean; + dir?: string; + maxAgeDays?: number; +} + +export interface BuildCache { + store: Store; + dir: string; + /** Resolved targets this run may skip entirely (all-or-nothing) */ + skippedTargets: Set; + /** Chain salt for per-item leaf-cache keys (code + config, no inputs) */ + chainSalt(specifier: string): Promise; + /** Content hash of the source file behind a module path (`/fs`), if known */ + sourceHash(modulePath: string): Promise; + /** Records an output file write (tracked writeFile / vite / copies) */ + recordOutput(absPath: string, content: Buffer | string): void; + /** Tracked write: records, skips byte-identical writes */ + writeTracked(file: string, data: unknown, rest: unknown[]): Promise; + /** Flushes writes, persists the profile, prunes, reports stats */ + finalize(success: boolean): Promise; +} diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index b079dc0d..a9b56262 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { setupBuildCache } from './cache/index.mjs'; import { createCache } from './caching.mjs'; import { loadGenerators, @@ -12,6 +13,14 @@ import { isAsyncIterable } from './utils/misc.mjs'; const generatorsLogger = logger.child('generators'); +/** + * Sentinel result for a target that was skipped because the durable cache + * verified its outputs are already on disk and up to date. Only ever returned + * for requested targets (whose results the CLI discards), never fed to a + * dependent generator. + */ +export const SKIPPED = Symbol('doc-kit:skipped'); + /** * Creates a generator orchestration system that manages the execution of * documentation generators in dependency order, with support for parallel @@ -107,36 +116,67 @@ const createGenerator = () => { const targets = target.map(resolveGeneratorSpecifier); const generators = await loadGenerators(targets); + // Durable cross-run cache: hashes inputs and decides whether this + // invocation's outputs are already on disk. Null when disabled. + const buildCache = await setupBuildCache( + configuration, + generators, + targets + ); + const skipped = buildCache?.skippedTargets ?? new Set(); + const active = targets.filter(specifier => !skipped.has(specifier)); + generatorsLogger.debug(`Starting pipeline`, { generators: targets.join(', '), + skipped: skipped.size, threads, }); - // Compute consumer counts up front so dependencies can be evicted as soon - // as their last consumer runs (must be ready before any generator starts). - cache.populateConsumerCounts(targets, specifier => { - const { dependsOn } = generators.get(specifier); + let success = false; - return dependsOn && resolveGeneratorSpecifier(dependsOn); - }); + try { + if (active.length === 0) { + success = true; - // Create worker pool - pool = createWorkerPool(threads); + return targets.map(() => SKIPPED); + } - // Schedule all generators - for (const specifier of targets) { - scheduleGenerator(specifier, generators, configuration); - } + // Compute consumer counts up front so dependencies can be evicted as + // soon as their last consumer runs (must be ready before any generator + // starts). Skipped targets never run, so only active ones count. + cache.populateConsumerCounts(active, specifier => { + const { dependsOn } = generators.get(specifier); - // Start all collections in parallel (don't await sequentially). Consuming - // through the shared path lets the final read also trigger eviction. - const results = await Promise.all( - targets.map(specifier => cache.consume(specifier)) - ); + return dependsOn && resolveGeneratorSpecifier(dependsOn); + }); + + // Create worker pool + pool = createWorkerPool(threads); + + // Schedule all generators + for (const specifier of active) { + scheduleGenerator(specifier, generators, configuration); + } - await pool.destroy(); + // Start all collections in parallel (don't await sequentially). + // Consuming through the shared path lets the final read also trigger + // eviction. + const results = await Promise.all( + targets.map(specifier => + skipped.has(specifier) ? SKIPPED : cache.consume(specifier) + ) + ); - return results; + await pool.destroy(); + + success = true; + + return results; + } finally { + // Persist per-run cache state even on failure paths (writes that + // completed stay valid); the profile is only marked complete on success. + await buildCache?.finalize(success); + } }; return { runGenerators }; diff --git a/packages/core/src/generators/ast-js/generate.mjs b/packages/core/src/generators/ast-js/generate.mjs index 303e5ec0..4ad8b25c 100644 --- a/packages/core/src/generators/ast-js/generate.mjs +++ b/packages/core/src/generators/ast-js/generate.mjs @@ -44,9 +44,11 @@ export async function processChunk(inputSlice, itemIndices) { export async function* generate(_, worker) { const config = getConfig('ast-js'); - const files = globSync(config.input, { ignore: config.ignore }).filter( - p => extname(p) === '.js' - ); + const files = globSync(config.input, { ignore: config.ignore }) + .filter(p => extname(p) === '.js') + // Glob traversal order is not guaranteed; api-links merges definitions + // last-write-wins, so file order must be deterministic. + .sort(); // Parse the Javascript sources into ASTs in parallel using worker threads // source is both the items list and the fullInput since we use sliceInput diff --git a/packages/core/src/generators/ast/generate.mjs b/packages/core/src/generators/ast/generate.mjs index 1808c6b1..093f5743 100644 --- a/packages/core/src/generators/ast/generate.mjs +++ b/packages/core/src/generators/ast/generate.mjs @@ -118,6 +118,10 @@ export async function* generate(_, worker) { ]); }); + // Glob traversal order is not guaranteed; sort so the pipeline sees files + // in a deterministic order regardless of filesystem or platform. + files.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + // Parse markdown files in parallel using worker threads for await (const chunkResult of worker.stream(files)) { yield chunkResult; diff --git a/packages/core/src/generators/json-simple/generate.mjs b/packages/core/src/generators/json-simple/generate.mjs index 116b9362..b62b2333 100644 --- a/packages/core/src/generators/json-simple/generate.mjs +++ b/packages/core/src/generators/json-simple/generate.mjs @@ -2,11 +2,15 @@ import { join } from 'node:path'; -import { remove } from 'unist-util-remove'; - import getConfig from '../../utils/configuration/index.mjs'; import { writeFile } from '../../utils/file.mjs'; import { UNIST } from '../../utils/queries/index.mjs'; +import { stringifyWithout } from '../../utils/unist.mjs'; + +// Nodes stripped from the simplified output. Filtered during stringification +// rather than removed from the trees: the entries are shared with every other +// generator running off the same metadata. +const EXCLUDED_NODES = [UNIST.isStabilityNode, UNIST.isHeading]; /** * Generates the simplified JSON version of the API docs @@ -16,16 +20,12 @@ import { UNIST } from '../../utils/queries/index.mjs'; export async function generate(input) { const config = getConfig('json-simple'); - input.forEach(node => - remove(node.content, [UNIST.isStabilityNode, UNIST.isHeading]) - ); - if (config.output) { // Writes all the API docs stringified content into one file // Note: The full JSON generator in the future will create one JSON file per top-level API doc file await writeFile( join(config.output, 'api-docs.json'), - config.minify ? JSON.stringify(input) : JSON.stringify(input, null, 2) + stringifyWithout(input, EXCLUDED_NODES, config.minify ? undefined : 2) ); } diff --git a/packages/core/src/generators/metadata/generate.mjs b/packages/core/src/generators/metadata/generate.mjs index b5691258..785a56f1 100644 --- a/packages/core/src/generators/metadata/generate.mjs +++ b/packages/core/src/generators/metadata/generate.mjs @@ -28,9 +28,13 @@ export async function processChunk(fullInput, itemIndices, typeMap) { export async function* generate(inputs, worker) { const { metadata: config } = getConfig(); - const typeMap = config.typeMap - ? JSON.parse(await loadFromURL(config.typeMap)) - : {}; + // The build cache resolves a string typeMap into its parsed object during + // setup (its content participates in cache keys); with the cache disabled + // the string arrives here untouched. + const typeMap = + typeof config.typeMap === 'string' + ? JSON.parse(await loadFromURL(config.typeMap)) + : (config.typeMap ?? {}); // Stream chunks as they complete - allows dependent generators // to start collecting/preparing while we're still processing diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index b0e59f6c..bb8256ae 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -98,34 +98,40 @@ export default function createParallelWorker( const runInOneGo = threads <= 1 || items.length <= 2; - // Submit all tasks to Piscina - each promise resolves to itself for removal - const pending = new Set( - chunks.map(indices => { - if (runInOneGo) { - const promise = generator - .processChunk(items, indices, extra) - .then(result => ({ promise, result })); - - return promise; - } - - const promise = pool - .run( + // In-process chunks get the same isolation the Piscina transfer gives + // worker chunks: sliced, remapped, and structured-cloned. Without the + // clone, generators sharing one process observe each other's (and their + // own cross-chunk) mutations of shared input — e.g. jsx-ast rewriting + // entry trees into JSX nodes that legacy-html then cannot compile. + const slices = runInOneGo + ? chunks.map(indices => structuredClone(indices.map(i => items[i]))) + : null; + + // Submit all tasks up front so every chunk is in flight at once + const tasks = chunks.map((indices, chunk) => + runInOneGo + ? generator.processChunk( + slices[chunk], + indices.map((_, i) => i), + structuredClone(extra) + ) + : pool.run( createTask(items, indices, extra, configuration, specifier, name) ) - .then(result => ({ promise, result })); - - return promise; - }) ); - // Yield results as they complete (true parallel collection) - let completed = 0; + // A chunk may reject while an earlier one is still being awaited; the + // rejection is re-observed at its `await` below. + tasks.forEach(task => task.catch(() => {})); - while (pending.size > 0) { - const { promise, result } = await Promise.race(pending); + // Yield in submission order so collected results are deterministic + // run to run. Parallelism is unaffected (all tasks already run above), + // and consumers collect the full stream anyway, so completed-but-unyielded + // chunks cost no memory the collector wasn't about to hold. + let completed = 0; - pending.delete(promise); + for (const task of tasks) { + const result = await task; completed++; diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index 5d00ee53..5fddedac 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -70,6 +70,11 @@ export const getDefaultConfig = (generators, config) => // See also https://github.com/nodejs/node/pull/60591 threads: process.arch === 'riscv64' ? 1 : cpus().length, chunkSize: 10, + + cache: { + enabled: true, + maxAgeDays: 7, + }, }) ); @@ -144,7 +149,7 @@ const transformConfig = async value => { /** * Converts CLI options into a config - * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options + * @param {import('../../../bin/utils.mjs').CLIOptions} options * @returns {import('./types').Configuration} */ export const createConfigFromCLIOptions = options => ({ @@ -164,6 +169,18 @@ export const createConfigFromCLIOptions = options => ({ target: options.target, threads: options.threads, chunkSize: options.chunkSize, + + // Only carry explicitly-passed cache flags, so config-file settings are + // not clobbered by CLI defaults during the merge. + ...(options.cache === false || options.force || options.cacheDir + ? { + cache: { + ...(options.cache === false ? { enabled: false } : {}), + ...(options.force ? { force: true } : {}), + ...(options.cacheDir ? { dir: options.cacheDir } : {}), + }, + } + : {}), }); /** @@ -189,7 +206,7 @@ export const assertRunnableOptions = config => { * Processes and validates configuration values including version coercion, changelog parsing, * and constraint enforcement for threads and chunk size. * - * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options - User-provided configuration options + * @param {import('../../../bin/utils.mjs').CLIOptions} options - User-provided configuration options * @returns {Promise} The configuration */ export const createRunConfiguration = async options => { @@ -246,7 +263,7 @@ let config; /** * Configuration setter - * @param {import('./types').Configuration | import('../../../bin/commands/generate.mjs').CLIOptions} options + * @param {import('./types').Configuration | import('../../../bin/utils.mjs').CLIOptions} options * @returns {Promise} */ export const setConfig = async options => diff --git a/packages/core/src/utils/file.mjs b/packages/core/src/utils/file.mjs index e7c284bb..59e94b5b 100644 --- a/packages/core/src/utils/file.mjs +++ b/packages/core/src/utils/file.mjs @@ -1,5 +1,7 @@ import fs from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { dirname, join, relative } from 'node:path'; + +import { getBuildCache } from '../cache/index.mjs'; /** * Returns the input string with the `ext` extension, replacing any pre-existing extension @@ -10,11 +12,53 @@ export const withExt = (str, ext) => `${str.replace(/\.[0-9a-z]+$/i, '')}${ext ? `.${ext}` : ''}`; /** - * Writes a file, recursively + * Writes a file, recursively. When a build cache is active, the write is + * recorded for output verification and skipped when byte-identical. * * @type {typeof fs.writeFile} */ -export const writeFile = (file, ...args) => - fs +export const writeFile = (file, ...args) => { + const cache = getBuildCache(); + + if (cache) { + return cache.writeTracked(file, args[0], args.slice(1)); + } + + return fs .mkdir(dirname(file), { recursive: true }) .then(() => fs.writeFile(file, ...args)); +}; + +/** + * Copies a file or directory recursively, recording every copied file with + * the build cache (when active) so copied assets participate in output + * verification. + * + * @param {string} src - Source file or directory + * @param {string} dest - Destination path + * @returns {Promise} + */ +export const copyPath = async (src, dest) => { + await fs.cp(src, dest, { recursive: true, force: true }); + + const cache = getBuildCache(); + + if (!cache) { + return; + } + + if ((await fs.stat(src)).isFile()) { + return cache.recordOutput(dest, await fs.readFile(dest)); + } + + for await (const entry of fs.glob('**/*', { + cwd: src, + withFileTypes: true, + })) { + if (entry.isFile()) { + const rel = relative(src, join(entry.parentPath, entry.name)); + + cache.recordOutput(join(dest, rel), await fs.readFile(join(dest, rel))); + } + } +}; diff --git a/packages/core/src/utils/unist.mjs b/packages/core/src/utils/unist.mjs index 51f222d6..7ff55be4 100644 --- a/packages/core/src/utils/unist.mjs +++ b/packages/core/src/utils/unist.mjs @@ -53,6 +53,32 @@ export const transformNodesToString = (nodes, escape) => { return mappedChildren.join(''); }; +/** + * Stringifies a value while dropping every node (in any `children`-style + * array) that matches one of the given tests, without mutating the input. + * Deep removal equivalent to `unist-util-remove`, expressed as a + * `JSON.stringify` replacer so shared trees stay intact for other consumers. + * + * @param {unknown} value The value to stringify + * @param {Array<(node: import('unist').Node) => boolean>} tests Nodes matching any test are omitted + * @param {string | number} [space] Forwarded to `JSON.stringify` + * @returns {string} The filtered JSON string + */ +export const stringifyWithout = (value, tests, space) => + JSON.stringify( + value, + (_, val) => + Array.isArray(val) + ? val.filter( + node => + node === null || + typeof node !== 'object' || + !tests.some(test => test(node)) + ) + : val, + space + ); + /** * This method is an utility that allows us to conditionally invoke/call a callback * based on test conditions related to a Node's position relative to another one diff --git a/packages/legacy/src/legacy-html-all/generate.mjs b/packages/legacy/src/legacy-html-all/generate.mjs index 192f18e4..63853303 100644 --- a/packages/legacy/src/legacy-html-all/generate.mjs +++ b/packages/legacy/src/legacy-html-all/generate.mjs @@ -1,9 +1,10 @@ 'use strict'; -import { readFile, writeFile } from 'node:fs/promises'; +import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; +import { writeFile } from '@nodejs/doc-kit/utils/file.mjs'; import { minifyHTML } from '@nodejs/doc-kit/utils/html-minifier.mjs'; import { getRemarkRehype as remark } from '@nodejs/doc-kit/utils/remark.mjs'; diff --git a/packages/legacy/src/legacy-html/generate.mjs b/packages/legacy/src/legacy-html/generate.mjs index b3f143c3..ecfe701d 100644 --- a/packages/legacy/src/legacy-html/generate.mjs +++ b/packages/legacy/src/legacy-html/generate.mjs @@ -1,10 +1,12 @@ 'use strict'; -import { readFile, cp } from 'node:fs/promises'; +import { readFile } from 'node:fs/promises'; import { basename, join } from 'node:path'; +import { combine, hashData, hashValue } from '@nodejs/doc-kit/cache/hash.mjs'; +import { getBuildCache } from '@nodejs/doc-kit/cache/index.mjs'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; -import { writeFile } from '@nodejs/doc-kit/utils/file.mjs'; +import { copyPath, writeFile } from '@nodejs/doc-kit/utils/file.mjs'; import { groupNodesByModule } from '@nodejs/doc-kit/utils/generators.mjs'; import { minifyHTML } from '@nodejs/doc-kit/utils/html-minifier.mjs'; import { getRemarkRehypeWithShiki as remark } from '@nodejs/doc-kit/utils/remark.mjs'; @@ -13,6 +15,9 @@ import buildContent from './utils/buildContent.mjs'; import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; import tableOfContents from './utils/tableOfContents.mjs'; +const GENERATOR_SPECIFIER = '@nodejs/doc-kit-generator-legacy/legacy-html'; +const ITEM_NAMESPACE = 'legacy-html:item'; + /** * Creates a heading object with the given name. * @param {string} name - The name of the heading @@ -21,19 +26,45 @@ import tableOfContents from './utils/tableOfContents.mjs'; const getHeading = name => ({ depth: 1, data: { name } }); /** - * Process a chunk of items in a worker thread. - * Builds HTML template objects - FS operations happen in generate(). + * Narrows the head nodes to exactly what per-module rendering reads from + * other modules (`buildExtraContent`'s stability overview): a projection small + * enough to hash and send to every worker, and stable under body-only edits — + * which is what lets every other module's cache entry survive such an edit. * - * Each item is pre-grouped {head, nodes, headNodes} - no need to - * recompute groupNodesByModule for every chunk. + * @param {Array} headNodes - All depth-1 entries, sorted + * @returns {Array} The narrowed projection + */ +const buildHeadNodesLite = headNodes => + headNodes.map(({ api, heading, stability }) => ({ + api, + heading: { data: { name: heading.data.name } }, + stability: stability && { + data: { + index: stability.data.index, + description: stability.data.description, + }, + }, + })); + +/** + * Process a chunk of items in a worker thread: renders the module content + * (Shiki highlighting included), populates the page template, and minifies — + * so a cached module costs the main thread nothing but a file write. + * + * Each item is a pre-grouped `{ head, nodes }`; the shared navigation, the + * head-node projection, and the page template arrive once per chunk as extra. * * @type {import('./types').Generator['processChunk']} */ -export async function processChunk(slicedInput, itemIndices, navigation) { +export async function processChunk(slicedInput, itemIndices, extra) { + const { navigation, headNodesLite, apiTemplate } = extra; + + const config = getConfig('legacy-html'); + const results = []; for (const idx of itemIndices) { - const { head, nodes, headNodes } = slicedInput[idx]; + const { head, nodes } = slicedInput[idx]; const nav = navigation.replace( `class="nav-${head.api}"`, @@ -49,7 +80,7 @@ export async function processChunk(slicedInput, itemIndices, navigation) { ) ); - const content = buildContent(headNodes, nodes); + const content = buildContent(headNodesLite, nodes); const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); @@ -63,7 +94,13 @@ export async function processChunk(slicedInput, itemIndices, navigation) { content, }; - results.push(template); + let html = replaceTemplateValues(apiTemplate, template, config); + + if (config.minify) { + html = await minifyHTML(html); + } + + results.push({ ...template, html }); } return results; @@ -107,33 +144,102 @@ export async function* generate(input, worker) { const assetsFolder = join(config.output, basename(path)); // Copy all files from assets folder to output - await cp(path, assetsFolder, { recursive: true }); + await copyPath(path, assetsFolder); } } - // Create sliced input: each item contains head + its module's entries + headNodes reference - // This avoids sending all ~4900 entries to every worker and recomputing groupings - const entries = headNodes.map(head => ({ + const headNodesLite = buildHeadNodesLite(headNodes); + const items = headNodes.map(head => ({ head, nodes: groupedModules.get(head.api), - headNodes, })); - // Stream chunks as they complete - HTML files are written immediately - for await (const chunkResult of worker.stream(entries, navigation)) { - // Write files for this chunk in the generate method (main thread) - if (config.output) { - for (const template of chunkResult) { - let result = replaceTemplateValues(apiTemplate, template, config); + // Per-module leaf cache: a module's page is a pure function of its own + // source file, the two global projections (navigation string, head-node + // projection), the page template, and the chain salt (code + config). A + // body-only edit leaves both projections unchanged, so every other module + // hits and never runs Shiki or the minifier. + const buildCache = getBuildCache(); + + /** @type {Map} Cached results by api */ + const cached = new Map(); + + /** @type {Array<{ item: object, key: string | null }>} */ + const misses = []; + + if (buildCache) { + const salt = await buildCache.chainSalt(GENERATOR_SPECIFIER); + const projectionHash = combine( + hashData(navigation), + hashValue(headNodesLite), + hashData(apiTemplate) + ); + + for (const item of items) { + const source = await buildCache.sourceHash(item.head.path); + + // Unknown provenance (no source file behind the entry) is uncacheable. + const key = source + ? combine(ITEM_NAMESPACE, salt, source, projectionHash) + : null; + + const hit = key && (await buildCache.store.get(key)); - if (config.minify) { - result = await minifyHTML(result); + if (hit) { + cached.set(item.head.api, JSON.parse(hit)); + } else { + misses.push({ item, key }); + } + } + } else { + misses.push(...items.map(item => ({ item, key: null }))); + } + + const extra = { navigation, headNodesLite, apiTemplate }; + + // Drive the worker stream concurrently, resolving each miss's deferred as + // its chunk lands; results are stored write-behind under the miss's key. + const deferreds = new Map( + misses.map(({ item }) => [item.head.api, Promise.withResolvers()]) + ); + + const pump = (async () => { + let index = 0; + + for await (const chunk of worker.stream( + misses.map(({ item }) => item), + extra + )) { + for (const result of chunk) { + const { key } = misses[index++]; + + if (buildCache && key) { + buildCache.store.put(key, JSON.stringify(result)); } - await writeFile(join(config.output, `${template.api}.html`), result); + deferreds.get(result.api).resolve(result); } } + })(); + + pump.catch(error => { + for (const { reject } of deferreds.values()) { + reject(error); + } + }); + + // Emit in canonical (sorted head-node) order regardless of the hit/miss + // split, so downstream aggregation (legacy-html-all) is byte-stable. + for (const head of headNodes) { + const result = + cached.get(head.api) ?? (await deferreds.get(head.api).promise); + + if (config.output) { + await writeFile(join(config.output, `${result.api}.html`), result.html); + } - yield chunkResult; + yield [result]; } + + await pump; } diff --git a/packages/legacy/src/legacy-json-all/generate.mjs b/packages/legacy/src/legacy-json-all/generate.mjs index 67b0be2a..e9f53c11 100644 --- a/packages/legacy/src/legacy-json-all/generate.mjs +++ b/packages/legacy/src/legacy-json-all/generate.mjs @@ -1,9 +1,9 @@ 'use strict'; -import { writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; +import { writeFile } from '@nodejs/doc-kit/utils/file.mjs'; import { legacyToJSON } from '../utils/legacyToJSON.mjs'; diff --git a/packages/react/src/html/bundlers/vite.mjs b/packages/react/src/html/bundlers/vite.mjs index 83906eb1..a56fb2b7 100644 --- a/packages/react/src/html/bundlers/vite.mjs +++ b/packages/react/src/html/bundlers/vite.mjs @@ -3,6 +3,9 @@ import { tmpdir } from 'node:os'; import { basename, isAbsolute, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { hashData } from '@nodejs/doc-kit/cache/hash.mjs'; +import { getBuildCache } from '@nodejs/doc-kit/cache/index.mjs'; +import { writeFile } from '@nodejs/doc-kit/utils/file.mjs'; import { minifyHTML } from '@nodejs/doc-kit/utils/html-minifier.mjs'; import { build as viteBuild, @@ -87,7 +90,9 @@ const createHTMLFinalizerPlugin = () => ({ name: 'doc-kit:finalize-html', /** * Minifies every generated HTML entry after Vite has injected its scripts, - * stylesheets, and module preloads. + * stylesheets, and module preloads. Minification is memoized in the durable + * build cache by pre-minify content, so on warm builds only pages whose + * content actually changed pay the minifier. */ generateBundle: { order: 'post', @@ -96,6 +101,8 @@ const createHTMLFinalizerPlugin = () => ({ * @param {Record} bundle */ async handler(_, bundle) { + const cache = getBuildCache(); + await Promise.all( Object.values(bundle) .filter( @@ -107,7 +114,11 @@ const createHTMLFinalizerPlugin = () => ({ ? asset.source : Buffer.from(asset.source).toString('utf8'); - asset.source = await minifyHTML(source); + asset.source = cache + ? await cache.store.memo('html:minify', hashData(source), () => + minifyHTML(source) + ) + : await minifyHTML(source); }) ); }, @@ -224,9 +235,12 @@ export const createViteConfig = ({ ...vite.build, // Both builds are complete Vite outputs. SSR uses a private directory - // because its entries can share chunks; the client writes the final site. + // because its entries can share chunks and is executed from disk. The + // client bundle is written by `build()` below through the tracked + // writeFile, so byte-identical files are skipped (stable mtimes) and + // every output participates in cache verification. outDir: server ? serverOutDir : resolve(webConfig.output), - write: true, + write: server, emptyOutDir: false, copyPublicDir: false, watch: null, @@ -374,7 +388,7 @@ export const build = async ({ sources.set(id, code); } - await viteBuild( + const result = await viteBuild( createViteConfig({ sources, input, @@ -383,6 +397,20 @@ export const build = async ({ vite, }) ); + + const outDir = resolve(config.output); + + // Write the bundle through the tracked writeFile: byte-identical files are + // skipped (unchanged pages and shared assets keep their mtimes) and every + // emitted file is recorded for whole-run cache verification. + for (const { output } of [result].flat()) { + for (const item of output) { + await writeFile( + join(outDir, item.fileName), + item.type === 'chunk' ? item.code : item.source + ); + } + } }; /** diff --git a/packages/react/src/html/generate.mjs b/packages/react/src/html/generate.mjs index 4e5c7cac..b3be85f0 100644 --- a/packages/react/src/html/generate.mjs +++ b/packages/react/src/html/generate.mjs @@ -5,15 +5,17 @@ import { readFile } from 'node:fs/promises'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; import { copyStaticAssets } from './utils/copying.mjs'; -import { createCodeConverter, processBundles } from './utils/processing.mjs'; +import { processBundles } from './utils/processing.mjs'; /** * Main generation function that sends per-page JSX code to the web bundler. * - * Receives `jsx-ast`'s output as `{ data, code }` items — the JSX AST was - * already serialized to `code` in the jsx-ast worker, so no AST is held here. - * Bundling and rendering then run once over the accumulated code, since shared - * component chunks, CSS, and the sidebar need every entry together. + * Receives `jsx-ast`'s output as `{ data, code }` or `{ data, codeRef }` + * items — the JSX AST was already serialized to `code` in the jsx-ast worker, + * and for cached pages even the code string stays on disk behind the lazy + * `codeRef` until (and unless) server rendering actually needs it. Bundling + * runs once over the accumulated entries, since shared component chunks, CSS, + * and the sidebar need every entry together. * * @type {import('./types').Generator['generate']} */ @@ -22,17 +24,9 @@ export async function generate(input) { const template = await readFile(config.templatePath, 'utf-8'); - const converter = createCodeConverter(); - - // Per-page metadata, in render order. Each item is already just - // `{ data, code }` — the heavy JSX AST was converted to `code` and discarded - // in the jsx-ast worker, so nothing large is held here. - const datas = []; - - for (const item of input) { - converter.add(item); - datas.push(item.data); - } + // Per-page metadata, in render order. Each item is small — the heavy JSX + // AST was converted to `code` and discarded in the jsx-ast worker. + const datas = input.map(item => item.data); // Sidebar lists only the real module pages. const sidebarEntries = datas @@ -40,8 +34,7 @@ export async function generate(input) { .map(data => ({ data })); await processBundles({ - serverCodeMap: converter.serverCodeMap, - clientCodeMap: converter.clientCodeMap, + items: input, datas, sidebarEntries, template, diff --git a/packages/react/src/html/index.mjs b/packages/react/src/html/index.mjs index 461e6387..7d78f0bc 100644 --- a/packages/react/src/html/index.mjs +++ b/packages/react/src/html/index.mjs @@ -17,9 +17,11 @@ import { generate } from './generate.mjs'; * * `jsx-ast` serializes each page's JSX AST to a `code` string inside its worker, * so this generator only ever handles small `{ data, code }` items — the heavy - * ASTs (notably the giant `all` page) never reach the main thread. Bundling and - * rendering run once over the accumulated code, since code-splitting and the - * sidebar need every entry together. + * ASTs (notably the giant `all` page) never reach the main thread. Cached pages + * arrive as `{ data, codeRef }` and their code is only read from disk when + * server rendering actually misses the durable cache. Bundling runs once over + * the accumulated entries, since code-splitting and the sidebar need every + * entry together. * * @type {import('./types').Generator} */ diff --git a/packages/react/src/html/utils/__tests__/copying.test.mjs b/packages/react/src/html/utils/__tests__/copying.test.mjs index 052a1559..52d86553 100644 --- a/packages/react/src/html/utils/__tests__/copying.test.mjs +++ b/packages/react/src/html/utils/__tests__/copying.test.mjs @@ -3,8 +3,8 @@ import { join } from 'node:path'; import { describe, it, mock, beforeEach } from 'node:test'; const mockCp = mock.fn(() => Promise.resolve()); -mock.module('node:fs/promises', { - namedExports: { cp: mockCp }, +mock.module('@nodejs/doc-kit/utils/file.mjs', { + namedExports: { copyPath: mockCp }, }); const mockLogError = mock.fn(); @@ -48,13 +48,11 @@ describe('copyStaticAssets', () => { assert.deepStrictEqual(mockCp.mock.calls[0].arguments, [ 'src/assets', join('/out', 'assets'), - { recursive: true, force: true }, ]); assert.deepStrictEqual(mockCp.mock.calls[1].arguments, [ 'docs/images', join('/out', 'images'), - { recursive: true, force: true }, ]); }); @@ -76,13 +74,11 @@ describe('copyStaticAssets', () => { assert.deepStrictEqual(mockCp.mock.calls[0].arguments, [ 'src/custom', join('/out', 'dest-folder/custom'), - { recursive: true, force: true }, ]); assert.deepStrictEqual(mockCp.mock.calls[1].arguments, [ 'src/another', join('/out', 'another-folder'), - { recursive: true, force: true }, ]); }); diff --git a/packages/react/src/html/utils/copying.mjs b/packages/react/src/html/utils/copying.mjs index 1c43a907..bf7360f9 100644 --- a/packages/react/src/html/utils/copying.mjs +++ b/packages/react/src/html/utils/copying.mjs @@ -1,7 +1,7 @@ -import { cp } from 'node:fs/promises'; import { join, basename } from 'node:path'; import logger from '@nodejs/doc-kit/logger/index.mjs'; +import { copyPath } from '@nodejs/doc-kit/utils/file.mjs'; /** * Copies static directories/files defined in `pathsToCopy` to the output directory. @@ -24,7 +24,7 @@ export async function copyStaticAssets(config) { for (const { src, dest } of copyTasks) { try { - await cp(src, dest, { recursive: true, force: true }); + await copyPath(src, dest); } catch (err) { if (err.code !== 'ENOENT') { logger.error( diff --git a/packages/react/src/html/utils/processing.mjs b/packages/react/src/html/utils/processing.mjs index 5fe51daa..0cc53562 100644 --- a/packages/react/src/html/utils/processing.mjs +++ b/packages/react/src/html/utils/processing.mjs @@ -1,3 +1,9 @@ +import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { isAbsolute, resolve as resolvePath } from 'node:path'; + +import { combine, hashData, hashValue } from '@nodejs/doc-kit/cache/hash.mjs'; +import { getBuildCache } from '@nodejs/doc-kit/cache/index.mjs'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; import { populate } from '@nodejs/doc-kit/utils/configuration/templates.mjs'; @@ -8,6 +14,59 @@ import { resolveBundler } from '../bundlers/index.mjs'; import { SPECULATION_RULES } from '../constants.mjs'; import { THEME_SCRIPT } from '../ui/theme-script.mjs'; +const GENERATOR_SPECIFIER = '@nodejs/doc-kit-generator-react/html'; + +/** + * Runtime dependencies whose versions shape server-rendered output but sit + * outside this package's own salt (installed separately, version ranges). + */ +const RUNTIME_DEPS = ['preact', 'preact-render-to-string', 'vite']; + +const require = createRequire(import.meta.url); + +/** + * Identity of everything the SSR output depends on beyond the page's own + * code: the server `#theme/config` virtual module (page list, versions, + * config), user virtual imports, theme-override import files (by content), + * and runtime dependency versions. Returns null when any theme import cannot + * be content-hashed (e.g. a directory alias) — unknown inputs make SSR + * results uncacheable, never silently stale. + * + * @param {import('../types').ResolvedWebConfiguration} config - The html configuration + * @param {Record} serverVirtualImports - Server virtual module sources + * @returns {Promise} SSR environment hash, or null + */ +const ssrEnvironmentHash = async (config, serverVirtualImports) => { + const parts = [hashValue(serverVirtualImports)]; + + for (const [name, target] of Object.entries(config.imports ?? {})) { + if ( + typeof target === 'string' && + (target.startsWith('.') || isAbsolute(target)) + ) { + const content = await readFile(resolvePath(target)).catch(() => null); + + if (content === null) { + return null; + } + + parts.push(`${name}:${hashData(content)}`); + } else { + parts.push(`${name}:${String(target)}`); + } + } + + for (const dep of RUNTIME_DEPS) { + try { + parts.push(`${dep}@${require(`${dep}/package.json`).version}`); + } catch { + parts.push(`${dep}@unresolved`); + } + } + + return hashValue(parts); +}; + /** * Creates the virtual imports for one bundle target. * @@ -87,71 +146,124 @@ export const buildHead = ({ meta = [], links = [], html = [] }) => ].join('\n '); /** - * Creates an accumulator that wraps per-page JSX code into server and client - * programs one at a time. The JSX AST has already been serialized to a code - * string upstream (in the `jsx-ast` worker), so the heavy AST never reaches - * the main thread — only the code string and page metadata stream in here. + * Server-renders every page, serving unchanged pages from the durable cache. + * + * A page's dehydrated HTML is a pure function of its code (identified by + * content hash, whether inline or behind a lazy `codeRef`) and the SSR + * environment. Only cache misses have their code materialized, get wrapped + * into server programs, and go through one batched subset SSR build — the + * SSR bundle has no cross-page hash coupling, so partial input sets are safe. * - * @returns {{ add: (item: { data: import('@nodejs/doc-kit/generators/metadata/types').MetadataEntry, code: string }) => void, serverCodeMap: Map, clientCodeMap: Map }} + * @param {Array<{ data: import('@nodejs/doc-kit/generators/metadata/types').MetadataEntry, code?: string, codeRef?: { contentHash: string, load: () => Promise } }>} items - jsx-ast output items + * @param {Record} serverVirtualImports - Server virtual module sources + * @param {import('../types').ResolvedWebConfiguration} config - The html configuration + * @param {import('../types').WebBundler} bundler - The resolved bundler + * @returns {Promise>} Dehydrated HTML per page api */ -export function createCodeConverter() { - const { buildServerProgram, clientProgram } = createProgramBuilder(); - - const serverCodeMap = new Map(); - const clientCodeMap = new Map(); - - return { - /** - * Records the server/client programs for a single page's JSX code. - * - * @param {{ data: import('@nodejs/doc-kit/generators/metadata/types').MetadataEntry, code: string }} item - */ - add: ({ data, code }) => { - const fileName = `${data.api}.jsx`; - - // Prepare code for server-side execution (wrapped for SSR) - serverCodeMap.set(fileName, buildServerProgram(code)); - - // Every page's entry is the same module; the bundler emits one chunk. - clientCodeMap.set(fileName, clientProgram); - }, - serverCodeMap, - clientCodeMap, - }; -} +const renderServerPages = async ( + items, + serverVirtualImports, + config, + bundler +) => { + const { buildServerProgram } = createProgramBuilder(); + const buildCache = getBuildCache(); + + const environment = buildCache + ? await ssrEnvironmentHash(config, serverVirtualImports) + : null; + + const salt = environment && (await buildCache.chainSalt(GENERATOR_SPECIFIER)); + + const serverPages = new Map(); + const missEntries = new Map(); + + /** @type {Map} api → ssr cache key */ + const missKeys = new Map(); + + for (const item of items) { + const { data } = item; + + const contentHash = + item.code != null ? hashData(item.code) : item.codeRef.contentHash; + + const key = salt && combine('html:ssr', salt, environment, contentHash); + const hit = key && (await buildCache.store.get(key)); + + if (hit !== null && hit !== false && hit !== undefined) { + serverPages.set(data.api, hit); + + continue; + } + + const code = item.code ?? (await item.codeRef.load()); + + if (code == null) { + // A concurrent prune raced the lazy reference away — vanishingly rare + // (touch() refreshes mtimes at check time). Failing loudly here is + // self-healing: the next run rebuilds the page from source. + throw new Error( + `Cached page code for "${data.api}" disappeared mid-run; rerun the build` + ); + } + + missEntries.set(`${data.api}.jsx`, buildServerProgram(code)); + + if (key) { + missKeys.set(data.api, key); + } + } + + if (missEntries.size > 0) { + const rendered = await bundler.render({ + entries: missEntries, + virtualImports: serverVirtualImports, + config, + }); + + for (const [api, html] of rendered) { + serverPages.set(api, html); + + const key = missKeys.get(api); + + if (key) { + buildCache.store.put(key, html); + } + } + } + + return serverPages; +}; /** - * Bundles pre-converted JSX code into complete HTML pages and client assets. - * Conversion (JSX AST → code) happens upstream via - * {@link createCodeConverter} so the heavy ASTs are already discarded; this - * step needs every entry together for code-splitting and the shared sidebar. + * Bundles per-page JSX into complete HTML pages and client assets. Server + * rendering is per-page cached (see {@link renderServerPages}); the client + * build always runs over every entry — partial client input sets measurably + * change chunk hashing, so the whole-graph build is the correctness path and + * is kept cheap by per-page minify memoization and skipped identical writes. * * @param {object} params - * @param {Map} params.serverCodeMap - Server-side code per page. - * @param {Map} params.clientCodeMap - Client-side code per page. + * @param {Array<{ data: import('@nodejs/doc-kit/generators/metadata/types').MetadataEntry, code?: string, codeRef?: object }>} params.items - jsx-ast output items, in render order. * @param {Array} params.datas - Per-page metadata, in render order. * @param {Array<{ data: import('@nodejs/doc-kit/generators/metadata/types').MetadataEntry }>} params.sidebarEntries - Entries used to build the sidebar page list (real module pages only). * @param {string} params.template - The HTML template string for the output pages. */ export async function processBundles({ - serverCodeMap, - clientCodeMap, + items, datas, sidebarEntries, template, }) { const config = getConfig('html'); const bundler = await resolveBundler(config.bundler); + const { clientProgram } = createProgramBuilder(); - const serverPages = await bundler.render({ - entries: serverCodeMap, - virtualImports: createVirtualImports( - sidebarEntries, - config.virtualImports, - true - ), + const serverPages = await renderServerPages( + items, + createVirtualImports(sidebarEntries, config.virtualImports, true), config, - }); + bundler + ); const titleSuffix = populate(config.title, { ...config, @@ -192,6 +304,12 @@ export async function processBundles({ }) ); + // Every page's client entry is the same program; the bundler emits shared + // chunks and per-page entry stubs from the full set. + const clientCodeMap = new Map( + datas.map(data => [`${data.api}.jsx`, clientProgram]) + ); + await bundler.build({ entries: clientCodeMap, virtualImports: createVirtualImports( diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index 9a21310d..d055f632 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -1,3 +1,5 @@ +import { combine, hashData, hashValue } from '@nodejs/doc-kit/cache/hash.mjs'; +import { getBuildCache } from '@nodejs/doc-kit/cache/index.mjs'; import getConfig from '@nodejs/doc-kit/utils/configuration/index.mjs'; import { groupNodesByModule } from '@nodejs/doc-kit/utils/generators.mjs'; import { jsx, toJs } from 'estree-util-to-js'; @@ -8,6 +10,8 @@ import { buildNotFoundPage } from './utils/synthetic/404.mjs'; import { buildAllPage } from './utils/synthetic/all.mjs'; import { buildIndexPage } from './utils/synthetic/index.mjs'; +const GENERATOR_SPECIFIER = '@nodejs/doc-kit-generator-react/jsx-ast'; + /** * Builds the `{ head, entries }` page descriptors for all configured synthetic * pages. The descriptors are cheap to build; the expensive `buildContent` step @@ -26,6 +30,52 @@ const buildSyntheticDescriptors = input => { ].filter(Boolean); }; +/** + * Derives the cache-key material for one descriptor, or null when the + * descriptor is uncacheable. + * + * Module pages key on their source file's content hash. The synthetic `index` + * page keys on a projection of exactly what it renders (sorted head names, + * apis, and stability data), so body-only edits keep it cached. The synthetic + * `all` page is deliberately uncacheable: it folds every entry, so its key + * would change on any edit — and when nothing changed at all, the whole run + * is skipped upstream — leaving no run that could ever hit; storing its huge + * code string would be pure cache churn. + * + * @param {{ head: object, entries?: object[] }} descriptor - Page descriptor + * @param {import('@nodejs/doc-kit/cache/types').BuildCache} buildCache - Active build cache + * @param {Array} moduleInput - All non-index entries + * @returns {Promise} Key material, or null + */ +const itemKeyBase = async (descriptor, buildCache, moduleInput) => { + const { head } = descriptor; + + if (!head.synthetic) { + return (await buildCache.sourceHash(head.path)) ?? null; + } + + if (head.api === 'index') { + return hashValue( + getSortedHeadNodes(moduleInput) + .filter(entry => entry.stability) + .map(({ api, heading, stability }) => ({ + api, + name: heading.data.name, + stability: { + index: stability.data.index, + description: stability.data.description, + }, + })) + ); + } + + if (head.api === '404') { + return 'synthetic:404'; + } + + return null; +}; + /** * Process a chunk of items in a worker thread. * @@ -57,6 +107,13 @@ export async function processChunk(slicedInput, itemIndices) { /** * Generates per-page JSX code from API metadata. * + * Cached pages yield `{ data, codeRef }` instead of `{ data, code }`: the + * code string stays on disk behind a lazy `codeRef.load()`, so downstream + * consumers that don't need it (a page whose SSR output is also cached) + * never pay to deserialize it. `codeRef.contentHash` identifies the code by + * content for downstream keying, identically whether the page was cached or + * freshly built. + * * @type {import('./types').Generator['generate']} */ export async function* generate(input, worker) { @@ -75,7 +132,98 @@ export async function* generate(input, worker) { // (potentially enormous) content is built and converted off the main thread. descriptors.push(...buildSyntheticDescriptors(moduleInput)); - for await (const chunkResult of worker.stream(descriptors)) { - yield chunkResult; + const buildCache = getBuildCache(); + + /** @type {Map} Cached results by descriptor index */ + const cached = new Map(); + + /** @type {Array<{ index: number, dataKey: string | null, codeKey: string | null }>} */ + const misses = []; + + if (buildCache) { + const salt = await buildCache.chainSalt(GENERATOR_SPECIFIER); + + for (const [index, descriptor] of descriptors.entries()) { + const base = await itemKeyBase(descriptor, buildCache, moduleInput); + const dataKey = base && combine('jsx-ast:data', salt, base); + const codeKey = base && combine('jsx-ast:code', salt, base); + + // The code object is only touched (existence + prune protection), not + // read — that is the whole point of the lazy reference. + const record = dataKey && (await buildCache.store.get(dataKey)); + const hasCode = codeKey && (await buildCache.store.touch(codeKey)); + + if (record !== null && record && hasCode) { + const { data, codeHash } = JSON.parse(record); + + cached.set(index, { + data, + codeRef: { + contentHash: codeHash, + /** + * Reads the page's code string from the store on demand. + * + * @returns {Promise} + */ + load: () => buildCache.store.get(codeKey), + }, + }); + } else { + misses.push({ index, dataKey, codeKey }); + } + } + } else { + misses.push( + ...descriptors.map((_, index) => ({ + index, + dataKey: null, + codeKey: null, + })) + ); } + + // Drive the worker stream concurrently, resolving each miss's deferred as + // its chunk lands; results are stored write-behind under the miss's keys. + const deferreds = new Map( + misses.map(({ index }) => [index, Promise.withResolvers()]) + ); + + const pump = (async () => { + let at = 0; + + for await (const chunk of worker.stream( + misses.map(({ index }) => descriptors[index]) + )) { + for (const result of chunk) { + const miss = misses[at++]; + + if (buildCache && miss.dataKey) { + buildCache.store.put( + miss.dataKey, + JSON.stringify({ + data: result.data, + codeHash: hashData(result.code), + }) + ); + buildCache.store.put(miss.codeKey, result.code); + } + + deferreds.get(miss.index).resolve(result); + } + } + })(); + + pump.catch(error => { + for (const { reject } of deferreds.values()) { + reject(error); + } + }); + + // Emit in canonical descriptor order regardless of the hit/miss split, so + // downstream page ordering is byte-stable. + for (const [index] of descriptors.entries()) { + yield [cached.get(index) ?? (await deferreds.get(index).promise)]; + } + + await pump; } diff --git a/packages/react/src/jsx-ast/index.mjs b/packages/react/src/jsx-ast/index.mjs index 9cd6a71a..8bbf100d 100644 --- a/packages/react/src/jsx-ast/index.mjs +++ b/packages/react/src/jsx-ast/index.mjs @@ -16,7 +16,7 @@ export default { defaultConfiguration: { ref: 'main', - generateAllPage: true, + generateAllPage: false, generateIndexPage: true, generateNotFoundPage: true, }, diff --git a/packages/react/src/orama-db/generate.mjs b/packages/react/src/orama-db/generate.mjs index 4d29e22b..57a9ae80 100644 --- a/packages/react/src/orama-db/generate.mjs +++ b/packages/react/src/orama-db/generate.mjs @@ -43,6 +43,11 @@ export async function generate(input) { }) ); + // Orama otherwise mints `${timestamp}-${counter}` ids, which would make the + // saved database differ between byte-identical builds. Document order is + // deterministic, so positions are stable ids. + documents.forEach((document, index) => (document.id = String(index))); + // Insert all documents await insertMultiple(db, documents); diff --git a/packages/react/src/sitemap/generate.mjs b/packages/react/src/sitemap/generate.mjs index 3fb63d12..3ec6fc60 100644 --- a/packages/react/src/sitemap/generate.mjs +++ b/packages/react/src/sitemap/generate.mjs @@ -27,7 +27,9 @@ export async function generate(entries) { 'utf-8' ); - const lastmod = new Date().toISOString().split('T')[0]; + // Only emitted when configured: stamping the build date would make every + // build differ and misstate when the content actually changed. + const lastmod = config.lastmod; const apiPages = entries .filter(entry => entry.heading.depth === 1) @@ -49,7 +51,10 @@ export async function generate(entries) { const urlset = apiPages .map(page => - entryTemplate + (page.lastmod + ? entryTemplate + : entryTemplate.replace(/\s*__LASTMOD__<\/lastmod>/, '') + ) .replace('__LOC__', page.loc) .replace('__LASTMOD__', page.lastmod) .replace('__CHANGEFREQ__', page.changefreq) diff --git a/packages/react/src/sitemap/types.d.ts b/packages/react/src/sitemap/types.d.ts index f1ca96f3..6ace81c3 100644 --- a/packages/react/src/sitemap/types.d.ts +++ b/packages/react/src/sitemap/types.d.ts @@ -18,6 +18,12 @@ export type Generator = GeneratorMetadata< { indexURL: string; pageURL: string; + /** + * `YYYY-MM-DD` date emitted as every entry's ``. When unset, the + * element is omitted entirely — the sitemap spec allows it, and stamping + * the build date would misstate when the content actually changed. + */ + lastmod?: string; }, Generate, Promise> >; diff --git a/www/doc-kit.config.mjs b/www/doc-kit.config.mjs index 14e3164e..63b47e24 100644 --- a/www/doc-kit.config.mjs +++ b/www/doc-kit.config.mjs @@ -97,6 +97,9 @@ export default { items: [ { label: 'CLI', link: '/cli' }, { label: 'Configuration', link: '/configuration' }, + { label: 'Caching', link: '/caching' }, + { label: 'Creating Commands', link: '/commands' }, + { label: 'Creating Generators', link: '/generators' }, { label: 'Specification', link: '/specification' }, ], }, From 6e4d3d6d664ad14dd5698785fb5bfea6dd5321b4 Mon Sep 17 00:00:00 2001 From: avivkeller Date: Fri, 7 Aug 2026 12:03:55 -0400 Subject: [PATCH 2/3] fixup! --- docs/caching.md | 44 +++--- .../core/src/cache/__tests__/hash.test.mjs | 4 +- .../core/src/cache/__tests__/store.test.mjs | 12 -- packages/core/src/cache/hash.mjs | 58 +++----- packages/core/src/cache/index.mjs | 113 +++++---------- packages/core/src/cache/salt.mjs | 136 +++--------------- packages/core/src/cache/store.mjs | 67 +++------ packages/core/src/cache/types.d.ts | 12 +- packages/core/src/generators.mjs | 25 ++-- packages/legacy/src/legacy-html/generate.mjs | 46 +++--- packages/react/src/html/utils/processing.mjs | 4 +- packages/react/src/jsx-ast/generate.mjs | 65 ++++----- 12 files changed, 171 insertions(+), 415 deletions(-) diff --git a/docs/caching.md b/docs/caching.md index b92b0217..7588251e 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -33,19 +33,20 @@ scripting around it. ## How invalidation works -Cache keys are content hashes — never timestamps. Every key is salted with: - -- each package's identity: the published version, or a content hash of its - `src/` tree when running from a workspace or `npm link` (so hacking on - doc-kit itself invalidates correctly); -- the resolved configuration (including the parsed changelog, index, and the - fetched `typeMap` bytes); -- the Node.js major version and a cache schema version. +Cache keys are content hashes — never timestamps. Every key is salted with +the resolved configuration (including the parsed changelog, index, and the +fetched `typeMap` bytes) and a cache schema version. Anything the cache cannot fully account for (for example a theme `imports` alias pointing at a directory) makes the affected entries uncacheable rather than possibly stale. +Generator _code_ is identified by the cache schema version alone: doc-kit +bumps it in releases whose generated output changes, so releases that don't +change output keep existing caches valid. When developing doc-kit or a custom +generator locally, code edits do not invalidate the cache on their own — +build with `--force` while iterating. + ## Configuration ```js @@ -58,17 +59,13 @@ export default { }; ``` -CLI flags and environment variables: +CLI flags: -| Surface | Effect | -| -------------------------- | --------------------------------------------- | -| `--no-cache` | Disable reads and writes for this run | -| `--force` | Ignore existing entries; still write new ones | -| `--cache-dir ` | Override the cache directory | -| `DOC_KIT_NO_CACHE=1` | Same as `--no-cache` | -| `DOC_KIT_CACHE_FORCE=1` | Same as `--force` | -| `DOC_KIT_CACHE_DIR` | Same as `--cache-dir` | -| `DOC_KIT_CACHE_STATS_FILE` | Write machine-readable run stats as JSON | +| Flag | Effect | +| -------------------- | --------------------------------------------- | +| `--no-cache` | Disable reads and writes for this run | +| `--force` | Ignore existing entries; still write new ones | +| `--cache-dir ` | Override the cache directory | ## CI usage @@ -87,10 +84,9 @@ content) turns unchanged-doc CI builds into sub-second no-ops: The output directory can always be deleted independently of the cache; a warm run regenerates it byte-for-byte. -## Guarantees and verification +## Guarantees -`scripts/cache-verify/index.mjs` asserts the invariants end to end on every -change: cold builds are byte-identical across runs and across -threading/chunking topologies; cached builds are byte-identical to -`--no-cache` builds; a wiped output directory or a corrupted cache silently -recovers; and one-file edits rebuild exactly the affected outputs. +Cold builds are byte-identical across runs and across threading/chunking +topologies; cached builds are byte-identical to `--no-cache` builds; a wiped +output directory or a corrupted cache silently recovers; and one-file edits +rebuild exactly the affected outputs. diff --git a/packages/core/src/cache/__tests__/hash.test.mjs b/packages/core/src/cache/__tests__/hash.test.mjs index e6ba410e..62878813 100644 --- a/packages/core/src/cache/__tests__/hash.test.mjs +++ b/packages/core/src/cache/__tests__/hash.test.mjs @@ -42,11 +42,11 @@ describe('canonicalJSON', () => { assert.equal(canonicalJSON(fn), JSON.stringify(String(fn))); }); - it('marks cycles instead of throwing', () => { + it('throws on cycles (callers disable caching)', () => { const value = { a: 1 }; value.self = value; - assert.equal(canonicalJSON(value), '{"a":1,"self":"[circular]"}'); + assert.throws(() => canonicalJSON(value)); }); it('allows shared (non-cyclic) references', () => { diff --git a/packages/core/src/cache/__tests__/store.test.mjs b/packages/core/src/cache/__tests__/store.test.mjs index 1c98753d..678b23bb 100644 --- a/packages/core/src/cache/__tests__/store.test.mjs +++ b/packages/core/src/cache/__tests__/store.test.mjs @@ -26,18 +26,6 @@ describe('store', () => { const store = await createTempStore(); assert.equal(await store.get(KEY_A), null); - assert.equal(store.stats.misses, 1); - }); - - it('counts hits and writes', async () => { - const store = await createTempStore(); - - store.put(KEY_A, 'value'); - await store.flush(); - await store.get(KEY_A); - - assert.equal(store.stats.hits, 1); - assert.equal(store.stats.writes, 1); }); it('memoizes: computes once, then serves from disk', async () => { diff --git a/packages/core/src/cache/hash.mjs b/packages/core/src/cache/hash.mjs index b05da938..ce0267a9 100644 --- a/packages/core/src/cache/hash.mjs +++ b/packages/core/src/cache/hash.mjs @@ -12,55 +12,29 @@ import { hash } from 'node:crypto'; export const hashData = data => hash('sha256', data, 'hex').slice(0, 32); /** - * Serializes a value into a canonical JSON string: object keys sorted, - * `undefined` treated as JSON does (omitted from objects, `null` in arrays), - * functions by source text, cycles marked. Two structurally equal values - * always produce the same string. + * Serializes a value into a canonical JSON string: a `JSON.stringify` pass + * whose replacer sorts object keys and serializes functions and bigints by + * source text. `undefined` behaves as JSON does (omitted from objects, `null` + * in arrays and at the top level). Cyclic values throw — callers treat any + * hashing failure as "disable caching", never "guess". * * @param {unknown} value - Value to serialize - * @param {Set} [seen] - Ancestry for cycle detection * @returns {string} Canonical JSON */ -export const canonicalJSON = (value, seen = new Set()) => { - if (value === undefined) { - return 'null'; - } - - if (typeof value === 'function') { - return JSON.stringify(String(value)); - } - - if (typeof value === 'bigint') { - return JSON.stringify(String(value)); - } - - if (value === null || typeof value !== 'object') { - return JSON.stringify(value); - } - - if (seen.has(value)) { - return '"[circular]"'; - } - - seen.add(value); - - try { - if (Array.isArray(value)) { - return `[${value.map(item => canonicalJSON(item, seen)).join(',')}]`; +export const canonicalJSON = value => + JSON.stringify(value, (_, val) => { + if (typeof val === 'function' || typeof val === 'bigint') { + return String(val); } - const entries = Object.entries(value) - .filter(([, val]) => val !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map( - ([key, val]) => `${JSON.stringify(key)}:${canonicalJSON(val, seen)}` - ); + if (val === null || typeof val !== 'object' || Array.isArray(val)) { + return val; + } - return `{${entries.join(',')}}`; - } finally { - seen.delete(value); - } -}; + return Object.fromEntries( + Object.entries(val).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ); + }) ?? 'null'; /** * Hashes any JSON-serializable value canonically. diff --git a/packages/core/src/cache/index.mjs b/packages/core/src/cache/index.mjs index 9a410527..14f83a10 100644 --- a/packages/core/src/cache/index.mjs +++ b/packages/core/src/cache/index.mjs @@ -28,23 +28,20 @@ let context = null; export const getBuildCache = () => context; /** - * Resolves the cache directory: explicit configuration, environment override, - * then `node_modules/.cache/doc-kit` when a `node_modules` exists, falling - * back to `.doc-kit-cache`. + * Resolves the cache directory: explicit configuration, then + * `node_modules/.cache/doc-kit` when a `node_modules` exists, falling back + * to `.doc-kit-cache`. * * @param {string} [explicit] - Configured directory, if any * @returns {string} Absolute cache directory */ -export const resolveCacheDir = explicit => { - const dir = +export const resolveCacheDir = explicit => + resolve( explicit || - process.env.DOC_KIT_CACHE_DIR || - (existsSync('node_modules') - ? join('node_modules', '.cache', 'doc-kit') - : '.doc-kit-cache'); - - return resolve(dir); -}; + (existsSync('node_modules') + ? join('node_modules', '.cache', 'doc-kit') + : '.doc-kit-cache') + ); /** * Checks that every output recorded by a previous run is still on disk with @@ -100,7 +97,8 @@ const resolveTypeMap = async configuration => { /** * Sets up the durable build cache for one run: hashes input snapshots, * computes every target's aggregate key, and decides whether this invocation - * can skip entirely because its outputs are already on disk and verified. + * can skip entirely because its outputs are already on disk and verified — + * in which case `{ skipped: true }` is returned and no cache is set up. * * Any failure disables the cache for the run (with a debug log) — the build * must never fail, or even get slower than a cold build, because of caching. @@ -108,17 +106,15 @@ const resolveTypeMap = async configuration => { * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration * @param {Map} generators - Loaded generators * @param {string[]} targets - Resolved target specifiers - * @returns {Promise} + * @returns {Promise} */ export const setupBuildCache = async (configuration, generators, targets) => { const settings = { enabled: true, maxAgeDays: 7, ...configuration.cache }; - if (settings.enabled === false || process.env.DOC_KIT_NO_CACHE) { + if (settings.enabled === false) { return (context = null); } - const force = settings.force || Boolean(process.env.DOC_KIT_CACHE_FORCE); - try { const dir = resolveCacheDir(settings.dir); const outputDir = configuration.global?.output; @@ -187,44 +183,18 @@ export const setupBuildCache = async (configuration, generators, targets) => { // cheap instead. const profile = manifest.profiles[profileKey]; - let skippedTargets = new Set(); - if ( - !force && + !settings.force && profile?.complete && targets.every(target => profile.targets[target] === targetKeys[target]) && (await verifyOutputs(profile, outputDir)) ) { - skippedTargets = new Set(targets); - } - - /** @type {Map>} */ - const chainSalts = new Map(); - - /** - * Memoized chain salt for a generator, for use in per-item leaf-cache - * keys: covers every code and configuration input in the generator's - * dependency chain, but not the input files (leaf keys add the specific - * file hashes they depend on). - * - * @param {string} specifier - Resolved generator specifier - * @returns {Promise} Chain salt - */ - const chainSaltFor = specifier => { - if (!chainSalts.has(specifier)) { - chainSalts.set( - specifier, - chainSalt( - specifier, - generators, - resolveDependency, - configuration - ).then(({ salt }) => salt) - ); - } + cacheLogger.info( + `All ${targets.length} target(s) up to date; outputs verified on disk` + ); - return chainSalts.get(specifier); - }; + return { skipped: true }; + } /** @type {Map | null} */ let sourceHashes = null; @@ -283,11 +253,21 @@ export const setupBuildCache = async (configuration, generators, targets) => { context = { store, dir, - skippedTargets, recordOutput, - chainSalt: chainSaltFor, sourceHash: sourceHashFor, + /** + * Chain salt for a generator, for use in per-item leaf-cache keys: + * covers every code and configuration input in the generator's + * dependency chain, but not the input files (leaf keys add the + * specific file hashes they depend on). + * + * @param {string} specifier - Resolved generator specifier + * @returns {string} Chain salt + */ + chainSalt: specifier => + chainSalt(specifier, generators, resolveDependency, configuration).salt, + /** * Tracked file write: records the output and skips byte-identical * writes so warm runs leave mtimes untouched. @@ -324,8 +304,7 @@ export const setupBuildCache = async (configuration, generators, targets) => { /** * Flushes pending object writes, persists the run's profile (only on - * success, and only when the run actually executed), prunes stale - * objects, and reports stats. + * success), and prunes stale objects. * * @param {boolean} success - Whether the run completed without error * @returns {Promise} @@ -334,7 +313,7 @@ export const setupBuildCache = async (configuration, generators, targets) => { await store.flush().catch(() => undefined); try { - if (success && skippedTargets.size === 0) { + if (success) { await saveManifest(dir, { [profileKey]: { completedAt: Date.now(), @@ -352,32 +331,6 @@ export const setupBuildCache = async (configuration, generators, targets) => { }); } - const stats = { - skippedTargets: [...skippedTargets], - targets: targetKeys, - outputsRecorded: outputs.size, - store: store.stats, - }; - - if (process.env.DOC_KIT_CACHE_STATS_FILE) { - await writeFile( - process.env.DOC_KIT_CACHE_STATS_FILE, - JSON.stringify(stats, null, 2) - ).catch(() => undefined); - } - - if (skippedTargets.size > 0) { - cacheLogger.info( - `Skipped ${skippedTargets.size} up-to-date target(s); outputs verified on disk` - ); - } else { - const { hits, misses, writes } = store.stats; - - cacheLogger.info( - `${hits} hits, ${misses} misses, ${writes} objects written, ${outputs.size} outputs recorded` - ); - } - context = null; }, }; diff --git a/packages/core/src/cache/salt.mjs b/packages/core/src/cache/salt.mjs index e1346288..65d88dd6 100644 --- a/packages/core/src/cache/salt.mjs +++ b/packages/core/src/cache/salt.mjs @@ -1,152 +1,58 @@ 'use strict'; -import { realpathSync } from 'node:fs'; -import { readFile } from 'node:fs/promises'; -import { dirname, join, sep } from 'node:path'; -import { fileURLToPath } from 'node:url'; - -import { globSync } from 'tinyglobby'; - -import { combine, hashData, hashValue } from './hash.mjs'; +import { combine, hashValue } from './hash.mjs'; /** - * Bump to invalidate every existing cache entry and manifest when the cache's - * own semantics change. + * The only code-identity salt. Bump it in any release whose generated output + * differs (and whenever the cache's own semantics change) to invalidate every + * existing entry; releases that don't change output leave caches valid. When + * hacking on generator code locally, build with `--force`. */ export const CACHE_SCHEMA = 1; -/** @type {Map>} */ -const packageSalts = new Map(); - -/** - * Finds the directory of the package that owns a resolved generator - * specifier, by resolving the module and walking up to its `package.json`. - * - * @param {string} specifier - Resolved generator import specifier - * @returns {Promise} Real (symlink-free) package directory - */ -const findPackageDir = async specifier => { - const modulePath = realpathSync( - fileURLToPath(import.meta.resolve(specifier)) - ); - - let dir = dirname(modulePath); - - while (true) { - try { - await readFile(join(dir, 'package.json')); - - return dir; - } catch { - const parent = dirname(dir); - - if (parent === dir) { - throw new Error(`No package.json found above ${modulePath}`); - } - - dir = parent; - } - } -}; - -/** - * Computes a salt identifying a package's code. Published installs (real path - * inside node_modules) are identified by name and version. Anything else — a - * workspace, an `npm link`, a raw checkout — is a development install whose - * version never changes while its code does, so the salt is a content hash of - * the package sources instead. - * - * @param {string} pkgDir - Real package directory - * @returns {Promise} Package salt - */ -const computePackageSalt = async pkgDir => { - const pkg = JSON.parse(await readFile(join(pkgDir, 'package.json'), 'utf-8')); - - if (pkgDir.split(sep).includes('node_modules')) { - return hashData(`${pkg.name}@${pkg.version}`); - } - - const sources = globSync('src/**/*', { - cwd: pkgDir, - onlyFiles: true, - ignore: ['**/__tests__/**', '**/*.test.mjs'], - }).sort(); - - const hashes = await Promise.all( - sources.map( - async rel => `${rel}:${hashData(await readFile(join(pkgDir, rel)))}` - ) - ); - - return hashData(`${pkg.name}@dev\n${hashes.join('\n')}`); -}; - /** - * Memoized package salt for a resolved generator specifier. + * Salt for one generator: its name plus its resolved configuration slice + * (which inherits the global configuration, so parsed changelog/index/version + * are covered). The output directory is excluded — where output lands does + * not change what it contains. * - * @param {string} specifier - Resolved generator import specifier - * @returns {Promise} Package salt - */ -export const packageSalt = specifier => { - if (!packageSalts.has(specifier)) { - packageSalts.set( - specifier, - findPackageDir(specifier).then(computePackageSalt) - ); - } - - return packageSalts.get(specifier); -}; - -/** - * Salt for one generator: its package's code identity plus its resolved - * configuration slice (which inherits the global configuration, so parsed - * changelog/index/version are covered). The output directory is excluded — - * where output lands does not change what it contains. - * - * @param {string} specifier - Resolved generator specifier * @param {GeneratorMetadata} generator - Loaded generator * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration - * @returns {Promise} Generator salt + * @returns {string} Generator salt */ -const generatorSalt = async (specifier, generator, configuration) => { +const generatorSalt = (generator, configuration) => { const slice = { ...(configuration[generator.name] ?? {}) }; delete slice.output; - return combine(await packageSalt(specifier), hashValue(slice)); + return hashValue({ name: generator.name, config: slice }); }; /** - * Salt for a target's whole dependency chain: the cache schema, the Node.js - * major (worker/serialization behavior), and the salt of every generator from - * the target down to its root. Everything code- and configuration-shaped that - * can affect the target's output — but not the input files themselves. + * Salt for a target's whole dependency chain: the cache schema plus the salt + * of every generator from the target down to its root. Everything + * configuration-shaped that can affect the target's output — but not the + * input files themselves. * * @param {string} target - Resolved target specifier * @param {Map} generators - Loaded generators * @param {(specifier: string) => string | undefined} resolveDependency - Maps a specifier to its resolved dependency * @param {import('../utils/configuration/types').Configuration} configuration - Resolved configuration - * @returns {Promise<{ salt: string, root: string }>} The chain salt and the chain's root specifier + * @returns {{ salt: string, root: string }} The chain salt and the chain's root specifier */ -export const chainSalt = async ( +export const chainSalt = ( target, generators, resolveDependency, configuration ) => { - const parts = [ - `schema:${CACHE_SCHEMA}`, - `node:${process.versions.node.split('.')[0]}`, - ]; + const parts = [`schema:${CACHE_SCHEMA}`]; let specifier = target; let root = target; while (specifier) { - const generator = generators.get(specifier); - - parts.push(await generatorSalt(specifier, generator, configuration)); + parts.push(generatorSalt(generators.get(specifier), configuration)); root = specifier; specifier = resolveDependency(specifier); @@ -174,7 +80,7 @@ export const aggregateKey = async ( configuration, snapshotFor ) => { - const { salt, root } = await chainSalt( + const { salt, root } = chainSalt( target, generators, resolveDependency, diff --git a/packages/core/src/cache/store.mjs b/packages/core/src/cache/store.mjs index 3a7e6bf0..b79bebc5 100644 --- a/packages/core/src/cache/store.mjs +++ b/packages/core/src/cache/store.mjs @@ -1,5 +1,6 @@ 'use strict'; +import { randomUUID } from 'node:crypto'; import { mkdir, readFile, @@ -33,10 +34,6 @@ export const createStore = dir => { /** @type {Set>} */ const pending = new Set(); - const stats = { hits: 0, misses: 0, writes: 0, bytesWritten: 0 }; - - let tmpCounter = 0; - /** * @param {string} key - Object key * @returns {string} Object path, sharded to keep directories small @@ -49,19 +46,7 @@ export const createStore = dir => { * @param {string} key - Object key * @returns {Promise} Stored value, or null */ - const get = async key => { - try { - const value = await readFile(pathFor(key), 'utf-8'); - - stats.hits++; - - return value; - } catch { - stats.misses++; - - return null; - } - }; + const get = key => readFile(pathFor(key), 'utf-8').catch(() => null); /** * Persists an object write-behind: failures are logged and swallowed — @@ -73,15 +58,12 @@ export const createStore = dir => { */ const put = (key, value) => { const write = (async () => { - const tmp = join(tmpDir, `${process.pid}-${tmpCounter++}`); + const tmp = join(tmpDir, randomUUID()); await mkdir(dirname(pathFor(key)), { recursive: true }); await mkdir(tmpDir, { recursive: true }); await writeFile(tmp, value); await rename(tmp, pathFor(key)); - - stats.writes++; - stats.bytesWritten += Buffer.byteLength(value); })().catch(error => storeLogger.debug(`Cache write failed for ${key}`, { error: error.message, @@ -95,7 +77,6 @@ export const createStore = dir => { return { get, put, - stats, /** * Checks that an object exists without reading it, refreshing its mtime @@ -158,30 +139,26 @@ export const createStore = dir => { prune: async maxAgeDays => { const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000; - let shards; - - try { - shards = await readdir(objectsDir); - } catch { - return; - } - - for (const shard of shards) { - const shardDir = join(objectsDir, shard); - const objects = await readdir(shardDir).catch(() => []); - - for (const object of objects) { - const path = join(shardDir, object); - - try { - if ((await stat(path)).mtimeMs < cutoff) { - await rm(path, { force: true }); + const entries = await readdir(objectsDir, { + recursive: true, + withFileTypes: true, + }).catch(() => []); + + await Promise.all( + entries + .filter(entry => entry.isFile()) + .map(async entry => { + const path = join(entry.parentPath, entry.name); + + try { + if ((await stat(path)).mtimeMs < cutoff) { + await rm(path, { force: true }); + } + } catch { + // A concurrent process may have pruned it already. } - } catch { - // A concurrent process may have pruned it already. - } - } - } + }) + ); }, }; }; diff --git a/packages/core/src/cache/types.d.ts b/packages/core/src/cache/types.d.ts index 0a0eb28d..e8325fd3 100644 --- a/packages/core/src/cache/types.d.ts +++ b/packages/core/src/cache/types.d.ts @@ -13,13 +13,6 @@ export interface Snapshot { digest: string; } -export interface StoreStats { - hits: number; - misses: number; - writes: number; - bytesWritten: number; -} - export interface Store { get(key: string): Promise; put(key: string, value: string): void; @@ -32,7 +25,6 @@ export interface Store { ): Promise; flush(): Promise; prune(maxAgeDays: number): Promise; - stats: StoreStats; } export interface OutputRecord { @@ -65,10 +57,8 @@ export interface CacheConfiguration { export interface BuildCache { store: Store; dir: string; - /** Resolved targets this run may skip entirely (all-or-nothing) */ - skippedTargets: Set; /** Chain salt for per-item leaf-cache keys (code + config, no inputs) */ - chainSalt(specifier: string): Promise; + chainSalt(specifier: string): string; /** Content hash of the source file behind a module path (`/fs`), if known */ sourceHash(modulePath: string): Promise; /** Records an output file write (tracked writeFile / vite / copies) */ diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index a9b56262..4f5d505f 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -123,28 +123,25 @@ const createGenerator = () => { generators, targets ); - const skipped = buildCache?.skippedTargets ?? new Set(); - const active = targets.filter(specifier => !skipped.has(specifier)); + + // Skip is all-or-nothing: the cache verified every target's outputs are + // already on disk, so there is nothing to run. + if (buildCache?.skipped) { + return targets.map(() => SKIPPED); + } generatorsLogger.debug(`Starting pipeline`, { generators: targets.join(', '), - skipped: skipped.size, threads, }); let success = false; try { - if (active.length === 0) { - success = true; - - return targets.map(() => SKIPPED); - } - // Compute consumer counts up front so dependencies can be evicted as // soon as their last consumer runs (must be ready before any generator - // starts). Skipped targets never run, so only active ones count. - cache.populateConsumerCounts(active, specifier => { + // starts). + cache.populateConsumerCounts(targets, specifier => { const { dependsOn } = generators.get(specifier); return dependsOn && resolveGeneratorSpecifier(dependsOn); @@ -154,7 +151,7 @@ const createGenerator = () => { pool = createWorkerPool(threads); // Schedule all generators - for (const specifier of active) { + for (const specifier of targets) { scheduleGenerator(specifier, generators, configuration); } @@ -162,9 +159,7 @@ const createGenerator = () => { // Consuming through the shared path lets the final read also trigger // eviction. const results = await Promise.all( - targets.map(specifier => - skipped.has(specifier) ? SKIPPED : cache.consume(specifier) - ) + targets.map(specifier => cache.consume(specifier)) ); await pool.destroy(); diff --git a/packages/legacy/src/legacy-html/generate.mjs b/packages/legacy/src/legacy-html/generate.mjs index ecfe701d..0479f610 100644 --- a/packages/legacy/src/legacy-html/generate.mjs +++ b/packages/legacy/src/legacy-html/generate.mjs @@ -168,7 +168,7 @@ export async function* generate(input, worker) { const misses = []; if (buildCache) { - const salt = await buildCache.chainSalt(GENERATOR_SPECIFIER); + const salt = buildCache.chainSalt(GENERATOR_SPECIFIER); const projectionHash = combine( hashData(navigation), hashValue(headNodesLite), @@ -197,42 +197,32 @@ export async function* generate(input, worker) { const extra = { navigation, headNodesLite, apiTemplate }; - // Drive the worker stream concurrently, resolving each miss's deferred as - // its chunk lands; results are stored write-behind under the miss's key. - const deferreds = new Map( - misses.map(({ item }) => [item.head.api, Promise.withResolvers()]) - ); - - const pump = (async () => { - let index = 0; + /** @type {Map} Freshly built results by api */ + const produced = new Map(); - for await (const chunk of worker.stream( - misses.map(({ item }) => item), - extra - )) { - for (const result of chunk) { - const { key } = misses[index++]; + let index = 0; - if (buildCache && key) { - buildCache.store.put(key, JSON.stringify(result)); - } + // The worker yields chunks in submission order (parallel.mjs), so results + // pair with misses positionally; results are stored write-behind. + for await (const chunk of worker.stream( + misses.map(({ item }) => item), + extra + )) { + for (const result of chunk) { + const { key } = misses[index++]; - deferreds.get(result.api).resolve(result); + if (key) { + buildCache.store.put(key, JSON.stringify(result)); } - } - })(); - pump.catch(error => { - for (const { reject } of deferreds.values()) { - reject(error); + produced.set(result.api, result); } - }); + } // Emit in canonical (sorted head-node) order regardless of the hit/miss // split, so downstream aggregation (legacy-html-all) is byte-stable. for (const head of headNodes) { - const result = - cached.get(head.api) ?? (await deferreds.get(head.api).promise); + const result = cached.get(head.api) ?? produced.get(head.api); if (config.output) { await writeFile(join(config.output, `${result.api}.html`), result.html); @@ -240,6 +230,4 @@ export async function* generate(input, worker) { yield [result]; } - - await pump; } diff --git a/packages/react/src/html/utils/processing.mjs b/packages/react/src/html/utils/processing.mjs index 0cc53562..3e053d7c 100644 --- a/packages/react/src/html/utils/processing.mjs +++ b/packages/react/src/html/utils/processing.mjs @@ -173,7 +173,7 @@ const renderServerPages = async ( ? await ssrEnvironmentHash(config, serverVirtualImports) : null; - const salt = environment && (await buildCache.chainSalt(GENERATOR_SPECIFIER)); + const salt = environment && buildCache.chainSalt(GENERATOR_SPECIFIER); const serverPages = new Map(); const missEntries = new Map(); @@ -190,7 +190,7 @@ const renderServerPages = async ( const key = salt && combine('html:ssr', salt, environment, contentHash); const hit = key && (await buildCache.store.get(key)); - if (hit !== null && hit !== false && hit !== undefined) { + if (hit) { serverPages.set(data.api, hit); continue; diff --git a/packages/react/src/jsx-ast/generate.mjs b/packages/react/src/jsx-ast/generate.mjs index d055f632..32880569 100644 --- a/packages/react/src/jsx-ast/generate.mjs +++ b/packages/react/src/jsx-ast/generate.mjs @@ -141,7 +141,7 @@ export async function* generate(input, worker) { const misses = []; if (buildCache) { - const salt = await buildCache.chainSalt(GENERATOR_SPECIFIER); + const salt = buildCache.chainSalt(GENERATOR_SPECIFIER); for (const [index, descriptor] of descriptors.entries()) { const base = await itemKeyBase(descriptor, buildCache, moduleInput); @@ -153,7 +153,7 @@ export async function* generate(input, worker) { const record = dataKey && (await buildCache.store.get(dataKey)); const hasCode = codeKey && (await buildCache.store.touch(codeKey)); - if (record !== null && record && hasCode) { + if (record && hasCode) { const { data, codeHash } = JSON.parse(record); cached.set(index, { @@ -182,48 +182,37 @@ export async function* generate(input, worker) { ); } - // Drive the worker stream concurrently, resolving each miss's deferred as - // its chunk lands; results are stored write-behind under the miss's keys. - const deferreds = new Map( - misses.map(({ index }) => [index, Promise.withResolvers()]) - ); - - const pump = (async () => { - let at = 0; - - for await (const chunk of worker.stream( - misses.map(({ index }) => descriptors[index]) - )) { - for (const result of chunk) { - const miss = misses[at++]; - - if (buildCache && miss.dataKey) { - buildCache.store.put( - miss.dataKey, - JSON.stringify({ - data: result.data, - codeHash: hashData(result.code), - }) - ); - buildCache.store.put(miss.codeKey, result.code); - } - - deferreds.get(miss.index).resolve(result); + /** @type {Map} Freshly built results by descriptor index */ + const produced = new Map(); + + let at = 0; + + // The worker yields chunks in submission order (parallel.mjs), so results + // pair with misses positionally; results are stored write-behind. + for await (const chunk of worker.stream( + misses.map(({ index }) => descriptors[index]) + )) { + for (const result of chunk) { + const miss = misses[at++]; + + if (miss.dataKey) { + buildCache.store.put( + miss.dataKey, + JSON.stringify({ + data: result.data, + codeHash: hashData(result.code), + }) + ); + buildCache.store.put(miss.codeKey, result.code); } - } - })(); - pump.catch(error => { - for (const { reject } of deferreds.values()) { - reject(error); + produced.set(miss.index, result); } - }); + } // Emit in canonical descriptor order regardless of the hit/miss split, so // downstream page ordering is byte-stable. for (const [index] of descriptors.entries()) { - yield [cached.get(index) ?? (await deferreds.get(index).promise)]; + yield [cached.get(index) ?? produced.get(index)]; } - - await pump; } From 103bd7364b87e7cf81ee6c9ed67f4b223afe19c4 Mon Sep 17 00:00:00 2001 From: avivkeller Date: Fri, 7 Aug 2026 20:31:22 -0400 Subject: [PATCH 3/3] fixup! --- packages/core/bin/commands/generate.mjs | 6 ++---- packages/core/bin/commands/watch.mjs | 5 ++--- packages/core/bin/utils.mjs | 15 +++++++-------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/core/bin/commands/generate.mjs b/packages/core/bin/commands/generate.mjs index 80aca46f..716ffccc 100644 --- a/packages/core/bin/commands/generate.mjs +++ b/packages/core/bin/commands/generate.mjs @@ -1,15 +1,13 @@ -import { Command } from 'commander'; - import createGenerator from '../../src/generators.mjs'; import { assertRunnableOptions, setConfig, } from '../../src/utils/configuration/index.mjs'; -import { errorWrap, insertCommonOptions } from '../utils.mjs'; +import { createCommonCommand, errorWrap } from '../utils.mjs'; const { runGenerators } = createGenerator(); -export default insertCommonOptions(new Command('generate')) +export default createCommonCommand('generate') .description('Generate API docs') .action( errorWrap(async opts => { diff --git a/packages/core/bin/commands/watch.mjs b/packages/core/bin/commands/watch.mjs index b598f3c1..765e0c66 100644 --- a/packages/core/bin/commands/watch.mjs +++ b/packages/core/bin/commands/watch.mjs @@ -2,7 +2,6 @@ import { matchesGlob, resolve } from 'node:path'; import process from 'node:process'; import { watch } from 'chokidar'; -import { Command } from 'commander'; import globParent from 'glob-parent'; import createGenerator, { SKIPPED } from '../../src/generators.mjs'; @@ -11,7 +10,7 @@ import { assertRunnableOptions, setConfig, } from '../../src/utils/configuration/index.mjs'; -import { errorWrap, insertCommonOptions } from '../utils.mjs'; +import { createCommonCommand, errorWrap } from '../utils.mjs'; const watchLogger = logger.child('watch'); @@ -95,7 +94,7 @@ const createScheduler = task => { }; }; -export default insertCommonOptions(new Command('watch')) +export default createCommonCommand('watch') .description('Generate API docs, rebuilding whenever an input file changes') .action( errorWrap(async opts => { diff --git a/packages/core/bin/utils.mjs b/packages/core/bin/utils.mjs index bf9c4bad..d43ec925 100644 --- a/packages/core/bin/utils.mjs +++ b/packages/core/bin/utils.mjs @@ -1,4 +1,4 @@ -import { Option } from 'commander'; +import { Command, Option } from 'commander'; import { publicGenerators } from '../src/generators/index.mjs'; import logger from '../src/logger/index.mjs'; @@ -43,15 +43,14 @@ export const errorWrap = }; /** - * Adds the options shared by every command that resolves a configuration and - * runs generators. + * Creates a command carrying the options shared by every command that resolves + * a configuration and runs generators. * - * @template {import('commander').Command} T - * @param {T} cmd - The command to extend - * @returns {T} The same command, for chaining + * @param {string} name - The command name + * @returns {import('commander').Command} The command, for chaining */ -export const insertCommonOptions = cmd => - cmd +export const createCommonCommand = name => + new Command(name) .addOption(new Option('--config-file ', 'Config file')) // Options that need to be converted into a configuration