build: Fix build cache of page content

AGENTS.md

@@ -10,9 +10,15 @@
 - **Lint**: `pnpm run lint` (ESLint 9 flat config in `eslint.config.mjs`; React
   Compiler rules are warnings until the vendored
   `src/components/search/docsearch` code is rewritten)
-- **Framework**: Next.js 16 with `--webpack` (`next-contentlayer2` has no
-  Turbopack plugin); the request rewrite lives in `src/proxy.ts`;
-  `/api/releases` and `/api/versions` opt into static caching with
+- **Framework**: Next.js 16. `next build` uses Turbopack; `dev/build-content.mjs`
+  runs `contentlayer2 build` first, with its cache under `.next/cache` so Vercel
+  keeps it between deploys, and with each `.mdx` file's mtime set from its
+  content, since contentlayer2 keys its cache on mtime and a fresh clone resets
+  those (workaround; drop once <https://github.com/timlrx/contentlayer2/pull/94>
+  ships). `next dev --webpack` still uses the `next-contentlayer2` webpack plugin
+  to regenerate content on change (`next.config.js` applies it only in the dev
+  phase). The request rewrite lives in `src/proxy.ts`; `/api/releases` and
+  `/api/versions` opt into static caching with
   `export const dynamic = 'force-static'`
 - **Checks**: `pnpm run check` runs the checks in `dev/checks.mjs` (links,
   filenames, images); `pnpm run build` runs filenames and images first, so a

contentlayer.config.ts

@@ -6,6 +6,7 @@ import rehypeSlug from 'rehype-slug';
 import remarkGfm from 'remark-gfm';
 import {MDXDocument, allCoreContent} from './src/utils/contentlayer';
 import {searchMetadata} from './src/data/search';
+import shadesOfPurple from './src/styles/shades-of-purple.json';
 import GithubSlugger from 'github-slugger';
 import {visit} from 'unist-util-visit';
 
@@ -96,15 +97,7 @@ function createSearchIndex(allPosts: MDXDocument[]) {
 
 const prettyCodeOptions = {
 	keepBackground: true,
-	theme: JSON.parse(
-		fs.readFileSync(
-			new URL(
-				'./../../../src/styles/shades-of-purple.json',
-				import.meta.url
-			),
-			'utf-8'
-		)
-	)
+	theme: shadesOfPurple
 };
 
 const rehypePlugins: any = [

cspell-allow-list.txt

@@ -320,6 +320,7 @@ mmap
 modelconfig
 mountpoint
 mpim
+mtimes
 multiplicatively
 multiqueue
 multiversion
@@ -543,6 +544,7 @@ thorstens
 threadcreate
 timedout
 timemachine
+timlrx
 tini
 tjdevries
 tolerations

dev/TODO.md

@@ -4,3 +4,13 @@
   React Compiler ESLint rules `react-hooks/refs`, `react-hooks/set-state-in-effect`
   and `react-hooks/static-components` can go back to `error` in `eslint.config.mjs`
   (31 of the 39 warnings from `pnpm run lint` are in that directory)
+- Move `pnpm run dev` from webpack to Turbopack, like `pnpm run build`. The only
+  thing keeping it on webpack is the `next-contentlayer2` plugin, which
+  regenerates `.contentlayer` when an `.mdx` file changes; Turbopack has no
+  plugin hook for that. Needs a way to start contentlayer's watcher
+  (`contentlayer2 dev`) automatically when `next dev` starts, then the plugin
+  and the `PHASE_DEVELOPMENT_SERVER` branch in `next.config.js` can go
+- Faster Vercel static generation: the 505 `/api/og` images are ~40% of the
+  "Generating static pages" time (locally 6.6s with them, 3.6s without).
+  Options: hoist the font and logo reads in `src/app/api/og/[...path]/route.tsx`
+  to module scope, or render one image per top-level section instead of per page

dev/build-content.mjs

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+/**
+ * Runs `contentlayer2 build` so that its cache works on Vercel.
+ *
+ * Without this every deploy re-renders all ~500 MDX pages (~25s on Vercel's
+ * 4-core build machine); with it, only the pages whose source changed. Two
+ * things stand in the way:
+ *
+ * - Contentlayer only writes its cache to .contentlayer/.cache, and Vercel only
+ *   keeps .next/cache between builds. So .contentlayer/.cache is a symlink into
+ *   .next/cache.
+ * - Contentlayer decides whether a cached document is current by comparing the
+ *   source file's mtime, and a fresh git clone sets every mtime to the clone
+ *   time, so on Vercel every entry misses. So each source file's mtime is set
+ *   from a hash of its content, which is the same wherever the same content is
+ *   checked out. This is a workaround for contentlayer2 keying its cache on
+ *   mtime (@contentlayer2/source-files, makeCacheItemFromFilePath.ts) and can
+ *   go once https://github.com/timlrx/contentlayer2/pull/94 ships.
+ */
+
+import {execFileSync} from 'child_process';
+import {createHash} from 'crypto';
+import {
+	lstatSync,
+	mkdirSync,
+	readdirSync,
+	readFileSync,
+	rmSync,
+	symlinkSync,
+	utimesSync
+} from 'fs';
+import path from 'path';
+
+const contentDir = 'docs';
+const cacheDir = path.join('.next', 'cache', 'contentlayer');
+const cacheLink = path.join('.contentlayer', '.cache');
+
+// Workaround: content-derived mtimes so contentlayer's cache hits on a fresh
+// clone. Remove once https://github.com/timlrx/contentlayer2/pull/94 ships.
+for (const entry of readdirSync(contentDir, {
+	recursive: true,
+	withFileTypes: true
+})) {
+	if (!entry.isFile() || !entry.name.endsWith('.mdx')) continue;
+	const file = path.join(entry.parentPath, entry.name);
+	// Whole seconds, so the value survives any filesystem's timestamp precision
+	const seconds = createHash('sha1')
+		.update(readFileSync(file))
+		.digest()
+		.readUInt32BE(0);
+	utimesSync(file, seconds, seconds);
+}
+
+mkdirSync(cacheDir, {recursive: true});
+mkdirSync('.contentlayer', {recursive: true});
+
+// A real directory here is a cache from an earlier `next build --webpack`.
+const existing = lstatSync(cacheLink, {throwIfNoEntry: false});
+if (existing && !existing.isSymbolicLink()) {
+	rmSync(cacheLink, {recursive: true});
+}
+if (!existing || !existing.isSymbolicLink()) {
+	symlinkSync(path.relative('.contentlayer', cacheDir), cacheLink);
+}
+
+execFileSync('contentlayer2', ['build'], {stdio: 'inherit'});

next.config.js

@@ -1,3 +1,4 @@
+const {PHASE_DEVELOPMENT_SERVER} = require('next/constants');
 const {withContentlayer} = require('next-contentlayer2');
 /** @type {import('next').NextConfig} */
 
@@ -35,4 +36,10 @@ const nextConfig = {
 	}
 };
 
-module.exports = withContentlayer(nextConfig);
+// withContentlayer is a webpack hook that regenerates .contentlayer when content
+// changes, which `next dev --webpack` needs. `next build` uses Turbopack, with
+// dev/build-content.mjs generating .contentlayer beforehand.
+module.exports = phase =>
+	phase === PHASE_DEVELOPMENT_SERVER
+		? withContentlayer(nextConfig)
+		: nextConfig;

package.json

@@ -16,7 +16,7 @@
 	},
 	"scripts": {
 		"dev": "next dev --webpack",
-		"build": "node dev/checks.mjs filenames images && node dev/generate-mermaid-icons.mjs && next build --webpack",
+		"build": "node dev/checks.mjs filenames images && node dev/generate-mermaid-icons.mjs && node dev/build-content.mjs && next build",
 		"start": "next start",
 		"lint": "eslint src",
 		"check": "node dev/checks.mjs",

src/data/redirects.ts

@@ -4287,6 +4287,4 @@ const updatedRedirectsData = redirectsData.map(redirect => {
 	};
 });
 
-module.exports = {
-	updatedRedirectsData
-};
+export {updatedRedirectsData};

src/proxy.ts

@@ -3,8 +3,7 @@ import {NextResponse} from 'next/server';
 import docsConfig from '../docs.config.js';
 
 import {TECHNICAL_CHANGELOG_RSS_URL} from './data/constants';
-
-const {updatedRedirectsData} = require('./data/redirects.ts');
+import {updatedRedirectsData} from './data/redirects';
 
 function createRedirectUrl(
 	request: NextRequest,