feat(llms-txt): generate llms.txt and per-page markdown at build time

cspell-wordlist.txt

@@ -43,6 +43,7 @@ fortawesome
 frontmatter
 fullscreen
 geolocation
+headerless
 iconset
 interactives
 isopen
@@ -53,6 +54,8 @@ jsdelivr
 keyframes
 keytool
 lifecycles
+llms
+llmstxt
 localstorage
 mobileweb
 phablet

docusaurus.config.js

@@ -404,8 +404,13 @@ module.exports = {
       'docusaurus-plugin-copy-page-button',
       {
         injectButton: false,
+        // docusaurus-plugin-llms-txt writes the markdown twins instead, reusing
+        // this package's converter after repairing the HTML it is given.
+        // Turning both on would have the two race for the same files.
+        generateMarkdownRoutes: false,
       },
     ],
+    path.resolve(__dirname, 'plugins', 'docusaurus-plugin-llms-txt'),
   ],
   customFields: {},
   themes: [],

plugins/docusaurus-plugin-llms-txt/README.md

@@ -0,0 +1,41 @@
+# docusaurus-plugin-llms-txt
+
+Writes `llms.txt` into the build output so it is served at https://ionicframework.com/docs/llms.txt, in the format described at [llmstxt.org](https://llmstxt.org).
+
+The plugin also writes a markdown twin of every docs page, which is what the index links to. Twins are written for every version, so a v8 page is readable as markdown too, but `llms.txt` itself covers only the current version in English.
+
+## `md` and `mdx`
+
+Authored docs stay `.mdx`, and nothing here changes that. Everything this plugin emits is plain `.md`, because it is written for agents rather than for the site: an agent fetching `/docs/api/button.md` gets markdown it can read straight off, with no JSX, no imports and no components to resolve. The `.md` files are build output only, so none of them are checked in and none of them should be edited by hand.
+
+## Serving the twins
+
+Vercel's default `Content-Disposition` on a `.md` response carries a `filename`, and some clients take that as a save hint, so an agent asking for `/docs/api/button.md` downloads the file instead of reading the body it just fetched. The `headers` block in `vercel.json` sets a bare `Content-Disposition: inline` for anything ending in `.md`, which drops the filename and leaves the markdown in the response. Nothing in the plugin depends on that header, so it is easy to lose in a `vercel.json` cleanup without anything failing.
+
+## Why the twins are written here
+
+The conversion comes from `docusaurus-plugin-copy-page-button`, which is already a dependency. Its own `generateMarkdownRoutes` option writes the same files, and is deliberately left off in `docusaurus.config.js`, because plugin `postBuild` hooks run concurrently under `Promise.all` and having both write the same paths would be a race. The converter is reused here instead, with the HTML repaired on the way through.
+
+Docusaurus emits minified HTML with the optional `</td>` and `</tr>` end tags left out, which that converter's parser does not account for, so every table used to collapse onto a single line. Separately, a `<Playground>` mounts its editor on the client, so the server-rendered HTML is an empty shell and the code examples went missing. Those snippets are on disk under `static/usage/`, so they get read from there and spliced back in. The smaller repairs are commented in `lib/markdown-twins.js`.
+
+There's one trap if you touch the path handling. The converter also has a client-side `getMarkdownRouteUrl` that disagrees with what it writes to disk for the site root, giving `/docs.md` where the file is `/docs/index.md`. Use `lib/markdown-path.js`, which follows the file on disk, and which the theme shares so the copy page button cannot drift from the generated files.
+
+## Which pages are covered
+
+Sections mirror the top-level categories of the `docs` sidebar. The generated reference pages (the `api`, `cli` and `native` sidebars) go under `## Optional`, the spec's reserved heading for links an agent can skip when it needs a shorter context.
+
+A page is included when some sidebar points at it, which keeps unreferenced pages out of both the index and the twins without a path list to maintain. Draft and unlisted pages are dropped too.
+
+The Japanese build is skipped. It gets its own `build/ja` output root so there is no clash with the English file, but the section labels come from the English sidebar and nothing would link the result.
+
+## Descriptions
+
+Bullet descriptions come from the docs plugin's resolved `description`. Almost no page sets one in frontmatter, so in practice this is Docusaurus's body excerpt, which for most pages is the SEO title out of the in-body `<head>` block and reads well enough. A few fall through to something useless, and `cleanDescription` drops those so the bullet ends up title-only. Setting a frontmatter `description` on a page beats the excerpt.
+
+## Layout and tests
+
+```bash
+npx vitest run plugins/docusaurus-plugin-llms-txt
+```
+
+The `index.js` hook owns the filesystem and everything under `lib/` is pure. `llms-txt.js` builds the index, `markdown-twins.js` the twins, `playground-code.js` reads a usage folder, `paths.js` maps a permalink to files on disk, and `markdown-path.js` holds the permalink-to-twin mapping that the theme shares.

plugins/docusaurus-plugin-llms-txt/index.js

@@ -0,0 +1,158 @@
+const fs = require('fs');
+const path = require('path');
+
+const { buildSections, getReferencedDocIds, renderLlmsTxt } = require('./lib/llms-txt');
+const { buildTwin, hasPlaygrounds, readUsageDirs } = require('./lib/markdown-twins');
+const { toMarkdownPath, toOutputPaths } = require('./lib/paths');
+
+const DOCS_PLUGIN_NAME = 'docusaurus-plugin-content-docs';
+const DOCS_PLUGIN_ID = 'default';
+const CURRENT_VERSION = 'current';
+
+const INTRO =
+  'Every page below links to its markdown source, and each one opens with the URL of the page it came from. ' +
+  'These pages document the current version of Ionic Framework in English.';
+
+/** Names the other versions that also have twins, so they are discoverable. */
+const olderVersionsNote = (loadedVersions) => {
+  const paths = loadedVersions
+    .filter((version) => version.versionName !== CURRENT_VERSION)
+    .map((version) => `${version.path.replace(/\/+$/, '')}/`);
+
+  return paths.length > 0 ? ` Pages for earlier versions are at the same paths under ${paths.join(' and ')}.` : '';
+};
+
+const firstExisting = (candidates) => candidates.find((candidate) => fs.existsSync(candidate));
+const withTrailingSlash = (baseUrl) => (baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`);
+
+/**
+ * Writes an llms.txt index (https://llmstxt.org) into the build output, plus a
+ * markdown twin of every docs page for it to link to. Both land at the docs
+ * root alongside sitemap.xml. See ./README.md.
+ */
+module.exports = function llmsTxtPlugin() {
+  return {
+    name: 'docusaurus-plugin-llms-txt',
+
+    async postBuild(props) {
+      const { outDir, siteDir, i18n, plugins, siteConfig, baseUrl } = props;
+
+      /**
+       * Each locale has its own outDir, so the Japanese build would write a
+       * `build/ja/llms.txt` that nothing links to, with section labels still
+       * taken from the English sidebar.
+       */
+      if (i18n.currentLocale !== i18n.defaultLocale) {
+        return;
+      }
+
+      const docsPlugin = plugins.find(
+        (plugin) => plugin.name === DOCS_PLUGIN_NAME && (plugin.options?.id ?? DOCS_PLUGIN_ID) === DOCS_PLUGIN_ID
+      );
+
+      if (!docsPlugin?.content?.loadedVersions) {
+        console.warn('[llms-txt] docs plugin content was not available, skipping llms.txt.');
+        return;
+      }
+
+      const { loadedVersions } = docsPlugin.content;
+      const staticDir = path.join(siteDir, 'static');
+      const warn = (message) => console.warn(`[llms-txt] ${message}`);
+
+      /** A missing or renamed source costs that page its snippets, not the build. */
+      const readSource = (doc) => {
+        try {
+          return fs.readFileSync(path.join(siteDir, doc.source.replace(/^@site\//, '')), 'utf8');
+        } catch {
+          warn(`could not read ${doc.source}, leaving its playground code out.`);
+          return '';
+        }
+      };
+
+      // Twins cover every version, not just the current one.
+      const referencedByVersion = new Map(
+        loadedVersions.map((loaded) => [loaded.versionName, getReferencedDocIds(loaded.sidebars)])
+      );
+
+      /**
+       * Every page that gets a twin, so a link between two of them can be
+       * rewritten to stay inside the markdown corpus.
+       */
+      const twinUrls = new Map();
+      for (const version of loadedVersions) {
+        for (const doc of version.docs) {
+          if (!doc.draft && !doc.unlisted && referencedByVersion.get(version.versionName).has(doc.id)) {
+            twinUrls.set(doc.permalink, `${withTrailingSlash(baseUrl)}${toMarkdownPath(doc.permalink, baseUrl)}`);
+          }
+        }
+      }
+
+      let twins = 0;
+      for (const version of loadedVersions) {
+        const referencedIds = referencedByVersion.get(version.versionName);
+
+        for (const doc of version.docs) {
+          // Matching what the index lists keeps scratch and redirected pages
+          // from being published as markdown nobody can reach.
+          if (doc.draft || doc.unlisted || !referencedIds.has(doc.id)) {
+            continue;
+          }
+
+          const { htmlCandidates, markdownPath } = toOutputPaths(doc.permalink, { outDir, baseUrl });
+          const htmlPath = firstExisting(htmlCandidates);
+          if (!htmlPath) {
+            warn(`no rendered HTML for ${doc.permalink}, skipping its markdown twin.`);
+            continue;
+          }
+
+          const html = fs.readFileSync(htmlPath, 'utf8');
+          const markdown = buildTwin({
+            html,
+            pageUrl: `${siteConfig.url.replace(/\/+$/, '')}${doc.permalink}`,
+            usageDirs: hasPlaygrounds(html) ? readUsageDirs(readSource(doc)) : [],
+            staticDir,
+            twinUrls,
+            onWarn: warn,
+          });
+
+          if (markdown) {
+            fs.mkdirSync(path.dirname(markdownPath), { recursive: true });
+            fs.writeFileSync(markdownPath, markdown);
+            twins += 1;
+          }
+        }
+      }
+
+      const version = loadedVersions.find((loaded) => loaded.versionName === CURRENT_VERSION);
+      if (!version) {
+        warn(`no "${CURRENT_VERSION}" docs version found, skipping llms.txt.`);
+        return;
+      }
+
+      const referencedIds = referencedByVersion.get(CURRENT_VERSION);
+      const docsById = new Map(
+        version.docs
+          .filter((doc) => !doc.draft && !doc.unlisted && referencedIds.has(doc.id))
+          .map((doc) => [doc.id, doc])
+      );
+
+      const { sections, optional } = buildSections({ sidebars: version.sidebars, docsById });
+
+      fs.writeFileSync(
+        path.join(outDir, 'llms.txt'),
+        renderLlmsTxt({
+          title: siteConfig.title,
+          tagline: siteConfig.tagline,
+          intro: `${INTRO}${olderVersionsNote(loadedVersions)}`,
+          sections,
+          optional,
+          siteUrl: siteConfig.url,
+          baseUrl,
+        })
+      );
+
+      const linkCount = sections.reduce((total, section) => total + section.docs.length, 0) + optional.length;
+      console.log(`[llms-txt] wrote ${twins} markdown twins and llms.txt with ${linkCount} links.`);
+    },
+  };
+};