From 9de4f8995858a8998406ee1c1ab57a60f1fd91a7 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Thu, 14 May 2026 17:09:57 -0700 Subject: [PATCH 01/10] feat: add markdown-pages plugin for per-page .md endpoints Adds a Docusaurus plugin that copies raw markdown source files to the build output at their corresponding URL paths with a .md extension. This lets AI agents fetch individual doc pages as markdown by appending .md to any docs URL. Pages can opt out by setting llm_exclude in frontmatter to a string explaining why, which is served in place of the raw content. Co-Authored-By: Claude Opus 4.6 (1M context) --- docusaurus.config.js | 6 ++++ plugins/markdown-pages/index.js | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 plugins/markdown-pages/index.js diff --git a/docusaurus.config.js b/docusaurus.config.js index 310f34e1d3..23d49f3ca1 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -377,6 +377,12 @@ module.exports = async function createConfigAsync() { routeBasePath: 'ai-cookbook', // change if you use a different base }, ], + [ + require.resolve('./plugins/markdown-pages'), + { + docsDir: 'docs', + }, + ], [ 'docusaurus-plugin-llms', { diff --git a/plugins/markdown-pages/index.js b/plugins/markdown-pages/index.js new file mode 100644 index 0000000000..12eeb7fd2d --- /dev/null +++ b/plugins/markdown-pages/index.js @@ -0,0 +1,63 @@ +const fs = require('fs'); +const path = require('path'); +const matter = require('gray-matter'); + +module.exports = function markdownPagesPlugin(context, options = {}) { + const docsDir = path.resolve(context.siteDir, options.docsDir || 'docs'); + const routeBasePath = options.routeBasePath || '/'; + + function walkDir(dir) { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir).flatMap((name) => { + const full = path.join(dir, name); + if (fs.statSync(full).isDirectory()) return walkDir(full); + if (/\.(md|mdx)$/i.test(name)) return [full]; + return []; + }); + } + + function resolveUrlPath(filePath, frontmatter) { + if (frontmatter.slug) { + const slug = frontmatter.slug.replace(/^\/+/, '').replace(/\/+$/, ''); + return slug || 'index'; + } + const rel = path.relative(docsDir, filePath).replace(/\\/g, '/'); + const withoutExt = rel.replace(/\.(md|mdx)$/i, ''); + const id = frontmatter.id || path.basename(withoutExt); + const dir = path.dirname(withoutExt); + if (dir === '.') return id; + return `${dir}/${id}`; + } + + return { + name: 'markdown-pages', + + async postBuild({ outDir }) { + const files = walkDir(docsDir); + let generated = 0; + let excluded = 0; + + for (const filePath of files) { + const raw = fs.readFileSync(filePath, 'utf8'); + const { data: frontmatter } = matter(raw); + + const urlPath = resolveUrlPath(filePath, frontmatter); + const outputPath = path.join(outDir, urlPath + '.md'); + + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + + if (frontmatter.llm_exclude) { + fs.writeFileSync(outputPath, frontmatter.llm_exclude + '\n'); + excluded++; + } else { + fs.writeFileSync(outputPath, raw); + generated++; + } + } + + console.log( + `[markdown-pages] Generated ${generated} markdown files, ${excluded} excluded` + ); + }, + }; +}; From 38fe74205886704708aa5bfa3cb21f0d8dcc532c Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 10:15:22 -0700 Subject: [PATCH 02/10] fix: resolve index pages to parent path for .md URLs encyclopedia/index.mdx now outputs encyclopedia.md instead of encyclopedia/index.md, matching how Docusaurus routes index pages. Co-Authored-By: Claude Opus 4.6 (1M context) --- plugins/markdown-pages/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/markdown-pages/index.js b/plugins/markdown-pages/index.js index 12eeb7fd2d..f5540e2a65 100644 --- a/plugins/markdown-pages/index.js +++ b/plugins/markdown-pages/index.js @@ -25,7 +25,8 @@ module.exports = function markdownPagesPlugin(context, options = {}) { const withoutExt = rel.replace(/\.(md|mdx)$/i, ''); const id = frontmatter.id || path.basename(withoutExt); const dir = path.dirname(withoutExt); - if (dir === '.') return id; + if (dir === '.') return id === 'index' ? 'index' : id; + if (id === 'index') return dir; return `${dir}/${id}`; } From 8ead0deaa1372177d8fd1c36d13dbb31f24071e1 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 10:23:43 -0700 Subject: [PATCH 03/10] docs: add .md URL instructions to llms.txt via rootContent Tells agents in llms.txt that they can append .md to any page URL to fetch raw Markdown source. Co-Authored-By: Claude Opus 4.6 (1M context) --- docusaurus.config.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docusaurus.config.js b/docusaurus.config.js index 23d49f3ca1..1713a0dbb5 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -393,6 +393,15 @@ module.exports = async function createConfigAsync() { // Exclude imported markdown partials that should not be published as standalone LLM docs. ignoreFiles: ['docs/cloud/references/regions/private-service.md', 'docs/cloud/references/regions/gcpregions.md'], + // Tell agents how to fetch individual pages as raw markdown + rootContent: + 'This file contains links to documentation sections following the llmstxt.org standard.\n\n' + + '## Fetching individual pages\n\n' + + 'To fetch any page as raw Markdown, append `.md` to its URL path. ' + + 'For example, `https://docs.temporal.io/encyclopedia.md` returns the raw Markdown source for the Encyclopedia page.\n\n' + + 'Some pages (interactive demos, landing pages) are not available as Markdown. ' + + 'Requesting `.md` for those pages returns a short explanation instead.', + // Clean up content for better LLM consumption excludeImports: true, removeDuplicateHeadings: true, From 1e754e151fd139ea84ede49f8ed89b1910831231 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 10:27:06 -0700 Subject: [PATCH 04/10] feat: add content negotiation for markdown via Accept header Adds Vercel rewrites that serve raw markdown when a request includes Accept: text/markdown. Agents can request any docs URL with this header and get the .md source instead of the rendered HTML page. Co-Authored-By: Claude Opus 4.6 (1M context) --- vercel.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/vercel.json b/vercel.json index 68698a992c..34d186d0ca 100644 --- a/vercel.json +++ b/vercel.json @@ -4,6 +4,30 @@ "github": { "silent": true }, + "rewrites": [ + { + "source": "/", + "has": [ + { + "type": "header", + "key": "accept", + "value": "(?:.*,\\s*)?text/markdown(?:\\s*;.*)?" + } + ], + "destination": "/index.md" + }, + { + "source": "/:path*", + "has": [ + { + "type": "header", + "key": "accept", + "value": "(?:.*,\\s*)?text/markdown(?:\\s*;.*)?" + } + ], + "destination": "/:path*.md" + } + ], "redirects": [ { "source": "/evaluate/serverless-workers-demo", From ceb1c91efc6d33d275e25e21523744518c775e91 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 10:32:55 -0700 Subject: [PATCH 05/10] feat: use Vercel Edge Middleware for content negotiation Replace vercel.json rewrites (which run after static file matching) with Vercel Routing Middleware that runs before. When a request includes Accept: text/markdown, the middleware rewrites to the .md file. Otherwise it passes through to the normal static HTML. Co-Authored-By: Claude Opus 4.6 (1M context) --- middleware.mjs | 19 +++++++++++++++++++ package.json | 1 + vercel.json | 24 ------------------------ yarn.lock | 12 ++++++++++++ 4 files changed, 32 insertions(+), 24 deletions(-) create mode 100644 middleware.mjs diff --git a/middleware.mjs b/middleware.mjs new file mode 100644 index 0000000000..44f34188db --- /dev/null +++ b/middleware.mjs @@ -0,0 +1,19 @@ +import { rewrite, next } from '@vercel/functions'; + +export default function middleware(request) { + const accept = request.headers.get('accept') || ''; + + if (accept.includes('text/markdown')) { + const url = new URL(request.url); + const path = url.pathname.replace(/\/+$/, '') || '/index'; + return rewrite(new URL(`${path}.md`, request.url)); + } + + return next(); +} + +export const config = { + matcher: [ + '/((?!_next/static|_next/image|assets|img|scripts|favicon\\.ico|llms|.*\\.md$).*)', + ], +}; diff --git a/package.json b/package.json index 58ca8f230d..81974b059d 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "@mdx-js/react": "^3.0.0", "@types/react": "^19.1.0", "@types/react-dom": "^19.1.0", + "@vercel/functions": "^3.5.1", "algoliasearch": "^5.40.0", "chart.js": "^4.4.1", "clsx": "^1.1.1", diff --git a/vercel.json b/vercel.json index 34d186d0ca..68698a992c 100644 --- a/vercel.json +++ b/vercel.json @@ -4,30 +4,6 @@ "github": { "silent": true }, - "rewrites": [ - { - "source": "/", - "has": [ - { - "type": "header", - "key": "accept", - "value": "(?:.*,\\s*)?text/markdown(?:\\s*;.*)?" - } - ], - "destination": "/index.md" - }, - { - "source": "/:path*", - "has": [ - { - "type": "header", - "key": "accept", - "value": "(?:.*,\\s*)?text/markdown(?:\\s*;.*)?" - } - ], - "destination": "/:path*.md" - } - ], "redirects": [ { "source": "/evaluate/serverless-workers-demo", diff --git a/yarn.lock b/yarn.lock index 3b086283cf..a7e8f92881 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3723,6 +3723,18 @@ resolved "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz" integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g== +"@vercel/functions@^3.5.1": + version "3.5.1" + resolved "https://registry.yarnpkg.com/@vercel/functions/-/functions-3.5.1.tgz#008c45551eacec2ea595c76d260581bf100aaa69" + integrity sha512-ndh5v+uhWqGA8033oD0i0KHvqUHcLlLCOaLOw5L+xx5zVsWUSQcZPKEYk2nm51aisnKhcnTylxnOmhx+w4UCRA== + dependencies: + "@vercel/oidc" "3.4.1" + +"@vercel/oidc@3.4.1": + version "3.4.1" + resolved "https://registry.yarnpkg.com/@vercel/oidc/-/oidc-3.4.1.tgz#7f7f0361da6e021273bbf5bf467f5c495488c91e" + integrity sha512-H6B+/ig/GoahccL3WZjiHayHw1H5KhvTJNceqYulwfK9kkz5iul2hTmYzcJ7tTCQzyd0dutuL9xYFZCyLUqsog== + "@webassemblyjs/ast@1.14.1", "@webassemblyjs/ast@^1.14.1": version "1.14.1" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz" From 07d61f24409f9cb384c7844b49d84cc89dcb37e9 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 11:01:53 -0700 Subject: [PATCH 06/10] fix: override framework detection to enable middleware Set framework: null with explicit build config so Vercel treats this as a generic project that supports Routing Middleware, while still building Docusaurus correctly. Co-Authored-By: Claude Opus 4.6 (1M context) --- vercel.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vercel.json b/vercel.json index 68698a992c..0018f8509d 100644 --- a/vercel.json +++ b/vercel.json @@ -1,6 +1,9 @@ { "public": true, "trailingSlash": false, + "framework": null, + "buildCommand": "docusaurus build", + "outputDirectory": "build", "github": { "silent": true }, From 32815baabb1d8beef0020183d08fab2942aa9c10 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 11:05:23 -0700 Subject: [PATCH 07/10] fix: rename middleware.mjs to middleware.js, revert framework override Try .js extension for middleware detection. Revert framework: null since the Docusaurus preset shouldn't block middleware. Co-Authored-By: Claude Opus 4.6 (1M context) --- middleware.mjs => middleware.js | 0 vercel.json | 3 --- 2 files changed, 3 deletions(-) rename middleware.mjs => middleware.js (100%) diff --git a/middleware.mjs b/middleware.js similarity index 100% rename from middleware.mjs rename to middleware.js diff --git a/vercel.json b/vercel.json index 0018f8509d..68698a992c 100644 --- a/vercel.json +++ b/vercel.json @@ -1,9 +1,6 @@ { "public": true, "trailingSlash": false, - "framework": null, - "buildCommand": "docusaurus build", - "outputDirectory": "build", "github": { "silent": true }, From 1e6e56bf771c7459ce747e28ef486dfbbcf8a1a5 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 11:22:37 -0700 Subject: [PATCH 08/10] no index on md URLs --- docusaurus.config.js | 1 + vercel.json | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/docusaurus.config.js b/docusaurus.config.js index 1713a0dbb5..09bdfebb0f 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -389,6 +389,7 @@ module.exports = async function createConfigAsync() { // Generate both llms.txt (index) and llms-full.txt (complete content) generateLLMsTxt: true, generateLLMsFullTxt: true, + generateMarkdownFiles: true, // Exclude imported markdown partials that should not be published as standalone LLM docs. ignoreFiles: ['docs/cloud/references/regions/private-service.md', 'docs/cloud/references/regions/gcpregions.md'], diff --git a/vercel.json b/vercel.json index 68698a992c..2eabcc399c 100644 --- a/vercel.json +++ b/vercel.json @@ -4,6 +4,17 @@ "github": { "silent": true }, + "headers": [ + { + "source": "/(.*)\\.md", + "headers": [ + { + "key": "X-Robots-Tag", + "value": "noindex, nofollow" + } + ] + } + ], "redirects": [ { "source": "/evaluate/serverless-workers-demo", From 45fef6737ede9e7900d2668424241821b646aec8 Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Fri, 15 May 2026 12:59:40 -0700 Subject: [PATCH 09/10] add llm.exclude field --- .../serverless-workers/demo.mdx | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/docs/evaluate/development-production-features/serverless-workers/demo.mdx b/docs/evaluate/development-production-features/serverless-workers/demo.mdx index 6b86f5a10f..832bf43701 100644 --- a/docs/evaluate/development-production-features/serverless-workers/demo.mdx +++ b/docs/evaluate/development-production-features/serverless-workers/demo.mdx @@ -15,36 +15,39 @@ tags: - Workers - Serverless - AWS Lambda -description: An interactive demo for Temporal Serverless Workers. Explore the configuration, generated code, and execution flow for running Workers on AWS Lambda. +description: + An interactive demo for Temporal Serverless Workers. Explore the configuration, generated code, and execution flow for + running Workers on AWS Lambda. +llm_exclude: + This page contains an interactive demo component and is not suitable for LLM consumption. Refer to the [Serverless + Workers documentation](/serverless-workers) for detailed information on related concepts. --- :::tip SUPPORT, STABILITY, and DEPENDENCY INFO -Serverless Workers are in [Pre-release](/evaluate/development-production-features/release-stages#pre-release) and available to select Temporal Cloud customers. -To request access during Pre-release, create a [support ticket](/cloud/support#support-ticket) or contact your account team. -APIs are experimental and may be subject to backwards-incompatible changes. -[Sign up for updates](https://temporal.io/pages/serverless-workers-updates) to be notified when Serverless Workers reach Public Preview. +Serverless Workers are in [Pre-release](/evaluate/development-production-features/release-stages#pre-release) and +available to select Temporal Cloud customers. To request access during Pre-release, create a +[support ticket](/cloud/support#support-ticket) or contact your account team. APIs are experimental and may be subject +to backwards-incompatible changes. [Sign up for updates](https://temporal.io/pages/serverless-workers-updates) to be +notified when Serverless Workers reach Public Preview. ::: -Serverless Workers let you run Temporal Workers on serverless compute like AWS Lambda. -There are no long-lived processes to provision or scale. -Temporal Cloud invokes your Worker when Tasks arrive, and the Worker shuts down when the work is done. - -Use the interactive demo below to explore how the configuration options affect the generated -Worker code, deployment script, and CLI commands. Click "Start Workflow" to simulate the -end-to-end Serverless Worker invocation flow. - +Serverless Workers let you run Temporal Workers on serverless compute like AWS Lambda. There are no long-lived processes +to provision or scale. Temporal Cloud invokes your Worker when Tasks arrive, and the Worker shuts down when the work is +done. +Use the interactive demo below to explore how the configuration options affect the generated Worker code, deployment +script, and CLI commands. Click "Start Workflow" to simulate the end-to-end Serverless Worker invocation flow. import { ServerlessWorkerDemo } from '@site/src/components'; - --- ## Next steps - [Serverless Workers](/serverless-workers) for concepts, autoscaling, and lifecycle details. -- [Deploy a Serverless Worker](/production-deployment/worker-deployments/serverless-workers) for the full end-to-end deployment guide. +- [Deploy a Serverless Worker](/production-deployment/worker-deployments/serverless-workers) for the full end-to-end + deployment guide. From 934fadc59554db98ca4de8c0f5ee2747172ee86c Mon Sep 17 00:00:00 2001 From: Lenny Chen Date: Wed, 20 May 2026 10:41:43 -0700 Subject: [PATCH 10/10] fix: disable llms plugin markdown generation, hide buttons on excluded pages - Set generateMarkdownFiles: false in docusaurus-plugin-llms to prevent it from overwriting our raw .md files with processed versions that strip frontmatter. - Hide "Copy for LLM" and "View as Markdown" buttons on pages with llm_exclude frontmatter. Co-Authored-By: Claude Opus 4.6 (1M context) --- docusaurus.config.js | 2 +- src/components/LLMActions/LLMActions.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docusaurus.config.js b/docusaurus.config.js index 09bdfebb0f..47c8ef42b6 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -389,7 +389,7 @@ module.exports = async function createConfigAsync() { // Generate both llms.txt (index) and llms-full.txt (complete content) generateLLMsTxt: true, generateLLMsFullTxt: true, - generateMarkdownFiles: true, + generateMarkdownFiles: false, // Exclude imported markdown partials that should not be published as standalone LLM docs. ignoreFiles: ['docs/cloud/references/regions/private-service.md', 'docs/cloud/references/regions/gcpregions.md'], diff --git a/src/components/LLMActions/LLMActions.tsx b/src/components/LLMActions/LLMActions.tsx index 7e7c8532c3..fe8bd44d7b 100644 --- a/src/components/LLMActions/LLMActions.tsx +++ b/src/components/LLMActions/LLMActions.tsx @@ -37,7 +37,7 @@ export default function LLMActions() { const [copied, setCopied] = useState(false); const [loading, setLoading] = useState(false); - const { metadata } = useDoc(); + const { metadata, frontMatter } = useDoc(); const { editUrl, slug } = metadata; // Try to get raw URL from editUrl first, then fall back to slug-based construction @@ -78,7 +78,7 @@ export default function LLMActions() { } }, [rawUrl]); - if (!rawUrl) { + if (!rawUrl || frontMatter.llm_exclude) { return null; }