feat: serve raw markdown at .md URLs for AI agents

docs/evaluate/development-production-features/serverless-workers/demo.mdx

@@ -15,36 +15,39 @@ tags:
   - Workers
   - Serverless
   - AWS Lambda
-description: An interactive demo for Temporal Serverless Workers. Explore the configuration, generated code, and execution flow for running Workers on AWS Lambda.
+description:
+  An interactive demo for Temporal Serverless Workers. Explore the configuration, generated code, and execution flow for
+  running Workers on AWS Lambda.
+llm_exclude:
+  This page contains an interactive demo component and is not suitable for LLM consumption. Refer to the [Serverless
+  Workers documentation](/serverless-workers) for detailed information on related concepts.
 ---
 
 :::tip SUPPORT, STABILITY, and DEPENDENCY INFO
 
-Serverless Workers are in [Pre-release](/evaluate/development-production-features/release-stages#pre-release) and available to select Temporal Cloud customers.
-To request access during Pre-release, create a [support ticket](/cloud/support#support-ticket) or contact your account team.
-APIs are experimental and may be subject to backwards-incompatible changes.
-[Sign up for updates](https://temporal.io/pages/serverless-workers-updates) to be notified when Serverless Workers reach Public Preview.
+Serverless Workers are in [Pre-release](/evaluate/development-production-features/release-stages#pre-release) and
+available to select Temporal Cloud customers. To request access during Pre-release, create a
+[support ticket](/cloud/support#support-ticket) or contact your account team. APIs are experimental and may be subject
+to backwards-incompatible changes. [Sign up for updates](https://temporal.io/pages/serverless-workers-updates) to be
+notified when Serverless Workers reach Public Preview.
 
 :::
 
-Serverless Workers let you run Temporal Workers on serverless compute like AWS Lambda.
-There are no long-lived processes to provision or scale.
-Temporal Cloud invokes your Worker when Tasks arrive, and the Worker shuts down when the work is done.
-
-Use the interactive demo below to explore how the configuration options affect the generated
-Worker code, deployment script, and CLI commands. Click "Start Workflow" to simulate the
-end-to-end Serverless Worker invocation flow.
-
+Serverless Workers let you run Temporal Workers on serverless compute like AWS Lambda. There are no long-lived processes
+to provision or scale. Temporal Cloud invokes your Worker when Tasks arrive, and the Worker shuts down when the work is
+done.
 
+Use the interactive demo below to explore how the configuration options affect the generated Worker code, deployment
+script, and CLI commands. Click "Start Workflow" to simulate the end-to-end Serverless Worker invocation flow.
 
 import { ServerlessWorkerDemo } from '@site/src/components';
 
 <ServerlessWorkerDemo />
 
-
 ---
 
 ## Next steps
 
 - [Serverless Workers](/serverless-workers) for concepts, autoscaling, and lifecycle details.
-- [Deploy a Serverless Worker](/production-deployment/worker-deployments/serverless-workers) for the full end-to-end deployment guide.
+- [Deploy a Serverless Worker](/production-deployment/worker-deployments/serverless-workers) for the full end-to-end
+  deployment guide.

docusaurus.config.js

@@ -377,16 +377,32 @@ module.exports = async function createConfigAsync() {
           routeBasePath: 'ai-cookbook', // change if you use a different base
         },
       ],
+      [
+        require.resolve('./plugins/markdown-pages'),
+        {
+          docsDir: 'docs',
+        },
+      ],
       [
         'docusaurus-plugin-llms',
         {
           // Generate both llms.txt (index) and llms-full.txt (complete content)
           generateLLMsTxt: true,
           generateLLMsFullTxt: true,
+          generateMarkdownFiles: true,
 
           // Exclude imported markdown partials that should not be published as standalone LLM docs.
           ignoreFiles: ['docs/cloud/references/regions/private-service.md', 'docs/cloud/references/regions/gcpregions.md'],
 
+          // Tell agents how to fetch individual pages as raw markdown
+          rootContent:
+            'This file contains links to documentation sections following the llmstxt.org standard.\n\n' +
+            '## Fetching individual pages\n\n' +
+            'To fetch any page as raw Markdown, append `.md` to its URL path. ' +
+            'For example, `https://docs.temporal.io/encyclopedia.md` returns the raw Markdown source for the Encyclopedia page.\n\n' +
+            'Some pages (interactive demos, landing pages) are not available as Markdown. ' +
+            'Requesting `.md` for those pages returns a short explanation instead.',
+
           // Clean up content for better LLM consumption
           excludeImports: true,
           removeDuplicateHeadings: true,

middleware.js

@@ -0,0 +1,19 @@
+import { rewrite, next } from '@vercel/functions';
+
+export default function middleware(request) {
+  const accept = request.headers.get('accept') || '';
+
+  if (accept.includes('text/markdown')) {
+    const url = new URL(request.url);
+    const path = url.pathname.replace(/\/+$/, '') || '/index';
+    return rewrite(new URL(`${path}.md`, request.url));
+  }
+
+  return next();
+}
+
+export const config = {
+  matcher: [
+    '/((?!_next/static|_next/image|assets|img|scripts|favicon\\.ico|llms|.*\\.md$).*)',
+  ],
+};

package.json

@@ -60,6 +60,7 @@
     "@mdx-js/react": "^3.0.0",
     "@types/react": "^19.1.0",
     "@types/react-dom": "^19.1.0",
+    "@vercel/functions": "^3.5.1",
     "algoliasearch": "^5.40.0",
     "chart.js": "^4.4.1",
     "clsx": "^1.1.1",

plugins/markdown-pages/index.js

@@ -25,7 +25,8 @@ module.exports = function markdownPagesPlugin(context, options = {}) {
     const withoutExt = rel.replace(/\.(md|mdx)$/i, '');
     const id = frontmatter.id || path.basename(withoutExt);
     const dir = path.dirname(withoutExt);
-    if (dir === '.') return id;
+    if (dir === '.') return id === 'index' ? 'index' : id;
+    if (id === 'index') return dir;
     return `${dir}/${id}`;
   }
 

vercel.json

@@ -4,6 +4,17 @@
   "github": {
     "silent": true
   },
+  "headers": [
+    {
+      "source": "/(.*)\\.md",
+      "headers": [
+        {
+          "key": "X-Robots-Tag",
+          "value": "noindex, nofollow"
+        }
+      ]
+    }
+  ],
   "redirects": [
     {
       "source": "/evaluate/serverless-workers-demo",

yarn.lock

@@ -3723,6 +3723,18 @@
   resolved "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz"
   integrity sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==
 
+"@vercel/functions@^3.5.1":
+  version "3.5.1"
+  resolved "https://registry.yarnpkg.com/@vercel/functions/-/functions-3.5.1.tgz#008c45551eacec2ea595c76d260581bf100aaa69"
+  integrity sha512-ndh5v+uhWqGA8033oD0i0KHvqUHcLlLCOaLOw5L+xx5zVsWUSQcZPKEYk2nm51aisnKhcnTylxnOmhx+w4UCRA==
+  dependencies:
+    "@vercel/oidc" "3.4.1"
+
+"@vercel/oidc@3.4.1":
+  version "3.4.1"
+  resolved "https://registry.yarnpkg.com/@vercel/oidc/-/oidc-3.4.1.tgz#7f7f0361da6e021273bbf5bf467f5c495488c91e"
+  integrity sha512-H6B+/ig/GoahccL3WZjiHayHw1H5KhvTJNceqYulwfK9kkz5iul2hTmYzcJ7tTCQzyd0dutuL9xYFZCyLUqsog==
+
 "@webassemblyjs/ast@1.14.1", "@webassemblyjs/ast@^1.14.1":
   version "1.14.1"
   resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz"