PRDCT-377: backfill frontmatter description across docs

package.json

@@ -10,7 +10,9 @@
     "preview": "astro preview",
     "astro": "astro",
     "gen:sidebar": "node scripts/convert-nav.mjs",
-    "check:cli": "node scripts/check-cli-reference.mjs"
+    "check:cli": "node scripts/check-cli-reference.mjs",
+    "lint:content": "node scripts/content-lint.mjs",
+    "backfill:desc": "node scripts/backfill-descriptions.mjs"
   },
   "dependencies": {
     "@astrojs/mdx": "^5.0.2",

scripts/backfill-descriptions.mjs

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+/**
+ * backfill-descriptions.mjs — add a draft `description:` to every page missing one.
+ *
+ * `description` feeds search snippets, the <meta> tag, the page lede, and Ask
+ * Kai / RAG context — yet all migrated pages lack it. This proposes a first
+ * draft from each page's opening prose so a human only has to refine, not write
+ * from scratch. Drafts are deliberately plain; treat them as a starting point.
+ *
+ * Usage:
+ *   node scripts/backfill-descriptions.mjs            # DRY RUN — preview only
+ *   node scripts/backfill-descriptions.mjs --apply    # write the frontmatter
+ *
+ * Idempotent: pages that already have a non-empty description are left untouched.
+ */
+import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
+import { join, relative, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const DOCS = join(ROOT, 'src', 'content', 'docs');
+const APPLY = process.argv.includes('--apply');
+const MAX = 160; // search-snippet sweet spot
+
+function findMd(dir, out = []) {
+  for (const e of readdirSync(dir)) {
+    const p = join(dir, e);
+    statSync(p).isDirectory() ? findMd(p, out) : /\.mdx?$/.test(e) && out.push(p);
+  }
+  return out;
+}
+
+const splitFm = (raw) => {
+  const m = raw.match(/^(---\r?\n)([\s\S]*?)(\r?\n---\r?\n?)/);
+  return m ? { open: m[1], fm: m[2], close: m[3], body: raw.slice(m[0].length) } : null;
+};
+
+/** Headings, images, HTML, blockquotes, lists, and tables are never prose. */
+const NOT_PROSE = /^(#|!\[|<|>|[-*]\s|\d+\.\s|\|)/;
+
+const LIST_ITEM = /^([-*]\s+|\d+\.\s+)/;
+
+/** A sentence is finished; anything else is still running. A trailing colon
+ *  counts as finished — it introduces an enumeration rather than continuing the
+ *  sentence, and `finish()` turns it into a period. */
+const ENDS_SENTENCE = (t) => /[.!?:]["'”’)\]]?$/.test(t.replace(/\s+$/, ''));
+
+const MIN = 25; // shorter than this is a label ("Video:"), not a description
+
+const cleanParagraph = (lines) =>
+  lines
+    .join(' ')
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, '') // images
+    .replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') // links → text
+    .replace(/<[^>]*>/g, ' ') // stray inline HTML tags
+    .replace(/[*_`]/g, '') // emphasis/code marks
+    .replace(/\s+/g, ' ')
+    .trim();
+
+/** Long enough, and not a side-note or visual deixis — those aren't intros. */
+const usable = (text) => {
+  const t = text.replace(/[\s,;:]+$/, '');
+  return t.length >= MIN && !/^note:/i.test(t) && !/^as you can see/i.test(t);
+};
+
+const VOID_TAG = /^<(br|hr|img|input|meta|link|source|wbr)\b/i;
+
+/** Net change in open-HTML-block depth contributed by one line. */
+const htmlDelta = (t) => {
+  const opens = (t.match(/<[a-z][^>]*>/gi) || []).filter(
+    (tag) => !/\/>$/.test(tag) && !VOID_TAG.test(tag)
+  ).length;
+  const closes = (t.match(/<\/[a-z][^>]*>/gi) || []).length;
+  return opens - closes;
+};
+
+/**
+ * First usable prose paragraph — skips headings, images, admonitions, HTML
+ * comments/blocks, lists, and tables. A non-prose line *ends* the current
+ * paragraph (a table or iframe is never absorbed as continuation), and an
+ * unusable paragraph falls through to the next one.
+ */
+function leadProse(body) {
+  // HTML comments span lines and leak template internals — drop them first.
+  const lines = body.replace(/<!--[\s\S]*?-->/g, ' ').split(/\r?\n/);
+  let para = [];
+  let inAside = false;
+  let inFence = false;
+  let htmlDepth = 0; // >0 = inside a multi-line HTML block (tables can contain blank lines)
+  const flush = () => {
+    const text = cleanParagraph(para);
+    para = [];
+    return usable(text) ? text : '';
+  };
+  for (let i = 0; i < lines.length; i++) {
+    const t = lines[i].trim();
+    if (/^```/.test(t)) { inFence = !inFence; continue; }
+    if (inFence) continue;
+    if (/^:::/.test(t)) { inAside = !inAside; continue; } // skip whole admonition
+    if (inAside) continue;
+    if (htmlDepth > 0) {
+      htmlDepth = Math.max(0, htmlDepth + htmlDelta(t));
+      continue;
+    }
+    if (!t) {
+      // A blank line usually ends the paragraph — unless the sentence is still
+      // open and a list follows, in which case the list *is* the rest of it.
+      if (para.length && !ENDS_SENTENCE(cleanParagraph(para))) {
+        const next = lines.slice(i + 1).find((l) => l.trim());
+        if (next && LIST_ITEM.test(next.trim())) continue;
+      }
+      const text = flush();
+      if (text) return text;
+      continue;
+    }
+    if (NOT_PROSE.test(t)) {
+      // A sentence whose object *is* the list ("…fetch data from the" / "- X or"
+      // / "- Y.") must absorb the items, or the description ends on a dangling
+      // preposition. Only while the sentence is still open, never from scratch.
+      if (LIST_ITEM.test(t) && para.length && !ENDS_SENTENCE(cleanParagraph(para))) {
+        para.push(t.replace(LIST_ITEM, ''));
+        continue;
+      }
+      const text = flush();
+      if (text) return text;
+      if (t.startsWith('<')) htmlDepth = Math.max(0, htmlDelta(t)); // enter the HTML block
+      continue;
+    }
+    para.push(t);
+  }
+  return flush();
+}
+
+/** Trim to <= MAX chars on a word/sentence boundary. */
+function clip(text) {
+  if (text.length <= MAX) return text;
+  const cut = text.slice(0, MAX);
+  const sentence = cut.lastIndexOf('. ');
+  if (sentence > MAX * 0.4) return cut.slice(0, sentence + 1);
+  return cut.slice(0, cut.lastIndexOf(' ')).replace(/[,;:]$/, '') + '…';
+}
+
+/** Final polish: a list-introducing colon becomes a period; a paragraph cut
+ *  mid-sentence (prose running into a list) retreats to the previous sentence
+ *  or gets an explicit ellipsis. */
+function finish(text) {
+  if (!text) return text;
+  let t = text.replace(/\s+$/, '').replace(/[,;:]$/, '.');
+  if (!/[.!?…"”')\]]$/.test(t)) {
+    const s = t.lastIndexOf('. ');
+    t = s >= MIN ? t.slice(0, s + 1) : `${t}…`;
+  }
+  return t;
+}
+
+const yamlEscape = (s) =>
+  /[:#'"\[\]{}&*!|>%@`\\]/.test(s) ? `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"` : s;
+
+const files = findMd(DOCS).sort();
+let filled = 0;
+let skippedNoProse = 0;
+
+for (const file of files) {
+  const raw = readFileSync(file, 'utf8');
+  const parts = splitFm(raw);
+  if (!parts) continue;
+  if (/^description:\s*\S/m.test(parts.fm)) continue; // already has one
+
+  const draft = finish(clip(leadProse(parts.body)));
+  if (!draft || draft.length < MIN) { skippedNoProse++; continue; }
+
+  const line = `description: ${yamlEscape(draft)}`;
+  // place after `title:` (or `slug:`) for readability; [^\n] (not `.`) so a
+  // CRLF file keeps its `\r` on the anchor line, and the new line gets one too
+  const anchored = parts.fm.match(/^(title:[^\n]*|slug:[^\n]*)$/m);
+  const cr = anchored?.[0].endsWith('\r') ? '\r' : '';
+  const newFm = anchored
+    ? parts.fm.replace(anchored[0], `${anchored[0]}\n${line}${cr}`)
+    : `${parts.fm}\n${line}`;
+
+  filled++;
+  if (filled <= 8 || !APPLY) {
+    console.log(`\n${relative(DOCS, file)}`);
+    console.log(`  + ${line}`);
+  }
+  if (APPLY) writeFileSync(file, parts.open + newFm + parts.close + parts.body);
+}
+
+console.log(`\n${'─'.repeat(50)}`);
+console.log(`${APPLY ? 'Wrote' : 'Would write'} description to ${filled} page(s).`);
+if (skippedNoProse) console.log(`Skipped ${skippedNoProse} page(s) with no usable opening prose — fill by hand.`);
+if (!APPLY) console.log('Dry run — re-run with --apply to write.');

scripts/content-lint.mjs

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+/**
+ * content-lint.mjs — content-quality linter for the docs.
+ *
+ * Walks every page in src/content/docs and reports issues that hurt search,
+ * Ask Kai, and reader trust — the things a Phase-3 content pass should clear:
+ *
+ *   • missing `description:`   — feeds search snippets, <meta>, the page lede,
+ *                                and Ask Kai/RAG context
+ *   • liquid           — leftover Jekyll `{% … %}` / `{{ … }}` template tags
+ *   • html-links       — links to `*.html` (old Jekyll URLs; should be clean paths)
+ *   • visual-deixis    — "as shown above/below", "see the screenshot above" — prose
+ *                        that assumes an image that may not be there / is being removed
+ *   • deprecated       — terms we've renamed/retired (configurable below)
+ *
+ * Usage:
+ *   node scripts/content-lint.mjs            # full report
+ *   node scripts/content-lint.mjs --rule=description   # one rule only
+ *   node scripts/content-lint.mjs --quiet    # counts only (good for CI)
+ *
+ * Exit code is non-zero when any issue is found, so it can gate CI later.
+ */
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, relative, resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const DOCS = join(ROOT, 'src', 'content', 'docs');
+
+const onlyRule = process.argv.find((a) => a.startsWith('--rule='))?.split('=')[1];
+const QUIET = process.argv.includes('--quiet');
+
+/** Terms we've renamed/retired. Keep conservative — add as the IA settles. */
+const DEPRECATED = [
+  { term: /\bOrchestr(ation|ator)s?\b/g, use: 'Flows' },
+  { term: /\bSandbox(es)?\b/g, use: 'Workspaces' },
+  { term: /\bProcessed Tags?\b/g, use: '(deprecated — file input mapping)' },
+];
+
+const VISUAL_DEIXIS =
+  /\b(as (shown|seen|illustrated|depicted) (above|below)|(screenshot|picture|image|figure) (above|below)|see (the )?(screenshot|picture|image|figure) (above|below)|in the (screenshot|picture|image) (above|below))\b/gi;
+
+const LIQUID = /\{%[^%]*%\}|\{\{[^}]*\}\}/g;
+const HTML_LINK = /\]\(([^)\s]*\.html(#[^)]*)?)\)/g;
+
+/** Internal links only: relative paths or help.keboola.com — external sites that
+ *  happen to end in `.html` (oracle.com/index.html, debezium.io/…​.html) are fine. */
+const isInternal = (url) =>
+  !/^https?:\/\//i.test(url) || /^https?:\/\/help\.keboola\.com/i.test(url);
+
+function findMd(dir, out = []) {
+  for (const e of readdirSync(dir)) {
+    const p = join(dir, e);
+    statSync(p).isDirectory() ? findMd(p, out) : /\.mdx?$/.test(e) && out.push(p);
+  }
+  return out;
+}
+
+function splitFrontmatter(raw) {
+  const m = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  return m ? { fm: m[1], body: raw.slice(m[0].length) } : { fm: '', body: raw };
+}
+
+function fmField(fm, key) {
+  const m = fm.match(new RegExp(`^${key}:\\s*(.*)$`, 'm'));
+  return m ? m[1].trim().replace(/^['"]|['"]$/g, '') : null;
+}
+
+/** strip fenced code blocks so we don't lint code samples */
+const stripCode = (body) => body.replace(/```[\s\S]*?```/g, '').replace(/`[^`]*`/g, '');
+
+const RULES = {
+  description(fm) {
+    const d = fmField(fm, 'description');
+    return !d || !d.length ? [{ msg: 'no description' }] : [];
+  },
+  liquid(_fm, body) {
+    // Only prose Liquid is a Jekyll artifact — `{{ }}`/`{% %}` inside code blocks
+    // are legit (dbt `env_var`, Streamlit/Jinja examples), so strip code first.
+    return [...stripCode(body).matchAll(LIQUID)].map((m) => ({ msg: m[0].slice(0, 40) }));
+  },
+  'html-links'(_fm, body) {
+    return [...body.matchAll(HTML_LINK)]
+      .filter((m) => isInternal(m[1]))
+      .map((m) => ({ msg: m[1] }));
+  },
+  'visual-deixis'(_fm, body) {
+    return [...stripCode(body).matchAll(VISUAL_DEIXIS)].map((m) => ({ msg: `“${m[0]}”` }));
+  },
+  deprecated(_fm, body) {
+    const text = stripCode(body);
+    const out = [];
+    for (const { term, use } of DEPRECATED) {
+      for (const m of text.matchAll(term)) out.push({ msg: `“${m[0]}” → ${use}` });
+    }
+    return out;
+  },
+};
+
+const ruleNames = Object.keys(RULES).filter((r) => !onlyRule || r === onlyRule);
+const findings = Object.fromEntries(ruleNames.map((r) => [r, []]));
+
+for (const file of findMd(DOCS).sort()) {
+  const raw = readFileSync(file, 'utf8');
+  const { fm, body } = splitFrontmatter(raw);
+  const rel = relative(DOCS, file);
+  for (const r of ruleNames) {
+    for (const hit of RULES[r](fm, body)) findings[r].push({ file: rel, ...hit });
+  }
+}
+
+// ── Report ────────────────────────────────────────────────────────────────
+let total = 0;
+console.log(`\nContent lint — ${findMd(DOCS).length} pages\n${'─'.repeat(50)}`);
+for (const r of ruleNames) {
+  const hits = findings[r];
+  total += hits.length;
+  const pages = new Set(hits.map((h) => h.file)).size;
+  console.log(`\n${r.toUpperCase()}  —  ${hits.length} issue(s) across ${pages} page(s)`);
+  if (!QUIET) {
+    for (const h of hits.slice(0, 60)) console.log(`  ${h.file}${h.msg ? `  ·  ${h.msg}` : ''}`);
+    if (hits.length > 60) console.log(`  …and ${hits.length - 60} more`);
+  }
+}
+console.log(`\n${'─'.repeat(50)}\nTOTAL: ${total} issue(s)\n`);
+process.exit(total > 0 ? 1 : 0);

src/content/docs/ai/ai-kit/index.md

@@ -1,5 +1,6 @@
 ---
 title: AI Kit
+description: AI Kit is a plugin marketplace for AI coding assistants that provides specialized agents, commands, and workflows for Keboola development.
 slug: 'ai/ai-kit'
 ---
 

src/content/docs/ai/index.md

@@ -1,5 +1,6 @@
 ---
 title: AI Features
+description: "Keboola's AI-powered capabilities help you build, optimize, and automate your data pipelines with intelligent assistance."
 slug: 'ai'
 ---
 

src/content/docs/ai/mcp-server/index.md

@@ -1,5 +1,6 @@
 ---
 title: Keboola Model Context Protocol (MCP) Server
+description: Connect your MCP clients and AI assistants to your Keboola Project and give them the powers of a Keboola Expert user.
 slug: 'ai/mcp-server'
 redirect_from:
     - /external-integrations/mcp-server/

src/content/docs/catalog/index.md

@@ -1,5 +1,6 @@
 ---
 title: Data Catalog
+description: The data catalog represents an overview of data shared to and from the project.
 slug: 'catalog'
 redirect_from:
   - /storage/buckets/sharing/

src/content/docs/catalog/multi-project/index.md

@@ -1,5 +1,6 @@
 ---
 title: Multi-Project Architecture
+description: Keboola is a very versatile platform and can be used in many different ways.
 slug: 'catalog/multi-project'
 ---
 

src/content/docs/components/applications/ai/generative-ai/index.md

@@ -1,5 +1,6 @@
 ---
 title: Generative AI
+description: The Generative AI application allows you to query both OpenAI and Azure OpenAI models using data from your Keboola project.
 slug: 'components/applications/ai/generative-ai'
 redirect_from:
     - /components/applications/ai/open-ai/

src/content/docs/components/applications/ai/index.md

@@ -1,5 +1,6 @@
 ---
 title: AI Applications
+description: AI applications are used to generate, process, or enrich data via custom models or 3rd-party AI API services. The following AI applications are available.
 slug: 'components/applications/ai'
 redirect_from:
     - /applications/ai/

src/content/docs/components/applications/custom-python/index.md

@@ -1,5 +1,6 @@
 ---
 title: Custom Python
+description: This component lets you run custom Python code directly in Keboola.
 slug: 'components/applications/custom-python'
 ---
 

src/content/docs/components/applications/data-gateway/index.md

@@ -1,5 +1,6 @@
 ---
 title: Data Gateway
+description: This application allows you to share data in read-only mode with third-party BI and visualization tools.
 slug: 'components/applications/data-gateway'
 redirect_from:
     - /applications/data-gateway/