Skip to content
Draft
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions __tests__/lib/stripComments.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,29 @@ end"`);
`);
});

it.each([
['mdx', { mdx: true }],
['mdxish', { mdxish: true }],
])('preserves comments inside MDX HTMLBlocks (%s)', async (_name, opts) => {
const input = `<HTMLBlock>{\`
<!-- comment -->
<div>Hello world</div>
\`}</HTMLBlock>`;
const output = await stripComments(input, opts);
expect(output).toContain('<!-- comment -->');
});

it('preserves HTMLBlock template literal expressions in mdxish mode', async () => {
const input = `<HTMLBlock>{\`
<div style="color: red">
<strong>Hello, World!</strong>
</div>
\`}</HTMLBlock>`;

const output = await stripComments(input, { mdxish: true });
expect(output).toBe(input);
});

it('preserves jsx tables in mdxish mode', async () => {
const input = `<Table align={["left","left"]}>
<thead>
Expand Down
21 changes: 17 additions & 4 deletions lib/stripComments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ import { stripCommentsTransformer } from '../processor/transform/stripComments';

import { jsxTableFromMarkdown } from './mdast-util/jsx-table';
import { jsxTable } from './micromark/jsx-table';
import { extractMagicBlocks, restoreMagicBlocks } from './utils/extractMagicBlocks';
import { protectHTMLBlockContent, restoreHTMLBlockContent } from './utils/extractors/html-blocks';
import { extractMagicBlocks, restoreMagicBlocks } from './utils/extractors/magic-blocks';

interface Opts {
mdx?: boolean;
Expand All @@ -23,7 +24,15 @@ interface Opts {
* Removes Markdown and MDX comments.
*/
async function stripComments(doc: string, { mdx, mdxish }: Opts = {}): Promise<string> {
const { replaced, blocks } = extractMagicBlocks(doc);
// Preprocessing step: Don't touch magic blocks and HTML block content
let preprocessedDoc = doc;
const { replaced, blocks } = extractMagicBlocks(preprocessedDoc);
preprocessedDoc = replaced;

// We only need to protect HTML block content if we're in MDXish mode, and no
Comment thread
maximilianfalco marked this conversation as resolved.
Outdated
if (mdxish) {
preprocessedDoc = protectHTMLBlockContent(preprocessedDoc);
}

const processor = unified();

Expand Down Expand Up @@ -75,10 +84,14 @@ async function stripComments(doc: string, { mdx, mdxish }: Opts = {}): Promise<s
},
);

const file = await processor.process(replaced);
const file = await processor.process(preprocessedDoc);
const stringified = String(file).trim();

const restored = restoreMagicBlocks(stringified, blocks);
let restored = stringified;
if (mdxish) {
restored = restoreHTMLBlockContent(restored, true);
}
restored = restoreMagicBlocks(restored, blocks);
return restored;
}

Expand Down
76 changes: 76 additions & 0 deletions lib/utils/extractors/html-blocks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Base64 encode (Node.js + browser compatible)
function base64Encode(str: string): string {
if (typeof Buffer !== 'undefined') {
return Buffer.from(str, 'utf-8').toString('base64');
}
return btoa(unescape(encodeURIComponent(str)));
}

function base64Decode(str: string): string {
if (typeof Buffer !== 'undefined') {
return Buffer.from(str, 'base64').toString('utf-8');
}
return decodeURIComponent(escape(atob(str)));
}

// Markers for protected HTMLBlock content
// Add some random characters to the markers to reduce likelihood of conflicts with other content
const HTML_BLOCK_CONTENT_START = 'RMDX-!#@-HTMLBLOCK-START%:';
const HTML_BLOCK_CONTENT_END = ':%RMDX-!#@-HTMLBLOCK-END';

/**
* Matches HTMLBlock template literal expressions: `<HTMLBlock>{` ... `}</HTMLBlock>`
*/
export const HTMLBLOCK_TEMPLATE_LITERAL_REGEX = /(<HTMLBlock[^>]*>)\{\s*`((?:[^`\\]|\\.)*)`\s*\}(<\/HTMLBlock>)/g;

/**
* Base64 encodes HTMLBlock template literal content to prevent markdown parser from consuming <script>/<style> tags.
*
* @param content
* @returns Content with HTMLBlock template literals base64 encoded in HTML comments
* @example
* ```typescript
* const input = '<HTMLBlock>{`<script>alert("xss")</script>`}</HTMLBlock>';
* protectHTMLBlockContent(input)
* // Returns: '<HTMLBlock>RDMX-HTMLBLOCK-START:PHNjcmlwdD5hbGVydCgieHNzIik8L3NjcmlwdD4=:RDMX-HTMLBLOCK-END</HTMLBlock>'
Comment thread
maximilianfalco marked this conversation as resolved.
Outdated
* ```
*/
export function protectHTMLBlockContent(content: string) {
return content.replace(
HTMLBLOCK_TEMPLATE_LITERAL_REGEX,
(_match, openTag: string, templateContent: string, closeTag: string) => {
const encoded = base64Encode(templateContent);
return `${openTag}${HTML_BLOCK_CONTENT_START}${encoded}${HTML_BLOCK_CONTENT_END}${closeTag}`;
},
);
}

/**
* Restores HTMLBlock content that was protected by `protectHTMLBlockContent`.
* When `withBackticks` is true, re-wraps the decoded body as `{`...`}` (used by `stripComments` to preserve
* original markup). When false or omitted, inserts decoded HTML only (typical for transformers).
*
* @param content
* @param withBackticks
* @returns Content with protected markers replaced by decoded HTML, optionally wrapped in template literal syntax
* @example
* ```typescript
* const input =
* '<HTMLBlock>RMDX-!#@-HTMLBLOCK-START%:PHNjcmlwdD5hbGVydCgieHNzIik8L3NjcmlwdD4=:RMDX-!#@-HTMLBLOCK-END</HTMLBlock>';
* restoreHTMLBlockContent(input, true);
* // Returns: '<HTMLBlock>{`<script>alert("xss")</script>`}</HTMLBlock>'
* restoreHTMLBlockContent(input);
* // Returns: '<HTMLBlock><script>alert("xss")</script></HTMLBlock>'
* ```
*/
export function restoreHTMLBlockContent(content: string, withBackticks?: boolean) {
const markerRegex = new RegExp(`${HTML_BLOCK_CONTENT_START}([A-Za-z0-9+/=]+)${HTML_BLOCK_CONTENT_END}`, 'g');
return content.replace(markerRegex, (_match, encoded: string) => {
try {
const decoded = base64Decode(encoded);
return withBackticks ? `{\`${decoded}\`}` : decoded;
} catch {
return encoded;
}
});
}
Comment thread
maximilianfalco marked this conversation as resolved.
Outdated
File renamed without changes.
29 changes: 5 additions & 24 deletions processor/transform/mdxish/mdxish-html-blocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,9 @@ import type { Transform } from 'mdast-util-from-markdown';
import { visit } from 'unist-util-visit';

import { NodeTypes } from '../../../enums';
import { restoreHTMLBlockContent } from '../../../lib/utils/extractors/html-blocks';
import { formatHtmlForMdxish } from '../../utils';

import { base64Decode, HTML_BLOCK_CONTENT_END, HTML_BLOCK_CONTENT_START } from './preprocess-jsx-expressions';

/**
* Decodes HTMLBlock content that was protected during preprocessing.
* Content is wrapped in <!--RDMX_HTMLBLOCK:base64:RDMX_HTMLBLOCK-->
*/
function decodeProtectedContent(content: string): string {
// Escape special regex characters in the markers
const startEscaped = HTML_BLOCK_CONTENT_START.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const endEscaped = HTML_BLOCK_CONTENT_END.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const markerRegex = new RegExp(`${startEscaped}([A-Za-z0-9+/=]+)${endEscaped}`, 'g');
return content.replace(markerRegex, (_match, encoded: string) => {
try {
return base64Decode(encoded);
} catch {
return encoded;
}
});
}

/**
* Collects text content from a node and its children recursively
*/
Expand Down Expand Up @@ -205,7 +186,7 @@ const mdxishHtmlBlocks = (): Transform => tree => {
let content = contentParts.join('');
content = content.replace(/^<HTMLBlock[^>]*>\s*\{?\s*`?/, '').replace(/`?\s*\}?\s*<\/HTMLBlock>$/, '');
// Decode protected content that was base64 encoded during preprocessing
content = decodeProtectedContent(content);
content = restoreHTMLBlockContent(content);

const htmlString = formatHtmlForMdxish(content);
const runScripts = extractRunScriptsAttr(attrs);
Expand Down Expand Up @@ -241,7 +222,7 @@ const mdxishHtmlBlocks = (): Transform => tree => {
// Remove template literal syntax if present: {`...`}
content = content.replace(/^\s*\{\s*`/, '').replace(/`\s*\}\s*$/, '');
// Decode protected content that was base64 encoded during preprocessing
content = decodeProtectedContent(content);
content = restoreHTMLBlockContent(content);

const htmlString = formatHtmlForMdxish(content);
const runScripts = extractRunScriptsAttr(attrs);
Expand Down Expand Up @@ -286,7 +267,7 @@ const mdxishHtmlBlocks = (): Transform => tree => {
}

// Decode protected content that was base64 encoded during preprocessing
const decodedContent = decodeProtectedContent(contentParts.join(''));
const decodedContent = restoreHTMLBlockContent(contentParts.join(''));
const htmlString = formatHtmlForMdxish(decodedContent);
const runScripts = extractRunScriptsAttr(value);
const safeMode = extractBooleanAttr(value, 'safeMode');
Expand Down Expand Up @@ -356,7 +337,7 @@ const mdxishHtmlBlocks = (): Transform => tree => {
}

// Decode protected content that was base64 encoded during preprocessing
const decodedContent = decodeProtectedContent(templateContent.join(''));
const decodedContent = restoreHTMLBlockContent(templateContent.join(''));
const htmlString = formatHtmlForMdxish(decodedContent);

const runScripts = openingTag.value ? extractRunScriptsAttr(openingTag.value) : undefined;
Expand Down
43 changes: 1 addition & 42 deletions processor/transform/mdxish/preprocess-jsx-expressions.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,11 @@
import { protectHTMLBlockContent } from '../../../lib/utils/extractors/html-blocks';
import {
type ProtectedCode,
protectCodeBlocks,
restoreCodeBlocks,
restoreInlineCode,
} from '../../../lib/utils/mdxish/protect-code-blocks';

// Base64 encode (Node.js + browser compatible)
function base64Encode(str: string): string {
if (typeof Buffer !== 'undefined') {
return Buffer.from(str, 'utf-8').toString('base64');
}
return btoa(unescape(encodeURIComponent(str)));
}

// Base64 decode (Node.js + browser compatible)
export function base64Decode(str: string): string {
if (typeof Buffer !== 'undefined') {
return Buffer.from(str, 'base64').toString('utf-8');
}
return decodeURIComponent(escape(atob(str)));
}

function escapeHtmlAttribute(value: string): string {
return value
.replace(/&/g, '&amp;')
Expand All @@ -34,10 +19,6 @@ function escapeHtmlAttribute(value: string): string {
// Using a prefix that won't conflict with regular string values
export const JSON_VALUE_MARKER = '__MDXISH_JSON__';

// Markers for protected HTMLBlock content (HTML comments avoid markdown parsing issues)
export const HTML_BLOCK_CONTENT_START = '<!--RDMX_HTMLBLOCK:';
export const HTML_BLOCK_CONTENT_END = ':RDMX_HTMLBLOCK-->';

/**
* Pre-processes JSX-like expressions before markdown parsing.
* Converts href={'value'} to href="value", evaluates {expressions}, etc.
Expand Down Expand Up @@ -65,28 +46,6 @@ export function evaluateExpression(expression: string, context: JSXContext) {
return func(...contextValues);
}

/**
* Base64 encodes HTMLBlock template literal content to prevent markdown parser from consuming <script>/<style> tags.
*
* @param content
* @returns Content with HTMLBlock template literals base64 encoded in HTML comments
* @example
* ```typescript
* const input = '<HTMLBlock>{`<script>alert("xss")</script>`}</HTMLBlock>';
* protectHTMLBlockContent(input)
* // Returns: '<HTMLBlock><!--RDMX_HTMLBLOCK:PHNjcmlwdD5hbGVydCgieHNzIik8L3NjcmlwdD4=:RDMX_HTMLBLOCK--></HTMLBlock>'
* ```
*/
function protectHTMLBlockContent(content: string): string {
return content.replace(
/(<HTMLBlock[^>]*>)\{\s*`((?:[^`\\]|\\.)*)`\s*\}(<\/HTMLBlock>)/g,
(_match, openTag: string, templateContent: string, closeTag: string) => {
const encoded = base64Encode(templateContent);
return `${openTag}${HTML_BLOCK_CONTENT_START}${encoded}${HTML_BLOCK_CONTENT_END}${closeTag}`;
},
);
}

/**
* Removes JSX-style comments (e.g., { /* comment *\/ }) from content.
*
Expand Down
Loading