diff --git a/__tests__/lib/stripComments.test.ts b/__tests__/lib/stripComments.test.ts index 6794ab628..37980fef1 100644 --- a/__tests__/lib/stripComments.test.ts +++ b/__tests__/lib/stripComments.test.ts @@ -382,6 +382,75 @@ end"`); expect(output).not.toContain(''); }); + describe('HTMLBlock handling in mdxish mode', () => { + it('does not crash on multiline HTMLBlock content', async () => { + const input = `{\` +
hello
+\`}
`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).toContain(''); + expect(output).toContain('
hello
'); + }); + + it('preserves HTMLBlock with attributes', async () => { + const input = `{\` +

content

+\`}
`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).toContain('content

'); + }); + + it('strips comments outside HTMLBlocks while preserving the block', async () => { + const input = ` + +{\` +
hello
+\`}
+ +`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).toContain(''); + expect(output).not.toContain(''); + expect(output).not.toContain(''); + }); + + it('strips HTML comments inside HTMLBlock content', async () => { + const input = `{\` + +
hello
+\`}
`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).not.toContain(''); + expect(output).toContain('
hello
'); + }); + + it('strips all comments including inside HTMLBlocks', async () => { + const input = `
+ +{\`

hi

\`}
+
`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).not.toContain(''); + expect(output).not.toContain(''); + expect(output).toContain(''); + }); + + it('handles nested HTMLBlocks', async () => { + const input = `{\` +{\`
nested
\`}
+\`}
`; + + const output = await stripComments(input, { mdxish: true }); + expect(output).toContain(''); + }); + }); + describe('strip comments edge cases', () => { it.each([ ['should return empty for empty string', '', undefined, ''], diff --git a/lib/mdast-util/html-block-component/index.ts b/lib/mdast-util/html-block-component/index.ts new file mode 100644 index 000000000..e5153ae4e --- /dev/null +++ b/lib/mdast-util/html-block-component/index.ts @@ -0,0 +1,53 @@ +import type { Html } from 'mdast'; +import type { CompileContext, Extension as FromMarkdownExtension, Handle, Token } from 'mdast-util-from-markdown'; + +const contextMap = new WeakMap(); + +function findHtmlBlockComponentToken(this: CompileContext): Token | undefined { + const events = this.tokenStack; + for (let i = events.length - 1; i >= 0; i -= 1) { + if (events[i][0].type === 'htmlBlockComponent') return events[i][0]; + } + return undefined; +} + +function enterHtmlBlockComponent(this: CompileContext, token: Parameters[0]): void { + contextMap.set(token, { chunks: [], lastEndLine: token.start.line }); + this.enter({ type: 'html', value: '' } as Html, token); +} + +function exitHtmlBlockComponentData(this: CompileContext, token: Parameters[0]): void { + const componentToken = findHtmlBlockComponentToken.call(this); + if (!componentToken) return; + const ctx = contextMap.get(componentToken); + if (ctx) { + const gap = token.start.line - ctx.lastEndLine; + if (ctx.chunks.length > 0 && gap > 0) { + ctx.chunks.push('\n'.repeat(gap)); + } + ctx.chunks.push(this.sliceSerialize(token)); + ctx.lastEndLine = token.end.line; + } +} + +function exitHtmlBlockComponent(this: CompileContext, token: Parameters[0]): void { + const ctx = contextMap.get(token); + const node = this.stack[this.stack.length - 1] as Html; + if (ctx) { + node.value = ctx.chunks.join(''); + contextMap.delete(token); + } + this.exit(token); +} + +export function htmlBlockComponentFromMarkdown(): FromMarkdownExtension { + return { + enter: { + htmlBlockComponent: enterHtmlBlockComponent, + }, + exit: { + htmlBlockComponentData: exitHtmlBlockComponentData, + htmlBlockComponent: exitHtmlBlockComponent, + }, + }; +} diff --git a/lib/micromark/html-block-component/index.ts b/lib/micromark/html-block-component/index.ts new file mode 100644 index 000000000..1a86842bd --- /dev/null +++ b/lib/micromark/html-block-component/index.ts @@ -0,0 +1 @@ +export { htmlBlockComponent } from './syntax'; diff --git a/lib/micromark/html-block-component/syntax.ts b/lib/micromark/html-block-component/syntax.ts new file mode 100644 index 000000000..53ed06924 --- /dev/null +++ b/lib/micromark/html-block-component/syntax.ts @@ -0,0 +1,321 @@ +/* eslint-disable @typescript-eslint/no-use-before-define */ +import type { Code, Construct, Effects, Extension, Resolver, State, TokenizeContext } from 'micromark-util-types'; + +import { markdownLineEnding } from 'micromark-util-character'; +import { codes, types } from 'micromark-util-symbol'; + +declare module 'micromark-util-types' { + interface TokenTypeMap { + htmlBlockComponent: 'htmlBlockComponent'; + htmlBlockComponentData: 'htmlBlockComponentData'; + } +} + +const TAG_SUFFIX: Code[] = [ + codes.uppercaseT, + codes.uppercaseM, + codes.uppercaseL, + codes.uppercaseB, + codes.lowercaseL, + codes.lowercaseO, + codes.lowercaseC, + codes.lowercaseK, +]; + +// --------------------------------------------------------------------------- +// Shared tokenizer factory +// --------------------------------------------------------------------------- + +/** + * Creates a tokenize function for `...`. + * + * - **flow** (block-level): supports multiline content via line continuations, + * consumes trailing whitespace after the closing tag. + * - **text** (inline): single-line only, exits immediately after the closing tag. + */ +function createTokenize(mode: 'flow' | 'text') { + return function tokenize(this: TokenizeContext, effects: Effects, ok: State, nok: State) { + let depth = 1; + + function matchChars(chars: Code[], onMatch: State, onFail: (code: Code) => State | undefined): State { + if (chars.length === 0) return onMatch; + const next: State = (code: Code): State | undefined => { + if (code === chars[0]) { + effects.consume(code); + return matchChars(chars.slice(1), onMatch, onFail); + } + return onFail(code); + }; + return next; + } + + function matchTagName(onMatch: State, onFail: (code: Code) => State | undefined): State { + const next: State = (code: Code): State | undefined => { + if (code === codes.uppercaseH) { + effects.consume(code); + return matchChars(TAG_SUFFIX, onMatch, onFail); + } + return onFail(code); + }; + return next; + } + + return start; + + function start(code: Code): State | undefined { + if (code !== codes.lessThan) return nok(code); + effects.enter('htmlBlockComponent'); + effects.enter('htmlBlockComponentData'); + effects.consume(code); + return matchTagName(afterTagName, nok); + } + + function afterTagName(code: Code): State | undefined { + if (code === codes.greaterThan) { + effects.consume(code); + return body; + } + if (code === codes.space || code === codes.horizontalTab) { + effects.consume(code); + return inAttributes; + } + if (mode === 'flow' && markdownLineEnding(code)) { + effects.exit('htmlBlockComponentData'); + return attributeContinuationStart(code); + } + if (code === codes.slash) { + effects.consume(code); + return selfClose; + } + return nok(code); + } + + function inAttributes(code: Code): State | undefined { + if (code === codes.greaterThan) { + effects.consume(code); + return body; + } + if (code === null) { + return nok(code); + } + if (markdownLineEnding(code)) { + if (mode === 'text') return nok(code); + effects.exit('htmlBlockComponentData'); + return attributeContinuationStart(code); + } + effects.consume(code); + return inAttributes; + } + + function attributeContinuationStart(code: Code): State | undefined { + return effects.check(nonLazyContinuationStart, attributeContinuationNonLazy, continuationAfter)(code); + } + + function attributeContinuationNonLazy(code: Code): State | undefined { + effects.enter(types.lineEnding); + effects.consume(code); + effects.exit(types.lineEnding); + return attributeContinuationBefore; + } + + function attributeContinuationBefore(code: Code): State | undefined { + if (code === null || markdownLineEnding(code)) { + return attributeContinuationStart(code); + } + effects.enter('htmlBlockComponentData'); + return inAttributes(code); + } + + function selfClose(code: Code): State | undefined { + if (code === codes.greaterThan) { + effects.consume(code); + return mode === 'flow' ? afterClose : done(code); + } + return nok(code); + } + + function body(code: Code): State | undefined { + if (code === null) return nok(code); + + if (markdownLineEnding(code)) { + if (mode === 'text') { + // Text constructs operate on paragraph content which spans lines + effects.consume(code); + return body; + } + effects.exit('htmlBlockComponentData'); + return continuationStart(code); + } + + if (code === codes.lessThan) { + effects.consume(code); + return closeSlash; + } + + effects.consume(code); + return body; + } + + function closeSlash(code: Code): State | undefined { + if (code === codes.slash) { + effects.consume(code); + return matchTagName(closeGt, body); + } + return matchTagName(openAfterTagName, body)(code); + } + + function openAfterTagName(code: Code): State | undefined { + if (code === codes.greaterThan || code === codes.space || code === codes.horizontalTab) { + depth += 1; + effects.consume(code); + return body; + } + return body(code); + } + + function closeGt(code: Code): State | undefined { + if (code === codes.greaterThan) { + depth -= 1; + effects.consume(code); + if (depth === 0) { + return mode === 'flow' ? afterClose : done(code); + } + return body; + } + return body(code); + } + + // -- flow-only states --------------------------------------------------- + + function afterClose(code: Code): State | undefined { + if (code === null || markdownLineEnding(code)) { + return done(code); + } + if (code === codes.space || code === codes.horizontalTab) { + effects.consume(code); + return afterClose; + } + // Reject so the block re-parses as a paragraph, deferring to the + // text tokenizer which preserves trailing content in the same line. + return nok(code); + } + + // -- flow-only: line continuation --------------------------------------- + + function continuationStart(code: Code): State | undefined { + return effects.check(nonLazyContinuationStart, continuationStartNonLazy, continuationAfter)(code); + } + + function continuationStartNonLazy(code: Code): State | undefined { + effects.enter(types.lineEnding); + effects.consume(code); + effects.exit(types.lineEnding); + return continuationBefore; + } + + function continuationBefore(code: Code): State | undefined { + if (code === null || markdownLineEnding(code)) { + return continuationStart(code); + } + effects.enter('htmlBlockComponentData'); + return body(code); + } + + function continuationAfter(code: Code): State | undefined { + if (code === null) return nok(code); + effects.exit('htmlBlockComponent'); + return ok(code); + } + + // -- shared exit -------------------------------------------------------- + + function done(_code: Code): State | undefined { + effects.exit('htmlBlockComponentData'); + effects.exit('htmlBlockComponent'); + return ok(_code); + } + }; +} + +// --------------------------------------------------------------------------- +// Flow construct (block-level) +// --------------------------------------------------------------------------- + +const nonLazyContinuationStart: Construct = { + tokenize: tokenizeNonLazyContinuationStart, + partial: true, +}; + +function resolveToHtmlBlockComponent(events: Parameters[0]) { + let index = events.length; + + while (index > 0) { + index -= 1; + if (events[index][0] === 'enter' && events[index][1].type === 'htmlBlockComponent') { + break; + } + } + + if (index > 1 && events[index - 2][1].type === types.linePrefix) { + events[index][1].start = events[index - 2][1].start; + events[index + 1][1].start = events[index - 2][1].start; + events.splice(index - 2, 2); + } + + return events; +} + +const htmlBlockComponentFlowConstruct: Construct = { + name: 'htmlBlockComponent', + tokenize: createTokenize('flow'), + resolveTo: resolveToHtmlBlockComponent, + concrete: true, +}; + +function tokenizeNonLazyContinuationStart(this: TokenizeContext, effects: Effects, ok: State, nok: State) { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + + return start; + + function start(code: Code): State | undefined { + if (markdownLineEnding(code)) { + effects.enter(types.lineEnding); + effects.consume(code); + effects.exit(types.lineEnding); + return after; + } + return nok(code); + } + + function after(code: Code): State | undefined { + if (self.parser.lazy[self.now().line]) { + return nok(code); + } + return ok(code); + } +} + +// --------------------------------------------------------------------------- +// Text construct (inline) +// --------------------------------------------------------------------------- + +const htmlBlockComponentTextConstruct: Construct = { + name: 'htmlBlockComponent', + tokenize: createTokenize('text'), +}; + +// --------------------------------------------------------------------------- +// Extension +// --------------------------------------------------------------------------- + +export function htmlBlockComponent(): Extension { + return { + flow: { + [codes.lessThan]: [htmlBlockComponentFlowConstruct], + }, + text: { + [codes.lessThan]: [htmlBlockComponentTextConstruct], + }, + }; +} diff --git a/lib/stripComments.ts b/lib/stripComments.ts index 5bd204141..727d92268 100644 --- a/lib/stripComments.ts +++ b/lib/stripComments.ts @@ -10,7 +10,9 @@ import { unified } from 'unified'; import normalizeEmphasisAST from '../processor/transform/mdxish/normalize-malformed-md-syntax'; import { stripCommentsTransformer } from '../processor/transform/stripComments'; +import { htmlBlockComponentFromMarkdown } from './mdast-util/html-block-component'; import { jsxTableFromMarkdown } from './mdast-util/jsx-table'; +import { htmlBlockComponent } from './micromark/html-block-component'; import { jsxTable } from './micromark/jsx-table'; import { extractMagicBlocks, restoreMagicBlocks } from './utils/extractMagicBlocks'; @@ -27,13 +29,14 @@ async function stripComments(doc: string, { mdx, mdxish }: Opts = {}): Promise before htmlFlow intercepts its inner HTML tags if (mdxish) { processor - .data('micromarkExtensions', [jsxTable(), mdxExpression({ allowEmpty: true })]) - .data('fromMarkdownExtensions', [jsxTableFromMarkdown(), mdxExpressionFromMarkdown()]) + .data('micromarkExtensions', [htmlBlockComponent(), jsxTable(), mdxExpression({ allowEmpty: true })]) + .data('fromMarkdownExtensions', [htmlBlockComponentFromMarkdown(), jsxTableFromMarkdown(), mdxExpressionFromMarkdown()]) .data('toMarkdownExtensions', [mdxExpressionToMarkdown()]); }