Skip to content
Merged
6 changes: 4 additions & 2 deletions docs/content/improve/rule-editor.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ The Rule Editor turns anti-pattern detection into an open, editable system. Ever

## Layout

The page lists every rule grouped by practice category. Each row shows the rule name, group, severity, scope (requests or sessions), and a layer badge indicating whether the rule is built-in, a user override, or a fresh user-authored rule stored under `.ai-engineer-coach/rules/` in your home directory or workspace root.
The page lists every rule grouped by practice category. Each row shows the rule name, group, severity, scope (requests, sessions, or both), and a layer badge indicating whether the rule is built-in, a user override, or a fresh user-authored rule stored under `.ai-engineer-coach/rules/` in your home directory or workspace root.

Use the filter bar to search by tag or group, and click any rule to open its detail view.

Expand All @@ -23,7 +23,7 @@ Click **Edit** on any rule to open the modal editor. The editor has two tabs:
| **Form** | Fill in structured fields -- id, name, group, severity, scope, tags, thresholds, description, examples, and detection logic |
| **Source** | View or edit the full markdown file directly. Useful for advanced edits or for pasting in rules from elsewhere |

Switching tabs syncs both directions, so you can round-trip between the form and the raw source.
Switching tabs syncs both directions, so you can round-trip between the form and the raw source. The form writes back only the fields you change; everything else in the rule stays as it is, including frontmatter keys and sections the form doesn't show, such as `# Tests`. Rules with Windows (CRLF) line endings load into the form like any other.

## Live Testing

Expand All @@ -35,6 +35,8 @@ Click **Test Rule** to run the current draft against your real session data usin

Threshold sliders appear above the result so you can sweep a value without retyping the markdown. Changes are applied to the draft in memory; nothing is persisted until you hit **Save**.

A draft with no detection logic (no detect block, `# Filter` section, metric or parent rule) shows an error instead of running, and isn't saved.

## AI Builder

Use the **Generate** button to describe the rule you want in natural language. The AI builder drafts a complete markdown rule -- frontmatter, detection logic, examples -- and loads it into the editor. The generator retries automatically if it produces invalid DSL. You can always review the output before saving.
Expand Down
2 changes: 1 addition & 1 deletion src/webview/dsl-cheatsheet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ id: rule-id
name: Human-Readable Name
group: prompt-quality | session-hygiene | code-review | tool-mastery | context-management
severity: low | medium | high
scope: requests | sessions
scope: requests | sessions | both
version: 1
tags: [tag1, tag2]
thresholds:
Expand Down
353 changes: 353 additions & 0 deletions src/webview/page-antipatterns-editor-form.test.ts

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions src/webview/page-antipatterns-editor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,4 +85,18 @@ describe('page-antipatterns-editor threshold helpers', () => {
expect(out).toContain('shortLen: 999');
});
});

describe('with CRLF line endings', () => {
const crlfMd = sampleMd.replaceAll('\n', '\r\n');

it('parseThresholdsFromMarkdown reads the thresholds', () => {
expect(parseThresholdsFromMarkdown(crlfMd)).toEqual({ shortLen: 50, ratioLimit: 0.25 });
});

it('applyThresholdOverrides applies the overrides', () => {
const out = applyThresholdOverrides(crlfMd, { shortLen: 100 });
expect(out).toContain(' shortLen: 100\n');
expect(parseThresholdsFromMarkdown(out)).toEqual({ shortLen: 100, ratioLimit: 0.25 });
});
});
});
165 changes: 69 additions & 96 deletions src/webview/page-antipatterns-editor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { DateFilter } from '../core/types';
import { rpc } from './shared';
import { html, render, type ComponentChildren } from './render';
import { DSL_CHEATSHEET } from './dsl-cheatsheet';
import { hasDetectionLogic, mergeRuleMarkdown, normalizeLineEndings, readRuleFields, type RuleFormField, type RuleFormFields } from './rule-form-markdown';

const RULE_EDITOR_SYSTEM_PROMPT = `You are an expert at writing detection rules for the AI Engineer Coach VS Code extension.
Rules are markdown files with YAML frontmatter and a Detection Logic block using a custom DSL.
Expand All @@ -17,8 +18,11 @@ ${DSL_CHEATSHEET}

Output ONLY the raw markdown rule. No code fences around the whole output. No explanation.`;

export function parseThresholdsFromMarkdown(md: string): Record<string, number> {
const NO_DETECTION_LOGIC = 'This rule has no detection logic. Add a detect block under Detection Logic first.';

export function parseThresholdsFromMarkdown(markdown: string): Record<string, number> {
const result: Record<string, number> = {};
const md = normalizeLineEndings(markdown);
const fmMatch = md.match(/^---\n([\s\S]*?)\n---/);
if (!fmMatch) return result;
const threshMatch = fmMatch[1].match(/thresholds:\n((?:\s{2,}\S.*\n?)*)/);
Expand All @@ -34,7 +38,7 @@ export function parseThresholdsFromMarkdown(md: string): Record<string, number>
}

export function applyThresholdOverrides(md: string, overrides: Record<string, number>): string {
return md.replace(/(thresholds:\n)((?:\s{2,}\S.*\n?)*)/, (_full, head: string, body: string) => {
return normalizeLineEndings(md).replace(/(thresholds:\n)((?:\s{2,}\S.*\n?)*)/, (_full, head: string, body: string) => {
const newBody = body.replaceAll(/^(\s+)(\w+):[ \t]*([-\d.]+)[ \t]*$/gm, (line, indent: string, key: string, _val: string) => {
if (overrides[key] !== undefined) {
const v = overrides[key];
Expand Down Expand Up @@ -73,95 +77,59 @@ function buildThresholdSliders(thresholds: Record<string, number>, overrides: Re
return html`<div class="rule-test-sliders"><div class="rule-test-sliders-title">Tune thresholds (live)</div>${rows}</div>`;
}

function formToMarkdown(c: HTMLElement): string {
const val = (id: string) => (c.querySelector(`#${id}`) as HTMLInputElement | HTMLTextAreaElement)?.value?.trim() || '';
const thresholds = val('rf-thresholds').split('\n').filter(l => l.includes(':')).map(l => ' ' + l.trim()).join('\n');
const tags = val('rf-tags').split(',').map(t => t.trim()).filter(Boolean);
const patternsRaw = val('rf-patterns');
const fileTypesRaw = val('rf-filetypes');
const extraFm = val('rf-extra-fm');
return `---
id: ${val('rf-id') || 'my-custom-rule'}
name: ${val('rf-name') || 'My Custom Rule'}
group: ${val('rf-group')}
severity: ${val('rf-severity')}
scope: ${val('rf-scope')}
version: ${val('rf-version') || '1'}
tags: [${tags.join(', ')}]
${thresholds ? `thresholds:\n${thresholds}` : 'thresholds: {}'}
${patternsRaw ? `patterns:\n${patternsRaw}` : ''}${fileTypesRaw ? `\nfileTypes:\n${fileTypesRaw}` : ''}${extraFm ? `\n${extraFm}` : ''}
---

# Description
${val('rf-description') || 'Describe what this rule detects.'}

# When Triggered
${val('rf-when-triggered') || '{{count}} occurrences detected ({{pct}} of requests).'}

# How to Improve
${val('rf-how-to-improve') || 'Explain how to fix this anti-pattern.'}

# Examples
${val('rf-examples') || '"{{message}}..."'}

# Detection Logic
\`\`\`detect
${val('rf-detect') || 'scan: requests\nmatch: messageLength > 0\naggregate: count\ncheck: count >= 1'}
\`\`\`
`;
type FieldInput = HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement;

const FIELD_INPUTS: Record<RuleFormField, string> = {
id: 'rf-id',
name: 'rf-name',
group: 'rf-group',
severity: 'rf-severity',
scope: 'rf-scope',
version: 'rf-version',
tags: 'rf-tags',
thresholds: 'rf-thresholds',
description: 'rf-description',
whenTriggered: 'rf-when-triggered',
howToImprove: 'rf-how-to-improve',
examples: 'rf-examples',
detect: 'rf-detect',
};
const FORM_FIELDS = Object.keys(FIELD_INPUTS) as RuleFormField[];

/** What the form was loaded from: the rule markdown, and the form's values then. */
const loadedRules = new WeakMap<HTMLElement, { markdown: string; fields: RuleFormFields }>();

function readForm(c: HTMLElement): RuleFormFields {
const fields = {} as RuleFormFields;
for (const field of FORM_FIELDS) fields[field] = c.querySelector<FieldInput>(`#${FIELD_INPUTS[field]}`)?.value.trim() ?? '';
return fields;
}

function markdownToForm(c: HTMLElement, md: string): void {
const set = (id: string, v: string) => {
const el = c.querySelector(`#${id}`) as HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement;
if (el) el.value = v;
};
const fmMatch = md.match(/^---\n([\s\S]*?)\n---/);
if (fmMatch) {
const fm = fmMatch[1];
const field = (key: string) => {
const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
return m ? m[1].trim() : '';
};
set('rf-id', field('id'));
set('rf-name', field('name'));
set('rf-group', field('group'));
set('rf-severity', field('severity'));
set('rf-scope', field('scope'));
set('rf-version', field('version') || '1');
const tagsMatch = field('tags');
const tags = tagsMatch.replaceAll(/^\[|\]$/g, '').split(',').map(t => t.trim()).filter(Boolean);
set('rf-tags', tags.join(', '));
const threshMatch = fm.match(/thresholds:\n((?:\s{2,}\S.*\n?)*)/);
if (threshMatch) {
set('rf-thresholds', threshMatch[1].replaceAll(/^ {2}/gm, '').trim());
} else {
set('rf-thresholds', '');
}
const patternsMatch = fm.match(/patterns:\n((?:\s{2,}\S.*\n?)*)/);
set('rf-patterns', patternsMatch ? patternsMatch[1].trim().replaceAll(/^/gm, ' ') : '');
const fileTypesMatch = fm.match(/fileTypes:\n((?:\s{2,}\S.*\n?)*)/);
set('rf-filetypes', fileTypesMatch ? fileTypesMatch[1].trim().replaceAll(/^/gm, ' ') : '');
const extraFmParts: string[] = [];
const reqIdeMatch = fm.match(/^requiresIdeContext:\s*(.+)$/m);
if (reqIdeMatch) extraFmParts.push(`requiresIdeContext: ${reqIdeMatch[1].trim()}`);
const extendsMatch = fm.match(/^extends:\s*(.+)$/m);
if (extendsMatch) extraFmParts.push(`extends: ${extendsMatch[1].trim()}`);
set('rf-extra-fm', extraFmParts.join('\n'));
/** Fills the form from rule markdown, and keeps that markdown as the Source text that formToMarkdown edits. */
export function markdownToForm(c: HTMLElement, markdown: string): void {
const fields = readRuleFields(markdown);
for (const field of FORM_FIELDS) {
const input = c.querySelector<FieldInput>(`#${FIELD_INPUTS[field]}`);
const value = fields[field];
if (input && value !== undefined) input.value = value;
}
const section = (heading: string) => {
const re = new RegExp(`# ${heading}\\n([\\s\\S]*?)(?=\\n# |$)`);
const m = md.match(re);
return m ? m[1].trim() : '';
};
set('rf-description', section('Description'));
set('rf-when-triggered', section('When Triggered'));
set('rf-how-to-improve', section('How to Improve'));
const exSection = section('Examples');
const exClean = exSection.replaceAll(/```detect[\s\S]*?```/g, '').trim();
set('rf-examples', exClean);
const detectMatch = md.match(/```detect\n([\s\S]*?)```/);
set('rf-detect', detectMatch ? detectMatch[1].trim() : '');
loadedRules.set(c, { markdown, fields: readForm(c) });
const source = c.querySelector<HTMLTextAreaElement>('#rule-editor-raw');
if (source) source.value = markdown;
}

/**
* Builds rule markdown from the form. A new rule gets the whole template; a loaded rule gets the fields changed
* since it was loaded written into its Source text (or, without a Source view, into the markdown it was loaded
* from), so everything the form doesn't show is kept.
*/
export function formToMarkdown(c: HTMLElement): string {
const loaded = loadedRules.get(c);
const source = c.querySelector<HTMLTextAreaElement>('#rule-editor-raw')?.value ?? loaded?.markdown ?? '';
const base = source.trim() ? source : '';
const fields = readForm(c);
const changed = FORM_FIELDS.filter(field => !base || fields[field] !== loaded?.fields[field]);
return mergeRuleMarkdown(base, fields, new Set(changed));
}

export function openRuleEditor(container: HTMLElement, existingRuleId: string | null, markdown?: string): void {
Expand All @@ -171,8 +139,7 @@ export function openRuleEditor(container: HTMLElement, existingRuleId: string |
title.textContent = existingRuleId ? 'Edit Rule' : 'New Rule';

const fields = ['rf-id', 'rf-name', 'rf-tags', 'rf-description', 'rf-when-triggered',
'rf-how-to-improve', 'rf-examples', 'rf-thresholds', 'rf-detect',
'rf-patterns', 'rf-filetypes', 'rf-extra-fm'];
'rf-how-to-improve', 'rf-examples', 'rf-thresholds', 'rf-detect', 'rule-editor-raw'];
for (const id of fields) {
const el = container.querySelector(`#${id}`) as HTMLInputElement | HTMLTextAreaElement;
if (el) el.value = '';
Expand All @@ -188,14 +155,10 @@ export function openRuleEditor(container: HTMLElement, existingRuleId: string |

if (markdown) {
markdownToForm(container, markdown);
(container.querySelector('#rule-editor-raw') as HTMLTextAreaElement).value = markdown;
} else if (existingRuleId) {
rpc<{ source: string }>('getRuleSource', { ruleId: existingRuleId }).then(res => {
markdownToForm(container, res.source);
(container.querySelector('#rule-editor-raw') as HTMLTextAreaElement).value = res.source;
}).catch(() => {});
} else {
(container.querySelector('#rule-editor-raw') as HTMLTextAreaElement).value = '';
}

switchEditorTab(container, 'form');
Expand All @@ -211,7 +174,7 @@ function switchEditorTab(container: HTMLElement, tab: 'form' | 'source'): void {
formPanel.style.display = '';
sourcePanel.style.display = 'none';
} else {
(container.querySelector('#rule-editor-raw') as HTMLTextAreaElement).value = formToMarkdown(container);
markdownToForm(container, formToMarkdown(container));
formPanel.style.display = 'none';
sourcePanel.style.display = '';
}
Expand Down Expand Up @@ -252,6 +215,10 @@ export function wireRuleEditorModal(
markdown = formToMarkdown(container);
}
if (!markdown.trim()) return;
if (!hasDetectionLogic(markdown)) {
alert(`Failed to save: ${NO_DETECTION_LOGIC}`);
return;
}
const ruleId = modal.dataset.ruleId || undefined;
try {
await rpc<{ ok: boolean }>('saveRule', { markdown, ruleId });
Expand All @@ -272,6 +239,13 @@ export function wireRuleEditorModal(
markdown = formToMarkdown(container);
}
if (!markdown.trim()) return;
if (!hasDetectionLogic(markdown)) {
const results = container.querySelector<HTMLElement>('#rule-test-results')!;
results.style.display = '';
results.className = 'rule-test-results rule-test-error';
render(html`<strong>Error:</strong> ${NO_DETECTION_LOGIC}`, results);
return;
}

if (overrides && Object.keys(overrides).length > 0) {
markdown = applyThresholdOverrides(markdown, overrides);
Expand Down Expand Up @@ -360,7 +334,6 @@ export function wireRuleEditorModal(
const result = await rpc<{ markdown: string }>('generateRule', { prompt });
const md = result.markdown;
markdownToForm(container, md);
(container.querySelector('#rule-editor-raw') as HTMLTextAreaElement).value = md;
const idField = container.querySelector('#rf-id') as HTMLInputElement;
if (!idField.value) {
idField.value = prompt.toLowerCase().replaceAll(/[^a-z0-9]+/g, '-').replaceAll(/^-|-$/g, '').substring(0, 40);
Expand Down
6 changes: 2 additions & 4 deletions src/webview/page-antipatterns.ts
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ export async function renderAntiPatterns(container: HTMLElement, currentFilter:
<select id="rf-scope">
<option value="requests">Requests</option>
<option value="sessions">Sessions</option>
<option value="both">Both</option>
</select>
</div>
<div class="rule-form-field">
Expand All @@ -424,7 +425,7 @@ export async function renderAntiPatterns(container: HTMLElement, currentFilter:
</div>
<div class="rule-form-field">
<label>Examples Template</label>
<input type="text" id="rf-examples" placeholder=${"\"{{message}}...\""} />
<textarea id="rf-examples" rows="2" placeholder=${"\"{{message}}...\""}></textarea>
</div>
<div class="rule-form-field">
<label>Thresholds <span class="muted">(key: value, one per line)</span></label>
Expand All @@ -434,9 +435,6 @@ export async function renderAntiPatterns(container: HTMLElement, currentFilter:
<label>Detection Logic <span class="muted">(DSL)</span></label>
<textarea id="rf-detect" rows="8" class="rule-mono" spellcheck="false" placeholder=${"scan: requests\nmatch: messageLength > 0\naggregate: count\ncheck: count >= thresholds.myThreshold\nexamples: \"{{messageText | truncate:60}}\""}></textarea>
</div>
<textarea id="rf-patterns" style="display:none"></textarea>
<textarea id="rf-filetypes" style="display:none"></textarea>
<textarea id="rf-extra-fm" style="display:none"></textarea>
</div>

<!-- Source view -->
Expand Down
24 changes: 24 additions & 0 deletions src/webview/panel-rpc.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,30 @@ describe('generateRuleMarkdown', () => {
expect(ask).toHaveBeenCalledTimes(3);
expect(ask.mock.calls[2][0]?.reply).toBe('draft two');
});

it('accepts a reply with CRLF line endings and returns it with LF', async () => {
const ask = asker().mockResolvedValueOnce(VALID_RULE.replaceAll('\n', '\r\n'));

await expect(generateRuleMarkdown(ask)).resolves.toEqual({ markdown: VALID_RULE });
expect(ask).toHaveBeenCalledTimes(1);
});

it.each(['scope: both', 'scope: "both"'])('accepts %s', async scope => {
const rule = VALID_RULE.replace('scope: sessions', scope);
const ask = asker().mockResolvedValueOnce(rule);

await expect(generateRuleMarkdown(ask)).resolves.toEqual({ markdown: rule });
expect(ask).toHaveBeenCalledTimes(1);
});

it('asks again when the scope only starts with an allowed one', async () => {
const ask = asker()
.mockResolvedValueOnce(VALID_RULE.replace('scope: sessions', 'scope: both requests and sessions'))
.mockResolvedValueOnce(VALID_RULE);

await expect(generateRuleMarkdown(ask)).resolves.toEqual({ markdown: VALID_RULE });
expect(ask.mock.calls[1][0]?.issues).toContain('Missing or invalid "scope:" (must be requests, sessions, or both)');
});
});

describe('generateRuleMarkdown failures', () => {
Expand Down
4 changes: 2 additions & 2 deletions src/webview/panel-rpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ examples: {{workspaceName}}: {{requestCount}} messages
Output ONLY the raw markdown rule. No code fences around the whole output. No explanation.`;

function cleanRuleMarkdown(raw: string): string {
let md = raw.trim();
let md = raw.replaceAll(/\r\n?/g, '\n').trim();
// Strip outer code fences
md = md.replace(/^```(?:markdown|md)?\s*\n?/i, '').replace(/\n?```\s*$/i, '');
Comment thread
san360 marked this conversation as resolved.
Resolved
// Strip leading prose before the frontmatter
Expand All @@ -626,7 +626,7 @@ function validateRuleMarkdown(md: string): string[] {
issues.push('Missing or invalid "group:" (must be one of: prompt-quality, session-hygiene, code-review, tool-mastery, context-management)');
}
if (!md.match(/^severity:\s*(low|medium|high)/m)) issues.push('Missing or invalid "severity:" (must be low, medium, or high)');
if (!md.match(/^scope:\s*(requests|sessions)/m)) issues.push('Missing or invalid "scope:" (must be requests or sessions)');
if (!md.match(/^scope:[ \t]*(['"]?)(requests|sessions|both)\1[ \t]*$/m)) issues.push('Missing or invalid "scope:" (must be requests, sessions, or both)');
if (!md.includes('# Description')) issues.push('Missing "# Description" section');
if (!md.includes('# When Triggered')) issues.push('Missing "# When Triggered" section');
if (!md.includes('# How to Improve')) issues.push('Missing "# How to Improve" section');
Expand Down
Loading
Loading