diff --git a/graphify/__init__.py b/graphify/__init__.py index 41d4b1f3d..fbec7daed 100644 --- a/graphify/__init__.py +++ b/graphify/__init__.py @@ -6,6 +6,7 @@ def __getattr__(name): _map = { "extract": ("graphify.extract", "extract"), "collect_files": ("graphify.extract", "collect_files"), + "structural_extensions": ("graphify.extract", "structural_extensions"), "build_from_json": ("graphify.build", "build_from_json"), "cluster": ("graphify.cluster", "cluster"), "score_all": ("graphify.cluster", "score_all"), diff --git a/graphify/extract.py b/graphify/extract.py index b29caec66..cf38d2992 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -4330,6 +4330,20 @@ def add_existing_edge(edge: dict) -> None: } +def structural_extensions() -> frozenset[str]: + """Extensions with a registered structural (AST) extractor - i.e. what + extract() can walk deterministically, with no LLM. + + Callers that need to decide "does this file qualify for structural + extraction" (the skill's Part A, `graphify update`, etc.) should call this + instead of hand-maintaining a second extension list: a duplicate list is + exactly the drift hazard `graphify.ids` was written to close for node-ID + recipes (#811, #550, #1033, #1104) - a new extractor's extension would + silently stay LLM-only here until someone remembered to update the copy. + """ + return frozenset(_DISPATCH) + + # Extensions whose extractor depends on an optional-dependency extra # (pyproject [project.optional-dependencies]) and hard-fails without it, # rather than falling back like Pascal does. Used by the #1745 warning in diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index 190827d9a..4c9d295fa 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-aider.md b/graphify/skill-aider.md index 4996beb78..688cdc25a 100644 --- a/graphify/skill-aider.md +++ b/graphify/skill-aider.md @@ -183,29 +183,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index 190827d9a..4c9d295fa 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index abd2811d2..59a8a0762 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index af3f723c7..3703e2f41 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index abd2811d2..59a8a0762 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-devin.md b/graphify/skill-devin.md index f9be846cb..3055bf1c8 100644 --- a/graphify/skill-devin.md +++ b/graphify/skill-devin.md @@ -196,29 +196,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index fd148d485..3fcdd636f 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 3e70b050a..dce93a8e7 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index abd2811d2..59a8a0762 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 91ced6067..c7a714633 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index abd2811d2..59a8a0762 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 050667bc2..96e2dd92f 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 20c7c0835..f8612ce65 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index d631821ec..a3006eff1 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -189,29 +189,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```powershell @' import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') '@ | & (Get-Content graphify-out\.graphify_python) - ``` diff --git a/graphify/skill.md b/graphify/skill.md index abd2811d2..59a8a0762 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/graphify/skills/agents/references/extraction-spec.md b/graphify/skills/agents/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/agents/references/extraction-spec.md +++ b/graphify/skills/agents/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index 388df7674..a22df1736 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index d3629a73c..d45a8acf5 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -82,6 +82,51 @@ def test_update_without_root_still_preserves_hyperedges(tmp_path): assert "he_b" not in ids +def test_stub_node_source_file_pointing_at_unextracted_file_prunes_it(tmp_path): + """Documents a real hazard, not a bug to fix here: replace-on-re-extract keys + off ANY node's source_file, with no concept of "this file was actually read + this run" versus "this node merely cites that file." A semantic subagent + that mislabels a whole-file reference/stub node's source_file as the file it + CITES (rather than the file it was extracting FROM) makes build_merge treat + the cited file as re-extracted and prune its real content - silently, on an + incremental update, for a file nobody actually re-read. + + This is exactly the failure this repo's extraction-spec.md now warns + against by name (the "Whole-file nodes" section's incremental-update + caveat) after it destroyed 293/460 nodes of a real docs corpus: updating + two changed files whose extraction created citation stub nodes for + ARCHITECTURE.md/DATABASE.md/etc. with THOSE files' own paths as + source_file, rather than the changed files' own paths, pruned every + untouched file mentioned. The fix lives in the prompt (never emit a node + whose source_file isn't your own FILE_LIST entry); this test exists so a + future change to build_merge's replace semantics doesn't accidentally + make the hazard worse without anyone noticing, and so the mechanism is + verifiable in code rather than only in prose. + """ + root, graph_path = _seed_two_file_graph(tmp_path) + # b.md is "re-extracted" this run, but its extraction wrongly stamps a stub + # node standing in for a.md with a.md's OWN path as source_file (the bug), + # instead of b.md's path (the correct rule). + new_chunk = { + "nodes": [ + {"id": "b1", "label": "b1", "file_type": "document", "source_file": "b.md"}, + {"id": "a1", "label": "A (cited)", "file_type": "document", "source_file": "a.md"}, + ], + "edges": [{"source": "b1", "target": "a1", "relation": "references", + "confidence": "EXTRACTED", "source_file": "b.md"}], + "hyperedges": [], + } + G = build_merge([new_chunk], graph_path, dedup=False, root=root) + # a.md was never actually re-extracted, but its real node was pruned anyway - + # replaced by the stub's thin re-statement. This is the data loss the spec + # rule prevents by keeping stub nodes' source_file on the citing file. + assert G.nodes["a1"]["label"] == "A (cited)" # NOT the original a.md content + # a.md's own hyperedge (he_a) is gone too, wrongly treated as re-extracted; + # only the source_file-less global hyperedge (never subject to replacement) + # and b.md's (legitimately re-extracted, if it had a hyperedge) could survive. + assert _he_ids(G) == {"he_global"} + + def test_deleted_file_hyperedges_are_pruned(tmp_path): root, graph_path = _seed_two_file_graph(tmp_path) deleted_abs = [str(root / "a.md")] diff --git a/tests/test_extraction_spec_ids.py b/tests/test_extraction_spec_ids.py index 46fabc3ba..c7e2ab713 100644 --- a/tests/test_extraction_spec_ids.py +++ b/tests/test_extraction_spec_ids.py @@ -30,6 +30,10 @@ # the examples never leaks in. _EXAMPLE_RE = re.compile(r"`([^`]+)`\s*\+\s*`([^`]+)`\s*→\s*`([^`]+)`") +# `path.md` → `id` — the whole-file-node convention (no `+`/entity group, so this +# never overlaps with `_EXAMPLE_RE` above). +_WHOLE_FILE_EXAMPLE_RE = re.compile(r"`([\w./-]+\.md)`\s*→\s*`([a-z0-9_]+)`") + def _spec_files() -> list[Path]: roots = [REPO_ROOT / "graphify" / "skills", REPO_ROOT / "tools" / "skillgen" / "fragments"] @@ -59,6 +63,23 @@ def _ast_symbol_id(path: str, entity: str) -> str: return _make_id(_file_stem(Path(path)), entity) +def _ast_file_id(path: str) -> str: + """Reproduce the AST extractor's whole-file node ID (no entity part) - what + `extract_markdown`'s `file_nid` canonicalizes to after extract()'s id-remap + post-pass, and what a semantic subagent must independently reproduce for a + whole-file node to merge instead of splitting (#1033-shaped bug class).""" + return _make_id(_file_stem(Path(path))) + + +def _whole_file_examples() -> list[tuple[Path, str, str]]: + out: list[tuple[Path, str, str]] = [] + for f in _spec_files(): + text = f.read_text(encoding="utf-8") + for path, expected in _WHOLE_FILE_EXAMPLE_RE.findall(text): + out.append((f, path, expected)) + return out + + def test_spec_files_are_discoverable(): """Guard the guard: if the spec moves or the example format changes so nothing parses, fail loudly rather than passing vacuously.""" @@ -94,3 +115,39 @@ def test_cautionary_wrong_forms_are_actually_wrong(): # are both wrong now that the stem is the full repo-relative path (#1504). assert _make_id("session", "ValidateToken") != correct assert _make_id("auth", "session", "ValidateToken") != correct + + +def test_whole_file_node_examples_are_discoverable(): + """Guard the guard, for the whole-file-node convention specifically: if the + spec wording changes so the example no longer parses, fail loudly.""" + examples = _whole_file_examples() + assert examples, ( + "no whole-file-node ID examples found across host specs — did the " + "'Whole-file nodes' wording change?" + ) + + +@pytest.mark.parametrize( + "path,expected", + [(p, x) for (_f, p, x) in _whole_file_examples()], + ids=[f"{f.parent.parent.name}:{p}" for (f, p, x) in _whole_file_examples()], +) +def test_whole_file_node_id_examples_match_ast_extractor(path, expected): + got = _ast_file_id(path) + assert got == expected, ( + f"whole-file node-ID spec drift: spec says `{path}` → `{expected}`, but " + f"extract._make_id(_file_stem(...)) produces `{got}`. Update the spec " + f"example and the ID functions together." + ) + + +def test_whole_file_node_id_has_no_document_suffix(): + """Regression test for the exact split-node bug this spec section closes: two + subagents extracting the same doc suite in parallel independently produced + `docs_architecture` and `docs_architecture_document` for one real file, + because nothing told either of them what a whole-file node's ID should be. + The correct, single ID is the bare stem — the same recipe as a symbol ID + with the entity part simply omitted, never a `_document`/`_file` suffix.""" + correct = _ast_file_id("docs/ARCHITECTURE.md") + assert correct == "docs_architecture" + assert correct != "docs_architecture_document" diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index cf116869f..901fe613a 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -109,7 +109,7 @@ def test_lean_core_runs_default_pipeline_with_zero_references(): "### Step 1 - Ensure graphify is installed", "### Step 2 - Detect files", "### Step 3 - Extract entities and relationships", - "#### Part A - Structural extraction for code files", + "#### Part A - Structural extraction for code and structurally-supported docs", "#### Part C - Merge AST + semantic into final extraction", "### Step 4 - Build graph, cluster, analyze, generate outputs", "### Step 5 - Label communities", @@ -363,7 +363,7 @@ def test_every_platform_query_has_expansion_and_fallback(): "### Step 1 - Ensure graphify is installed", "### Step 2 - Detect files", "### Step 3 - Extract entities and relationships", - "#### Part A - Structural extraction for code files", + "#### Part A - Structural extraction for code and structurally-supported docs", "#### Part B - Semantic extraction (parallel subagents)", "#### Part C - Merge AST + semantic into final extraction", "### Step 4 - Build graph, cluster, analyze, generate outputs", @@ -886,7 +886,7 @@ def test_audit_allowlist_documents_only_consolidations(): A genuine drop (trae's native AGENTS.md integration) must never be in the allowlist, or the guard would rubber-stamp the regression it exists to catch. """ - all_allowlisted = set(gen.SHARED_INTRO_ALLOWLIST) + all_allowlisted = set(gen.SHARED_CORE_ALLOWLIST) for hs in gen._CONSOLIDATION_ALLOWLIST.values(): all_allowlisted |= set(hs) assert "## For native AGENTS.md integration (Trae)" not in all_allowlisted diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 190827d9a..4c9d295fa 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 4996beb78..688cdc25a 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -183,29 +183,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 190827d9a..4c9d295fa 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index abd2811d2..59a8a0762 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index af3f723c7..3703e2f41 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index abd2811d2..59a8a0762 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index f9be846cb..3055bf1c8 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -196,29 +196,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index fd148d485..3fcdd636f 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 3e70b050a..dce93a8e7 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index abd2811d2..59a8a0762 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 91ced6067..c7a714633 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index abd2811d2..59a8a0762 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 050667bc2..96e2dd92f 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 20c7c0835..f8612ce65 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index d631821ec..a3006eff1 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -189,29 +189,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```powershell @' import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding="utf-8")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8") print(f'AST: {len(result["nodes"])} nodes, {len(result["edges"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding="utf-8") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') '@ | & (Get-Content graphify-out\.graphify_python) - ``` diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index abd2811d2..59a8a0762 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -167,29 +167,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 4b278b28d..93a7251fe 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 4996beb78..688cdc25a 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -183,29 +183,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index c527a1256..ee26e8d64 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -126,29 +126,51 @@ Print it once, then continue — do not wait for the user to supply a key. If `G Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. + +This matters beyond the free structure: it gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention. Skipping this step is how a +real corpus produced `docs_architecture` and `docs_architecture_document` as two +separate nodes for one file — two subagents extracting the same doc suite in +parallel, with nothing upstream having already decided the canonical ID, guessed +differently. Part C's merge already dedupes AST + semantic nodes by id, so once +Part A mints the doc's node first, that dedup closes the gap for free. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files, cache_root=Path('INPUT_PATH')) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files, cache_root=Path('INPUT_PATH')) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\") print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\") - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index f9be846cb..3055bf1c8 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -196,29 +196,45 @@ This step has two parts: **structural extraction** (deterministic, free) and **s Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers. -#### Part A - Structural extraction for code files - -For any code files detected, run AST extraction in parallel with Part B subagents: +#### Part A - Structural extraction for code and structurally-supported docs + +Run AST extraction in parallel with Part B subagents, over every code file plus any +document/paper file whose extension has a registered structural extractor +(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that +mints one whole-file node and one node per heading, no LLM involved). Doc files +without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected +and still go through Part B only. This also gives every Part B subagent a +pre-existing, deterministic ID for "the node representing this file" to reference, +instead of each subagent inventing its own convention and splitting one file into +two disconnected nodes. ```bash $(cat graphify-out/.graphify_python) -c " import sys, json -from graphify.extract import collect_files, extract +from graphify.extract import collect_files, extract, structural_extensions from pathlib import Path import json -code_files = [] +exts = structural_extensions() +structural_files = [] detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) for f in detect.get('files', {}).get('code', []): - code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) - -if code_files: - result = extract(code_files) + structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)]) +for cat in ('document', 'paper'): + for f in detect.get('files', {}).get(cat, []): + p = Path(f) + if p.is_dir(): + structural_files.extend(x for x in collect_files(p) if x.suffix in exts) + elif p.suffix in exts: + structural_files.append(p) + +if structural_files: + result = extract(structural_files) Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2)) print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges') else: Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0})) - print('No code files - skipping AST extraction') + print('No structurally-supported files - skipping AST extraction') " ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 4b278b28d..93a7251fe 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -15,6 +15,7 @@ Rules: - AMBIGUOUS: uncertain — flag it, do not omit - Code files: semantic edges AST cannot find. Do not re-extract imports. When adding `calls` edges: source is the caller, target is the callee, never reversed; keep `calls` within one language. - Doc/paper files: named concepts, entities, citations. Store rationale (WHY decisions were made) as a `rationale` attribute on the relevant node, not as a separate node. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms) and `file_type:"concept"` for named concepts. `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +- Doc/paper files: `references`/`cites` edges between whole-file document nodes go citer→citee (source = the file in YOUR FILE_LIST where you found the mention, target = the file cited), never reversed. If an edge's `source_file` matches its `target` node's file instead of its `source` node's file, the direction is inverted — fix it. - Image files: use vision — understand what the image IS, not just OCR - DEEP_MODE (if --mode deep): be aggressive with INFERRED edges — indirect deps, shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting. - Semantic similarity: if two concepts solve the same problem or represent the same idea without a structural link (no import, call, or citation), add a `semantically_similar_to` edge marked INFERRED with confidence_score 0.6-0.95. Non-obvious cross-file links only. @@ -22,7 +23,7 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Whole-file nodes (representing an entire file, including one you only cite, not extract) use the bare stem with no entity and no suffix: `docs/ARCHITECTURE.md` → `docs_architecture`, never `docs_architecture_document` — this must also merge into whatever a structural extractor already minted for that file. The source_file RULE below still applies unchanged even to a whole-file node standing in for a file you only cite — never point its source_file at that other file (an incremental --update would read that as proof the file was re-extracted and prune its real content). Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index 388df7674..a22df1736 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -17,6 +17,7 @@ Rules: Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns). Do not re-extract imports - AST already has those. Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected. +Doc/paper files: when adding a `references`/`cites` edge between two whole-file document nodes (e.g. a spec citing a companion spec, or an ADR citing a product-requirements section), source MUST be the file doing the citing — the file in YOUR CURRENT FILE_LIST where you found the mention — target MUST be the file or section being cited. Never reverse this direction, mirroring the `calls` rule below. Self-check: the edge's own `source_file` should equal (or be a location within) the `source` node's file; if `source_file` instead matches the `target` node's file, the direction is almost certainly inverted. Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them. Image files: use vision to understand what the image IS - do not just OCR. UI screenshot: layout patterns, design decisions, key elements, purpose. @@ -60,11 +61,14 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Whole-file nodes: when you need a node ID that represents an entire file — a file in your own FILE_LIST, or another file you are only citing — use the bare stem with no entity part: `{stem}` (e.g. `docs/ARCHITECTURE.md` → `docs_architecture`). Never append `_document`, `_file`, or any other suffix, and never invent a different form for "the file itself" than you would for a symbol inside it. This is also the exact ID a structural extractor mints for that file when one exists (currently Markdown) — it must merge into that node, not create a second one. Skipping this is how a real corpus once produced two disconnected nodes, `docs_architecture` and `docs_architecture_document`, for a single file. The **source_file RULE below still applies unchanged to a whole-file node representing a file you are only citing** — do NOT set its source_file to that other file's path; an incremental `--update` treats any node's source_file as proof that file was re-extracted this run and prunes its prior content, so pointing a stub node's source_file at a file you didn't actually read would silently delete that file's real nodes on the next update. + Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0} source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. + Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH ``` diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 09e19ede0..c0e6a6da7 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -197,22 +197,31 @@ def _v8_baseline_ref(platform_key: str) -> str: "agents-md": "AGENTS.md", } -# Allowlist for the per-host coverage audit (waves 2-3 consolidations). +# Allowlist for the per-host coverage audit (waves 2-3 consolidations, and later +# shared-core heading renames). # # The lean core is one shared template across every split host, so a few v8 # headings deliberately do NOT survive verbatim in a given host's render. These # are intentional consolidations, not content drops, and the audit must not flag -# them. Two classes: +# them. Three classes: # -# 1. SHARED_INTRO_ALLOWLIST — the lean intro consolidation. "## What graphify is -# for" is the lean intro the core carries; the minimal v8 bodies (kilo, vscode) -# had verbose intro prose with no such heading, while the richer v8 bodies -# already had it. Listing it documents the wave-2/3 intro consolidation; it -# single-homes in every render, so it is never itself a coverage hole. The enum -# unification (Decision A) is prose, not a heading, and is guarded separately by -# schema-singleton. +# 1. Intro consolidation — "## What graphify is for" is the lean intro the core +# carries; the minimal v8 bodies (kilo, vscode) had verbose intro prose with no +# such heading, while the richer v8 bodies already had it. Listing it documents +# the wave-2/3 intro consolidation; it single-homes in every render, so it is +# never itself a coverage hole. The enum unification (Decision A) is prose, not +# a heading, and is guarded separately by schema-singleton. # -# 2. _CONSOLIDATION_ALLOWLIST[host] — per-host v8 headings the shared lean core +# 2. Shared-core heading renames — a heading in the shared core is later reworded +# (not removed; the section and its content stay, just under a new name) for +# every split host at once. E.g. "Part A - Structural extraction for code +# files" -> "...for code and structurally-supported docs" when Part A grew to +# also cover doc files with a registered structural extractor. +# +# SHARED_CORE_ALLOWLIST holds both classes 1 and 2 (they apply uniformly across +# every split host). _CONSOLIDATION_ALLOWLIST[host] below is class 3. +# +# 3. _CONSOLIDATION_ALLOWLIST[host] — per-host v8 headings the shared lean core # re-homes under a reworded or re-leveled heading while preserving (or # enriching) the content. The two minimal v8 bodies, kilo (414 L) and vscode # (258 L), are the only hosts affected: the shared core is a richer superset @@ -223,9 +232,11 @@ def _v8_baseline_ref(platform_key: str) -> str: # trae's native AGENTS.md integration) still fails loudly. # # Adding a heading here is a deliberate, reviewed act: it asserts "this v8 -# heading was consolidated on purpose and its content is covered elsewhere." -SHARED_INTRO_ALLOWLIST: frozenset[str] = frozenset({ +# heading was consolidated/renamed on purpose and its content is covered +# elsewhere under the new heading." +SHARED_CORE_ALLOWLIST: frozenset[str] = frozenset({ "## What graphify is for", # lean intro; v8 hosts had verbose intro prose, no heading. + "#### Part A - Structural extraction for code files", # renamed; Part A now also covers docs. }) _CONSOLIDATION_ALLOWLIST: dict[str, frozenset[str]] = { @@ -258,7 +269,7 @@ def _v8_baseline_ref(platform_key: str) -> str: def _audit_allowlist(platform_key: str) -> frozenset[str]: """The full set of v8 headings the audit may skip for this host.""" - return SHARED_INTRO_ALLOWLIST | _CONSOLIDATION_ALLOWLIST.get(platform_key, frozenset()) + return SHARED_CORE_ALLOWLIST | _CONSOLIDATION_ALLOWLIST.get(platform_key, frozenset()) @dataclass(frozen=True) @@ -1142,6 +1153,57 @@ def _is_community_label_export_fix_line(line: str) -> bool: ) +def _is_structural_docs_fix_line(line: str) -> bool: + """Whether a line belongs to the Part A doc-structural-extraction change. + + Part A used to structurally extract (AST, no LLM) only code files. It now + also covers document/paper files whose extension has a registered + structural extractor (currently Markdown), so every Part B subagent gets a + pre-existing, deterministic ID for "the node representing this file" to + reference instead of inventing its own convention. Skipping this is how a + real corpus produced two disconnected nodes, `docs_architecture` and + `docs_architecture_document`, for one file. Both the old code-only body + (removed) and the new code-plus-docs body (added) are listed here, along + with the renamed heading (also allowlisted in SHARED_CORE_ALLOWLIST for the + split-host coverage audit). + """ + stripped = line.strip() + return stripped in { + "#### Part A - Structural extraction for code and structurally-supported docs", + "#### Part A - Structural extraction for code files", + "Run AST extraction in parallel with Part B subagents, over every code file plus any", + "document/paper file whose extension has a registered structural extractor", + "(currently `.md`/`.mdx`/`.qmd`/`.skill`, via a deterministic Markdown parser that", + "mints one whole-file node and one node per heading, no LLM involved). Doc files", + "without a registered extractor (`.txt`, `.rst`, `.html`, `.pdf`, ...) are unaffected", + "and still go through Part B only. This also gives every Part B subagent a", + 'pre-existing, deterministic ID for "the node representing this file" to reference,', + "instead of each subagent inventing its own convention and splitting one file into", + "two disconnected nodes.", + "For any code files detected, run AST extraction in parallel with Part B subagents:", + "from graphify.extract import collect_files, extract, structural_extensions", + "from graphify.extract import collect_files, extract", + "exts = structural_extensions()", + "structural_files = []", + "code_files = []", + "structural_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)])", + "code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)])", + "for cat in ('document', 'paper'):", + "for f in detect.get('files', {}).get(cat, []):", + "p = Path(f)", + "if p.is_dir():", + "structural_files.extend(x for x in collect_files(p) if x.suffix in exts)", + "elif p.suffix in exts:", + "structural_files.append(p)", + "if structural_files:", + "if code_files:", + "result = extract(structural_files)", + "result = extract(code_files)", + "print('No structurally-supported files - skipping AST extraction')", + "print('No code files - skipping AST extraction')", + } + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -1163,6 +1225,7 @@ def _is_community_label_export_fix_line(line: str) -> bool: _is_uv_from_interpreter_fix_line, _is_semantic_cache_scope_fix_line, _is_community_label_export_fix_line, + _is_structural_docs_fix_line, )