diff --git a/docs/_docset.yml b/docs/_docset.yml index 33dfda75e4..8861722d3b 100644 --- a/docs/_docset.yml +++ b/docs/_docset.yml @@ -181,6 +181,7 @@ toc: children: - file: installation.md - file: shell-autocompletion.md + - file: docset/list-dependents.md - folder: schema-support children: - file: index.md diff --git a/docs/cli/docset/list-dependents.md b/docs/cli/docset/list-dependents.md new file mode 100644 index 0000000000..62931ae058 --- /dev/null +++ b/docs/cli/docset/list-dependents.md @@ -0,0 +1,63 @@ +# list-dependents + +Resolves the markdown pages that transitively include the given files. + +A page-level dependent is any non-snippet markdown file that — directly or through a +chain of `_snippets/`-to-`_snippets/` includes — pulls in the input file via an +`{{include}}` or `{{csv-include}}` directive. + +This is intended for the docs preview workflow: when a PR only edits snippets or +CSV data, the preview comment would otherwise have no specific pages to link to. +Feeding the changed files through `list-dependents` produces the list of pages +that would re-render, so the comment can point at them instead. + +## Usage + +```bash +docs-builder list-dependents --files [options...] [-h|--help] [--version] +``` + +## Options + +`--files ` +: Comma-separated list of file paths to resolve dependents for. Paths can be + git-relative (for example `docs/_snippets/foo.md`) or absolute. + +`-p|--path ` +: Defaults to the `{pwd}/docs` folder (optional) + +`--format ` +: Output format: `json` (default) or `text`. + +## Output + +`json` (default) emits a single object on stdout: + +```json +{ + "results": [ + { + "input": "docs/_snippets/applies-switch.md", + "resolved": "_snippets/applies-switch.md", + "found": true, + "reason": null, + "dependents": [ + "testing/index.md" + ] + }, + { + "input": "docs/_snippets/missing.md", + "resolved": "_snippets/missing.md", + "found": false, + "reason": "no consumers found", + "dependents": [] + } + ] +} +``` + +`text` emits a human-readable summary, one input per block. + +An entry with `found: false` means either the file has no consumers in the +documentation set, or its path resolves outside the documentation source +directory. diff --git a/src/Elastic.Markdown/IO/IncludeGraph.cs b/src/Elastic.Markdown/IO/IncludeGraph.cs new file mode 100644 index 0000000000..ea813afb4b --- /dev/null +++ b/src/Elastic.Markdown/IO/IncludeGraph.cs @@ -0,0 +1,123 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Collections.Concurrent; +using System.Collections.Frozen; +using Elastic.Markdown.Myst.Directives.CsvInclude; +using Elastic.Markdown.Myst.Directives.Include; +using Markdig.Syntax; + +namespace Elastic.Markdown.IO; + +/// +/// Reverse-dependency lookup over {{include}} and {{csv-include}} directives +/// across a . Resolves "which pages would re-render if this +/// snippet/CSV changes?" — used by the preview workflow to surface meaningful preview URLs +/// when only non-.md files (or files under _snippets/) are edited. +/// +public sealed class IncludeGraph +{ + private readonly FrozenDictionary> _consumersByTarget; + private readonly FrozenSet _snippetPaths; + + private IncludeGraph( + FrozenDictionary> consumersByTarget, + FrozenSet snippetPaths) + { + _consumersByTarget = consumersByTarget; + _snippetPaths = snippetPaths; + } + + public static async Task BuildAsync(DocumentationSet set, Cancel ctx) + { + var edges = new ConcurrentBag<(string Target, string Consumer)>(); + + // Pages live in DocumentationSet.MarkdownFiles; snippets are stored separately in + // DocumentationSet.Files as SnippetFile records. Both can host {{include}} directives, + // so we parse both to capture page→snippet AND snippet→snippet edges. + var pages = set.MarkdownFiles.Cast(); + var snippets = set.Files.Values.OfType().Cast(); + + await Parallel.ForEachAsync(pages.Concat(snippets), ctx, async (file, token) => + { + var document = file switch + { + MarkdownFile md => await md.MinimalParseAsync(set.TryFindDocumentByRelativePath, token), + SnippetFile sn => await set.MarkdownParser.MinimalParseAsync(sn.SourceFile, token), + _ => null + }; + if (document is null) + return; + + var consumer = NormalizePath(file.RelativePath); + + foreach (var include in document.Descendants()) + { + if (include is { Found: true, IncludePathRelativeToSource: { } target }) + edges.Add((NormalizePath(target), consumer)); + } + + foreach (var csv in document.Descendants()) + { + if (csv is { Found: true, CsvFilePathRelativeToSource: { } target }) + edges.Add((NormalizePath(target), consumer)); + } + }).ConfigureAwait(false); + + var consumersByTarget = edges + .GroupBy(e => e.Target, StringComparer.OrdinalIgnoreCase) + .ToFrozenDictionary( + g => g.Key, + g => g.Select(e => e.Consumer).ToFrozenSet(StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); + + var snippetPaths = set.Files.Values + .OfType() + .Select(f => NormalizePath(f.RelativePath)) + .ToFrozenSet(StringComparer.OrdinalIgnoreCase); + + return new IncludeGraph(consumersByTarget, snippetPaths); + } + + /// + /// Walks consumers of upward through any intermediate + /// snippet-includes-snippet hops and returns the set of *page-level* dependents (non-snippet + /// markdown files). The input itself is not included in the result. + /// + public IReadOnlySet ResolvePageDependents(string targetRelativePath) + { + var pages = new HashSet(StringComparer.OrdinalIgnoreCase); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var queue = new Queue(); + queue.Enqueue(NormalizePath(targetRelativePath)); + + while (queue.Count > 0) + { + var current = queue.Dequeue(); + if (!_consumersByTarget.TryGetValue(current, out var consumers)) + continue; + + foreach (var consumer in consumers) + { + if (!seen.Add(consumer)) + continue; + if (_snippetPaths.Contains(consumer)) + queue.Enqueue(consumer); + else + _ = pages.Add(consumer); + } + } + + return pages; + } + + /// + /// Returns true if the graph has at least one consumer recorded for the given target. + /// Lets callers distinguish "snippet exists but is unused" from "snippet path unknown". + /// + public bool HasConsumers(string targetRelativePath) => + _consumersByTarget.ContainsKey(NormalizePath(targetRelativePath)); + + private static string NormalizePath(string path) => path.Replace('\\', '/'); +} diff --git a/src/authoring/Elastic.Documentation.Refactor/Tracking/ListDependentsService.cs b/src/authoring/Elastic.Documentation.Refactor/Tracking/ListDependentsService.cs new file mode 100644 index 0000000000..2396b5bf25 --- /dev/null +++ b/src/authoring/Elastic.Documentation.Refactor/Tracking/ListDependentsService.cs @@ -0,0 +1,128 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Text.Json; +using System.Text.Json.Serialization; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Extensions; +using Elastic.Documentation.Links.CrossLinks; +using Elastic.Documentation.Services; +using Elastic.Markdown.IO; +using Microsoft.Extensions.Logging; +using Nullean.ScopedFileSystem; + +namespace Elastic.Documentation.Refactor.Tracking; + +public class ListDependentsService( + ILoggerFactory logFactory, + IConfigurationContext configurationContext +) : IService +{ + private readonly ILogger _logger = logFactory.CreateLogger(); + + public async Task ListDependents( + IDiagnosticsCollector collector, + ScopedFileSystem fs, + string? path, + IReadOnlyList files, + string format, + Cancel ctx) + { + if (files.Count == 0) + { + collector.EmitGlobalError("list-dependents requires at least one file argument."); + return false; + } + + if (!format.Equals("json", StringComparison.OrdinalIgnoreCase) && + !format.Equals("text", StringComparison.OrdinalIgnoreCase)) + { + collector.EmitGlobalError($"Unsupported format '{format}'. Expected 'json' or 'text'."); + return false; + } + + var context = new BuildContext(collector, fs, fs, configurationContext, ExportOptions.MetadataOnly, path, null); + var set = new DocumentationSet(context, logFactory, NoopCrossLinkResolver.Instance); + + var graph = await IncludeGraph.BuildAsync(set, ctx); + + var sourceDir = context.DocumentationSourceDirectory; + var gitRoot = Paths.FindGitRoot(sourceDir); + + var results = files + .Select(input => ResolveOne(input, sourceDir.FullName, gitRoot?.FullName, graph)) + .ToArray(); + + var output = format.Equals("text", StringComparison.OrdinalIgnoreCase) + ? RenderText(results) + : RenderJson(results); + Console.Out.WriteLine(output); + + var totalDependents = results.Sum(r => r.Dependents.Count); + _logger.LogInformation( + "Resolved {Inputs} input(s) → {Pages} page dependents", results.Length, totalDependents); + return true; + } + + private static DependentsResult ResolveOne(string input, string sourceDirFullName, string? gitRootFullName, IncludeGraph graph) + { + var absolute = Path.IsPathRooted(input) + ? input + : gitRootFullName is not null + ? Path.GetFullPath(Path.Combine(gitRootFullName, input)) + : Path.GetFullPath(input); + + var sourceRelative = Path.GetRelativePath(sourceDirFullName, absolute); + if (sourceRelative.StartsWith("..", StringComparison.Ordinal)) + { + return new DependentsResult(input, sourceRelative.OptionalWindowsReplace(), false, "outside documentation source", []); + } + + var normalized = sourceRelative.Replace('\\', '/'); + if (!graph.HasConsumers(normalized)) + return new DependentsResult(input, normalized, false, "no consumers found", []); + + var dependents = graph.ResolvePageDependents(normalized).OrderBy(p => p, StringComparer.OrdinalIgnoreCase).ToArray(); + return new DependentsResult(input, normalized, true, null, dependents); + } + + private static string RenderJson(IReadOnlyList results) + { + var payload = new DependentsPayload(results); + return JsonSerializer.Serialize(payload, ListDependentsJsonContext.Default.DependentsPayload); + } + + private static string RenderText(IReadOnlyList results) + { + var sb = new System.Text.StringBuilder(); + foreach (var r in results) + { + if (!r.Found) + { + _ = sb.Append(r.Input).Append(" → no dependents (").Append(r.Reason ?? "unknown").AppendLine(")"); + continue; + } + _ = sb.Append(r.Input).Append(" → ").Append(r.Dependents.Count).AppendLine(" dependent page(s):"); + foreach (var dep in r.Dependents) + _ = sb.Append(" ").AppendLine(dep); + } + return sb.ToString().TrimEnd(); + } +} + +public sealed record DependentsPayload( + [property: JsonPropertyName("results")] IReadOnlyList Results); + +public sealed record DependentsResult( + [property: JsonPropertyName("input")] string Input, + [property: JsonPropertyName("resolved")] string Resolved, + [property: JsonPropertyName("found")] bool Found, + [property: JsonPropertyName("reason")] string? Reason, + [property: JsonPropertyName("dependents")] IReadOnlyList Dependents); + +[JsonSourceGenerationOptions(WriteIndented = true)] +[JsonSerializable(typeof(DependentsPayload))] +internal sealed partial class ListDependentsJsonContext : JsonSerializerContext; diff --git a/src/tooling/docs-builder/Commands/ListDependentsCommand.cs b/src/tooling/docs-builder/Commands/ListDependentsCommand.cs new file mode 100644 index 0000000000..ced89a7d79 --- /dev/null +++ b/src/tooling/docs-builder/Commands/ListDependentsCommand.cs @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.Refactor.Tracking; +using Elastic.Documentation.Services; +using Microsoft.Extensions.Logging; +using Nullean.Argh; + +namespace Documentation.Builder.Commands; + +internal sealed class ListDependentsCommand( + ILoggerFactory logFactory, + IDiagnosticsCollector collector, + IConfigurationContext configurationContext +) +{ + /// Lists the markdown pages that transitively include the given files. + /// + /// + /// Resolves snippets, CSVs, or other inputs to {{include}} / {{csv-include}} + /// directives to the non-snippet pages that pull them in, following transitive chains. + /// Intended for the docs preview workflow: when a PR only edits non-page files, feed the + /// changed files through this command to get the page URLs to link in the preview comment. + /// + /// + /// Comma-separated list of file paths (git-relative or absolute) to resolve dependents for. + /// -p, Documentation source directory. Defaults to the cwd/docs folder. + /// Output format: json (default) or text. + [CommandName("list-dependents")] + public async Task ListDependents( + GlobalCliOptions _, + string files, + string? path = null, + string format = "json", + CancellationToken ct = default + ) + { + await using var serviceInvoker = new ServiceInvoker(collector); + + var fileList = files.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var service = new ListDependentsService(logFactory, configurationContext); + var fs = FileSystemFactory.RealGitRootForPath(path); + + serviceInvoker.AddCommand(service, (path, fs, fileList, format), + async static (s, collector, state, ctx) => await s.ListDependents( + collector, state.fs, state.path, state.fileList, state.format, ctx) + ); + return await serviceInvoker.InvokeAsync(ct); + } +} diff --git a/src/tooling/docs-builder/Program.cs b/src/tooling/docs-builder/Program.cs index 98eb0bc178..cbb417bf19 100644 --- a/src/tooling/docs-builder/Program.cs +++ b/src/tooling/docs-builder/Program.cs @@ -51,6 +51,7 @@ _ = app.Map(); _ = app.MapNamespace("changelog"); _ = app.MapNamespace("inbound-links"); + _ = app.Map(); _ = app.Map(); diff --git a/tests/Elastic.Markdown.Tests/FileInclusion/IncludeGraphTests.cs b/tests/Elastic.Markdown.Tests/FileInclusion/IncludeGraphTests.cs new file mode 100644 index 0000000000..68878b1aa7 --- /dev/null +++ b/tests/Elastic.Markdown.Tests/FileInclusion/IncludeGraphTests.cs @@ -0,0 +1,116 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using AwesomeAssertions; +using Elastic.Documentation.Configuration; +using Elastic.Markdown.IO; +using Nullean.ScopedFileSystem; + +namespace Elastic.Markdown.Tests.FileInclusion; + +public class IncludeGraphTests(ITestOutputHelper output) +{ + [Fact] + public async Task ResolvesDirectAndTransitiveDependents() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["docs/docset.yml"] = new(""" + project: test + toc: + - file: index.md + - file: page-a.md + - file: page-b.md + - file: page-c.md + """), + ["docs/index.md"] = new("# Home"), + ["docs/page-a.md"] = new(""" + # Page A + + :::{include} _snippets/foo.md + ::: + """), + ["docs/page-b.md"] = new(""" + # Page B + + :::{include} _snippets/bar.md + ::: + """), + ["docs/page-c.md"] = new("# Page C with no includes"), + ["docs/_snippets/foo.md"] = new("foo content"), + ["docs/_snippets/bar.md"] = new(""" + :::{include} /_snippets/foo.md + ::: + """) + }, new MockFileSystemOptions { CurrentDirectory = Paths.WorkingDirectoryRoot.FullName }); + + var graph = await BuildGraph(fileSystem); + + var fooDependents = graph.ResolvePageDependents("_snippets/foo.md"); + fooDependents.Should().BeEquivalentTo("page-a.md", "page-b.md"); + + var barDependents = graph.ResolvePageDependents("_snippets/bar.md"); + barDependents.Should().BeEquivalentTo("page-b.md"); + + graph.HasConsumers("_snippets/foo.md").Should().BeTrue(); + graph.HasConsumers("_snippets/bar.md").Should().BeTrue(); + } + + [Fact] + public async Task UnreferencedSnippetReportsNoConsumers() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["docs/docset.yml"] = new(""" + project: test + toc: + - file: index.md + """), + ["docs/index.md"] = new("# Home"), + ["docs/_snippets/orphan.md"] = new("nobody includes me") + }, new MockFileSystemOptions { CurrentDirectory = Paths.WorkingDirectoryRoot.FullName }); + + var graph = await BuildGraph(fileSystem); + + graph.HasConsumers("_snippets/orphan.md").Should().BeFalse(); + graph.ResolvePageDependents("_snippets/orphan.md").Should().BeEmpty(); + } + + [Fact] + public async Task ResolvesCsvIncludeDependents() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["docs/docset.yml"] = new(""" + project: test + toc: + - file: index.md + - file: data-page.md + """), + ["docs/index.md"] = new("# Home"), + ["docs/data-page.md"] = new(""" + # Data Page + + :::{csv-include} data/values.csv + ::: + """), + ["docs/data/values.csv"] = new("a,b\n1,2") + }, new MockFileSystemOptions { CurrentDirectory = Paths.WorkingDirectoryRoot.FullName }); + + var graph = await BuildGraph(fileSystem); + + graph.ResolvePageDependents("data/values.csv").Should().BeEquivalentTo("data-page.md"); + } + + private async Task BuildGraph(MockFileSystem fileSystem) + { + var collector = new TestDiagnosticsCollector(output); + _ = collector.StartAsync(TestContext.Current.CancellationToken); + var configurationContext = TestHelpers.CreateConfigurationContext(fileSystem); + var context = new BuildContext(collector, FileSystemFactory.ScopeCurrentWorkingDirectory(fileSystem), configurationContext); + var set = new DocumentationSet(context, new TestLoggerFactory(output), new TestCrossLinkResolver()); + return await IncludeGraph.BuildAsync(set, TestContext.Current.CancellationToken); + } +}