Improve diagnostics when Codex cross-link index fetch fails

src/Elastic.Codex/Building/CodexBuildService.cs

@@ -212,6 +212,7 @@ public async Task<CodexBuildResult> BuildAll(
 					buildContext.Configuration,
 					codexLinkIndexReader: buildContext.Configuration.Registry != DocSetRegistry.Public ? codexLinkIndexReader : null);
 				var crossLinks = await fetcher.FetchCrossLinks(ctx);
+				CrossLinkFetchDiagnostics.EmitFetchFailures(context.Collector, buildContext.ConfigurationPath.FullName, crossLinks);
 				if (crossLinks.CodexRepositories is not null)
 					codexRepos.UnionWith(crossLinks.CodexRepositories);
 				var uriResolver = new CodexAwareUriResolver(codexRepos.ToFrozenSet(), useRelativePaths: true);

src/Elastic.Documentation.LinkIndex/GitLinkIndexReader.cs

@@ -4,6 +4,7 @@
 
 using System.Diagnostics;
 using System.IO.Abstractions;
+using Elastic.Documentation;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.FileSystems;
 using Elastic.Documentation.Links;
@@ -24,18 +25,24 @@ public class GitLinkIndexReader : ILinkIndexReader, IDisposable
 
 	private readonly string _environment;
 	private readonly IFileSystem _fileSystem;
+	private readonly IEnvironmentVariables _environmentVariables;
 	private readonly bool _skipFetch;
 	private readonly SemaphoreSlim _cloneLock = new(1, 1);
 	private bool _ensuredClone;
 
-	public GitLinkIndexReader(string environment, ApplicationDataFileSystem? fileSystem = null, bool skipFetch = false)
+	public GitLinkIndexReader(
+		string environment,
+		ApplicationDataFileSystem? fileSystem = null,
+		bool skipFetch = false,
+		IEnvironmentVariables? environmentVariables = null)
 	{
 		if (string.IsNullOrWhiteSpace(environment))
 			throw new ArgumentException("Environment must be specified in the codex configuration (e.g., 'internal', 'security').", nameof(environment));
 
 		_environment = environment;
 		_fileSystem = fileSystem ?? new ApplicationDataFileSystem();
 		_skipFetch = skipFetch;
+		_environmentVariables = environmentVariables ?? SystemEnvironmentVariables.Instance;
 	}
 
 	/// <inheritdoc />
@@ -123,11 +130,11 @@ private async Task EnsureCloneAsync(Cancel cancellationToken)
 		}
 	}
 
-	private static string GetCodexLinkIndexGitUrl()
+	private string GetCodexLinkIndexGitUrl()
 	{
-		if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("GITHUB_ACTIONS")))
+		if (_environmentVariables.IsRunningOnCI)
 		{
-			var token = Environment.GetEnvironmentVariable("GITHUB_TOKEN");
+			var token = _environmentVariables.GetEnvironmentVariable("GITHUB_TOKEN");
 			return !string.IsNullOrEmpty(token)
 				? $"https://oauth2:{token}@github.com/{LinkIndexOrigin}.git"
 				: $"https://github.com/{LinkIndexOrigin}.git";
@@ -136,7 +143,7 @@ private static string GetCodexLinkIndexGitUrl()
 		return $"git@github.com:{LinkIndexOrigin}.git";
 	}
 
-	private static void RunGit(string workingDirectory, params string[] args)
+	private void RunGit(string workingDirectory, params string[] args)
 	{
 		var startInfo = new ProcessStartInfo
 		{
@@ -158,6 +165,24 @@ private static void RunGit(string workingDirectory, params string[] args)
 		process.WaitForExit();
 
 		if (process.ExitCode != 0)
-			throw new InvalidOperationException($"Git command failed (exit {process.ExitCode}): {stderr.Trim()}");
+			throw new InvalidOperationException(DescribeCloneFailure(
+				stderr,
+				_environmentVariables.IsRunningOnCI,
+				!string.IsNullOrEmpty(_environmentVariables.GetEnvironmentVariable("GITHUB_TOKEN"))));
+	}
+
+	private static string DescribeCloneFailure(string gitStderr, bool onActions, bool hasToken)
+	{
+		var message = $"Git clone failed: {gitStderr.Trim()}";
+
+		if (onActions && !hasToken)
+			return $"{message}{Environment.NewLine}{Environment.NewLine}"
+				+ "GitHub Actions did not provide GITHUB_TOKEN for the private Elastic Internal Docs link index."
+				+ $"{Environment.NewLine}Fork pull_request jobs do not receive the OIDC token needed to fetch this token. Push fork branches to the upstream repository."
+				+ $"{Environment.NewLine}For same-repository jobs, confirm permissions.id-token: write and the catalog-info token policy.";
+
+		return !onActions
+			? $"{message}{Environment.NewLine}{Environment.NewLine}Run 'docs-builder codex clone' first, or ensure SSH access to github.com works for git@github.com:elastic/codex-link-index.git."
+			: message;
 	}
 }

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetchDiagnostics.cs

@@ -0,0 +1,42 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Configuration.Builder;
+using Elastic.Documentation.Diagnostics;
+
+namespace Elastic.Documentation.Links.CrossLinks;
+
+public static class CrossLinkFetchDiagnostics
+{
+	private const string CodexLinkIndexUrl = "https://github.com/elastic/codex-link-index";
+
+	public static void EmitFetchFailures(IDiagnosticsCollector collector, string configurationPath, FetchedCrossLinks crossLinks)
+	{
+		if (crossLinks.FetchFailures.Count == 0)
+			return;
+
+		collector.EmitError(configurationPath, FormatSummary(crossLinks));
+	}
+
+	private static string FormatSummary(FetchedCrossLinks crossLinks)
+	{
+		var fetchFailures = crossLinks.FetchFailures;
+		var repositories = fetchFailures.Keys.Order(StringComparer.Ordinal).ToArray();
+		var isCodexFailure = repositories.All(repository =>
+			crossLinks.RegistryByRepository?.GetValueOrDefault(repository) is { } registry
+			&& registry != DocSetRegistry.Public);
+		var distinctReasons = fetchFailures.Values.Distinct(StringComparer.Ordinal).ToArray();
+		var heading = isCodexFailure && distinctReasons.Length == 1
+			? $"Could not fetch the Elastic Internal Docs link index from {CodexLinkIndexUrl}."
+			: $"Could not fetch cross-link index data for: {string.Join(", ", repositories)}.";
+		var details = distinctReasons.Length == 1
+			? distinctReasons[0]
+			: string.Join(Environment.NewLine, repositories.Select(repository => $"{repository}: {fetchFailures[repository]}"));
+		var validation = repositories.Length == 1
+			? $"Cross-links to {repositories[0]} were not validated."
+			: $"Cross-links to these repositories were not validated: {string.Join(", ", repositories)}.";
+
+		return $"{heading}{Environment.NewLine}{Environment.NewLine}{details}{Environment.NewLine}{Environment.NewLine}{validation}";
+	}
+}

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs

@@ -46,7 +46,13 @@ public record FetchedCrossLinks
 	/// True when all declared repositories resolved without falling back to placeholder data.
 	/// When false, callers should avoid caching so a subsequent reload retries the fetch.
 	/// </summary>
-	public bool IsComplete { get; init; } = true;
+	public bool IsComplete => FetchFailures.Count == 0;
+
+	/// <summary>
+	/// Repositories whose link index could not be fetched, mapped to a human-readable reason.
+	/// Callers emit one summary diagnostic and suppress per-link path errors for these repositories.
+	/// </summary>
+	public FrozenDictionary<string, string> FetchFailures { get; init; } = FrozenDictionary<string, string>.Empty;
 
 	public static FetchedCrossLinks Empty { get; } = new()
 	{
@@ -55,7 +61,8 @@ public record FetchedCrossLinks
 		LinkIndexEntries = new Dictionary<string, LinkRegistryEntry>().ToFrozenDictionary(),
 		RegistryUrlsByRepository = null,
 		RegistryByRepository = null,
-		CodexRepositories = null
+		CodexRepositories = null,
+		FetchFailures = FrozenDictionary<string, string>.Empty
 	};
 }
 

src/Elastic.Documentation.Links/CrossLinks/CrossLinkResolver.cs

@@ -75,6 +75,9 @@ public static bool TryResolve(
 	{
 		resolvedUri = null;
 
+		if (fetchedCrossLinks.FetchFailures.ContainsKey(crossLinkUri.Scheme))
+			return false;
+
 		// First, check if the repository is in the declared repositories list, even if it's not in the link references
 		var isDeclaredRepo = fetchedCrossLinks.DeclaredRepositories.Contains(crossLinkUri.Scheme);
 

src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs

@@ -28,16 +28,19 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 		var linkIndexEntries = new Dictionary<string, LinkRegistryEntry>();
 		var registryUrlsByRepository = new Dictionary<string, string>();
 		var registryByRepository = new Dictionary<string, DocSetRegistry>();
+		var fetchFailures = new Dictionary<string, string>();
 		var codexRepositories = new HashSet<string>();
 		var declaredRepositories = new HashSet<string>();
 
 		var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous();
 		var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null;
 
 		// Fetch each registry once up front so per-repository lookups don't trigger N S3 round-trips.
-		var publicRegistry = await TryGetRegistry(publicReader, ctx);
-		var codexRegistry = useDualRegistry ? await TryGetRegistry(_codexReader!, ctx) : null;
-		var hadFetchFailures = false;
+		var (publicRegistry, publicRegistryFailure) = await TryGetRegistry(publicReader, ctx);
+		LinkRegistry? codexRegistry = null;
+		string? codexRegistryFailure = null;
+		if (useDualRegistry)
+			(codexRegistry, codexRegistryFailure) = await TryGetRegistry(_codexReader!, ctx);
 
 		foreach (var entry in configuration.CrossLinkEntries)
 		{
@@ -46,44 +49,52 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 			var isCodexEntry = useDualRegistry && entry.Registry != DocSetRegistry.Public;
 			var reader = isCodexEntry ? _codexReader! : publicReader;
 			var registry = isCodexEntry ? codexRegistry : publicRegistry;
+			var registryFailure = isCodexEntry ? codexRegistryFailure : publicRegistryFailure;
+			registryUrlsByRepository[entry.Repository] = reader.RegistryUrl;
 
 			if (isCodexEntry)
 				_ = codexRepositories.Add(entry.Repository);
 
-			try
+			if (registry is null)
 			{
-				if (registry is null || !registry.Repositories.TryGetValue(entry.Repository, out var repoBranches))
-					throw new Exception($"Repository {entry.Repository} not found in link index");
+				fetchFailures[entry.Repository] =
+					registryFailure ?? $"Failed to fetch link index registry from {reader.RegistryUrl}";
+			}
+			else
+			{
+				try
+				{
+					if (!registry.Repositories.TryGetValue(entry.Repository, out var repoBranches))
+						throw new Exception($"Repository {entry.Repository} not found in link index");
 
-				var linkIndexEntry = GetNextContentSourceLinkIndexEntry(repoBranches, entry.Repository);
-				var linkReference = await FetchLinkIndexEntryFromReader(reader, entry.Repository, linkIndexEntry, ctx);
+					var linkIndexEntry = GetNextContentSourceLinkIndexEntry(repoBranches, entry.Repository);
+					var linkReference = await FetchLinkIndexEntryFromReader(reader, entry.Repository, linkIndexEntry, ctx);
 
-				linkReferences.Add(entry.Repository, linkReference);
-				linkIndexEntries.Add(entry.Repository, linkIndexEntry);
-				registryUrlsByRepository[entry.Repository] = reader.RegistryUrl;
+					linkReferences.Add(entry.Repository, linkReference);
+					linkIndexEntries.Add(entry.Repository, linkIndexEntry);
+				}
+				catch (Exception ex)
+				{
+					fetchFailures[entry.Repository] = ex.Message;
+					_logger.LogWarning(ex, "Error fetching link data for repository '{Repository}'. Cross-links to this repository may not resolve correctly.", entry.Repository);
+				}
 			}
-			catch (Exception ex)
-			{
-				hadFetchFailures = true;
-				_logger.LogWarning(ex, "Error fetching link data for repository '{Repository}'. Cross-links to this repository may not resolve correctly.", entry.Repository);
-				_ = registryUrlsByRepository.TryAdd(entry.Repository, reader.RegistryUrl);
 
-				if (!linkReferences.ContainsKey(entry.Repository))
+			if (!linkReferences.ContainsKey(entry.Repository))
+			{
+				linkReferences.Add(entry.Repository, new RepositoryLinks
 				{
-					linkReferences.Add(entry.Repository, new RepositoryLinks
+					Links = [],
+					Origin = new GitCheckoutInformation
 					{
-						Links = [],
-						Origin = new GitCheckoutInformation
-						{
-							Branch = "main",
-							RepositoryName = entry.Repository,
-							Remote = "origin",
-							Ref = "refs/heads/main"
-						},
-						UrlPathPrefix = "",
-						CrossLinks = []
-					});
-				}
+						Branch = "main",
+						RepositoryName = entry.Repository,
+						Remote = "origin",
+						Ref = "refs/heads/main"
+					},
+					UrlPathPrefix = "",
+					CrossLinks = []
+				});
 			}
 		}
 
@@ -95,15 +106,15 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 			RegistryUrlsByRepository = registryUrlsByRepository.ToFrozenDictionary(),
 			RegistryByRepository = registryByRepository.ToFrozenDictionary(),
 			CodexRepositories = codexRepositories.Count > 0 ? codexRepositories.ToFrozenSet() : null,
-			IsComplete = !hadFetchFailures,
+			FetchFailures = fetchFailures.ToFrozenDictionary(),
 		};
 	}
 
-	private async Task<LinkRegistry?> TryGetRegistry(ILinkIndexReader reader, Cancel ctx)
+	private async Task<(LinkRegistry? Registry, string? FailureReason)> TryGetRegistry(ILinkIndexReader reader, Cancel ctx)
 	{
 		try
 		{
-			return await reader.GetRegistry(ctx);
+			return (await reader.GetRegistry(ctx), null);
 		}
 		catch (OperationCanceledException)
 		{
@@ -112,7 +123,7 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 		catch (Exception ex)
 		{
 			_logger.LogWarning(ex, "Failed to fetch link index registry from {RegistryUrl}", reader.RegistryUrl);
-			return null;
+			return (null, ex.Message);
 		}
 	}
 }

src/services/Elastic.Documentation.Isolated/IsolatedBuildService.cs

@@ -139,6 +139,7 @@ public async Task<bool> Build(
 				context.Configuration,
 				codexLinkIndexReader: codexReader);
 			var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx);
+			CrossLinkFetchDiagnostics.EmitFetchFailures(context.Collector, context.ConfigurationPath.FullName, crossLinks);
 			IUriEnvironmentResolver? uriResolver = crossLinks.CodexRepositories is not null
 				? new CodexAwareUriResolver(crossLinks.CodexRepositories)
 				: null;

src/tooling/docs-builder/Http/ReloadableGeneratorState.cs

@@ -89,6 +89,7 @@ public async Task ReloadAsync(Cancel ctx, bool reloadConfiguration = true)
 		if (crossLinks is null || reloadConfiguration)
 		{
 			crossLinks = await _crossLinkFetcher.FetchCrossLinks(ctx);
+			CrossLinkFetchDiagnostics.EmitFetchFailures(_context.Collector, _context.ConfigurationPath.FullName, crossLinks);
 			// Only cache successful fetches so transient failures get retried on the next reload.
 			_cachedCrossLinks = crossLinks.IsComplete ? crossLinks : null;
 		}