Source search-labs content from Contentstack instead of crawling

docs/development/essc.md

@@ -17,11 +17,16 @@ existing interface.
 
 Current sources:
 
-| Source       | Namespace           | Description                                          |
-| ------------ | ------------------- | ---------------------------------------------------- |
-| Contentstack | `contentstack`      | elastic.co marketing, blog, product, and event pages |
-| Labs         | `labs`              | Search, security, and observability labs properties  |
-| Legacy docs  | `guide` _(planned)_ | `/guide` legacy documentation                        |
+| Source       | Namespace           | Description                                                             |
+| ------------ | ------------------- | ------------------------------------------------------------------------ |
+| Contentstack | `contentstack`      | elastic.co marketing, blog, product, event, and Search Labs pages       |
+| Labs         | `labs`              | Security and observability labs properties                             |
+| Legacy docs  | `guide` _(planned)_ | `/guide` legacy documentation                                           |
+
+Search Labs (`/search-labs/*` — blog, tutorials, notebooks, integrations) is sourced from
+Contentstack rather than crawled: it moved off the `labs` HTML crawler and is now indexed by
+`contentstack sync` alongside the rest of the marketing site. Security Labs and Observability Labs
+are still crawled by `labs sync`.
 
 ## Installation
 

src/tooling/essc/Commands/LabsCommands.cs

@@ -70,10 +70,11 @@ internal sealed class LabsSyncOptions
 }
 
 /// <summary>
-/// Crawl elastic.co labs properties (search, security, observability) into <c>labs-*</c> Elasticsearch indices.
+/// Crawl elastic.co labs properties (security, observability) into <c>labs-*</c> Elasticsearch indices.
 /// </summary>
 /// <remarks>
-/// Independent of Contentstack <c>contentstack</c> commands targeting <c>site-*</c> indices.
+/// Independent of Contentstack <c>contentstack</c> commands targeting <c>site-*</c> indices. Search Labs
+/// is sourced from Contentstack (see <c>essc contentstack sync</c>) and is not crawled here.
 /// Discovery starts from published labs sitemap URLs; use <c>--dry-run</c> to validate discovery only.
 /// </remarks>
 internal sealed class LabsCommands(
@@ -134,7 +135,7 @@ public async Task Sync([AsParameters] LabsSyncOptions options, Cancel ct = defau
 		var env = config.ElasticsearchEnvironment;
 		var indexAlias = LabsSiteCrawlPlanner.ResolveLexicalReadAlias(buildType, env);
 
-		AnsiConsole.MarkupLine("[aqua bold]Labs crawl[/] — [dim]search-labs, security-labs, observability-labs[/]");
+		AnsiConsole.MarkupLine("[aqua bold]Labs crawl[/] — [dim]security-labs, observability-labs[/]");
 		AnsiConsole.MarkupLine($"[dim]Elasticsearch:{Markup.Escape(cfg.Uri.ToString())}[/]");
 		AnsiConsole.MarkupLine($"[dim]Incremental cache alias:[/] [white]{Markup.Escape(indexAlias)}[/]");
 		if (force)

src/tooling/essc/ContentStack/ContentStackMapper.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Linq;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
@@ -139,9 +140,61 @@ internal static partial class ContentStackMapper
 		if (!string.IsNullOrWhiteSpace(v5Notes))
 			return v5Notes;
 
+		// Strategy 7: main_content.content_l10n (tutorial, tutorial_page, tutorial_chapter,
+		// labs_integration) or main_content.body.content_l10n (blog_v3) — a rich-text JSON AST
+		// (ProseMirror-style { type, children, text }), not an HTML string like the strategies above.
+		var richTextBody = ExtractRichTextBody(data);
+		if (!string.IsNullOrWhiteSpace(richTextBody))
+			return richTextBody;
+
+		return null;
+	}
+
+	private static string? ExtractRichTextBody(JsonElement data)
+	{
+		if (!data.TryGetProperty("main_content", out var mainContent) || mainContent.ValueKind != JsonValueKind.Object)
+			return null;
+
+		if (mainContent.TryGetProperty("content_l10n", out var direct) && direct.ValueKind == JsonValueKind.Object)
+			return RenderRichTextRoot(direct);
+
+		if (mainContent.TryGetProperty("body", out var body) && body.ValueKind == JsonValueKind.Object
+			&& body.TryGetProperty("content_l10n", out var nested) && nested.ValueKind == JsonValueKind.Object)
+			return RenderRichTextRoot(nested);
+
 		return null;
 	}
 
+	/// <summary>
+	/// Renders a ContentStack rich-text (ProseMirror-style) JSON AST — <c>{ type, children: [...], text? }</c>
+	/// — into a lightweight HTML-ish string so it flows through the same <see cref="StripHtml"/> /
+	/// <see cref="ExtractHeadings"/> regex pipeline as the legacy HTML-string strategies above, instead of
+	/// needing a parallel JSON-aware text/heading extractor.
+	/// </summary>
+	private static string RenderRichText(JsonElement node)
+	{
+		if (node.TryGetProperty("text", out var textProp) && textProp.ValueKind == JsonValueKind.String)
+			return textProp.GetString() ?? "";
+
+		if (!node.TryGetProperty("children", out var children) || children.ValueKind != JsonValueKind.Array)
+			return "";
+
+		var inner = string.Concat(children.EnumerateArray().Select(RenderRichText));
+		var type = GetString(node, "type");
+
+		return type switch
+		{
+			"h1" or "h2" or "h3" or "h4" or "h5" or "h6" or "p" or "li" or "ul" or "ol" => $"<{type}>{inner}</{type}>",
+			_ => inner
+		};
+	}
+
+	private static string? RenderRichTextRoot(JsonElement root)
+	{
+		var rendered = RenderRichText(root);
+		return string.IsNullOrWhiteSpace(StripHtml(rendered)) ? null : rendered;
+	}
+
 	private static string? ExtractModularBlocks(JsonElement data)
 	{
 		if (!data.TryGetProperty("modular_blocks", out var blocks) || blocks.ValueKind != JsonValueKind.Array)
@@ -232,6 +285,36 @@ internal static partial class ContentStackMapper
 		if (!string.IsNullOrWhiteSpace(intro))
 			return StripHtml(intro);
 
+		// description_l10n (notebook, series, labs_category, labs_homepage, glossary, examples_landing) —
+		// a plain string on most content types, but a nested rich-text object on labs_integration, so this
+		// simply returns null there (GetString ignores non-string values) and falls through below.
+		var descriptionL10n = GetString(data, "description_l10n");
+		if (!string.IsNullOrWhiteSpace(descriptionL10n))
+			return descriptionL10n;
+
+		// description_l10n.content_simple_l10n (labs_integration)
+		var richDescription = GetNestedRichText(data, "description_l10n", "content_simple_l10n");
+		if (!string.IsNullOrWhiteSpace(richDescription))
+			return StripHtml(richDescription);
+
+		// page_info.subheading_l10n (tutorials_landing, integrations_landing, blog_landing)
+		var subheading = GetNestedString(data, "page_info", "subheading_l10n");
+		if (!string.IsNullOrWhiteSpace(subheading))
+			return subheading;
+
+		// page_info.description.content_simple_l10n (tutorials_landing; often empty on other landing pages)
+		if (data.TryGetProperty("page_info", out var pageInfo) && pageInfo.ValueKind == JsonValueKind.Object)
+		{
+			var pageInfoRichDescription = GetNestedRichText(pageInfo, "description", "content_simple_l10n");
+			if (!string.IsNullOrWhiteSpace(pageInfoRichDescription))
+				return StripHtml(pageInfoRichDescription);
+		}
+
+		// summary_l10n (blog_v3)
+		var summary = GetString(data, "summary_l10n");
+		if (!string.IsNullOrWhiteSpace(summary))
+			return summary;
+
 		// SEO description fallback
 		return GetSeoString(data, "seo_description_l10n") ?? GetSeoString(data, "seo_description");
 	}
@@ -354,6 +437,12 @@ private static int ComputeNavigationDepth(string url)
 
 	internal static string GetNavigationSection(string url, string? contentTypeUid = null)
 	{
+		// Search Labs is sourced from Contentstack; classify consistently with the label
+		// LabsHtmlExtractor.GetNavigationSection assigns to the same URLs when crawled.
+		if (url.Contains("/search-labs", StringComparison.OrdinalIgnoreCase))
+			return "search-labs";
+		if (url.Contains("/glossary", StringComparison.OrdinalIgnoreCase))
+			return "glossary";
 		if (url.Contains("/blog/", StringComparison.OrdinalIgnoreCase))
 			return "blog";
 		if (url.Contains("/what-is/", StringComparison.OrdinalIgnoreCase))
@@ -465,6 +554,19 @@ private static string ComputeHash(string content)
 		return null;
 	}
 
+	/// <summary>
+	/// Reads a rich-text (ProseMirror-style) JSON AST nested at <c>el.{parent}.{child}</c> — e.g.
+	/// <c>description_l10n.content_simple_l10n</c> — and renders it via <see cref="RenderRichText"/>.
+	/// </summary>
+	private static string? GetNestedRichText(JsonElement el, string parent, string child)
+	{
+		if (!el.TryGetProperty(parent, out var p) || p.ValueKind != JsonValueKind.Object)
+			return null;
+		if (!p.TryGetProperty(child, out var richTextRoot) || richTextRoot.ValueKind != JsonValueKind.Object)
+			return null;
+		return RenderRichTextRoot(richTextRoot);
+	}
+
 	private static string? GetSeoString(JsonElement data, string field)
 	{
 		if (data.TryGetProperty("seo", out var seo) && seo.ValueKind == JsonValueKind.Object)

src/tooling/essc/ContentStack/SourcingState.cs

@@ -125,17 +125,8 @@ internal static class PageContentTypes
 		"demo_gallery_overview",
 		"timeline",
 		"events_overview",
-		"blog_archive_overview"
-	];
-
-	/// <summary>
-	/// Content types that exist in Contentstack but must never be synced or indexed — e.g. content
-	/// types still being authored that aren't ready to appear in search yet.
-	/// </summary>
-	public static readonly string[] Blocked =
-	[
+		"blog_archive_overview",
 		"blog_landing",
-		"example_link",
 		"glossary",
 		"examples_landing",
 		"integrations_landing",
@@ -151,6 +142,12 @@ internal static class PageContentTypes
 		"blog_v3"
 	];
 
+	/// <summary>
+	/// Content types that exist in Contentstack but must never be synced or indexed — e.g. content
+	/// types still being authored that aren't ready to appear in search yet.
+	/// </summary>
+	public static readonly string[] Blocked = [];
+
 	/// <summary>
 	/// Content types that are never expected to represent a standalone page — reusable components,
 	/// taxonomy/tags, navigation config, redirects, etc. Unlike <see cref="Blocked"/>, these aren't
@@ -187,6 +184,7 @@ internal static class PageContentTypes
 		"customer_industry",
 		"customer_use_case",
 		"date_field",
+		"example_link",
 		"featured_split_listing",
 		"features",
 		"footer",

src/tooling/essc/LabsCrawl/LabsSiteCrawlPlanner.cs

@@ -10,9 +10,9 @@ namespace Elastic.SiteSearch.Cli.LabsCrawl;
 /// <summary>Labs-only sitemap URLs and crawl planning (vendored from site crawler logic).</summary>
 public static class LabsSiteCrawlPlanner
 {
+	// search-labs is sourced from Contentstack (see essc contentstack sync) and is no longer crawled here.
 	public static readonly string[] LabsSitemapUrls =
 	[
-		"https://www.elastic.co/search-labs/sitemap.xml",
 		"https://www.elastic.co/security-labs/sitemap.xml",
 		"https://www.elastic.co/observability-labs/sitemap.xml"
 	];
@@ -32,7 +32,6 @@ public sealed record LabsSitemapDiscoveryResult(
 
 	private static readonly (string Pattern, string Label)[] PathEntries =
 	[
-		("/search-labs/", "/search-labs/"),
 		("/security-labs/", "/security-labs/"),
 		("/observability-labs/", "/observability-labs/")
 	];

src/tooling/essc/Program.cs

@@ -11,6 +11,7 @@
 using Elastic.SiteSearch.Cli.Logging;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Http.Resilience;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Console;
 using Nullean.Argh.Hosting;
@@ -56,6 +57,12 @@
 	o.Retry.UseJitter = true;
 	o.Retry.BackoffType = Polly.DelayBackoffType.Exponential;
 	o.Retry.Delay = TimeSpan.FromSeconds(2);
+	// Contentstack's sync endpoint intermittently returns a transient 422 (Unprocessable Entity)
+	// under load — not handled by the default transient-status predicate — which otherwise aborts
+	// the whole sync run before FinalizeAsync. See production run 32113750025.
+	o.Retry.ShouldHandle = args => ValueTask.FromResult(
+		HttpClientResiliencePredicates.IsTransient(args.Outcome) ||
+		args.Outcome.Result?.StatusCode == HttpStatusCode.UnprocessableEntity);
 });
 csHttpClient.AddHttpMessageHandler(() => RateLimitingHandler.CreateForContentStack());
 

tests/Elastic.SiteSearch.Tests/ContentStackMappingTests.cs

@@ -250,6 +250,102 @@ public void MinimalPage_Maps_TitleUrlSeo()
 		doc.Hash.Should().NotBeNullOrEmpty();
 	}
 
+	/// <summary>Covers: tutorial, tutorial_page, tutorial_chapter, labs_integration (rich-text JSON AST body + description)</summary>
+	[Fact]
+	public void RichTextPage_Maps_ProseMirrorBodyAndDescription()
+	{
+		var item = LoadFixture("rich_text_page.json", "labs_integration");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Path.Should().Be("/search-labs/integrations/amazon-bedrock");
+		doc.Section.Should().Be("search-labs");
+		doc.Body.Should().Contain("Amazon Bedrock makes leading foundation models available");
+		doc.Body.Should().Contain("RAG using Bedrock");
+		doc.Body.Should().NotContain("<p>");
+		doc.Body.Should().NotContain("<h3>");
+		doc.Headings.Should().Contain("Blogs to get started");
+		doc.Description.Should().Contain("fully managed service offering foundation models");
+		doc.Hash.Should().NotBeNullOrEmpty();
+	}
+
+	/// <summary>Covers: blog_v3 (rich-text body nested under main_content.body.content_l10n + summary_l10n description)</summary>
+	[Fact]
+	public void BlogV3_Maps_NestedRichTextBodyAndSummary()
+	{
+		var item = LoadFixture("rich_text_blog_v3.json", "blog_v3");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Path.Should().Be("/search-labs/blog/hybrid-search-with-elasticsearch");
+		doc.Section.Should().Be("search-labs");
+		doc.Body.Should().Contain("Hybrid search combines BM25 lexical scoring");
+		doc.Body.Should().NotContain("<p>");
+		doc.Headings.Should().Contain("Why hybrid search");
+		doc.Description.Should().Be("Combine lexical and semantic search for better relevance.");
+		doc.PublishedDate.Should().NotBeNull();
+	}
+
+	/// <summary>Covers: tutorials_landing, integrations_landing, blog_landing (page_info.subheading_l10n description, no body)</summary>
+	[Fact]
+	public void LandingPage_Maps_SubheadingDescription_NoBody()
+	{
+		var item = LoadFixture("landing_page.json", "tutorials_landing");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Path.Should().Be("/search-labs/tutorials");
+		doc.Section.Should().Be("search-labs");
+		doc.Description.Should().Be("Long-form guides to get you started");
+		// No main_content on landing pages — body falls back to the description.
+		doc.Body.Should().Be(doc.Description);
+	}
+
+	/// <summary>Covers: notebook, series, labs_category, labs_homepage, glossary, examples_landing (plain description_l10n, no body)</summary>
+	[Fact]
+	public void DescriptionOnly_Maps_PlainDescription_NoBody()
+	{
+		var item = LoadFixture("description_only.json", "notebook");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Section.Should().Be("search-labs");
+		doc.Description.Should().Contain("Learn how to build a RAG system");
+		doc.Body.Should().Be(doc.Description);
+		doc.Headings.Should().BeEmpty();
+	}
+
+	/// <summary>Empty page_info.description.content_simple_l10n must not win over the SEO fallback.</summary>
+	[Fact]
+	public void ToSiteDocument_LandingPage_WithEmptyRichDescription_FallsBackToSeo()
+	{
+		var item = LoadFromJson(/*lang=json,strict*/ """
+			{ "title": "Integrations", "url": "/search-labs/integrations", "locale": "en-us",
+			  "page_info": { "description": { "content_simple_l10n": { "type": "doc", "children": [] } } },
+			  "seo": { "seo_description_l10n": "Browse Search Labs integrations." } }
+			""", "integrations_landing");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Description.Should().Be("Browse Search Labs integrations.");
+	}
+
+	/// <summary>page_info.description.content_simple_l10n is used when there's no subheading_l10n.</summary>
+	[Fact]
+	public void ToSiteDocument_GlossaryPage_UsesPageInfoRichTextDescription_WhenNoSubheading()
+	{
+		var item = LoadFromJson(/*lang=json,strict*/ """
+			{ "title": "Glossary", "url": "/glossary", "locale": "en-us",
+			  "page_info": { "description": { "content_simple_l10n": { "type": "doc",
+			    "children": [ { "type": "p", "children": [ { "text": "All the terms, concepts, and abbreviations." } ] } ] } } } }
+			""", "glossary");
+		var doc = ContentStackMapper.ToSiteDocument(item);
+
+		doc.Should().NotBeNull();
+		doc.Description.Should().Contain("terms, concepts, and abbreviations");
+		doc.Section.Should().Be("glossary");
+	}
+
 	// --- Body projection helper tests ---
 
 	[Fact]
@@ -292,6 +388,9 @@ public void ExtractHeadings_Returns_Empty_For_No_Headings()
 	[Fact]
 	public void GetNavigationSection_Classifies_Known_Paths()
 	{
+		ContentStackMapper.GetNavigationSection("/search-labs").Should().Be("search-labs");
+		ContentStackMapper.GetNavigationSection("/search-labs/blog/some-post").Should().Be("search-labs");
+		ContentStackMapper.GetNavigationSection("/glossary").Should().Be("glossary");
 		ContentStackMapper.GetNavigationSection("/blog/some-post").Should().Be("blog");
 		ContentStackMapper.GetNavigationSection("/what-is/elasticsearch").Should().Be("concept");
 		ContentStackMapper.GetNavigationSection("/webinars/live-event").Should().Be("webinar");

tests/Elastic.SiteSearch.Tests/Fixtures/ContentStack/description_only.json

@@ -0,0 +1,12 @@
+{
+  "title": "Using Cohere for RAG and Rerank in Elasticsearch",
+  "title_l10n": "Using Cohere for RAG and Rerank in Elasticsearch",
+  "url": "/search-labs/tutorials/examples/cohere-rag-rerank-elasticsearch-tutorial",
+  "locale": "en-us",
+  "description_l10n": "Learn how to build a RAG system using the Cohere Inference API and Elasticsearch with embeddings, hybrid search, and advanced reranking.",
+  "seo": {
+    "seo_title_l10n": "Cohere RAG and Rerank with Elasticsearch",
+    "seo_description_l10n": "Learn how to build a RAG system using the Cohere Inference API."
+  },
+  "publish_details": { "locale": "en-us" }
+}