From 983b1cdfa30756e15293ebec79c2da4879658ba9 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Wed, 13 May 2026 18:50:31 +0200 Subject: [PATCH 01/13] feat: CommunityToolkit.Aspire.Hosting.K3s --- .github/workflows/tests.yaml | 1 + CommunityToolkit.Aspire.slnx | 5 + Directory.Packages.props | 1 + README.md | 6 + ...yToolkit.Aspire.Hosting.K3s.AppHost.csproj | 14 + .../Program.cs | 34 ++ .../Properties/launchSettings.json | 30 ++ .../Annotations/HelmReleaseAnnotation.cs | 35 ++ .../KubernetesDashboardAnnotation.cs | 11 + .../Annotations/KustomizeAnnotation.cs | 11 + ...CommunityToolkit.Aspire.Hosting.K3s.csproj | 19 + .../HelmReleaseHealthCheck.cs | 19 + .../HelmReleaseResource.cs | 45 ++ .../K3sAgentResource.cs | 22 + .../K3sBuilderExtensions.Helm.cs | 427 ++++++++++++++++ .../K3sBuilderExtensions.Manifest.cs | 400 +++++++++++++++ .../K3sBuilderExtensions.cs | 464 ++++++++++++++++++ .../K3sClusterOptions.cs | 40 ++ .../K3sClusterResource.cs | 63 +++ .../K3sContainerImageTags.cs | 13 + .../K3sInProcessPortForwarder.cs | 108 ++++ .../K3sReadinessHealthCheck.cs | 190 +++++++ .../K8sManifestResource.cs | 47 ++ .../KubeconfigInjectionStrategy.cs | 33 ++ .../README.md | 67 +++ .../CommunityToolkit.Aspire.Hosting.K3s.cs | 73 +++ ...ityToolkit.Aspire.Hosting.K3s.Tests.csproj | 10 + .../HelmReleaseResourceTests.cs | 406 +++++++++++++++ .../K3sAgentNodeTests.cs | 201 ++++++++ .../K3sClusterResourceTests.cs | 295 +++++++++++ .../K3sPublicApiTests.cs | 109 ++++ .../K8sManifestResourceTests.cs | 191 +++++++ 32 files changed, 3390 insertions(+) create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs create mode 100644 examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/README.md create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ddf4f4c47..4fae786ae 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -37,6 +37,7 @@ jobs: Hosting.Golang.Tests, Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, + Hosting.K3s.Tests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 1233718eb..3382bdbb0 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -59,6 +59,9 @@ + + + @@ -214,6 +217,7 @@ + @@ -276,6 +280,7 @@ + diff --git a/Directory.Packages.props b/Directory.Packages.props index 668a4938a..66d3b15ac 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -93,6 +93,7 @@ + diff --git a/README.md b/README.md index ce0170510..e9d3b7336 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This repository contains the source code for the Aspire Community Toolkit, a col | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | - **Learn More**: [`Hosting.Golang`][golang-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields]][golang-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields-preview]][golang-nuget-preview] | A hosting integration Golang apps. | | - **Learn More**: [`Hosting.Java`][java-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields]][java-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields-preview]][java-nuget-preview] | An integration for running Java code in Aspire either using the local JDK or using a container. | +| - **Learn More**: [`Hosting.K3s`][k3s-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields]][k3s-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields-preview]][k3s-nuget-preview] | An Aspire hosting integration for [k3s](https://k3s.io/), a lightweight Kubernetes distribution by Rancher. | | - **Learn More**: [`Hosting.NodeJS.Extensions`][nodejs-ext-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.NodeJS.Extensions][nodejs-ext-shields]][nodejs-ext-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.Hosting.JavaScript.Extensions][nodejs-ext-shields-preview]][nodejs-ext-nuget-preview] | An integration that contains some additional extensions for running Node.js applications | | - **Learn More**: [`Hosting.Ollama`][ollama-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields]][ollama-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields-preview]][ollama-nuget-preview] | An Aspire hosting integration leveraging the [Ollama](https://ollama.com) container with support for downloading a model on startup. | | - **Learn More**: [`OllamaSharp`][ollama-integration-docs]
- Stable πŸ“¦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields]][ollamasharp-nuget]
- Preview πŸ“¦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields-preview]][ollamasharp-nuget-preview] | An Aspire client integration for the [OllamaSharp](https://github.com/awaescher/OllamaSharp) package. | @@ -106,6 +107,11 @@ This project is supported by the [.NET Foundation](https://dotnetfoundation.org) [java-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/ [java-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.Java?label=nuget%20(preview) [java-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/absoluteLatest +[k3s-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s +[k3s-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.K3s +[k3s-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/ +[k3s-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.K3s?label=nuget%20(preview) +[k3s-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/absoluteLatest [nodejs-ext-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-nodejs-extensions [nodejs-ext-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions [nodejs-ext-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions/ diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj new file mode 100644 index 000000000..65e9770a0 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -0,0 +1,14 @@ + + + + Exe + enable + enable + true + + + + + + + diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs new file mode 100644 index 000000000..00f476326 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -0,0 +1,34 @@ +// K3s hosting example +// ────────────────────────────────────────────────────────────────────────────── +// Prerequisites (host machine): +// β€’ Docker with --privileged support +// β€’ helm β†’ https://helm.sh/docs/intro/install/ +// β€’ kubectl β†’ https://kubernetes.io/docs/tasks/tools/ +// +// What this demonstrates: +// 1. A k3s cluster starts inside a Docker container. +// 2. Headlamp (https://headlamp.dev) is installed as a child Helm release β€” +// a clickable http URL appears in the Aspire dashboard. +// 3. podinfo is installed β€” a lightweight demo app that shows the helm lifecycle. +// 4. Both releases are children of k8s in the Aspire resource tree. +// 5. WithPersistentState keeps the cluster data alive across AppHost restarts. +// ────────────────────────────────────────────────────────────────────────────── + +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder.AddK3sCluster("k8s", configure: opts => + { + opts.AgentCount = 2; + }) + .WithLifetime(ContainerLifetime.Persistent); + +// cluster.AddHelmRelease( +// name: "podinfo", +// chart: "podinfo", +// repo: "https://stefanprodan.github.io/podinfo", +// version: "6.7.1", +// @namespace: "podinfo") +// .WithHelmValue("service.type", "NodePort") +// .WithEndpoint("podinfo", servicePort: 9898, name: "web"); + +builder.Build().Run(); diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json new file mode 100644 index 000000000..dcfbae059 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:53201;http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:53203", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:53204" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:53205", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:53206" + } + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs new file mode 100644 index 000000000..b1d44bf9c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs @@ -0,0 +1,35 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Helm chart release to install into the cluster. +/// +/// The Helm release name. +/// The chart name or local path. +/// The Kubernetes namespace to install into. +/// Optional Helm repository URL containing the chart. +/// Optional chart version. +public sealed class HelmReleaseAnnotation( + string releaseName, + string chart, + string @namespace, + string? repoUrl, + string? version) : IResourceAnnotation +{ + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName; + + /// Gets the chart name or local path. + public string Chart { get; } = chart; + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace; + + /// Gets the optional Helm repository URL. + public string? RepoUrl { get; } = repoUrl; + + /// Gets the optional chart version. + public string? Version { get; } = version; + + /// Gets the extra --set values passed to helm install. + public IDictionary Values { get; } = new Dictionary(); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs new file mode 100644 index 000000000..635eebd5e --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that signals the Kubernetes Dashboard should be installed alongside the cluster. +/// +/// The Kubernetes Dashboard chart version to install. +public sealed class KubernetesDashboardAnnotation(string? version = null) : IResourceAnnotation +{ + /// Gets the dashboard chart version, or to use the latest. + public string? Version { get; } = version; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs new file mode 100644 index 000000000..4525fa5a1 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Kustomize overlay to apply into the cluster. +/// +/// Path to the kustomization directory or remote URL. +public sealed class KustomizeAnnotation(string path) : IResourceAnnotation +{ + /// Gets the kustomization path. + public string Path { get; } = path; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj new file mode 100644 index 000000000..1800d1555 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -0,0 +1,19 @@ + + + + kubernetes k3s hosting cluster + An Aspire hosting integration for k3s β€” a lightweight Kubernetes distribution. + + + + + + + + + + + + + + diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs new file mode 100644 index 000000000..eb0845e2d --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs @@ -0,0 +1,19 @@ +using Aspire.Hosting.ApplicationModel; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Lightweight health check that gates WaitFor(helmRelease) on completion of +/// the helm install lifecycle. Returns only +/// after is set by RunReleaseAsync. +/// +internal sealed class HelmReleaseHealthCheck(HelmReleaseResource release) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(release.IsReady + ? HealthCheckResult.Healthy("Helm release is running") + : HealthCheckResult.Unhealthy("Helm release not yet ready")); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs new file mode 100644 index 000000000..247f26e22 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -0,0 +1,45 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a Helm chart release deployed to a k3s cluster. +/// Appears as a distinct dashboard entry; transitions Starting β†’ Running +/// when all pods reach the Ready state. +/// +/// The Aspire resource name (also used as the Helm release name). +/// The Helm release name passed to helm upgrade --install. +/// The Kubernetes namespace to install into. +/// The parent k3s cluster resource. +public sealed class HelmReleaseResource( + string name, + string releaseName, + string @namespace, + K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName ?? throw new ArgumentNullException(nameof(releaseName)); + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + internal string? Chart { get; set; } + internal string? RepoUrl { get; set; } + internal string? Version { get; set; } + internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); + internal List EndpointDefinitions { get; } = []; + + /// + /// Set to by the lifecycle when the helm install completes and + /// all pods are ready. The WaitFor(helmRelease) health check polls this flag. + /// + internal volatile bool IsReady; +} + +/// Describes a Kubernetes service endpoint to expose from a Helm release. +/// The Kubernetes service name. +/// The service port number. +/// A friendly name shown in the dashboard. +internal sealed record HelmEndpointDefinition(string ServiceName, int ServicePort, string EndpointName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs new file mode 100644 index 000000000..f6850faaa --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs @@ -0,0 +1,22 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s agent (worker) node that is a child of a . +/// +/// Agent nodes run k3s agent and join the cluster by connecting to the server's API +/// server at https://{serverName}:6443, resolved via DCP's built-in Docker DNS. +/// Agents start immediately alongside the server (no WaitFor dependency) and use +/// k3s's built-in retry loop to connect once the server becomes reachable. The cluster's +/// health check waits for all 1 + nodes +/// to reach Ready state before transitioning to Running. +/// +/// +/// The resource name (e.g. k8s-agent-0). +/// The parent k3s cluster resource. +public sealed class K3sAgentResource(string name, K3sClusterResource cluster) + : ContainerResource(name), IResourceWithParent +{ + /// + public K3sClusterResource Parent { get; } = cluster + ?? throw new ArgumentNullException(nameof(cluster)); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs new file mode 100644 index 000000000..156c83cdb --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -0,0 +1,427 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for adding Helm release resources to a k3s cluster. +/// +public static class K3sHelmBuilderExtensions +{ + /// + /// Adds a Helm release as a child resource of the k3s cluster. + /// + /// Follows the same pattern as PostgresServerResource.AddDatabase: the release is + /// registered on the parent cluster, and the cluster's own + /// handler drives the install lifecycle for all registered releases. Helm output streams to + /// each release's individual log tab in the dashboard. + /// + /// + /// The k3s cluster resource builder. + /// Resource name β€” also the Helm release name. + /// Chart name. Add for remote charts. + /// Optional Helm repository URL (passed as --repo). + /// Optional chart version (--version). + /// Target namespace (created automatically). + /// A builder for the . + [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] + public static IResourceBuilder AddHelmRelease( + this IResourceBuilder builder, + [ResourceName] string name, + string chart, + string? repo = null, + string? version = null, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(chart); + + var cluster = builder.Resource; + + var release = new HelmReleaseResource(name, releaseName: name, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + + // Register the release on the parent cluster β€” mirrors PostgresServerResource.AddDatabase(). + cluster.AddHelmRelease(release.Name, release.ReleaseName); + + // Health check that satisfies WaitFor(helmRelease) on dependent resources. + // Returns Healthy only after the install lifecycle sets release.IsReady = true. + var healthCheckKey = $"helm_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new HelmReleaseHealthCheck(release), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(release) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "Helm Release", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ReleaseName", name), + new ResourcePropertySnapshot("Chart", chart), + new ResourcePropertySnapshot("Namespace", @namespace), + new ResourcePropertySnapshot("Version", version ?? "latest"), + ], + }); + } + + /// + /// Adds a Helm --set key=value argument to this release. + /// + public static IResourceBuilder WithHelmValue( + this IResourceBuilder builder, + string key, + string value) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(value); + + builder.Resource.HelmValues[key] = value; + return builder; + } + + /// + /// Exposes a Kubernetes service from this release as a clickable endpoint in the dashboard. + /// The NodePort is auto-discovered and forwarded in-process via the KubernetesClient WebSocket API. + /// + public static IResourceBuilder WithEndpoint( + this IResourceBuilder builder, + string serviceName, + int servicePort, + string name) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(serviceName); + ArgumentNullException.ThrowIfNull(name); + + builder.Resource.EndpointDefinitions.Add( + new HelmEndpointDefinition(serviceName, servicePort, name)); + + return builder; + } + + // ── Lifecycle (driven from the parent cluster's ResourceReadyEvent) ──────── + + /// + /// Runs the full install lifecycle for : + /// helm install β†’ NodePort discovery β†’ in-process port-forward β†’ Running. + /// Called by the parent cluster's handler β€” + /// the same pattern Postgres uses for database creation in AddPostgres. + /// + internal static async Task RunReleaseAsync( + HelmReleaseResource release, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(release, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); + + if (kubeconfigYaml is null) + { + throw new InvalidOperationException( + "k3s kubeconfig is not yet available. " + + "The cluster ResourceReadyEvent fired before the health check populated the kubeconfig."); + } + + await RunHelmAsync(release, kubeconfigYaml, logger, ct).ConfigureAwait(false); + + var urls = release.EndpointDefinitions.Count > 0 + ? await DiscoverAndStartPortForwardAsync(release, kubeconfigYaml, logger, ct) + .ConfigureAwait(false) + : ImmutableArray.Empty; + + // Set before PublishUpdateAsync so the health check unblocks WaitFor callers + // as soon as the Running state notification is processed. + release.IsReady = true; + + await notifications.PublishUpdateAsync(release, state => state with + { + State = KnownResourceStates.Running, + Urls = urls, + // Merge: keep all existing properties (the orchestrator injects ParentName which + // drives parent-child display in the dashboard) and only update/add our own. + // Replacing the entire Properties array would wipe out ParentName, causing the + // release to lose its parent and appear at the top level after going Running. + Properties = + [ + .. state.Properties.Where(p => + p.Name is not ("ReleaseName" or "Chart" or "Namespace" or "Version" or "ChartVersion")), + new ResourcePropertySnapshot("ReleaseName", release.ReleaseName), + new ResourcePropertySnapshot("Chart", release.Chart!), + new ResourcePropertySnapshot("ChartVersion", release.Version ?? "latest"), + new ResourcePropertySnapshot("Namespace", release.Namespace), + ], + }).ConfigureAwait(false); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Helm release '{Name}' failed.", release.ReleaseName); + + await notifications.PublishUpdateAsync(release, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + // ── helm subprocess ─────────────────────────────────────────────────────── + + private static async Task RunHelmAsync( + HelmReleaseResource release, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var tempKubeconfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-helm-{Environment.ProcessId}-{release.ReleaseName}.yaml"); + + await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + // When a repo URL is provided, add/update the repo first. + // The --repo shorthand in helm upgrade --install does not work reliably for + // all chart repositories (e.g. kubernetes-dashboard returns 404 via --repo). + // Official docs always show: helm repo add β†’ helm repo update β†’ helm install. + string? repoAlias = null; + if (release.RepoUrl is not null) + { + repoAlias = $"aspire-k3s-{release.ReleaseName}"; + await RunHelmCommandAsync( + logger, + ["repo", "add", "--force-update", repoAlias, release.RepoUrl], + ct).ConfigureAwait(false); + + await RunHelmCommandAsync( + logger, + ["repo", "update", repoAlias], + ct).ConfigureAwait(false); + } + + var args = BuildHelmInstallArgs( + release.ReleaseName, release.Chart!, repoAlias, + release.Version, release.Namespace, release.HelmValues, + tempKubeconfig); + + logger.LogInformation("Running: helm {Args}", string.Join(' ', args)); + + await RunHelmCommandAsync(logger, args, ct).ConfigureAwait(false); + } + finally + { + try { File.Delete(tempKubeconfig); } catch { /* best effort */ } + } + } + + // ── Argument builder ────────────────────────────────────────────────────── + + /// Runs a helm subcommand, logging output and failing on non-zero exit. + private static async Task RunHelmCommandAsync( + ILogger logger, + IEnumerable args, + CancellationToken ct) + { + var psi = new ProcessStartInfo("helm") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (var arg in args) + { + psi.ArgumentList.Add(arg); + } + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start helm process."); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogInformation("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogWarning("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"helm {string.Join(' ', psi.ArgumentList)} failed (exit code {process.ExitCode})."); + } + } + + internal static IReadOnlyList BuildHelmInstallArgs( + string releaseName, + string chart, + string? repoAlias, // null = no repo, non-null = "{alias}/{chart}" notation + string? version, + string @namespace, + IReadOnlyDictionary? values, + string kubeconfigPath) + { + // When a repo alias was registered via `helm repo add`, use "alias/chart" notation. + // Otherwise treat the chart as a path or OCI reference. + var chartRef = repoAlias is not null ? $"{repoAlias}/{chart}" : chart; + + var args = new List + { + "upgrade", "--install", + releaseName, + chartRef, + "--namespace", @namespace, + "--create-namespace", + "--wait", + "--timeout", "10m", + $"--kubeconfig={kubeconfigPath}", + }; + + if (version is not null) + { + args.Add("--version"); + args.Add(version); + } + + if (values is not null) + { + foreach (var (key, value) in values) + { + args.Add("--set"); + args.Add($"{key}={value}"); + } + } + + return args; + } + + // ── Port-forward ────────────────────────────────────────────────────────── + + private static async Task> DiscoverAndStartPortForwardAsync( + HelmReleaseResource release, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var urls = ImmutableArray.CreateBuilder(); + + using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); + using var k8sClient = new Kubernetes(config); + + foreach (var ep in release.EndpointDefinitions) + { + var nodePort = await DiscoverNodePortAsync( + k8sClient, release.ReleaseName, release.Namespace, + ep.ServiceName, ep.ServicePort, logger, ct).ConfigureAwait(false); + + var hostPort = AllocateHostPort(); + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigYaml, + release.Namespace, + ep.ServiceName, // look up by service name, not release label + hostPort, + ep.ServicePort); + + _ = forwarder.RunAsync(logger, ct); + + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); + + if (nodePort.HasValue) + { + urls.Add(new UrlSnapshot( + $"{ep.EndpointName} (container)", + $"{scheme}://{release.Parent.Name}:{nodePort.Value}", + IsInternal: true)); + } + } + + return urls.ToImmutable(); + } + + private static async Task DiscoverNodePortAsync( + Kubernetes k8sClient, + string releaseName, + string @namespace, + string serviceName, + int servicePort, + ILogger logger, + CancellationToken ct) + { + try + { + var services = await k8sClient.CoreV1.ListNamespacedServiceAsync( + @namespace, + labelSelector: $"app.kubernetes.io/instance={releaseName}", + cancellationToken: ct).ConfigureAwait(false); + + var port = services.Items + .FirstOrDefault(s => string.Equals( + s.Metadata.Name, serviceName, StringComparison.OrdinalIgnoreCase)) + ?.Spec.Ports + .FirstOrDefault(p => p.Port == servicePort); + + if (port?.NodePort is null) + { + logger.LogWarning( + "NodePort for {ServiceName}:{ServicePort} not found; container URL omitted.", + serviceName, servicePort); + } + + return port?.NodePort; + } + catch (Exception ex) + { + logger.LogWarning(ex, "NodePort discovery failed for service '{ServiceName}'.", serviceName); + return null; + } + } + + private static int AllocateHostPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs new file mode 100644 index 000000000..f50f70ba4 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -0,0 +1,400 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using k8s.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for applying Kubernetes YAML manifests to a k3s cluster. +/// +public static class K3sManifestBuilderExtensions +{ + /// + /// Applies one or more Kubernetes YAML files to the cluster via + /// kubectl apply --server-side (Server-Side Apply). No bind-mount is required. + /// + /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") + /// A directory: all .yaml/.yml files applied lexicographically. + /// A glob: "./k8s/crds/*.yaml" + /// + /// CRDs are detected automatically; the resource waits for the Established + /// condition via the KubernetesClient before transitioning to Running, so + /// WaitFor(manifest) correctly gates dependent operators. + /// + [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + public static IResourceBuilder AddK8sManifest( + this IResourceBuilder builder, + [ResourceName] string name, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(path); + + var cluster = builder.Resource; + var manifest = new K8sManifestResource(name, path, cluster); + + cluster.AddManifest(manifest.Name); + + var healthCheckKey = $"manifest_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new K8sManifestHealthCheck(manifest), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(manifest) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K8s Manifest", + State = KnownResourceStates.NotStarted, + Properties = [new ResourcePropertySnapshot("Path", path)], + }); + } + + /// + /// Exposes a Kubernetes service from this manifest as a clickable endpoint in the dashboard. + /// Traffic is forwarded in-process via the KubernetesClient WebSocket API. + /// + /// The manifest resource builder. + /// The Kubernetes service name. + /// The service port number. + /// Friendly name shown in the dashboard. + /// + /// The namespace containing the service. Defaults to "default". + /// For remote manifests (HTTP URLs) the namespace must be specified explicitly. + /// + public static IResourceBuilder WithEndpoint( + this IResourceBuilder builder, + string serviceName, + int servicePort, + string name, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(serviceName); + ArgumentNullException.ThrowIfNull(name); + + builder.Resource.EndpointDefinitions.Add( + new ManifestEndpointDefinition(serviceName, servicePort, name, @namespace)); + + return builder; + } + + // ── Lifecycle (called from the cluster's ResourceReadyEvent) ────────────── + + internal static async Task RunManifestAsync( + K8sManifestResource manifest, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(manifest, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); + + if (kubeconfigYaml is null) + { + throw new InvalidOperationException( + "k3s kubeconfig is not yet available when applying manifest."); + } + + // Write a temp kubeconfig file β€” kubectl requires a path, same as helm. + var tempKubeconfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-manifest-{Environment.ProcessId}-{manifest.Name}.yaml"); + + await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + var files = ResolveFiles(manifest.Path); + + logger.LogInformation( + "Applying {Count} manifest file(s) from '{Path}'", files.Count, manifest.Path); + + // kubectl apply --server-side (SSA) for each file. + foreach (var file in files) + { + await KubectlApplyAsync(file, tempKubeconfig, logger, ct) + .ConfigureAwait(false); + } + + // Wait for any CRDs to reach Established using the KubernetesClient. + await WaitForCrdsEstablishedAsync( + files, kubeconfigYaml, logger, ct).ConfigureAwait(false); + } + finally + { + try { File.Delete(tempKubeconfig); } catch { /* best effort */ } + } + + manifest.IsReady = true; + + // Start in-process port-forwards for any declared endpoints. + var urls = ImmutableArray.Empty; + if (manifest.EndpointDefinitions.Count > 0) + { + urls = await StartPortForwardAsync(manifest, kubeconfigYaml, logger, ct) + .ConfigureAwait(false); + } + + await notifications.PublishUpdateAsync(manifest, state => state with + { + State = KnownResourceStates.Running, + Urls = urls, + Properties = + [ + .. state.Properties.Where(p => p.Name is not "Path"), + new ResourcePropertySnapshot("Path", manifest.Path), + ], + }).ConfigureAwait(false); + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Failed to apply manifest '{Name}'.", manifest.Name); + + await notifications.PublishUpdateAsync(manifest, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + // ── kubectl apply ───────────────────────────────────────────────────────── + + private static async Task KubectlApplyAsync( + string file, + string kubeconfigPath, + ILogger logger, + CancellationToken ct) + { + logger.LogInformation("Applying {File}", file); + + var psi = new ProcessStartInfo("kubectl") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + psi.ArgumentList.Add("apply"); + psi.ArgumentList.Add("-f"); + psi.ArgumentList.Add(file); + psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); + psi.ArgumentList.Add("--server-side"); + psi.ArgumentList.Add("--field-manager=aspire-k3s"); + psi.ArgumentList.Add("--force-conflicts"); + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start kubectl process."); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogInformation("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogWarning("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"kubectl apply failed for '{file}' (exit code {process.ExitCode})."); + } + } + + // ── Port-forward for manifest endpoints ────────────────────────────────── + + private static async Task> StartPortForwardAsync( + K8sManifestResource manifest, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var urls = ImmutableArray.CreateBuilder(); + + foreach (var ep in manifest.EndpointDefinitions) + { + var hostPort = AllocateHostPort(); + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigYaml, + ep.Namespace, + ep.ServiceName, + hostPort, + ep.ServicePort); + + _ = forwarder.RunAsync(logger, ct); + + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); + } + + return urls.ToImmutable(); + } + + private static int AllocateHostPort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } + + // ── CRD readiness (in-process via KubernetesClient) ────────────────────── + + private static async Task WaitForCrdsEstablishedAsync( + IReadOnlyList files, + string kubeconfigYaml, + ILogger logger, + CancellationToken ct) + { + var crdNames = new List(); + + foreach (var file in files) + { + // Skip remote URLs β€” kubectl downloaded and applied them, but we can't parse + // them with KubernetesYaml locally. CRD detection for remote files is skipped; + // if you need CRD readiness gating, use a local file path instead. + if (file.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + file.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var objects = await KubernetesYaml.LoadAllFromFileAsync(file) + .ConfigureAwait(false); + + foreach (var obj in objects) + { + if (obj is V1CustomResourceDefinition crd && crd.Metadata?.Name is { } crdName) + { + crdNames.Add(crdName); + } + } + } + + if (crdNames.Count == 0) + { + return; + } + + using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); + using var k8sClient = new Kubernetes(config); + + foreach (var crdName in crdNames) + { + await WaitForCrdEstablishedAsync(k8sClient, crdName, logger, ct) + .ConfigureAwait(false); + } + } + + private static async Task WaitForCrdEstablishedAsync( + Kubernetes k8sClient, + string crdName, + ILogger logger, + CancellationToken ct) + { + logger.LogInformation("Waiting for CRD '{Crd}' to reach Established...", crdName); + + while (!ct.IsCancellationRequested) + { + var crd = await k8sClient.ApiextensionsV1 + .ReadCustomResourceDefinitionAsync(crdName, cancellationToken: ct) + .ConfigureAwait(false); + + var established = crd.Status?.Conditions?.Any(c => + c.Type == "Established" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; + + if (established) + { + logger.LogInformation("CRD '{Crd}' is Established.", crdName); + return; + } + + await Task.Delay(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); + } + } + + // ── File resolution ─────────────────────────────────────────────────────── + + // Exposed for unit tests via InternalsVisibleTo. + internal static IReadOnlyList ResolveFilesForTest(string path) => + ResolveFiles(path); + + private static IReadOnlyList ResolveFiles(string path) + { + // kubectl apply -f supports HTTPS URLs natively β€” pass through as-is. + if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + return [path]; + } + + if (Directory.Exists(path)) + { + return [ + ..Directory.GetFiles(path, "*.yaml", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(path, "*.yml", SearchOption.TopDirectoryOnly)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + ]; + } + + var dir = System.IO.Path.GetDirectoryName(path) ?? "."; + var pattern = System.IO.Path.GetFileName(path); + + if (pattern.Contains('*') || pattern.Contains('?')) + { + return [..Directory.GetFiles(dir, pattern).Order(StringComparer.OrdinalIgnoreCase)]; + } + + return [path]; + } +} + +/// +/// Health check that satisfies WaitFor(manifest). +/// Returns once all files are applied +/// and any CRDs have reached the Established condition. +/// +internal sealed class K8sManifestHealthCheck(K8sManifestResource manifest) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(manifest.IsReady + ? HealthCheckResult.Healthy("Manifests applied") + : HealthCheckResult.Unhealthy("Manifests not yet applied")); +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs new file mode 100644 index 000000000..3399474a0 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -0,0 +1,464 @@ +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using k8s; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using System.Net; +using System.Net.Sockets; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental +#pragma warning disable ASPIRECERTIFICATES001 // WithHttpsDeveloperCertificate is experimental + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding k3s cluster resources to an . +/// +public static class K3sBuilderExtensions +{ + /// + /// Adds a k3s Kubernetes cluster resource to the distributed application. + /// + /// The distributed application builder. + /// The resource name used for DNS resolution within the DCP network. + /// + /// Optional host port to bind the Kubernetes API server (port 6443) to. + /// When a random available port is assigned. + /// + /// Optional callback to configure . + /// A builder for the . + [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + public static IResourceBuilder AddK3sCluster( + this IDistributedApplicationBuilder builder, + [ResourceName] string name, + int? apiServerPort = null, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + + var options = new K3sClusterOptions(); + configure?.Invoke(options); + + var resource = new K3sClusterResource(name); + var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + + var resourceBuilder = builder.AddResource(resource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + + // ── k3d-style init entrypoint ───────────────────────────────────── + // Runs mount --make-rshared / and the cgroupsv2 evacuation fix before + // k3s starts β€” exactly what k3d does via /bin/k3d-entrypoint*.sh. + // WithContainerFiles injects the script via `docker cp` β€” no bind mounts, + // no host-side temp files, works on all platforms. + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + + // ── k3s server command ──────────────────────────────────────────── + .WithArgs("server") + + // NOTE: --cluster-init (embedded etcd) is intentionally NOT used. + // etcd stores the container's IP in its peer URL. When Docker assigns a new IP + // on container restart, etcd refuses to start ("not a member of the etcd cluster"). + // k3s uses SQLite by default for single-server clusters β€” it handles restarts and + // IP changes gracefully. k3d only adds --cluster-init for HA multi-server setups. + + // Add TLS SANs so the API server certificate is valid for all addresses + // that clients use to reach it: host-side (localhost), DCP-network containers + // ({resourceName}), and any forwarded address (0.0.0.0). + .WithArgs($"--tls-san=127.0.0.1") + .WithArgs($"--tls-san=localhost") + .WithArgs($"--tls-san={name}") + .WithArgs("--tls-san=0.0.0.0") + + // Disable components not needed for local development. + // servicelb and metrics-server are the biggest resource consumers and slowest + // to start; disabling them speeds up cluster readiness significantly. + .WithArgs("--disable=servicelb") + .WithArgs("--disable=metrics-server") + + // Reduce log verbosity β€” k3s writes everything to stderr which DCP surfaces + // as "error" level entries in the dashboard log viewer. + .WithArgs("-v", "0") + .WithArgs("--kube-apiserver-arg=v=0") + .WithArgs("--kube-controller-manager-arg=v=0") + .WithArgs("--kube-scheduler-arg=v=0") + // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning + // "Failed to kill all the processes attached to cgroup / os: process not initialized". + // This is a known benign race condition in Docker-in-Docker: the kubelet tries to + // force-kill pod cgroup processes that are already dead. The cgroup is still cleaned + // up correctly; only the redundant kill attempt fails. + .WithArgs("--kubelet-arg=v=0") + + // ── API server endpoint ─────────────────────────────────────────── + .WithHttpsEndpoint( + targetPort: 6443, + port: apiServerPort, + name: K3sClusterResource.ApiServerEndpointName) + + // ── Docker / container runtime flags (mirrors k3d) ──────────────── + // Privileged mode is mandatory for iptables, network namespaces, and cgroups. + .WithContainerRuntimeArgs("--privileged") + // k3d uses Docker's --init (tini) so that k3s's child processes are properly + // reaped and signals are forwarded correctly. Without it, zombie processes + // accumulate and shutdown becomes unreliable. + .WithContainerRuntimeArgs("--init") + // Use the host user namespace β€” required when Docker is configured with userns-remap; + // a no-op otherwise. k3d always passes this flag. + .WithContainerRuntimeArgs("--userns=host") + // Share the host's (Docker Desktop VM's) cgroup namespace instead of creating + // a new one. Without this, the k3s kubelet fails to create the "kubepods" cgroup + // hierarchy because the new isolated namespace has domain controllers in an invalid + // state for cgroupsv2. k3d always passes --cgroupns=host for this reason. + .WithContainerRuntimeArgs("--cgroupns=host") + // Bind-mount the cgroup filesystem from the Docker Desktop VM into the container + // as read-write. With --cgroupns=host the container sees the host's cgroup namespace, + // but the mount is still read-only by default; making it rw lets the kubelet create + // sub-cgroups for pods (kubepods/besteffort/...). k3d always mounts this as rw. + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + // tmpfs mounts for runtime sockets and PIDs β€” same as k3d defaults. + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + + // ── Environment ─────────────────────────────────────────────────── + // Set an explicit cluster token (k3d always sets K3S_TOKEN). + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + // World-readable kubeconfig so docker exec can read it without root. + .WithEnvironment("K3S_KUBECONFIG_MODE", "644") + + .WithIconName("Kubernetes") + .WithHttpsDeveloperCertificate(); + + if (options.ClusterCidr is not null) + { + resourceBuilder.WithArgs($"--cluster-cidr={options.ClusterCidr}"); + } + + if (options.ServiceCidr is not null) + { + resourceBuilder.WithArgs($"--service-cidr={options.ServiceCidr}"); + } + + foreach (var component in options.DisabledComponents) + { + resourceBuilder.WithArgs($"--disable={component}"); + } + + foreach (var arg in options.ExtraArgs) + { + resourceBuilder.WithArgs(arg); + } + + // Create agent nodes specified via options.AgentCount. + // Agents use DCP DNS: K3S_URL=https://{name}:6443 resolves to the server container. + // NO WaitFor β€” k3s agent retries indefinitely until the server is reachable. + // This avoids a deadlock where the cluster health check waits for nodes to be Ready + // while nodes wait for the cluster to be healthy. + for (var i = 0; i < options.AgentCount; i++) + { + resource.AgentCount++; + var agentName = $"{name}-agent-{i}"; + var agentResource = new K3sAgentResource(agentName, resource); + resource.AddAgentResource(agentResource); + + builder.AddResource(agentResource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + .WithArgs("agent") + .WithArgs("-v", "0") + .WithArgs("--kubelet-arg=v=0") + .WithEnvironment("K3S_URL", $"https://{name}:6443") + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + .WithEnvironment("K3S_NODE_NAME", agentName) + .WithContainerRuntimeArgs("--privileged") + .WithContainerRuntimeArgs("--init") + .WithContainerRuntimeArgs("--userns=host") + .WithContainerRuntimeArgs("--cgroupns=host") + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Agent", + State = KnownResourceStates.Starting, + Properties = [new ResourcePropertySnapshot("Cluster", name)], + }); + } + + resourceBuilder.WithHealthCheck($"k3s_{name}_ready"); + + builder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + $"k3s_{name}_ready", + sp => new K3sReadinessHealthCheck(resource, resource.ApiEndpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + // Postgres pattern: the parent cluster's ResourceReadyEvent drives ALL registered + // child lifecycles β€” both HelmReleases and K8sManifests β€” in parallel. + builder.Eventing.Subscribe(resource, (@event, ct) => + { + var appModel = @event.Services.GetRequiredService(); + var notifications = @event.Services.GetRequiredService(); + var loggerService = @event.Services.GetRequiredService(); + + // Start all Helm release installs concurrently. + foreach (var release in appModel.Resources + .OfType() + .Where(r => ReferenceEquals(r.Parent, resource))) + { + var logger = loggerService.GetLogger(release); + _ = Task.Run(() => K3sHelmBuilderExtensions.RunReleaseAsync( + release, resource, notifications, logger, ct), ct); + } + + // Start all manifest applies concurrently. + foreach (var manifest in appModel.Resources + .OfType() + .Where(m => ReferenceEquals(m.Parent, resource))) + { + var logger = loggerService.GetLogger(manifest); + _ = Task.Run(() => K3sManifestBuilderExtensions.RunManifestAsync( + manifest, resource, notifications, logger, ct), ct); + } + + return Task.CompletedTask; + }); + + return resourceBuilder; + } + + /// Overrides the k3s server image version. + [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + public static IResourceBuilder WithK3sVersion( + this IResourceBuilder builder, + string tag) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(tag); + + return builder.WithImageTag(tag); + } + + /// Sets the pod subnet CIDR (--cluster-cidr). + [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithPodSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--cluster-cidr={cidr}"); + } + + /// Sets the service subnet CIDR (--service-cidr). + [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithServiceSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--service-cidr={cidr}"); + } + + /// Disables a built-in k3s component (e.g. traefik). + [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + public static IResourceBuilder WithDisabledComponent( + this IResourceBuilder builder, + string component) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(component); + + return builder.WithArgs($"--disable={component}"); + } + + /// Appends a raw argument to the k3s server command. + [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + public static IResourceBuilder WithExtraArg( + this IResourceBuilder builder, + string arg) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(arg); + + return builder.WithArgs(arg); + } + + /// + /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) + /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. + /// + /// The cluster resource builder. + /// + /// The volume name. When , an auto-generated name is used in the form + /// {appName}-{sha256}-{resourceName}-data β€” the same scheme used by + /// PostgresServerResource.WithDataVolume and all other Aspire hosting integrations. + /// + /// The resource builder for chaining. + public static IResourceBuilder WithDataVolume( + this IResourceBuilder builder, + string? name = null) + { + ArgumentNullException.ThrowIfNull(builder); + + return builder + .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") + // Auto-restart on crash so a persistent cluster survives transient failures + // without requiring AppHost intervention. Docker will not restart the container + // on explicit stop (DCP shutdown), only on unexpected exits. + .WithContainerRuntimeArgs("--restart=unless-stopped"); + } + + /// + /// Injects the k3s kubeconfig into as a + /// KUBECONFIG_DATA environment variable (base-64-encoded YAML) β€” no files are written. + /// + /// + /// s (Auto or InlineData) receive the + /// container-network kubeconfig + /// (server: https://{resourceName}:6443). + /// + /// + /// Projects and executables (Auto or HostPath) receive the + /// host kubeconfig + /// (server: https://localhost:{allocatedPort}). + /// + /// + /// Consuming code reads the variable and builds a client without touching disk: + /// + /// var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); + /// using var stream = new MemoryStream(bytes); + /// var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); + /// + /// The variable is populated only after the cluster health check passes; use + /// WaitFor(cluster) on the dependent resource to guarantee ordering. + /// + [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source, + KubeconfigInjectionStrategy strategy = KubeconfigInjectionStrategy.Auto) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var cluster = source.Resource; + + return destination.WithEnvironment(ctx => + { + // Select the right kubeconfig variant based on where the resource runs: + // β€’ ContainerResource β†’ container-network URL (reaches API server via DCP DNS) + // β€’ Project/Executable β†’ host URL (reaches API server via localhost port-mapping) + var useContainerVariant = + strategy == KubeconfigInjectionStrategy.ContainerNetwork + || (strategy == KubeconfigInjectionStrategy.Auto + && destination.Resource is ContainerResource); + + var cfg = useContainerVariant + ? cluster.ContainerKubeconfig + : cluster.AdminKubeconfig; + + if (cfg is not null) + { + var yaml = KubernetesYaml.Serialize(cfg); + ctx.EnvironmentVariables["KUBECONFIG_DATA"] = + Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); + } + }); + } + + /// + /// Serialises the admin kubeconfig (server: https://localhost:{port}) to YAML. + /// Returns if the health check has not yet populated the config. + /// + internal static string? GetAdminKubeconfigYaml(K3sClusterResource cluster) => + cluster.AdminKubeconfig is null + ? null + : KubernetesYaml.Serialize(cluster.AdminKubeconfig); + + /// + /// Sets the container lifetime for the k3s cluster and all its agent nodes. + /// When is used the agents must also be + /// persistent β€” otherwise they are recreated on every AppHost restart while the server + /// retains its state, causing the node re-join sequence to fail. + /// + public static IResourceBuilder WithLifetime( + this IResourceBuilder builder, + ContainerLifetime lifetime) + { + ArgumentNullException.ThrowIfNull(builder); + + // Apply to the cluster container (identical to the built-in generic WithLifetime). + builder.WithAnnotation( + new ContainerLifetimeAnnotation { Lifetime = lifetime }, + ResourceAnnotationMutationBehavior.Replace); + + // Propagate to every agent β€” they share the same cluster state volume and must + // have the same lifetime so nodes survive AppHost restarts in sync with the server. + foreach (var agent in builder.Resource.AgentResources) + { + var existing = agent.Annotations.OfType().ToList(); + foreach (var ann in existing) + { + agent.Annotations.Remove(ann); + } + + agent.Annotations.Add(new ContainerLifetimeAnnotation { Lifetime = lifetime }); + } + + return builder; + } + + // cgroupsv2 fix adapted from moby/moby (Apache-2.0, used with permission by k3d). + // See: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh + private const string K3sInitEntrypointScript = """ + #!/bin/sh + # Aspire k3s init entrypoint β€” adapted from k3d (https://github.com/k3d-io/k3d) + # cgroupsv2 fix adapted from moby/moby (Apache-2.0), used with permission. + + # Make mountpoints recursively shared β€” required for volume propagation in Docker-in-Docker. + mount --make-rshared / 2>/dev/null || true + + # cgroupsv2: evacuate root cgroup so k3s kubelet can create pod sub-cgroups. + # Without this, writing to cgroup.subtree_control fails with EBUSY because + # init processes still live in the root cgroup. + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + if command -v xargs >/dev/null 2>&1; then + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + else + busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + fi + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true + fi + + exec k3s "$@" + """; +} + +#pragma warning restore ASPIREATS001 +#pragma warning restore ASPIRECERTIFICATES001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs new file mode 100644 index 000000000..cb33f7aa7 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -0,0 +1,40 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Configuration options for a k3s cluster resource. +/// +public sealed class K3sClusterOptions +{ + /// + /// Gets or sets the CIDR range for pod IPs (passed as --cluster-cidr). + /// + public string? ClusterCidr { get; set; } + + /// + /// Gets or sets the CIDR range for service IPs (passed as --service-cidr). + /// + public string? ServiceCidr { get; set; } + + /// + /// Gets the list of k3s components to disable (each passed as --disable=<component>). + /// + public IList DisabledComponents { get; } = new List(); + + /// + /// Gets the list of raw extra arguments appended to the k3s server command. + /// + public IList ExtraArgs { get; } = new List(); + + /// + /// Gets or sets the number of agent (worker) nodes to add to the cluster. + /// Equivalent to k3d's --agents N flag. + /// Defaults to 0 (single-node cluster). + /// + public int AgentCount { get; set; } + + /// + /// Gets or sets the k3s image tag (e.g. v1.31.4-k3s1). + /// When , the default tag embedded in the package is used. + /// + public string? ImageTag { get; set; } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs new file mode 100644 index 000000000..f7b983b5a --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -0,0 +1,63 @@ +using k8s.KubeConfigModels; + +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s Kubernetes cluster running as a privileged container resource. +/// +/// The resource name. +public sealed class K3sClusterResource(string name) : ContainerResource(name) +{ + internal const string ApiServerEndpointName = "api"; + + /// + /// Gets the admin kubeconfig for host-side processes + /// (server: https://localhost:{allocatedPort}). + /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// Serialise with KubernetesYaml.Serialize(AdminKubeconfig) when needed. + /// + internal K8SConfiguration? AdminKubeconfig { get; set; } + + /// + /// Gets the kubeconfig for containers on the DCP Docker network + /// (server: https://{resourceName}:6443). + /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// + internal K8SConfiguration? ContainerKubeconfig { get; set; } + + private EndpointReference? _apiEndpoint; + + /// Gets the endpoint reference for the k3s API server (port 6443). + public EndpointReference ApiEndpoint => _apiEndpoint ??= new(this, ApiServerEndpointName); + + // ── Child resource tracking (Postgres pattern) ──────────────────────────── + + /// + /// Number of agent (worker) nodes. The health check waits for all + /// 1 + AgentCount nodes to reach Ready state before marking the cluster healthy. + /// + internal int AgentCount { get; set; } + + private readonly List _agentResources = []; + + /// Agent resource instances, used to propagate annotations (e.g. lifetime) to all nodes. + internal IReadOnlyList AgentResources => _agentResources; + + internal void AddAgentResource(K3sAgentResource agent) => _agentResources.Add(agent); + + private readonly Dictionary _helmReleases = + new(StringComparer.OrdinalIgnoreCase); + + /// A dictionary of registered Helm releases keyed by resource name. + public IReadOnlyDictionary HelmReleases => _helmReleases; + + internal void AddHelmRelease(string resourceName, string releaseName) => + _helmReleases.TryAdd(resourceName, releaseName); + + private readonly List _manifests = []; + + /// Names of registered children. + public IReadOnlyList Manifests => _manifests; + + internal void AddManifest(string resourceName) => _manifests.Add(resourceName); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs new file mode 100644 index 000000000..605e5c65c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -0,0 +1,13 @@ +namespace Aspire.Hosting.ApplicationModel; + +internal static class K3sContainerImageTags +{ + /// docker.io + public const string Registry = "docker.io"; + + /// rancher/k3s + public const string Image = "rancher/k3s"; + + /// v1.31.4-k3s1 + public const string Tag = "v1.31.4-k3s1"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs new file mode 100644 index 000000000..1fa3eb074 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -0,0 +1,108 @@ +using System.Diagnostics; +using System.Text; +using Microsoft.Extensions.Logging; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Forwards a local TCP port to a Kubernetes service by running +/// kubectl port-forward service/{name} {localPort}:{servicePort} -n {namespace} +/// as a managed subprocess. +/// +/// Mirrors what a developer types in a terminal β€” the most reliable approach for +/// k3s-in-Docker because kubectl handles WebSocket negotiation, kubelet routing, +/// and reconnect logic internally. +/// +/// +internal sealed class K3sInProcessPortForwarder( + string kubeconfigYaml, + string @namespace, + string serviceName, + int localPort, + int servicePort) +{ + public async Task RunAsync(ILogger logger, CancellationToken ct) + { + logger.LogInformation( + "Port-forward: localhost:{Local} β†’ svc/{Service}.{Ns}:{Port}", + localPort, serviceName, @namespace, servicePort); + + var tempConfig = Path.Combine( + Path.GetTempPath(), + $"aspire-k3s-pf-{Environment.ProcessId}-{serviceName}.yaml"); + + await File.WriteAllTextAsync(tempConfig, kubeconfigYaml, Encoding.UTF8, ct) + .ConfigureAwait(false); + + try + { + while (!ct.IsCancellationRequested) + { + try + { + await RunKubectlAsync(tempConfig, logger, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, + "Port-forward for svc/{Service} exited; restarting in 5 s…", serviceName); + + await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false); + } + } + } + finally + { + try { File.Delete(tempConfig); } catch { /* best-effort cleanup */ } + } + } + + private async Task RunKubectlAsync(string kubeconfigPath, ILogger logger, CancellationToken ct) + { + var psi = new ProcessStartInfo("kubectl") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + psi.ArgumentList.Add("port-forward"); + psi.ArgumentList.Add($"service/{serviceName}"); + psi.ArgumentList.Add($"{localPort}:{servicePort}"); + psi.ArgumentList.Add("-n"); + psi.ArgumentList.Add(@namespace); + psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); + + using var process = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start kubectl port-forward."); + + using var reg = ct.Register(() => + { + try { process.Kill(entireProcessTree: true); } catch { /* already exited */ } + }); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogDebug("{Line}", e.Data); + }; + process.ErrorDataReceived += (_, e) => + { + if (e.Data is not null) logger.LogDebug("{Line}", e.Data); + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(ct).ConfigureAwait(false); + + if (process.ExitCode != 0 && !ct.IsCancellationRequested) + { + throw new InvalidOperationException( + $"kubectl port-forward exited with code {process.ExitCode}."); + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs new file mode 100644 index 000000000..f0cc8f1fd --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -0,0 +1,190 @@ +using System.Diagnostics; +using Aspire.Hosting.ApplicationModel; +using k8s; +using k8s.KubeConfigModels; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Health check for . +/// +/// Instead of probing GET /healthz (which requires authentication in Kubernetes 1.28+ +/// because anonymous-auth defaults to false), this check runs +/// docker exec {container} kubectl get nodes inside the k3s container. +/// kubectl inside the container uses the default in-cluster kubeconfig, so no external +/// credentials are needed and the result is authoritative: a node in Ready state +/// proves the API server, scheduler, and kubelet are all functional. +/// +/// +/// On first success the kubeconfig is read from the container via docker exec cat, +/// parsed into two variants, and stored on the resource: +/// +/// β€” server: https://localhost:{port} +/// β€” server: https://{name}:6443 +/// +/// +/// +internal sealed class K3sReadinessHealthCheck : IHealthCheck +{ + private readonly K3sClusterResource _resource; + private readonly EndpointReference _endpoint; + private bool _kubeconfigRead; + + internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) + { + _resource = resource; + _endpoint = endpoint; + } + + /// + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + if (!_endpoint.IsAllocated) + { + return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + } + + var port = _endpoint.Port; + + try + { + var containerId = await FindContainerIdAsync(cancellationToken); + + if (containerId is null) + { + return HealthCheckResult.Unhealthy("k3s container not yet found via docker ps"); + } + + // Run kubectl get nodes inside the container where the default kubeconfig is + // already configured β€” avoids any authentication issue from the outside. + var nodesOutput = await RunDockerAsync( + ["exec", containerId, + "kubectl", "get", "nodes", + "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", + "--no-headers"], + cancellationToken); + + if (nodesOutput is null) + { + return HealthCheckResult.Unhealthy( + "kubectl get nodes failed β€” k3s API server not yet ready"); + } + + // Count nodes actually in Ready state (excluding NotReady ones). + var readyNodeLines = nodesOutput + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.Contains("Ready") && !line.Contains("NotReady")) + .ToArray(); + + // For multi-node clusters (WithAgentNodes), wait for server + all agents. + // For single-node, 1 Ready node is sufficient. + var expectedNodes = 1 + _resource.AgentCount; + + if (readyNodeLines.Length < expectedNodes) + { + return HealthCheckResult.Unhealthy( + $"k3s cluster: {readyNodeLines.Length}/{expectedNodes} nodes Ready"); + } + + if (!_kubeconfigRead) + { + var rawYaml = await RunDockerAsync( + ["exec", containerId, "cat", "/etc/rancher/k3s/k3s.yaml"], + cancellationToken); + + if (rawYaml is null) + { + return HealthCheckResult.Unhealthy( + "k3s kubeconfig not yet available inside the container"); + } + + var parsed = KubernetesYaml.Deserialize(rawYaml); + + _resource.AdminKubeconfig = + BuildConfig(parsed, $"https://localhost:{port}"); + _resource.ContainerKubeconfig = + BuildConfig(parsed, $"https://{_resource.Name}:6443"); + _kubeconfigRead = true; + } + + return HealthCheckResult.Healthy("k3s cluster is ready"); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy(ex.Message, ex); + } + } + + private static K8SConfiguration BuildConfig(K8SConfiguration source, string serverUrl) + { + var yaml = KubernetesYaml.Serialize(source); + var copy = KubernetesYaml.Deserialize(yaml); + + foreach (var cluster in copy.Clusters ?? []) + { + if (cluster.ClusterEndpoint is not null) + { + cluster.ClusterEndpoint.Server = serverUrl; + } + } + + return copy; + } + + private async Task FindContainerIdAsync(CancellationToken ct) + { + // docker ps --filter name=VALUE uses substring matching: "name=k8s" also matches + // "k8s-agent-0", "k8s-agent-1", etc. Use --format to get names alongside IDs and + // exclude agent containers whose names contain "-agent-". + var output = await RunDockerAsync( + ["ps", + "--filter", $"name={_resource.Name}", + "--format", "{{.ID}}\t{{.Names}}", + "--no-trunc"], + ct); + + if (output is null) + { + return null; + } + + return output + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(line => line.Split('\t', 2)) + .Where(parts => parts.Length == 2 && !parts[1].Contains("-agent-")) + .Select(parts => parts[0].Trim()) + .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id)); + } + + private static async Task RunDockerAsync(string[] args, CancellationToken ct) + { + var psi = new ProcessStartInfo("docker") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + foreach (var arg in args) + { + psi.ArgumentList.Add(arg); + } + + using var process = Process.Start(psi); + if (process is null) + { + return null; + } + + var output = await process.StandardOutput.ReadToEndAsync(ct); + await process.WaitForExitAsync(ct); + + return process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output) + ? output + : null; + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs new file mode 100644 index 000000000..f0a6a361b --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -0,0 +1,47 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via +/// Server-Side Apply. This is a child resource of , following +/// the same parent-child pattern as . +/// +/// No kubectl binary is required β€” the KubernetesClient library handles the apply. +/// CRDs reach the Established condition before the resource transitions to +/// Running, so dependent resources can safely WaitFor the manifest. +/// +/// +/// The Aspire resource name. +/// +/// Path to a single YAML file, a directory, or a glob pattern (*.yaml). +/// Directories and globs are expanded lexicographically. +/// +/// The parent k3s cluster resource. +public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the manifest path, directory, or glob. + public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); + + /// + /// Set to by the lifecycle after all objects are applied and + /// (for CRDs) the Established condition is confirmed. + /// + internal volatile bool IsReady; + + /// Services to expose via in-process port-forward after the manifest is applied. + internal List EndpointDefinitions { get; } = []; +} + +/// Describes a service endpoint to expose from a . +/// Kubernetes service name. +/// Service port number. +/// Friendly name shown in the dashboard. +/// Kubernetes namespace where the service lives. +internal sealed record ManifestEndpointDefinition( + string ServiceName, + int ServicePort, + string EndpointName, + string Namespace); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs new file mode 100644 index 000000000..b747206b1 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs @@ -0,0 +1,33 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Controls which kubeconfig server-URL variant is injected via KUBECONFIG_DATA. +/// All variants are delivered as base-64-encoded YAML without writing any file. +/// +public enum KubeconfigInjectionStrategy +{ + /// + /// Selects the server URL automatically based on the resource type: + /// + /// Container resources receive the + /// container-network URL (https://{resourceName}:6443). + /// Projects and executables receive the host URL + /// (https://localhost:{allocatedPort}). + /// + /// + Auto, + + /// + /// Always inject the host-network kubeconfig (server: https://localhost:{port}). + /// Use when a container is launched with --network=host or when the caller + /// explicitly needs host-side connectivity. + /// + HostNetwork, + + /// + /// Always inject the DCP-network kubeconfig + /// (server: https://{resourceName}:6443). + /// Use when a host process needs to reach the cluster the same way containers do. + /// + ContainerNetwork, +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md new file mode 100644 index 000000000..f6a5d8150 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -0,0 +1,67 @@ +# CommunityToolkit.Aspire.Hosting.K3s + +An Aspire hosting integration for [k3s](https://k3s.io/) β€” a lightweight, certified Kubernetes distribution by Rancher/SUSE. + +## Getting started + +### Prerequisites + +- Docker with support for `--privileged` containers (Linux host or Docker Desktop on macOS/Windows) + +### Installation + +```sh +dotnet add package CommunityToolkit.Aspire.Hosting.K3s +``` + +## Usage + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder.AddK3sCluster("k8s") + .WithPersistentState(); + +builder.AddProject("api") + .WithReference(cluster) + .WaitFor(cluster); + +builder.Build().Run(); +``` + +### Kubeconfig injection + +`WithReference(cluster)` automatically selects the injection mode: + +| Resource type | Environment variable set | +|---|---| +| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=/tmp/aspire-k3s-k8s/admin.yaml` | +| `ContainerResource` | `KUBECONFIG_DATA=` | + +### Configuration options + +```csharp +builder.AddK3sCluster("k8s", configure: opts => +{ + opts.ClusterCidr = "10.42.0.0/16"; + opts.ServiceCidr = "10.43.0.0/16"; + opts.DisabledComponents.Add("traefik"); +}); +``` + +Or use the fluent API: + +```csharp +builder.AddK3sCluster("k8s") + .WithK3sVersion("v1.32.3-k3s1") + .WithPodSubnet("10.42.0.0/16") + .WithServiceSubnet("10.43.0.0/16") + .WithDisabledComponent("traefik") + .WithPersistentState(); +``` + +## Known limitations + +- Requires a privileged Docker runtime; `--privileged` is passed automatically. +- On Linux hosts the `/lib/modules` directory should be present for CNI networking. +- The first cluster start can take 30–60 seconds while container images and CNI plugins are initialised. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs new file mode 100644 index 000000000..50701b0d7 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Aspire.Hosting +{ + public static partial class K3sHelmBuilderExtensions + { + public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } + public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } + public static ApplicationModel.IResourceBuilder WithEndpoint(this ApplicationModel.IResourceBuilder builder, string serviceName, int servicePort, string name) { throw null; } + } + + public static partial class K3sBuilderExtensions + { + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source, CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy strategy = CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy.Auto) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } + public static ApplicationModel.IResourceBuilder WithPersistentState(this ApplicationModel.IResourceBuilder builder, string? volumeName = null) { throw null; } + } +} + +namespace Aspire.Hosting.ApplicationModel +{ + public sealed partial class HelmReleaseResource : Resource, IResourceWithParent, IResourceWithWaitSupport + { + public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!) { } + public K3sClusterResource Parent { get { throw null; } } + public string ReleaseName { get { throw null; } } + public string Namespace { get { throw null; } } + } + + public sealed partial class K3sClusterResource : ContainerResource + { + public K3sClusterResource(string name) : base(default!) { } + public EndpointReference ApiEndpoint { get { throw null; } } + } +} + +namespace CommunityToolkit.Aspire.Hosting +{ + public sealed partial class K3sClusterOptions + { + public K3sClusterOptions() { } + public string? ClusterCidr { get { throw null; } set { } } + public string? ImageTag { get { throw null; } set { } } + public string? ServiceCidr { get { throw null; } set { } } + public System.Collections.Generic.IList DisabledComponents { get { throw null; } } + public System.Collections.Generic.IList ExtraArgs { get { throw null; } } + } + + public enum KubeconfigInjectionStrategy + { + Auto = 0, + HostNetwork = 1, + ContainerNetwork = 2, + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj new file mode 100644 index 000000000..bad157c30 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs new file mode 100644 index 000000000..1269e0a34 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -0,0 +1,406 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class HelmReleaseResourceTests +{ + [Fact] + public void AddHelmReleaseAddsHelmReleaseResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Name); + } + + [Fact] + public void AddHelmReleaseDoesNotCreateSeparateInstallResource() + { + // helm upgrade --install runs internally inside the HelmReleaseResource lifecycle; + // no separate ExecutableResource is added to the application model. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AddHelmReleaseDefaultsReleaseName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("my-release", "my-chart"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("my-release", resource.ReleaseName); + } + + [Fact] + public void AddHelmReleaseDefaultsNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("default", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseWithExplicitNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", @namespace: "argocd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseStoresRepoUrl() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + "argocd", "argo-cd", + repo: "https://argoproj.github.io/argo-helm"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("https://argoproj.github.io/argo-helm", resource.RepoUrl); + } + + [Fact] + public void AddHelmReleaseStoresVersion() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", version: "7.8.0"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("7.8.0", resource.Version); + } + + [Fact] + public void HelmReleaseParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + } + + [Fact] + public void HelmReleaseImplementsNonGenericIResourceWithParent() + { + // The Aspire dashboard uses the non-generic IResourceWithParent to group + // child resources under their parent. Verify both the generic and non-generic + // interfaces are satisfied and point to the same cluster resource. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + + // Non-generic IResourceWithParent (used by the dashboard) + var nonGeneric = resource as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(cluster.Resource, nonGeneric.Parent); + } + + [Fact] + public void WithHelmValueAccumulatesValues() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValue("server.service.type", "NodePort") + .WithHelmValue("server.insecure", "true"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("NodePort", resource.HelmValues["server.service.type"]); + Assert.Equal("true", resource.HelmValues["server.insecure"]); + } + + [Fact] + public void WithEndpointAccumulatesEndpoints() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithEndpoint("argocd-server", servicePort: 443, name: "ui"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var ep = Assert.Single(resource.EndpointDefinitions); + Assert.Equal("argocd-server", ep.ServiceName); + Assert.Equal(443, ep.ServicePort); + Assert.Equal("ui", ep.EndpointName); + } + + [Fact] + public void WithEndpointMultipleEndpoints() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithEndpoint("argocd-server", 443, "ui") + .WithEndpoint("argocd-server", 80, "http"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal(2, resource.EndpointDefinitions.Count); + } + + [Fact] + public void HelmReleaseIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + // ── BuildHelmInstallArgs tests (pure logic, no DI needed) ───────────────── + + [Fact] + public void BuildHelmInstallArgsIncludesUpgradeInstall() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "argocd", "argo-cd", null, null, "argocd", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("upgrade", list); + Assert.Contains("--install", list); + Assert.Contains("argocd", list); + Assert.Contains("argo-cd", list); + } + + [Fact] + public void BuildHelmInstallArgsIncludesKubeconfig() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.Contains("--kubeconfig=/tmp/admin.yaml", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesWait() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.Contains("--wait", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesNamespace() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "my-ns", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--namespace", list); + Assert.Contains("my-ns", list); + Assert.Contains("--create-namespace", list); + } + + [Fact] + public void BuildHelmInstallArgsWithRepoAliasUsesPrefixedChartRef() + { + // When a repo alias is pre-registered via `helm repo add`, BuildHelmInstallArgs + // uses "{alias}/{chart}" notation β€” NOT the --repo flag (which is unreliable). + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", "my-repo-alias", null, "default", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.DoesNotContain("--repo", list); + Assert.Contains("my-repo-alias/chart", list); + } + + [Fact] + public void BuildHelmInstallArgsWithNullAliasUsesChartDirectly() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "oci://registry/chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--repo", args); + Assert.Contains("oci://registry/chart", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesVersion() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, "7.8.0", "default", null, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--version", list); + Assert.Contains("7.8.0", list); + } + + [Fact] + public void BuildHelmInstallArgsOmitsRepoWhenNull() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--repo", args); + } + + [Fact] + public void BuildHelmInstallArgsOmitsVersionWhenNull() + { + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + + Assert.DoesNotContain("--version", args); + } + + [Fact] + public void BuildHelmInstallArgsIncludesSetValues() + { + var values = new Dictionary + { + ["service.type"] = "NodePort", + ["replicaCount"] = "2", + }; + var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( + "r", "chart", null, null, "default", values, "/tmp/admin.yaml"); + + var list = args.ToArray(); + Assert.Contains("--set", list); + Assert.Contains("service.type=NodePort", list); + Assert.Contains("replicaCount=2", list); + } + + // ── WaitFor support ─────────────────────────────────────────────────────── + + [Fact] + public void HelmReleaseHasHealthCheckForWaitForSupport() + { + // WaitFor(helmRelease) is satisfied by the HelmReleaseHealthCheck, + // which flips IsReady once RunReleaseAsync completes. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(resource.Annotations.OfType(), a => + a.Key == "helm_argocd_ready"); + } + + [Fact] + public void HelmReleaseIsReadyFlagStartsFalse() + { + var resource = new HelmReleaseResource( + "argocd", "argocd", "default", new K3sClusterResource("k8s")); + + Assert.False(resource.IsReady); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddHelmReleaseShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddHelmRelease("argocd", "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease(null!, "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenChartIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease("argocd", null!); + Assert.Throws(action); + } + + [Fact] + public void WithHelmValueShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValue("key", "value"); + Assert.Throws(action); + } + + [Fact] + public void WithEndpointShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithEndpoint("svc", 443, "ui"); + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs new file mode 100644 index 000000000..b9dcf1f0c --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs @@ -0,0 +1,201 @@ +using Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sAgentNodeTests +{ + // ── Agent creation via K3sClusterOptions ───────────────────────────────── + + [Fact] + public void AgentCountInOptionsCreatesK3sAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + + Assert.Equal(2, agents.Count); + Assert.Contains(agents, a => a.Name == "k8s-agent-0"); + Assert.Contains(agents, a => a.Name == "k8s-agent-1"); + } + + [Fact] + public void AgentNodesAreChildrenOfCluster() + { + // Implements IResourceWithParent so they appear nested under k8s in the dashboard. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, agent.Parent); + + // Non-generic IResourceWithParent β€” used by the dashboard for grouping. + var nonGeneric = agent as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(cluster.Resource, nonGeneric.Parent); + } + + [Fact] + public void AgentCountZeroProducesNoAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 0); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentCountUpdatesClusterAgentCount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 3); + + Assert.Equal(3, cluster.Resource.AgentCount); + } + + [Fact] + public void AgentNodesDoNotHaveWaitForDependencyOnCluster() + { + // Agents must NOT WaitFor the cluster β€” that would create a deadlock because the + // cluster health check waits for all nodes (including agents) to be Ready. + // Instead, k3s agent retries connecting to the server independently. + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var waitAnnotations = agent.Annotations.OfType().ToList(); + + Assert.DoesNotContain(waitAnnotations, w => w.Resource is K3sClusterResource); + } + + [Fact] + public void AgentNodesUseSameImageAsServer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var img = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(K3sContainerImageTags.Image, img.Image); + Assert.Equal(K3sContainerImageTags.Registry, img.Registry); + } + + [Fact] + public void AgentNodesAreExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, agent.Annotations); + } + + [Fact] + public void AgentNodesHaveEnvironmentAnnotations() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var envCallbacks = agent.Annotations.OfType().ToList(); + Assert.True(envCallbacks.Count >= 3, + $"Expected at least 3 env annotations (K3S_URL, K3S_TOKEN, K3S_NODE_NAME), got {envCallbacks.Count}"); + } + + [Fact] + public void DefaultClusterHasNoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentNodeNamesFollowConvention() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("mycluster", configure: opts => opts.AgentCount = 3); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-0"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-1"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-2"); + } + + [Fact] + public void NegativeAgentCountIsIgnored() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = -1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void WithLifetimePersistentPropagatestoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .WithLifetime(ContainerLifetime.Persistent); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + Assert.Equal(2, agents.Count); + + foreach (var agent in agents) + { + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Persistent, annotation.Lifetime); + } + } + + [Fact] + public void WithLifetimeSessionDoesNotAddPersistentAnnotationToAgents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1) + .WithLifetime(ContainerLifetime.Session); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Session, annotation.Lifetime); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs new file mode 100644 index 000000000..9d1811255 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -0,0 +1,295 @@ +using System.Net.Sockets; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sClusterResourceTests +{ + [Fact] + public void AddK3sClusterAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.Equal("k8s", resource.Name); + } + + [Fact] + public void AddK3sClusterAddsCorrectContainerImage() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal(K3sContainerImageTags.Image, annotation.Image); + Assert.Equal(K3sContainerImageTags.Tag, annotation.Tag); + Assert.Equal(K3sContainerImageTags.Registry, annotation.Registry); + } + + [Fact] + public void AddK3sClusterAddsApiServerEndpointOnPort6443() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(ProtocolType.Tcp, endpoint.Protocol); + Assert.Equal("https", endpoint.UriScheme); + } + + [Fact] + public void AddK3sClusterWithExplicitPortBindsToThatPort() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", apiServerPort: 16443); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(16443, endpoint.Port); + } + + [Fact] + public void AddK3sClusterAddsServerArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var commandLineArgs = resource.Annotations.OfType(); + Assert.NotEmpty(commandLineArgs); + } + + [Fact] + public void WithK3sVersionOverridesImageTag() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithK3sVersion("v1.32.3-k3s1"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal("v1.32.3-k3s1", annotation.Tag); + } + + [Fact] + public void AddK3sClusterHasNoVolumeByDefault() + { + // Persistence is opt-in via WithPersistentState(). + // No volume is mounted by default so the cluster is ephemeral. + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.DoesNotContain( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + } + + [Fact] + public void WithDataVolumeAddsSingleVolumeMount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume(); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + // VolumeNameGenerator format: {appName}-{sha256}-{resourceName}-data + Assert.EndsWith("-k8s-data", volume.Source); + } + + [Fact] + public void WithDataVolumeUsesCustomName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume("my-k3s-data"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + Assert.Equal("my-k3s-data", volume.Source); + } + + [Fact] + public void AddK3sClusterWithClusterCidrViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.ClusterCidr = "10.99.0.0/16"; + }); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.NotNull(resource); + } + + [Fact] + public void WithReferenceSetsKubeconfigEnvForProject() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + + // ProjectResource would need a project file; use ExecutableResource as a proxy + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(cluster); + + // Verify the environment callback was added (no exception thrown) + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void WithReferenceSetsKubeconfigDataEnvForContainer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + var container = appBuilder.AddContainer("operator", "myorg/operator"); + container.WithReference(cluster); + + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void K3sClusterResourceHasNoKubeconfigDirectoryByDefault() + { + // Kubeconfig is now stored in-memory (K8SConfiguration objects) and + // never written to disk by the resource itself β€” docker exec reads it. + var resource = new K3sClusterResource("k8s"); + Assert.Null(resource.AdminKubeconfig); + Assert.Null(resource.ContainerKubeconfig); + } + + [Fact] + public void WithPodSubnetAddsClusterCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithPodSubnet("10.88.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithServiceSubnetAddsServiceCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithServiceSubnet("10.89.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithDisabledComponentAddsDisableArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDisabledComponent("traefik"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithExtraArgAddsRawArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithExtraArg("--write-kubeconfig-mode=644"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void ApiEndpointReturnsEndpointReferenceWithCorrectName() + { + var resource = new K3sClusterResource("k8s"); + var endpoint = resource.ApiEndpoint; + + Assert.NotNull(endpoint); + Assert.Equal(K3sClusterResource.ApiServerEndpointName, endpoint.EndpointName); + } + + [Fact] + public void ApiEndpointIsCached() + { + var resource = new K3sClusterResource("k8s"); + + var first = resource.ApiEndpoint; + var second = resource.ApiEndpoint; + + Assert.Same(first, second); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs new file mode 100644 index 000000000..f803d976f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs @@ -0,0 +1,109 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sPublicApiTests +{ + [Fact] + public void AddK3sClusterShouldThrowWhenBuilderIsNull() + { + IDistributedApplicationBuilder builder = null!; + + var action = () => builder.AddK3sCluster("k8s"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void AddK3sClusterShouldThrowWhenNameIsNull() + { + IDistributedApplicationBuilder builder = new DistributedApplicationBuilder([]); + string name = null!; + + var action = () => builder.AddK3sCluster(name); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(name), exception.ParamName); + } + + [Fact] + public void WithK3sVersionShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithK3sVersion("v1.32.3-k3s1"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithPodSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithPodSubnet("10.42.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithServiceSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithServiceSubnet("10.43.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithDisabledComponentShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithDisabledComponent("traefik"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithExtraArgShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithExtraArg("--write-kubeconfig-mode=644"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithReferenceShouldThrowWhenDestinationIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + IResourceBuilder destination = null!; + + var action = () => destination.WithReference(cluster); + + Assert.Throws(action); + } + + [Fact] + public void WithReferenceShouldThrowWhenSourceIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var container = appBuilder.AddContainer("app", "myimage"); + IResourceBuilder source = null!; + + var action = () => container.WithReference(source); + + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs new file mode 100644 index 000000000..906d590fd --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -0,0 +1,191 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K8sManifestResourceTests +{ + // ── AddK8sManifest registration ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./k8s/widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("widget-crd", resource.Name); + } + + [Fact] + public void AddK8sManifestStoresPath() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crds", "./k8s/crds/"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("./k8s/crds/", resource.Path); + } + + [Fact] + public void AddK8sManifestParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + } + + [Fact] + public void AddK8sManifestImplementsIResourceWithParent() + { + var resource = new K8sManifestResource( + "crd", "./crd.yaml", new K3sClusterResource("k8s")); + + var nonGeneric = resource as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(resource.Parent, nonGeneric.Parent); + } + + [Fact] + public void AddK8sManifestImplementsIResourceWithWaitSupport() + { + var resource = new K8sManifestResource( + "crd", "./crd.yaml", new K3sClusterResource("k8s")); + + Assert.IsAssignableFrom(resource); + } + + [Fact] + public void AddK8sManifestIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + [Fact] + public void ClusterTracksRegisteredManifests() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + cluster.AddK8sManifest("rbac", "./rbac.yaml"); + + Assert.Contains("widget-crd", cluster.Resource.Manifests); + Assert.Contains("rbac", cluster.Resource.Manifests); + Assert.Equal(2, cluster.Resource.Manifests.Count); + } + + [Fact] + public void MultipleManifestsCanBeRegisteredOnSameCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crd1", "./crd1.yaml"); + cluster.AddK8sManifest("crd2", "./crd2.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Equal(2, model.Resources.OfType().Count()); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddK8sManifest("crd", "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest(null!, "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenPathIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest("crd", null!); + Assert.Throws(action); + } + + // ── File resolution ─────────────────────────────────────────────────────── + + [Fact] + public void ResolveFilesSingleFile() + { + // Create a temp file to test with + var file = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(file, "apiVersion: v1\nkind: ConfigMap"); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(file); + Assert.Single(files, f => f == file); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public void ResolveFilesDirectory() + { + var dir = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "b.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "a.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "c.yml"), ""); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(dir); + + // Lexicographic order, all YAML extensions + Assert.Equal(3, files.Count); + Assert.Equal("a.yaml", Path.GetFileName(files[0])); + Assert.Equal("b.yaml", Path.GetFileName(files[1])); + Assert.Equal("c.yml", Path.GetFileName(files[2])); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } +} From 4dda24bf0f2e1743eef8fd678d05e2a7297c31af Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 14 May 2026 01:27:32 +0200 Subject: [PATCH 02/13] feat: proper support for service endpoints, helm and manifests resources --- .github/workflows/tests.yaml | 1 + .gitignore | 2 + CommunityToolkit.Aspire.slnx | 1 + .../Program.cs | 39 +- .../HelmContainerImageTags.cs | 8 + .../HelmReleaseHealthCheck.cs | 19 - .../HelmReleaseResource.cs | 23 +- .../K3sBuilderExtensions.Helm.cs | 396 +++-------------- .../K3sBuilderExtensions.Manifest.cs | 402 ++++-------------- .../K3sBuilderExtensions.ServiceEndpoint.cs | 218 ++++++++++ .../K3sBuilderExtensions.cs | 177 ++++---- .../K3sClusterOptions.cs | 40 ++ .../K3sClusterResource.cs | 29 +- .../K3sInProcessPortForwarder.cs | 169 ++++---- .../K3sReadinessHealthCheck.cs | 209 ++++----- .../K3sServiceEndpointResource.cs | 47 ++ .../K8sManifestResource.cs | 38 +- .../KubeconfigInjectionStrategy.cs | 33 -- .../KubectlContainerImageTags.cs | 11 + ...Aspire.Hosting.K3s.IntegrationTests.csproj | 8 + .../K3sIntegrationTests.cs | 158 +++++++ .../HelmReleaseResourceTests.cs | 210 ++++----- .../K3sClusterResourceTests.cs | 13 +- .../K8sManifestResourceTests.cs | 26 +- 24 files changed, 1027 insertions(+), 1250 deletions(-) create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs create mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj create mode 100644 tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4fae786ae..f24815979 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -38,6 +38,7 @@ jobs: Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, Hosting.K3s.Tests, + Hosting.K3s.IntegrationTests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, diff --git a/.gitignore b/.gitignore index 4795a4c5d..eb34c74c4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ examples/perl/**/local/* **cpanfile.snapshot **/.modules/ **/*.AppHost.TypeScript/nuget.config + +**/.k3s/ diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 3382bdbb0..2d71685f9 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -281,6 +281,7 @@ + diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 00f476326..527bf16d0 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -1,34 +1,37 @@ // K3s hosting example // ────────────────────────────────────────────────────────────────────────────── // Prerequisites (host machine): -// β€’ Docker with --privileged support +// β€’ Docker with --privileged support (Linux or Docker Desktop on Mac/Windows) // β€’ helm β†’ https://helm.sh/docs/intro/install/ // β€’ kubectl β†’ https://kubernetes.io/docs/tasks/tools/ // // What this demonstrates: // 1. A k3s cluster starts inside a Docker container. -// 2. Headlamp (https://headlamp.dev) is installed as a child Helm release β€” -// a clickable http URL appears in the Aspire dashboard. -// 3. podinfo is installed β€” a lightweight demo app that shows the helm lifecycle. -// 4. Both releases are children of k8s in the Aspire resource tree. -// 5. WithPersistentState keeps the cluster data alive across AppHost restarts. +// 2. podinfo is installed via Helm β€” a lightweight demo app. +// 3. A K3sServiceEndpointResource exposes the podinfo service: +// β€’ Host processes reach it at http://localhost:{port} +// β€’ DCP-network containers reach it at http://host.docker.internal:{port} +// 4. WithDataVolume keeps the cluster state alive across AppHost restarts. // ────────────────────────────────────────────────────────────────────────────── var builder = DistributedApplication.CreateBuilder(args); -var cluster = builder.AddK3sCluster("k8s", configure: opts => - { - opts.AgentCount = 2; - }) +var cluster = builder + .AddK3sCluster("k8s") + .WithDataVolume() .WithLifetime(ContainerLifetime.Persistent); -// cluster.AddHelmRelease( -// name: "podinfo", -// chart: "podinfo", -// repo: "https://stefanprodan.github.io/podinfo", -// version: "6.7.1", -// @namespace: "podinfo") -// .WithHelmValue("service.type", "NodePort") -// .WithEndpoint("podinfo", servicePort: 9898, name: "web"); +var podinfo = cluster.AddHelmRelease( + name: "podinfo", + chart: "podinfo", + repo: "https://stefanprodan.github.io/podinfo", + version: "6.7.1", + @namespace: "podinfo"); + +// Expose the podinfo service as an Aspire endpoint resource. +// WaitForCompletion waits for the helm install container to exit with code 0 +// before starting the port-forward β€” no NodePort required. +cluster.AddServiceEndpoint("podinfo-web", "podinfo", servicePort: 9898, @namespace: "podinfo") + .WaitForCompletion(podinfo); builder.Build().Run(); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs new file mode 100644 index 000000000..885c77724 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs @@ -0,0 +1,8 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class HelmContainerImageTags +{ + internal const string Registry = "docker.io"; + internal const string Image = "alpine/helm"; + internal const string Tag = "3.17.3"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs deleted file mode 100644 index eb0845e2d..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseHealthCheck.cs +++ /dev/null @@ -1,19 +0,0 @@ -using Aspire.Hosting.ApplicationModel; -using Microsoft.Extensions.Diagnostics.HealthChecks; - -namespace CommunityToolkit.Aspire.Hosting; - -/// -/// Lightweight health check that gates WaitFor(helmRelease) on completion of -/// the helm install lifecycle. Returns only -/// after is set by RunReleaseAsync. -/// -internal sealed class HelmReleaseHealthCheck(HelmReleaseResource release) : IHealthCheck -{ - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default) - => Task.FromResult(release.IsReady - ? HealthCheckResult.Healthy("Helm release is running") - : HealthCheckResult.Unhealthy("Helm release not yet ready")); -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index 247f26e22..b5de3d52b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -2,8 +2,12 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents a Helm chart release deployed to a k3s cluster. -/// Appears as a distinct dashboard entry; transitions Starting β†’ Running -/// when all pods reach the Ready state. +/// +/// Runs as an alpine/helm container on the DCP network. The container polls for the +/// cluster kubeconfig (written when the cluster health check first passes), executes +/// helm upgrade --install --wait, and exits with code 0 on success. Use +/// WaitForCompletion(helmRelease) on resources that depend on the release being installed. +/// /// /// The Aspire resource name (also used as the Helm release name). /// The Helm release name passed to helm upgrade --install. @@ -14,7 +18,7 @@ public sealed class HelmReleaseResource( string releaseName, string @namespace, K3sClusterResource cluster) - : Resource(name), IResourceWithParent, IResourceWithWaitSupport + : ContainerResource(name), IResourceWithParent { /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); @@ -29,17 +33,4 @@ public sealed class HelmReleaseResource( internal string? RepoUrl { get; set; } internal string? Version { get; set; } internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); - internal List EndpointDefinitions { get; } = []; - - /// - /// Set to by the lifecycle when the helm install completes and - /// all pods are ready. The WaitFor(helmRelease) health check polls this flag. - /// - internal volatile bool IsReady; } - -/// Describes a Kubernetes service endpoint to expose from a Helm release. -/// The Kubernetes service name. -/// The service port number. -/// A friendly name shown in the dashboard. -internal sealed record HelmEndpointDefinition(string ServiceName, int ServicePort, string EndpointName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 156c83cdb..bd8dbb780 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -1,14 +1,6 @@ -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; #pragma warning disable ASPIREATS001 // AspireExport is experimental @@ -22,17 +14,17 @@ public static class K3sHelmBuilderExtensions /// /// Adds a Helm release as a child resource of the k3s cluster. /// - /// Follows the same pattern as PostgresServerResource.AddDatabase: the release is - /// registered on the parent cluster, and the cluster's own - /// handler drives the install lifecycle for all registered releases. Helm output streams to - /// each release's individual log tab in the dashboard. + /// The release runs as a bitnami/helm container on the DCP network, executing + /// helm upgrade --install --wait then exiting. No host-side helm binary + /// is required. Use WaitForCompletion(helmRelease) on resources that depend on + /// the release being fully installed. /// /// /// The k3s cluster resource builder. - /// Resource name β€” also the Helm release name. + /// Resource name β€” also used as the Helm release name. /// Chart name. Add for remote charts. - /// Optional Helm repository URL (passed as --repo). - /// Optional chart version (--version). + /// Optional Helm repository URL. + /// Optional chart version. /// Target namespace (created automatically). /// A builder for the . [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] @@ -57,22 +49,43 @@ public static IResourceBuilder AddHelmRelease( Version = version, }; - // Register the release on the parent cluster β€” mirrors PostgresServerResource.AddDatabase(). cluster.AddHelmRelease(release.Name, release.ReleaseName); - // Health check that satisfies WaitFor(helmRelease) on dependent resources. - // Returns Healthy only after the install lifecycle sets release.IsReady = true. - var healthCheckKey = $"helm_{name}_ready"; - builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( - healthCheckKey, - sp => new HelmReleaseHealthCheck(release), - failureStatus: HealthStatus.Unhealthy, - tags: null)); + // The helm installer container mounts container/kubeconfig.yaml so it can reach + // the k3s API via DCP DNS (https://{clusterName}:6443). The directory is created + // by AddK3sCluster; the kubeconfig file is written by K3sReadinessHealthCheck on + // first successful health check. WaitFor(cluster) guarantees the file exists. + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + var (helmRegistry, helmImage, helmTag) = cluster.HelmImageInfo; return builder.ApplicationBuilder .AddResource(release) + .WithImage(helmImage, helmTag) + .WithImageRegistry(helmRegistry) + .WithEntrypoint("/bin/sh") + + // The install script is injected as /helm-install.sh via WithContainerFiles. + // The callback fires when the container is being started (after WaitFor(cluster) + // is satisfied), so all WithHelmValue() calls have been made by then. + .WithContainerFiles("/", async (ctx, ct) => + { + var script = BuildHelmScript(release); + return [new ContainerFile + { + Name = "helm-install.sh", + Contents = script, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]; + }) + .WithArgs("/helm-install.sh") + .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Rocket") .ExcludeFromManifest() - .WithHealthCheck(healthCheckKey) .WithInitialState(new CustomResourceSnapshot { ResourceType = "Helm Release", @@ -103,324 +116,43 @@ public static IResourceBuilder WithHelmValue( return builder; } - /// - /// Exposes a Kubernetes service from this release as a clickable endpoint in the dashboard. - /// The NodePort is auto-discovered and forwarded in-process via the KubernetesClient WebSocket API. - /// - public static IResourceBuilder WithEndpoint( - this IResourceBuilder builder, - string serviceName, - int servicePort, - string name) - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(serviceName); - ArgumentNullException.ThrowIfNull(name); - - builder.Resource.EndpointDefinitions.Add( - new HelmEndpointDefinition(serviceName, servicePort, name)); - - return builder; - } - - // ── Lifecycle (driven from the parent cluster's ResourceReadyEvent) ──────── + // ── Script generation ───────────────────────────────────────────────────── - /// - /// Runs the full install lifecycle for : - /// helm install β†’ NodePort discovery β†’ in-process port-forward β†’ Running. - /// Called by the parent cluster's handler β€” - /// the same pattern Postgres uses for database creation in AddPostgres. - /// - internal static async Task RunReleaseAsync( - HelmReleaseResource release, - K3sClusterResource cluster, - ResourceNotificationService notifications, - ILogger logger, - CancellationToken ct) + // Visible for testing. + internal static string BuildHelmScript(HelmReleaseResource release) { - await notifications.PublishUpdateAsync(release, - state => state with { State = KnownResourceStates.Starting }) - .ConfigureAwait(false); - - try - { - var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); - - if (kubeconfigYaml is null) - { - throw new InvalidOperationException( - "k3s kubeconfig is not yet available. " + - "The cluster ResourceReadyEvent fired before the health check populated the kubeconfig."); - } + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - await RunHelmAsync(release, kubeconfigYaml, logger, ct).ConfigureAwait(false); + // Poll until the k3s health check writes the kubeconfig β€” the file appears only + // after all nodes are Ready. This replaces WaitFor(cluster) since a container + // cannot WaitFor its IResourceWithParent. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); - var urls = release.EndpointDefinitions.Count > 0 - ? await DiscoverAndStartPortForwardAsync(release, kubeconfigYaml, logger, ct) - .ConfigureAwait(false) - : ImmutableArray.Empty; - - // Set before PublishUpdateAsync so the health check unblocks WaitFor callers - // as soon as the Running state notification is processed. - release.IsReady = true; - - await notifications.PublishUpdateAsync(release, state => state with - { - State = KnownResourceStates.Running, - Urls = urls, - // Merge: keep all existing properties (the orchestrator injects ParentName which - // drives parent-child display in the dashboard) and only update/add our own. - // Replacing the entire Properties array would wipe out ParentName, causing the - // release to lose its parent and appear at the top level after going Running. - Properties = - [ - .. state.Properties.Where(p => - p.Name is not ("ReleaseName" or "Chart" or "Namespace" or "Version" or "ChartVersion")), - new ResourcePropertySnapshot("ReleaseName", release.ReleaseName), - new ResourcePropertySnapshot("Chart", release.Chart!), - new ResourcePropertySnapshot("ChartVersion", release.Version ?? "latest"), - new ResourcePropertySnapshot("Namespace", release.Namespace), - ], - }).ConfigureAwait(false); - } - catch (Exception ex) when (!ct.IsCancellationRequested) + if (release.RepoUrl is not null) { - logger.LogError(ex, "Helm release '{Name}' failed.", release.ReleaseName); - - await notifications.PublishUpdateAsync(release, - state => state with { State = KnownResourceStates.FailedToStart }) - .ConfigureAwait(false); + var alias = $"aspire-k3s-{release.ReleaseName}"; + sb.AppendLine($"helm repo add --force-update \"{alias}\" \"{release.RepoUrl}\""); + sb.AppendLine($"helm repo update \"{alias}\""); } - } - // ── helm subprocess ─────────────────────────────────────────────────────── + var chartRef = release.RepoUrl is not null + ? $"aspire-k3s-{release.ReleaseName}/{release.Chart}" + : release.Chart!; - private static async Task RunHelmAsync( - HelmReleaseResource release, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var tempKubeconfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-helm-{Environment.ProcessId}-{release.ReleaseName}.yaml"); + sb.Append($"helm upgrade --install \"{release.ReleaseName}\" \"{chartRef}\""); + sb.Append($" --namespace \"{release.Namespace}\" --create-namespace"); + sb.Append(" --wait --timeout 10m"); - await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try - { - // When a repo URL is provided, add/update the repo first. - // The --repo shorthand in helm upgrade --install does not work reliably for - // all chart repositories (e.g. kubernetes-dashboard returns 404 via --repo). - // Official docs always show: helm repo add β†’ helm repo update β†’ helm install. - string? repoAlias = null; - if (release.RepoUrl is not null) - { - repoAlias = $"aspire-k3s-{release.ReleaseName}"; - await RunHelmCommandAsync( - logger, - ["repo", "add", "--force-update", repoAlias, release.RepoUrl], - ct).ConfigureAwait(false); + if (release.Version is not null) + sb.Append($" --version \"{release.Version}\""); - await RunHelmCommandAsync( - logger, - ["repo", "update", repoAlias], - ct).ConfigureAwait(false); - } + foreach (var (key, value) in release.HelmValues) + sb.Append($" --set \"{key}={value}\""); - var args = BuildHelmInstallArgs( - release.ReleaseName, release.Chart!, repoAlias, - release.Version, release.Namespace, release.HelmValues, - tempKubeconfig); - - logger.LogInformation("Running: helm {Args}", string.Join(' ', args)); - - await RunHelmCommandAsync(logger, args, ct).ConfigureAwait(false); - } - finally - { - try { File.Delete(tempKubeconfig); } catch { /* best effort */ } - } - } - - // ── Argument builder ────────────────────────────────────────────────────── - - /// Runs a helm subcommand, logging output and failing on non-zero exit. - private static async Task RunHelmCommandAsync( - ILogger logger, - IEnumerable args, - CancellationToken ct) - { - var psi = new ProcessStartInfo("helm") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - foreach (var arg in args) - { - psi.ArgumentList.Add(arg); - } - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start helm process."); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogInformation("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogWarning("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - await process.WaitForExitAsync(ct).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - throw new InvalidOperationException( - $"helm {string.Join(' ', psi.ArgumentList)} failed (exit code {process.ExitCode})."); - } - } - - internal static IReadOnlyList BuildHelmInstallArgs( - string releaseName, - string chart, - string? repoAlias, // null = no repo, non-null = "{alias}/{chart}" notation - string? version, - string @namespace, - IReadOnlyDictionary? values, - string kubeconfigPath) - { - // When a repo alias was registered via `helm repo add`, use "alias/chart" notation. - // Otherwise treat the chart as a path or OCI reference. - var chartRef = repoAlias is not null ? $"{repoAlias}/{chart}" : chart; - - var args = new List - { - "upgrade", "--install", - releaseName, - chartRef, - "--namespace", @namespace, - "--create-namespace", - "--wait", - "--timeout", "10m", - $"--kubeconfig={kubeconfigPath}", - }; - - if (version is not null) - { - args.Add("--version"); - args.Add(version); - } - - if (values is not null) - { - foreach (var (key, value) in values) - { - args.Add("--set"); - args.Add($"{key}={value}"); - } - } - - return args; - } - - // ── Port-forward ────────────────────────────────────────────────────────── - - private static async Task> DiscoverAndStartPortForwardAsync( - HelmReleaseResource release, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var urls = ImmutableArray.CreateBuilder(); - - using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); - using var k8sClient = new Kubernetes(config); - - foreach (var ep in release.EndpointDefinitions) - { - var nodePort = await DiscoverNodePortAsync( - k8sClient, release.ReleaseName, release.Namespace, - ep.ServiceName, ep.ServicePort, logger, ct).ConfigureAwait(false); - - var hostPort = AllocateHostPort(); - - var forwarder = new K3sInProcessPortForwarder( - kubeconfigYaml, - release.Namespace, - ep.ServiceName, // look up by service name, not release label - hostPort, - ep.ServicePort); - - _ = forwarder.RunAsync(logger, ct); - - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; - urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); - - if (nodePort.HasValue) - { - urls.Add(new UrlSnapshot( - $"{ep.EndpointName} (container)", - $"{scheme}://{release.Parent.Name}:{nodePort.Value}", - IsInternal: true)); - } - } - - return urls.ToImmutable(); - } - - private static async Task DiscoverNodePortAsync( - Kubernetes k8sClient, - string releaseName, - string @namespace, - string serviceName, - int servicePort, - ILogger logger, - CancellationToken ct) - { - try - { - var services = await k8sClient.CoreV1.ListNamespacedServiceAsync( - @namespace, - labelSelector: $"app.kubernetes.io/instance={releaseName}", - cancellationToken: ct).ConfigureAwait(false); - - var port = services.Items - .FirstOrDefault(s => string.Equals( - s.Metadata.Name, serviceName, StringComparison.OrdinalIgnoreCase)) - ?.Spec.Ports - .FirstOrDefault(p => p.Port == servicePort); - - if (port?.NodePort is null) - { - logger.LogWarning( - "NodePort for {ServiceName}:{ServicePort} not found; container URL omitted.", - serviceName, servicePort); - } - - return port?.NodePort; - } - catch (Exception ex) - { - logger.LogWarning(ex, "NodePort discovery failed for service '{ServiceName}'.", serviceName); - return null; - } - } - - private static int AllocateHostPort() - { - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; + return sb.ToString(); } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index f50f70ba4..40a911e55 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -1,15 +1,6 @@ -using System.Collections.Immutable; -using System.Diagnostics; -using System.Net; -using System.Net.Sockets; using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; -using k8s.Models; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Diagnostics.HealthChecks; -using Microsoft.Extensions.Logging; #pragma warning disable ASPIREATS001 // AspireExport is experimental @@ -22,15 +13,17 @@ public static class K3sManifestBuilderExtensions { /// /// Applies one or more Kubernetes YAML files to the cluster via - /// kubectl apply --server-side (Server-Side Apply). No bind-mount is required. + /// kubectl apply --server-side running inside a bitnami/kubectl container. + /// No host-side kubectl binary is required. + /// + /// After applying the manifests the container waits for any CRDs to reach the + /// Established condition, then exits with code 0. Use + /// WaitForCompletion(manifest) on dependent resources. + /// /// /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") - /// A directory: all .yaml/.yml files applied lexicographically. - /// A glob: "./k8s/crds/*.yaml" + /// A directory: all .yaml/.yml files applied (kubectl handles ordering). /// - /// CRDs are detected automatically; the resource waits for the Established - /// condition via the KubernetesClient before transitioning to Running, so - /// WaitFor(manifest) correctly gates dependent operators. /// [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] public static IResourceBuilder AddK8sManifest( @@ -43,323 +36,91 @@ public static IResourceBuilder AddK8sManifest( ArgumentNullException.ThrowIfNull(path); var cluster = builder.Resource; - var manifest = new K8sManifestResource(name, path, cluster); - cluster.AddManifest(manifest.Name); - - var healthCheckKey = $"manifest_{name}_ready"; - builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( - healthCheckKey, - sp => new K8sManifestHealthCheck(manifest), - failureStatus: HealthStatus.Unhealthy, - tags: null)); - - return builder.ApplicationBuilder - .AddResource(manifest) - .ExcludeFromManifest() - .WithHealthCheck(healthCheckKey) - .WithInitialState(new CustomResourceSnapshot - { - ResourceType = "K8s Manifest", - State = KnownResourceStates.NotStarted, - Properties = [new ResourcePropertySnapshot("Path", path)], - }); - } - - /// - /// Exposes a Kubernetes service from this manifest as a clickable endpoint in the dashboard. - /// Traffic is forwarded in-process via the KubernetesClient WebSocket API. - /// - /// The manifest resource builder. - /// The Kubernetes service name. - /// The service port number. - /// Friendly name shown in the dashboard. - /// - /// The namespace containing the service. Defaults to "default". - /// For remote manifests (HTTP URLs) the namespace must be specified explicitly. - /// - public static IResourceBuilder WithEndpoint( - this IResourceBuilder builder, - string serviceName, - int servicePort, - string name, - string @namespace = "default") - { - ArgumentNullException.ThrowIfNull(builder); - ArgumentNullException.ThrowIfNull(serviceName); - ArgumentNullException.ThrowIfNull(name); - - builder.Resource.EndpointDefinitions.Add( - new ManifestEndpointDefinition(serviceName, servicePort, name, @namespace)); - - return builder; - } + // Resolve to an absolute path so the bind-mount and container path are stable. + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine(builder.ApplicationBuilder.AppHostDirectory, path)); - // ── Lifecycle (called from the cluster's ResourceReadyEvent) ────────────── + string hostBindDir; + string containerManifestPath; - internal static async Task RunManifestAsync( - K8sManifestResource manifest, - K3sClusterResource cluster, - ResourceNotificationService notifications, - ILogger logger, - CancellationToken ct) - { - await notifications.PublishUpdateAsync(manifest, - state => state with { State = KnownResourceStates.Starting }) - .ConfigureAwait(false); - - try + if (Directory.Exists(absolutePath)) { - var kubeconfigYaml = K3sBuilderExtensions.GetAdminKubeconfigYaml(cluster); - - if (kubeconfigYaml is null) - { - throw new InvalidOperationException( - "k3s kubeconfig is not yet available when applying manifest."); - } - - // Write a temp kubeconfig file β€” kubectl requires a path, same as helm. - var tempKubeconfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-manifest-{Environment.ProcessId}-{manifest.Name}.yaml"); - - await File.WriteAllTextAsync(tempKubeconfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try - { - var files = ResolveFiles(manifest.Path); - - logger.LogInformation( - "Applying {Count} manifest file(s) from '{Path}'", files.Count, manifest.Path); - - // kubectl apply --server-side (SSA) for each file. - foreach (var file in files) - { - await KubectlApplyAsync(file, tempKubeconfig, logger, ct) - .ConfigureAwait(false); - } - - // Wait for any CRDs to reach Established using the KubernetesClient. - await WaitForCrdsEstablishedAsync( - files, kubeconfigYaml, logger, ct).ConfigureAwait(false); - } - finally - { - try { File.Delete(tempKubeconfig); } catch { /* best effort */ } - } - - manifest.IsReady = true; - - // Start in-process port-forwards for any declared endpoints. - var urls = ImmutableArray.Empty; - if (manifest.EndpointDefinitions.Count > 0) - { - urls = await StartPortForwardAsync(manifest, kubeconfigYaml, logger, ct) - .ConfigureAwait(false); - } - - await notifications.PublishUpdateAsync(manifest, state => state with - { - State = KnownResourceStates.Running, - Urls = urls, - Properties = - [ - .. state.Properties.Where(p => p.Name is not "Path"), - new ResourcePropertySnapshot("Path", manifest.Path), - ], - }).ConfigureAwait(false); + hostBindDir = absolutePath; + containerManifestPath = "/k8s-manifests"; } - catch (Exception ex) when (!ct.IsCancellationRequested) + else { - logger.LogError(ex, "Failed to apply manifest '{Name}'.", manifest.Name); - - await notifications.PublishUpdateAsync(manifest, - state => state with { State = KnownResourceStates.FailedToStart }) - .ConfigureAwait(false); + hostBindDir = System.IO.Path.GetDirectoryName(absolutePath)!; + containerManifestPath = $"/k8s-manifests/{System.IO.Path.GetFileName(absolutePath)}"; } - } - - // ── kubectl apply ───────────────────────────────────────────────────────── - private static async Task KubectlApplyAsync( - string file, - string kubeconfigPath, - ILogger logger, - CancellationToken ct) - { - logger.LogInformation("Applying {File}", file); - - var psi = new ProcessStartInfo("kubectl") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - psi.ArgumentList.Add("apply"); - psi.ArgumentList.Add("-f"); - psi.ArgumentList.Add(file); - psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); - psi.ArgumentList.Add("--server-side"); - psi.ArgumentList.Add("--field-manager=aspire-k3s"); - psi.ArgumentList.Add("--force-conflicts"); - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start kubectl process."); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogInformation("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogWarning("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - await process.WaitForExitAsync(ct).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - throw new InvalidOperationException( - $"kubectl apply failed for '{file}' (exit code {process.ExitCode})."); - } - } - - // ── Port-forward for manifest endpoints ────────────────────────────────── - - private static async Task> StartPortForwardAsync( - K8sManifestResource manifest, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var urls = ImmutableArray.CreateBuilder(); - - foreach (var ep in manifest.EndpointDefinitions) - { - var hostPort = AllocateHostPort(); - - var forwarder = new K3sInProcessPortForwarder( - kubeconfigYaml, - ep.Namespace, - ep.ServiceName, - hostPort, - ep.ServicePort); - - _ = forwarder.RunAsync(logger, ct); - - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; - urls.Add(new UrlSnapshot(ep.EndpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false)); - } - - return urls.ToImmutable(); - } - - private static int AllocateHostPort() - { - using var listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - var port = ((IPEndPoint)listener.LocalEndpoint).Port; - listener.Stop(); - return port; - } + var manifest = new K8sManifestResource(name, absolutePath, cluster); + cluster.AddManifest(manifest.Name); - // ── CRD readiness (in-process via KubernetesClient) ────────────────────── + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); - private static async Task WaitForCrdsEstablishedAsync( - IReadOnlyList files, - string kubeconfigYaml, - ILogger logger, - CancellationToken ct) - { - var crdNames = new List(); + var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; - foreach (var file in files) - { - // Skip remote URLs β€” kubectl downloaded and applied them, but we can't parse - // them with KubernetesYaml locally. CRD detection for remote files is skipped; - // if you need CRD readiness gating, use a local file path instead. - if (file.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - file.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var objects = await KubernetesYaml.LoadAllFromFileAsync(file) - .ConfigureAwait(false); - - foreach (var obj in objects) + return builder.ApplicationBuilder + .AddResource(manifest) + .WithImage(kubectlImage, kubectlTag) + .WithImageRegistry(kubectlRegistry) + .WithEntrypoint("/bin/sh") + .WithContainerFiles("/", async (ctx, ct) => { - if (obj is V1CustomResourceDefinition crd && crd.Metadata?.Name is { } crdName) + var script = BuildManifestScript(containerManifestPath); + return [new ContainerFile { - crdNames.Add(crdName); - } - } - } - - if (crdNames.Count == 0) - { - return; - } - - using var configStream = new MemoryStream(Encoding.UTF8.GetBytes(kubeconfigYaml)); - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(configStream); - using var k8sClient = new Kubernetes(config); - - foreach (var crdName in crdNames) - { - await WaitForCrdEstablishedAsync(k8sClient, crdName, logger, ct) - .ConfigureAwait(false); - } - } - - private static async Task WaitForCrdEstablishedAsync( - Kubernetes k8sClient, - string crdName, - ILogger logger, - CancellationToken ct) - { - logger.LogInformation("Waiting for CRD '{Crd}' to reach Established...", crdName); - - while (!ct.IsCancellationRequested) - { - var crd = await k8sClient.ApiextensionsV1 - .ReadCustomResourceDefinitionAsync(crdName, cancellationToken: ct) - .ConfigureAwait(false); - - var established = crd.Status?.Conditions?.Any(c => - c.Type == "Established" && - string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; - - if (established) + Name = "kubectl-apply.sh", + Contents = script, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]; + }) + .WithArgs("/kubectl-apply.sh") + .WithBindMount(hostBindDir, "/k8s-manifests") + .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Code") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot { - logger.LogInformation("CRD '{Crd}' is Established.", crdName); - return; - } - - await Task.Delay(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); - } + ResourceType = "K8s Manifest", + State = KnownResourceStates.NotStarted, + Properties = [new ResourcePropertySnapshot("Path", absolutePath)], + }); } - // ── File resolution ─────────────────────────────────────────────────────── - - // Exposed for unit tests via InternalsVisibleTo. - internal static IReadOnlyList ResolveFilesForTest(string path) => - ResolveFiles(path); + // ── Script generation ───────────────────────────────────────────────────── - private static IReadOnlyList ResolveFiles(string path) + internal static string BuildManifestScript(string containerManifestPath) { - // kubectl apply -f supports HTTPS URLs natively β€” pass through as-is. - if (path.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || - path.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - return [path]; - } + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); + + // Poll until the k3s health check writes the kubeconfig β€” same pattern as the + // helm installer. Replaces WaitFor(cluster) for child resources. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); + + sb.AppendLine($"kubectl apply -f \"{containerManifestPath}\" --server-side --field-manager=aspire-k3s --force-conflicts"); + // Wait for CRD Established condition if any CRDs are present. + // The check guard prevents failure when no CRDs were applied. + sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); + sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); + sb.AppendLine("fi"); + return sb.ToString(); + } + // Keep for unit tests β€” file resolution logic is the same. + internal static IReadOnlyList ResolveFilesForTest(string path) + { if (Directory.Exists(path)) { return [ @@ -382,19 +143,4 @@ private static IReadOnlyList ResolveFiles(string path) } } -/// -/// Health check that satisfies WaitFor(manifest). -/// Returns once all files are applied -/// and any CRDs have reached the Established condition. -/// -internal sealed class K8sManifestHealthCheck(K8sManifestResource manifest) : IHealthCheck -{ - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default) - => Task.FromResult(manifest.IsReady - ? HealthCheckResult.Healthy("Manifests applied") - : HealthCheckResult.Unhealthy("Manifests not yet applied")); -} - #pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs new file mode 100644 index 000000000..dfbe5ecb8 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -0,0 +1,218 @@ +using System.Collections.Immutable; +using System.Net; +using System.Net.Sockets; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for exposing Kubernetes services from a k3s cluster into the Aspire network. +/// +public static class K3sServiceEndpointExtensions +{ + /// + /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. + /// + /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, + /// binding to 0.0.0.0:{hostPort}. Use WaitFor to sequence after a + /// or that deploys the service. + /// + /// + /// + /// + /// var nginx = cluster.AddHelmRelease("nginx", "nginx", repo: "https://charts.bitnami.com/bitnami"); + /// var ui = cluster.AddServiceEndpoint("nginx-ui", "nginx", servicePort: 80) + /// .WaitFor(nginx); + /// builder.AddProject<Projects.Api>("api") + /// .WaitFor(ui) + /// .WithReference(ui); + /// + /// + [AspireExport("addServiceEndpoint", + Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + public static IResourceBuilder AddServiceEndpoint( + this IResourceBuilder builder, + [ResourceName] string name, + string serviceName, + int servicePort, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(serviceName); + + var cluster = builder.Resource; + var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); + + var healthCheckKey = $"k3s_endpoint_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new K3sServiceEndpointHealthCheck(endpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(endpoint) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithIconName("ArrowRouting") + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Service Endpoint", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ServiceName", serviceName), + new ResourcePropertySnapshot("ServicePort", servicePort.ToString()), + new ResourcePropertySnapshot("Namespace", @namespace), + ], + }); + } + + /// + /// Injects the service URL exposed by into + /// using the Aspire services__{name}__url convention. + /// + /// Host processes receive https://localhost:{port}. + /// Container resources receive https://host.docker.internal:{port}. + /// On Linux without Docker Desktop, add + /// --add-host=host.docker.internal:host-gateway to the container runtime args. + /// + /// + [AspireExport("withReference", + Description = "Injects the k3s service URL into a dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var ep = source.Resource; + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + var envKey = $"services__{ep.Name}__url"; + + if (destination.Resource is ContainerResource) + { + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://host.docker.internal:{ep.HostPort}"; + }); + } + + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://localhost:{ep.HostPort}"; + }); + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + internal static async Task RunEndpointAsync( + K3sServiceEndpointResource endpoint, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigPath = K3sBuilderExtensions.GetLocalKubeconfigPath(cluster); + if (kubeconfigPath is null || !File.Exists(kubeconfigPath)) + { + throw new InvalidOperationException( + "k3s local kubeconfig is not yet available for service endpoint."); + } + + // Allocate a host port by opening a listener, reading the OS-assigned port, + // then closing it before the forwarder binds β€” the port stays reserved in the + // kernel TIME_WAIT for long enough that the forwarder wins the race. + var hostPort = AllocatePort(); + endpoint.HostPort = hostPort; + + var scheme = endpoint.ServicePort is 443 or 8443 ? "https" : "http"; + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigPath, + endpoint.Namespace, + endpoint.ServiceName, + hostPort, + endpoint.ServicePort, + isReady => + { + endpoint.IsReady = isReady; + var state = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy; + var urls = isReady + ? BuildUrls(scheme, endpoint.Name, hostPort, cluster.Name) + : ImmutableArray.Empty; + + _ = notifications.PublishUpdateAsync(endpoint, s => s with + { + State = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy, + Urls = urls, + }); + }); + + _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); + + // Wait for the forwarder to signal ready (IsReady set via callback above). + // The health check also reads IsReady, so WaitFor on dependent resources + // naturally blocks until the port-forward is accepting connections. + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Service endpoint '{Name}' failed to start.", endpoint.Name); + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + private static ImmutableArray BuildUrls( + string scheme, string endpointName, int hostPort, string clusterName) + => [ + new UrlSnapshot(endpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false), + new UrlSnapshot( + $"{endpointName} (container)", + $"{scheme}://host.docker.internal:{hostPort}", + IsInternal: true), + ]; + + private static int AllocatePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} + +/// +/// Health check that satisfies WaitFor(serviceEndpoint). +/// Returns once the port-forward is active. +/// +internal sealed class K3sServiceEndpointHealthCheck(K3sServiceEndpointResource endpoint) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(endpoint.IsReady + ? HealthCheckResult.Healthy("Port-forward is active") + : HealthCheckResult.Unhealthy("Port-forward not yet active")); +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 3399474a0..6e87500dc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -1,11 +1,8 @@ using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; -using k8s; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; -using System.Net; -using System.Net.Sockets; #pragma warning disable ASPIREATS001 // AspireExport is experimental #pragma warning disable ASPIRECERTIFICATES001 // WithHttpsDeveloperCertificate is experimental @@ -41,9 +38,24 @@ public static IResourceBuilder AddK3sCluster( var options = new K3sClusterOptions(); configure?.Invoke(options); - var resource = new K3sClusterResource(name); + var resource = new K3sClusterResource(name) + { + HelmImageInfo = (options.HelmRegistry, options.HelmImage, options.HelmTag), + KubectlImageInfo = (options.KubectlRegistry, options.KubectlImage, options.KubectlTag), + }; var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + // ── Kubeconfig directory on the host ────────────────────────────────── + // AppHostDirectory/.k3s/{name}/ holds three sub-directories: + // cluster/ β€” bind-mounted into the k3s container; k3s writes kubeconfig.yaml here + // local/ β€” rewritten by the health check with server: https://localhost:{port} + // container/ β€” rewritten by the health check with server: https://{name}:6443 + var kubeconfigDir = Path.Combine(builder.AppHostDirectory, ".k3s", name); + var clusterDir = Path.Combine(kubeconfigDir, "cluster"); + Directory.CreateDirectory(clusterDir); + + resource.KubeconfigDirectory = kubeconfigDir; + var resourceBuilder = builder.AddResource(resource) .WithImage(K3sContainerImageTags.Image, tag) .WithImageRegistry(K3sContainerImageTags.Registry) @@ -93,11 +105,7 @@ public static IResourceBuilder AddK3sCluster( .WithArgs("--kube-apiserver-arg=v=0") .WithArgs("--kube-controller-manager-arg=v=0") .WithArgs("--kube-scheduler-arg=v=0") - // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning - // "Failed to kill all the processes attached to cgroup / os: process not initialized". - // This is a known benign race condition in Docker-in-Docker: the kubelet tries to - // force-kill pod cgroup processes that are already dead. The cgroup is still cleaned - // up correctly; only the redundant kill attempt fails. + // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning. .WithArgs("--kubelet-arg=v=0") // ── API server endpoint ─────────────────────────────────────────── @@ -107,33 +115,24 @@ public static IResourceBuilder AddK3sCluster( name: K3sClusterResource.ApiServerEndpointName) // ── Docker / container runtime flags (mirrors k3d) ──────────────── - // Privileged mode is mandatory for iptables, network namespaces, and cgroups. .WithContainerRuntimeArgs("--privileged") - // k3d uses Docker's --init (tini) so that k3s's child processes are properly - // reaped and signals are forwarded correctly. Without it, zombie processes - // accumulate and shutdown becomes unreliable. .WithContainerRuntimeArgs("--init") - // Use the host user namespace β€” required when Docker is configured with userns-remap; - // a no-op otherwise. k3d always passes this flag. .WithContainerRuntimeArgs("--userns=host") - // Share the host's (Docker Desktop VM's) cgroup namespace instead of creating - // a new one. Without this, the k3s kubelet fails to create the "kubepods" cgroup - // hierarchy because the new isolated namespace has domain controllers in an invalid - // state for cgroupsv2. k3d always passes --cgroupns=host for this reason. .WithContainerRuntimeArgs("--cgroupns=host") - // Bind-mount the cgroup filesystem from the Docker Desktop VM into the container - // as read-write. With --cgroupns=host the container sees the host's cgroup namespace, - // but the mount is still read-only by default; making it rw lets the kubelet create - // sub-cgroups for pods (kubepods/besteffort/...). k3d always mounts this as rw. .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") - // tmpfs mounts for runtime sockets and PIDs β€” same as k3d defaults. .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + // ── Kubeconfig bind-mount ───────────────────────────────────────── + // Mounts AppHostDirectory/.k3s/{name}/cluster/ into the container so k3s + // writes its kubeconfig into a host-accessible directory. + // K3S_KUBECONFIG_OUTPUT tells k3s where to write the kubeconfig file. + // The health check polls File.Exists on the host side β€” no docker exec needed. + .WithBindMount(clusterDir, "/tmp/k3s-kubeconfig") + // ── Environment ─────────────────────────────────────────────────── - // Set an explicit cluster token (k3d always sets K3S_TOKEN). .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") - // World-readable kubeconfig so docker exec can read it without root. .WithEnvironment("K3S_KUBECONFIG_MODE", "644") + .WithEnvironment("K3S_KUBECONFIG_OUTPUT", "/tmp/k3s-kubeconfig/kubeconfig.yaml") .WithIconName("Kubernetes") .WithHttpsDeveloperCertificate(); @@ -212,32 +211,23 @@ public static IResourceBuilder AddK3sCluster( failureStatus: HealthStatus.Unhealthy, tags: null)); - // Postgres pattern: the parent cluster's ResourceReadyEvent drives ALL registered - // child lifecycles β€” both HelmReleases and K8sManifests β€” in parallel. + // The cluster's ResourceReadyEvent drives service endpoint port-forwards. + // HelmReleaseResource and K8sManifestResource containers are managed directly + // by DCP β€” they WaitFor the cluster and exit when their work completes. builder.Eventing.Subscribe(resource, (@event, ct) => { var appModel = @event.Services.GetRequiredService(); var notifications = @event.Services.GetRequiredService(); var loggerService = @event.Services.GetRequiredService(); - // Start all Helm release installs concurrently. - foreach (var release in appModel.Resources - .OfType() - .Where(r => ReferenceEquals(r.Parent, resource))) + // Start all service endpoint forwarders concurrently. + foreach (var ep in appModel.Resources + .OfType() + .Where(e => ReferenceEquals(e.Parent, resource))) { - var logger = loggerService.GetLogger(release); - _ = Task.Run(() => K3sHelmBuilderExtensions.RunReleaseAsync( - release, resource, notifications, logger, ct), ct); - } - - // Start all manifest applies concurrently. - foreach (var manifest in appModel.Resources - .OfType() - .Where(m => ReferenceEquals(m.Parent, resource))) - { - var logger = loggerService.GetLogger(manifest); - _ = Task.Run(() => K3sManifestBuilderExtensions.RunManifestAsync( - manifest, resource, notifications, logger, ct), ct); + var logger = loggerService.GetLogger(ep); + _ = Task.Run(() => K3sServiceEndpointExtensions.RunEndpointAsync( + ep, resource, notifications, logger, ct), ct); } return Task.CompletedTask; @@ -310,13 +300,6 @@ public static IResourceBuilder WithExtraArg( /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. /// - /// The cluster resource builder. - /// - /// The volume name. When , an auto-generated name is used in the form - /// {appName}-{sha256}-{resourceName}-data β€” the same scheme used by - /// PostgresServerResource.WithDataVolume and all other Aspire hosting integrations. - /// - /// The resource builder for chaining. public static IResourceBuilder WithDataVolume( this IResourceBuilder builder, string? name = null) @@ -325,41 +308,30 @@ public static IResourceBuilder WithDataVolume( return builder .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") - // Auto-restart on crash so a persistent cluster survives transient failures - // without requiring AppHost intervention. Docker will not restart the container - // on explicit stop (DCP shutdown), only on unexpected exits. .WithContainerRuntimeArgs("--restart=unless-stopped"); } /// - /// Injects the k3s kubeconfig into as a - /// KUBECONFIG_DATA environment variable (base-64-encoded YAML) β€” no files are written. + /// Injects the k3s kubeconfig into so it can authenticate + /// to the cluster. The injection method is selected automatically based on the resource type: /// /// - /// s (Auto or InlineData) receive the - /// container-network kubeconfig - /// (server: https://{resourceName}:6443). + /// s receive KUBECONFIG_DATA (base-64-encoded YAML + /// of the container-network kubeconfig) and KUBECONFIG=/var/k3s/kubeconfig.yaml + /// once the file is injected at container-start time. /// /// - /// Projects and executables (Auto or HostPath) receive the - /// host kubeconfig - /// (server: https://localhost:{allocatedPort}). + /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml + /// pointing to a file that is accessible directly on the host filesystem. /// /// - /// Consuming code reads the variable and builds a client without touching disk: - /// - /// var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); - /// using var stream = new MemoryStream(bytes); - /// var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); - /// /// The variable is populated only after the cluster health check passes; use /// WaitFor(cluster) on the dependent resource to guarantee ordering. /// [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] public static IResourceBuilder WithReference( this IResourceBuilder destination, - IResourceBuilder source, - KubeconfigInjectionStrategy strategy = KubeconfigInjectionStrategy.Auto) + IResourceBuilder source) where TDestination : IResourceWithEnvironment { ArgumentNullException.ThrowIfNull(destination); @@ -367,43 +339,37 @@ public static IResourceBuilder WithReference( var cluster = source.Resource; - return destination.WithEnvironment(ctx => + if (destination.Resource is ContainerResource) { - // Select the right kubeconfig variant based on where the resource runs: - // β€’ ContainerResource β†’ container-network URL (reaches API server via DCP DNS) - // β€’ Project/Executable β†’ host URL (reaches API server via localhost port-mapping) - var useContainerVariant = - strategy == KubeconfigInjectionStrategy.ContainerNetwork - || (strategy == KubeconfigInjectionStrategy.Auto - && destination.Resource is ContainerResource); - - var cfg = useContainerVariant - ? cluster.ContainerKubeconfig - : cluster.AdminKubeconfig; - - if (cfg is not null) + // Containers receive KUBECONFIG_DATA (base64-encoded container-network kubeconfig). + // KubernetesClient reads this via: + // var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); + // using var stream = new MemoryStream(bytes); + // var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); + // For standard kubectl, use WithContainerFiles to inject a real file instead. + return destination.WithEnvironment(ctx => { - var yaml = KubernetesYaml.Serialize(cfg); + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "container", "kubeconfig.yaml"); + if (!File.Exists(path)) return; + var yaml = File.ReadAllText(path); ctx.EnvironmentVariables["KUBECONFIG_DATA"] = Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); - } + }); + } + + // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. + // This file is regenerated on every AppHost start (port may change). + return destination.WithEnvironment(ctx => + { + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + ctx.EnvironmentVariables["KUBECONFIG"] = path; }); } - /// - /// Serialises the admin kubeconfig (server: https://localhost:{port}) to YAML. - /// Returns if the health check has not yet populated the config. - /// - internal static string? GetAdminKubeconfigYaml(K3sClusterResource cluster) => - cluster.AdminKubeconfig is null - ? null - : KubernetesYaml.Serialize(cluster.AdminKubeconfig); - /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. - /// When is used the agents must also be - /// persistent β€” otherwise they are recreated on every AppHost restart while the server - /// retains its state, causing the node re-join sequence to fail. /// public static IResourceBuilder WithLifetime( this IResourceBuilder builder, @@ -411,13 +377,10 @@ public static IResourceBuilder WithLifetime( { ArgumentNullException.ThrowIfNull(builder); - // Apply to the cluster container (identical to the built-in generic WithLifetime). builder.WithAnnotation( new ContainerLifetimeAnnotation { Lifetime = lifetime }, ResourceAnnotationMutationBehavior.Replace); - // Propagate to every agent β€” they share the same cluster state volume and must - // have the same lifetime so nodes survive AppHost restarts in sync with the server. foreach (var agent in builder.Resource.AgentResources) { var existing = agent.Annotations.OfType().ToList(); @@ -432,6 +395,16 @@ public static IResourceBuilder WithLifetime( return builder; } + /// + /// Returns the path to the local/kubeconfig.yaml file for this cluster, + /// used by helm and manifest runners that invoke host-side tools. + /// Returns if the cluster directory is not yet configured. + /// + internal static string? GetLocalKubeconfigPath(K3sClusterResource cluster) => + cluster.KubeconfigDirectory is null + ? null + : Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + // cgroupsv2 fix adapted from moby/moby (Apache-2.0, used with permission by k3d). // See: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh private const string K3sInitEntrypointScript = """ diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index cb33f7aa7..ab67ca836 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -37,4 +37,44 @@ public sealed class K3sClusterOptions /// When , the default tag embedded in the package is used. /// public string? ImageTag { get; set; } + + // ── Helm installer image ────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the Helm installer container image. + /// Defaults to docker.io. + /// + public string HelmRegistry { get; set; } = HelmContainerImageTags.Registry; + + /// + /// Gets or sets the Helm installer container image name. + /// Defaults to alpine/helm. + /// + public string HelmImage { get; set; } = HelmContainerImageTags.Image; + + /// + /// Gets or sets the Helm installer container image tag. + /// Defaults to 3.17.3. + /// + public string HelmTag { get; set; } = HelmContainerImageTags.Tag; + + // ── kubectl image ───────────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the kubectl container image used by manifest applies. + /// Defaults to docker.io. + /// + public string KubectlRegistry { get; set; } = KubectlContainerImageTags.Registry; + + /// + /// Gets or sets the kubectl container image name used by manifest applies. + /// Defaults to alpine/k8s. + /// + public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; + + /// + /// Gets or sets the kubectl container image tag used by manifest applies. + /// Defaults to 1.32.3. + /// + public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index f7b983b5a..4e121d7f9 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -1,5 +1,3 @@ -using k8s.KubeConfigModels; - namespace Aspire.Hosting.ApplicationModel; /// @@ -10,20 +8,25 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) { internal const string ApiServerEndpointName = "api"; - /// - /// Gets the admin kubeconfig for host-side processes - /// (server: https://localhost:{allocatedPort}). - /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. - /// Serialise with KubernetesYaml.Serialize(AdminKubeconfig) when needed. - /// - internal K8SConfiguration? AdminKubeconfig { get; set; } + /// Container image settings for the Helm installer, resolved from cluster options. + internal (string Registry, string Image, string Tag) HelmImageInfo { get; set; } + = ("docker.io", "alpine/helm", "3.17.3"); + + /// Container image settings for the kubectl manifest applier, resolved from cluster options. + internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } + = ("docker.io", "alpine/k8s", "1.32.3"); /// - /// Gets the kubeconfig for containers on the DCP Docker network - /// (server: https://{resourceName}:6443). - /// Populated by K3sReadinessHealthCheck after the cluster passes /healthz. + /// Host-side directory that holds all kubeconfig variants for this cluster. + /// Set by AddK3sCluster to AppHostDirectory/.k3s/{name}/. + /// Sub-directories: + /// + /// cluster/kubeconfig.yaml β€” raw file written by k3s (bind-mounted) + /// local/kubeconfig.yaml β€” server: https://localhost:{port} (host processes) + /// container/kubeconfig.yaml β€” server: https://{name}:6443 (DCP-network containers) + /// /// - internal K8SConfiguration? ContainerKubeconfig { get; set; } + internal string? KubeconfigDirectory { get; set; } private EndpointReference? _apiEndpoint; diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 1fa3eb074..1d704399b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -1,108 +1,127 @@ -using System.Diagnostics; -using System.Text; +using System.Collections.Generic; +using System.Net; +using System.Net.Sockets; +using k8s; using Microsoft.Extensions.Logging; namespace CommunityToolkit.Aspire.Hosting; /// -/// Forwards a local TCP port to a Kubernetes service by running -/// kubectl port-forward service/{name} {localPort}:{servicePort} -n {namespace} -/// as a managed subprocess. +/// Forwards a local TCP port to a Kubernetes service using the KubernetesClient +/// WebSocket port-forward API β€” no kubectl binary required. /// -/// Mirrors what a developer types in a terminal β€” the most reliable approach for -/// k3s-in-Docker because kubectl handles WebSocket negotiation, kubelet routing, -/// and reconnect logic internally. +/// The listener binds to 0.0.0.0:{localPort} so both host processes +/// (localhost:{port}) and DCP-network containers +/// (host.docker.internal:{port}) can reach the service. /// /// internal sealed class K3sInProcessPortForwarder( - string kubeconfigYaml, + string kubeconfigPath, string @namespace, string serviceName, int localPort, - int servicePort) + int servicePort, + Action onReadyChanged) { public async Task RunAsync(ILogger logger, CancellationToken ct) { - logger.LogInformation( - "Port-forward: localhost:{Local} β†’ svc/{Service}.{Ns}:{Port}", - localPort, serviceName, @namespace, servicePort); + var backoff = TimeSpan.FromSeconds(2); - var tempConfig = Path.Combine( - Path.GetTempPath(), - $"aspire-k3s-pf-{Environment.ProcessId}-{serviceName}.yaml"); - - await File.WriteAllTextAsync(tempConfig, kubeconfigYaml, Encoding.UTF8, ct) - .ConfigureAwait(false); - - try + while (!ct.IsCancellationRequested) { - while (!ct.IsCancellationRequested) + var listener = new TcpListener(IPAddress.Any, localPort); + try { - try - { - await RunKubectlAsync(tempConfig, logger, ct).ConfigureAwait(false); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - logger.LogWarning(ex, - "Port-forward for svc/{Service} exited; restarting in 5 s…", serviceName); + listener.Start(); + onReadyChanged(true); + + logger.LogInformation( + "Port-forward: 0.0.0.0:{Local} β†’ svc/{Service}.{Ns}:{Port}", + localPort, serviceName, @namespace, servicePort); - await Task.Delay(TimeSpan.FromSeconds(5), ct).ConfigureAwait(false); + while (!ct.IsCancellationRequested) + { + var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); + _ = Task.Run( + () => ForwardConnectionAsync(tcp, logger, ct), + CancellationToken.None); } } - } - finally - { - try { File.Delete(tempConfig); } catch { /* best-effort cleanup */ } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, + "Port-forward for svc/{Service} failed; retrying in {Delay}s…", + serviceName, backoff.TotalSeconds); + onReadyChanged(false); + } + finally + { + listener.Stop(); + } + + if (ct.IsCancellationRequested) break; + + try { await Task.Delay(backoff, ct).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 30)); } } - private async Task RunKubectlAsync(string kubeconfigPath, ILogger logger, CancellationToken ct) + private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, CancellationToken ct) { - var psi = new ProcessStartInfo("kubectl") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - psi.ArgumentList.Add("port-forward"); - psi.ArgumentList.Add($"service/{serviceName}"); - psi.ArgumentList.Add($"{localPort}:{servicePort}"); - psi.ArgumentList.Add("-n"); - psi.ArgumentList.Add(@namespace); - psi.ArgumentList.Add($"--kubeconfig={kubeconfigPath}"); - - using var process = Process.Start(psi) - ?? throw new InvalidOperationException("Failed to start kubectl port-forward."); - - using var reg = ct.Register(() => + using var _ = tcp; + try { - try { process.Kill(entireProcessTree: true); } catch { /* already exited */ } - }); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); - process.OutputDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogDebug("{Line}", e.Data); - }; - process.ErrorDataReceived += (_, e) => - { - if (e.Data is not null) logger.LogDebug("{Line}", e.Data); - }; - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); + // Resolve the service to a running pod. + var svc = await k8sClient.CoreV1 + .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) + .ConfigureAwait(false); + + var selector = string.Join(",", + (svc.Spec.Selector ?? new Dictionary()).Select(kv => $"{kv.Key}={kv.Value}")); - await process.WaitForExitAsync(ct).ConfigureAwait(false); + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ConfigureAwait(false); - if (process.ExitCode != 0 && !ct.IsCancellationRequested) + var pod = pods.Items.FirstOrDefault(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (pod is null) + { + logger.LogWarning( + "No ready pod found for service {Service}/{Ns} β€” connection dropped.", + serviceName, @namespace); + return; + } + + // Open WebSocket port-forward to the pod. + using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( + pod.Metadata.Name, @namespace, [servicePort], + cancellationToken: ct).ConfigureAwait(false); + + using var demuxer = new StreamDemuxer(ws, StreamType.PortForward); + demuxer.Start(); + + using var k8sStream = demuxer.GetStream((byte?)0, (byte?)0); + using var tcpStream = tcp.GetStream(); + + // Bidirectional byte pump until either side closes. + await Task.WhenAny( + tcpStream.CopyToAsync(k8sStream, ct), + k8sStream.CopyToAsync(tcpStream, ct)).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { } + catch (Exception ex) { - throw new InvalidOperationException( - $"kubectl port-forward exited with code {process.ExitCode}."); + logger.LogDebug(ex, "Port-forward connection for svc/{Service} closed.", serviceName); } } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index f0cc8f1fd..a51a9e2e2 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -1,4 +1,3 @@ -using System.Diagnostics; using Aspire.Hosting.ApplicationModel; using k8s; using k8s.KubeConfigModels; @@ -9,27 +8,23 @@ namespace CommunityToolkit.Aspire.Hosting; /// /// Health check for . /// -/// Instead of probing GET /healthz (which requires authentication in Kubernetes 1.28+ -/// because anonymous-auth defaults to false), this check runs -/// docker exec {container} kubectl get nodes inside the k3s container. -/// kubectl inside the container uses the default in-cluster kubeconfig, so no external -/// credentials are needed and the result is authoritative: a node in Ready state -/// proves the API server, scheduler, and kubelet are all functional. -/// -/// -/// On first success the kubeconfig is read from the container via docker exec cat, -/// parsed into two variants, and stored on the resource: +/// Polls for the kubeconfig file written by k3s into the bind-mounted +/// AppHostDirectory/.k3s/{name}/cluster/kubeconfig.yaml. On first appearance +/// it rewrites the server URL for two variants: /// -/// β€” server: https://localhost:{port} -/// β€” server: https://{name}:6443 +/// local/kubeconfig.yaml β€” server: https://localhost:{allocatedPort} (host processes) +/// container/kubeconfig.yaml β€” server: https://{name}:6443 (DCP-network containers) /// +/// Then uses a cached client to call ListNodeAsync, +/// confirming that all expected nodes (server + agents) are in Ready state. +/// No docker exec is involved β€” works with any container runtime. /// /// internal sealed class K3sReadinessHealthCheck : IHealthCheck { private readonly K3sClusterResource _resource; private readonly EndpointReference _endpoint; - private bool _kubeconfigRead; + private Kubernetes? _cachedClient; internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) { @@ -43,148 +38,102 @@ public async Task CheckHealthAsync( CancellationToken cancellationToken = default) { if (!_endpoint.IsAllocated) - { return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); - } - var port = _endpoint.Port; + var dir = _resource.KubeconfigDirectory; + if (dir is null) + return HealthCheckResult.Unhealthy("Kubeconfig directory not configured on resource"); + + var rawPath = Path.Combine(dir, "cluster", "kubeconfig.yaml"); + if (!File.Exists(rawPath)) + return HealthCheckResult.Unhealthy("Waiting for k3s to write kubeconfig"); try { - var containerId = await FindContainerIdAsync(cancellationToken); - - if (containerId is null) - { - return HealthCheckResult.Unhealthy("k3s container not yet found via docker ps"); - } - - // Run kubectl get nodes inside the container where the default kubeconfig is - // already configured β€” avoids any authentication issue from the outside. - var nodesOutput = await RunDockerAsync( - ["exec", containerId, - "kubectl", "get", "nodes", - "--kubeconfig", "/etc/rancher/k3s/k3s.yaml", - "--no-headers"], - cancellationToken); - - if (nodesOutput is null) - { - return HealthCheckResult.Unhealthy( - "kubectl get nodes failed β€” k3s API server not yet ready"); - } - - // Count nodes actually in Ready state (excluding NotReady ones). - var readyNodeLines = nodesOutput - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Where(line => line.Contains("Ready") && !line.Contains("NotReady")) - .ToArray(); - - // For multi-node clusters (WithAgentNodes), wait for server + all agents. - // For single-node, 1 Ready node is sufficient. - var expectedNodes = 1 + _resource.AgentCount; - - if (readyNodeLines.Length < expectedNodes) - { - return HealthCheckResult.Unhealthy( - $"k3s cluster: {readyNodeLines.Length}/{expectedNodes} nodes Ready"); - } - - if (!_kubeconfigRead) - { - var rawYaml = await RunDockerAsync( - ["exec", containerId, "cat", "/etc/rancher/k3s/k3s.yaml"], - cancellationToken); - - if (rawYaml is null) - { - return HealthCheckResult.Unhealthy( - "k3s kubeconfig not yet available inside the container"); - } - - var parsed = KubernetesYaml.Deserialize(rawYaml); - - _resource.AdminKubeconfig = - BuildConfig(parsed, $"https://localhost:{port}"); - _resource.ContainerKubeconfig = - BuildConfig(parsed, $"https://{_resource.Name}:6443"); - _kubeconfigRead = true; - } + var client = await EnsureClientAsync(rawPath, cancellationToken).ConfigureAwait(false); + + var nodes = await client.CoreV1 + .ListNodeAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var readyCount = nodes.Items.Count(n => + n.Status?.Conditions?.Any(c => + c.Type == "Ready" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true); + + var expected = 1 + _resource.AgentCount; + if (readyCount < expected) + return HealthCheckResult.Unhealthy($"k3s cluster: {readyCount}/{expected} nodes Ready"); return HealthCheckResult.Healthy("k3s cluster is ready"); } + catch (Exception ex) when (IsTlsOrAuthFailure(ex)) + { + // Stale kubeconfig β€” cluster was recreated with new certs (e.g. data volume wiped). + // Invalidate everything; k3s will overwrite cluster/kubeconfig.yaml on next start. + _cachedClient?.Dispose(); + _cachedClient = null; + TryDelete(rawPath); + TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); + TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); + return HealthCheckResult.Unhealthy("k3s kubeconfig is stale β€” waiting for cluster refresh"); + } catch (Exception ex) { return HealthCheckResult.Unhealthy(ex.Message, ex); } } - private static K8SConfiguration BuildConfig(K8SConfiguration source, string serverUrl) + private async Task EnsureClientAsync(string rawPath, CancellationToken ct) + { + if (_cachedClient is not null) + return _cachedClient; + + var port = _endpoint.Port; + var dir = _resource.KubeconfigDirectory!; + + var rawYaml = await File.ReadAllTextAsync(rawPath, ct).ConfigureAwait(false); + var parsed = KubernetesYaml.Deserialize(rawYaml); + + var localDir = Path.Combine(dir, "local"); + Directory.CreateDirectory(localDir); + var localPath = Path.Combine(localDir, "kubeconfig.yaml"); + await File.WriteAllTextAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{port}"), ct) + .ConfigureAwait(false); + + var containerDir = Path.Combine(dir, "container"); + Directory.CreateDirectory(containerDir); + await File.WriteAllTextAsync( + Path.Combine(containerDir, "kubeconfig.yaml"), + BuildConfigYaml(parsed, $"https://{_resource.Name}:6443"), + ct).ConfigureAwait(false); + + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); + _cachedClient = new Kubernetes(config); + return _cachedClient; + } + + private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) { var yaml = KubernetesYaml.Serialize(source); var copy = KubernetesYaml.Deserialize(yaml); - foreach (var cluster in copy.Clusters ?? []) { if (cluster.ClusterEndpoint is not null) - { cluster.ClusterEndpoint.Server = serverUrl; - } } - return copy; + return KubernetesYaml.Serialize(copy); } - private async Task FindContainerIdAsync(CancellationToken ct) - { - // docker ps --filter name=VALUE uses substring matching: "name=k8s" also matches - // "k8s-agent-0", "k8s-agent-1", etc. Use --format to get names alongside IDs and - // exclude agent containers whose names contain "-agent-". - var output = await RunDockerAsync( - ["ps", - "--filter", $"name={_resource.Name}", - "--format", "{{.ID}}\t{{.Names}}", - "--no-trunc"], - ct); - - if (output is null) - { - return null; - } - - return output - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(line => line.Split('\t', 2)) - .Where(parts => parts.Length == 2 && !parts[1].Contains("-agent-")) - .Select(parts => parts[0].Trim()) - .FirstOrDefault(id => !string.IsNullOrWhiteSpace(id)); - } + private static bool IsTlsOrAuthFailure(Exception ex) => + ex is System.Security.Authentication.AuthenticationException + || ex.InnerException is System.Security.Authentication.AuthenticationException + || (ex is k8s.Autorest.HttpOperationException op && + op.Response?.StatusCode == System.Net.HttpStatusCode.Unauthorized); - private static async Task RunDockerAsync(string[] args, CancellationToken ct) + private static void TryDelete(string path) { - var psi = new ProcessStartInfo("docker") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true, - }; - - foreach (var arg in args) - { - psi.ArgumentList.Add(arg); - } - - using var process = Process.Start(psi); - if (process is null) - { - return null; - } - - var output = await process.StandardOutput.ReadToEndAsync(ct); - await process.WaitForExitAsync(ct); - - return process.ExitCode == 0 && !string.IsNullOrWhiteSpace(output) - ? output - : null; + try { File.Delete(path); } catch { /* best effort */ } } } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs new file mode 100644 index 000000000..95d4c9f9a --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -0,0 +1,47 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Exposes a Kubernetes service running inside a k3s cluster as an Aspire endpoint resource. +/// +/// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready. +/// The forwarder binds to 0.0.0.0:{hostPort} so both host processes and DCP-network +/// containers can reach the service. +/// +/// +/// Host consumers receive services__{name}__url=https://localhost:{port}. +/// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. +/// +/// +public sealed class K3sServiceEndpointResource( + string name, + string serviceName, + int servicePort, + string @namespace, + K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Kubernetes service name. + public string ServiceName { get; } = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); + + /// Gets the service port number. + public int ServicePort { get; } = servicePort; + + /// Gets the Kubernetes namespace containing the service. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + /// + /// The host port allocated for the port-forward listener. + /// Set by RunEndpointAsync before the resource transitions to Running. + /// Consumers can use this to construct the service URL directly when needed. + /// + public int HostPort { get; internal set; } + + /// + /// when the port-forward is active and accepting connections. + /// Set by K3sInProcessPortForwarder; read by the health check. + /// + internal volatile bool IsReady; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index f0a6a361b..a87a687ee 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -2,46 +2,22 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// Server-Side Apply. This is a child resource of , following -/// the same parent-child pattern as . +/// kubectl apply --server-side running inside a bitnami/kubectl container. /// -/// No kubectl binary is required β€” the KubernetesClient library handles the apply. -/// CRDs reach the Established condition before the resource transitions to -/// Running, so dependent resources can safely WaitFor the manifest. +/// The container polls for the cluster kubeconfig (written when the cluster health check +/// first passes), applies the manifests, waits for any CRDs to reach Established, +/// then exits with code 0. Use WaitForCompletion(manifest) on dependent resources. /// /// /// The Aspire resource name. -/// -/// Path to a single YAML file, a directory, or a glob pattern (*.yaml). -/// Directories and globs are expanded lexicographically. -/// +/// Absolute path to a single YAML file or a directory. /// The parent k3s cluster resource. public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) - : Resource(name), IResourceWithParent, IResourceWithWaitSupport + : ContainerResource(name), IResourceWithParent { /// public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); - /// Gets the manifest path, directory, or glob. + /// Gets the manifest path or directory on the host. public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); - - /// - /// Set to by the lifecycle after all objects are applied and - /// (for CRDs) the Established condition is confirmed. - /// - internal volatile bool IsReady; - - /// Services to expose via in-process port-forward after the manifest is applied. - internal List EndpointDefinitions { get; } = []; } - -/// Describes a service endpoint to expose from a . -/// Kubernetes service name. -/// Service port number. -/// Friendly name shown in the dashboard. -/// Kubernetes namespace where the service lives. -internal sealed record ManifestEndpointDefinition( - string ServiceName, - int ServicePort, - string EndpointName, - string Namespace); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs deleted file mode 100644 index b747206b1..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/KubeconfigInjectionStrategy.cs +++ /dev/null @@ -1,33 +0,0 @@ -namespace CommunityToolkit.Aspire.Hosting; - -/// -/// Controls which kubeconfig server-URL variant is injected via KUBECONFIG_DATA. -/// All variants are delivered as base-64-encoded YAML without writing any file. -/// -public enum KubeconfigInjectionStrategy -{ - /// - /// Selects the server URL automatically based on the resource type: - /// - /// Container resources receive the - /// container-network URL (https://{resourceName}:6443). - /// Projects and executables receive the host URL - /// (https://localhost:{allocatedPort}). - /// - /// - Auto, - - /// - /// Always inject the host-network kubeconfig (server: https://localhost:{port}). - /// Use when a container is launched with --network=host or when the caller - /// explicitly needs host-side connectivity. - /// - HostNetwork, - - /// - /// Always inject the DCP-network kubeconfig - /// (server: https://{resourceName}:6443). - /// Use when a host process needs to reach the cluster the same way containers do. - /// - ContainerNetwork, -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs new file mode 100644 index 000000000..eb6eedb68 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -0,0 +1,11 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class KubectlContainerImageTags +{ + internal const string Registry = "docker.io"; + // alpine/k8s: lightweight Alpine-based image that includes kubectl and other k8s tools. + // Same organisation as alpine/helm β€” consistent image family. + internal const string Image = "alpine/k8s"; + // Matches the Kubernetes version shipped by the default k3s tag. + internal const string Tag = "1.32.3"; +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj new file mode 100644 index 000000000..95df55b8f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs new file mode 100644 index 000000000..f08fa0588 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -0,0 +1,158 @@ +using Aspire.Components.Common.Tests; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Utils; +using CommunityToolkit.Aspire.Testing; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; + +/// +/// End-to-end integration tests that spin up a real k3s cluster inside Docker. +/// +/// Requirements: +/// +/// Linux with Docker (privileged containers required by k3s). +/// helm on PATH β€” used by AddHelmRelease. +/// kubectl on PATH β€” used by AddK8sManifest. +/// +/// The tests are gated by [RequiresDocker] and intended for the +/// ubuntu-latest-only CI job in tests.yaml. +/// +/// +[RequiresDocker] +public class K3sIntegrationTests : IAsyncLifetime +{ + private DistributedApplication? _app; + private IDistributedApplicationTestingBuilder? _builder; + + public async ValueTask InitializeAsync() + { + _builder = TestDistributedApplicationBuilder.Create(); + await ValueTask.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + if (_app is not null) + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } + + _builder?.Dispose(); + } + + [Fact] + public async Task ClusterReachesRunningAndKubeconfigIsValid() + { + var cluster = _builder!.AddK3sCluster("k8s"); + _app = _builder.Build(); + + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // local/kubeconfig.yaml must exist on the host. + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + Assert.True(File.Exists(kubeconfigPath), + $"Expected local kubeconfig at {kubeconfigPath}"); + + // container/kubeconfig.yaml must also exist. + var containerKubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); + Assert.True(File.Exists(containerKubeconfigPath), + $"Expected container kubeconfig at {containerKubeconfigPath}"); + } + + [Fact] + public async Task HelmReleaseReachesRunning() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == KnownResourceStates.Running, cts.Token); + } + + [Fact] + public async Task ServiceEndpointExposesHttpPort() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + var nginx = cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + cluster.AddServiceEndpoint("nginx-web", "nginx", servicePort: 80, @namespace: "nginx") + .WaitForCompletion(nginx); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + // nginx is a run-to-completion container β€” wait for it to exit (Exited state) + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == "Exited", cts.Token); + await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); + + // Find the allocated port from the endpoint resource. + var model = _app.Services.GetRequiredService(); + var ep = model.Resources.OfType().Single(); + + Assert.True(ep.HostPort > 0, "HostPort should be allocated"); + + using var http = new HttpClient(); + var response = await http.GetAsync($"http://localhost:{ep.HostPort}", cts.Token); + Assert.True(response.IsSuccessStatusCode, + $"Expected HTTP 200 from nginx at localhost:{ep.HostPort}, got {response.StatusCode}"); + } + + [Fact] + public async Task WithReferenceInjectsKubeconfigForProject() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + // WithReference on a project injects KUBECONFIG pointing to local/kubeconfig.yaml. + // We verify the env var would be set by checking the cluster state. + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + var yaml = await File.ReadAllTextAsync(kubeconfigPath, cts.Token); + Assert.Contains("localhost", yaml); + Assert.DoesNotContain("127.0.0.1:6443", yaml); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 1269e0a34..02d3fc3c4 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -21,10 +21,10 @@ public void AddHelmReleaseAddsHelmReleaseResourceWithCorrectName() } [Fact] - public void AddHelmReleaseDoesNotCreateSeparateInstallResource() + public void HelmReleaseResourceIsContainerResource() { - // helm upgrade --install runs internally inside the HelmReleaseResource lifecycle; - // no separate ExecutableResource is added to the application model. + // HelmReleaseResource extends ContainerResource β€” it runs bitnami/helm in Docker. + // No host-side helm binary required. WaitForCompletion waits for exit code 0. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -33,7 +33,8 @@ public void AddHelmReleaseDoesNotCreateSeparateInstallResource() using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - Assert.Empty(model.Resources.OfType()); + var resource = Assert.Single(model.Resources.OfType()); + Assert.IsAssignableFrom(resource); } [Fact] @@ -118,7 +119,6 @@ public void HelmReleaseParentIsCluster() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd"); using var app = appBuilder.Build(); @@ -126,27 +126,7 @@ public void HelmReleaseParentIsCluster() var resource = Assert.Single(model.Resources.OfType()); Assert.Same(cluster.Resource, resource.Parent); - } - - [Fact] - public void HelmReleaseImplementsNonGenericIResourceWithParent() - { - // The Aspire dashboard uses the non-generic IResourceWithParent to group - // child resources under their parent. Verify both the generic and non-generic - // interfaces are satisfied and point to the same cluster resource. - var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd"); - - using var app = appBuilder.Build(); - var model = app.Services.GetRequiredService(); - - var resource = Assert.Single(model.Resources.OfType()); - - // Non-generic IResourceWithParent (used by the dashboard) - var nonGeneric = resource as IResourceWithParent; - Assert.NotNull(nonGeneric); - Assert.Same(cluster.Resource, nonGeneric.Parent); + Assert.IsAssignableFrom>(resource); } [Fact] @@ -168,39 +148,36 @@ public void WithHelmValueAccumulatesValues() } [Fact] - public void WithEndpointAccumulatesEndpoints() + public void AddServiceEndpointAddsEndpointResource() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); - cluster.AddHelmRelease("argocd", "argo-cd") - .WithEndpoint("argocd-server", servicePort: 443, name: "ui"); + cluster.AddServiceEndpoint("argocd-ui", "argocd-server", servicePort: 443, @namespace: "argocd"); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var resource = Assert.Single(model.Resources.OfType()); - var ep = Assert.Single(resource.EndpointDefinitions); + var ep = Assert.Single(model.Resources.OfType()); Assert.Equal("argocd-server", ep.ServiceName); Assert.Equal(443, ep.ServicePort); - Assert.Equal("ui", ep.EndpointName); + Assert.Equal("argocd", ep.Namespace); } [Fact] - public void WithEndpointMultipleEndpoints() + public void AddServiceEndpointMultipleEndpointsAllRegistered() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); - cluster.AddHelmRelease("argocd", "argo-cd") - .WithEndpoint("argocd-server", 443, "ui") - .WithEndpoint("argocd-server", 80, "http"); + cluster.AddServiceEndpoint("ui", "argocd-server", 443); + cluster.AddServiceEndpoint("http", "argocd-server", 80); using var app = appBuilder.Build(); var model = app.Services.GetRequiredService(); - var resource = Assert.Single(model.Resources.OfType()); - Assert.Equal(2, resource.EndpointDefinitions.Count); + Assert.Equal(2, model.Resources.OfType().Count()); } [Fact] @@ -218,127 +195,106 @@ public void HelmReleaseIsExcludedFromManifest() Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); } - // ── BuildHelmInstallArgs tests (pure logic, no DI needed) ───────────────── - - [Fact] - public void BuildHelmInstallArgsIncludesUpgradeInstall() - { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "argocd", "argo-cd", null, null, "argocd", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("upgrade", list); - Assert.Contains("--install", list); - Assert.Contains("argocd", list); - Assert.Contains("argo-cd", list); - } + // ── BuildHelmScript tests (pure logic, no DI needed) ────────────────────── - [Fact] - public void BuildHelmInstallArgsIncludesKubeconfig() + private static HelmReleaseResource MakeRelease( + string releaseName, string chart, string? repo, string? version, + string @namespace, Dictionary? values = null) { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); - - Assert.Contains("--kubeconfig=/tmp/admin.yaml", args); + var cluster = new K3sClusterResource("k8s"); + var r = new HelmReleaseResource(releaseName, releaseName, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + foreach (var kv in values ?? []) + r.HelmValues[kv.Key] = kv.Value; + return r; } [Fact] - public void BuildHelmInstallArgsIncludesWait() + public void BuildHelmScriptIncludesUpgradeInstall() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("argocd", "argo-cd", null, null, "argocd")); - Assert.Contains("--wait", args); + Assert.Contains("helm upgrade --install", script); + Assert.Contains("\"argocd\"", script); + Assert.Contains("\"argo-cd\"", script); } [Fact] - public void BuildHelmInstallArgsIncludesNamespace() + public void BuildHelmScriptIncludesWaitAndNamespace() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "my-ns", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("--namespace", list); - Assert.Contains("my-ns", list); - Assert.Contains("--create-namespace", list); - } + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "my-ns")); - [Fact] - public void BuildHelmInstallArgsWithRepoAliasUsesPrefixedChartRef() - { - // When a repo alias is pre-registered via `helm repo add`, BuildHelmInstallArgs - // uses "{alias}/{chart}" notation β€” NOT the --repo flag (which is unreliable). - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", "my-repo-alias", null, "default", null, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.DoesNotContain("--repo", list); - Assert.Contains("my-repo-alias/chart", list); + Assert.Contains("--wait", script); + Assert.Contains("--namespace \"my-ns\"", script); + Assert.Contains("--create-namespace", script); } [Fact] - public void BuildHelmInstallArgsWithNullAliasUsesChartDirectly() + public void BuildHelmScriptWithRepoAddsRepoSteps() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "oci://registry/chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", "https://my-repo.example.com", null, "default")); - Assert.DoesNotContain("--repo", args); - Assert.Contains("oci://registry/chart", args); + Assert.Contains("helm repo add", script); + Assert.Contains("helm repo update", script); + Assert.Contains("aspire-k3s-r/chart", script); } [Fact] - public void BuildHelmInstallArgsIncludesVersion() + public void BuildHelmScriptWithoutRepoSkipsRepoSteps() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, "7.8.0", "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "oci://registry/chart", null, null, "default")); - var list = args.ToArray(); - Assert.Contains("--version", list); - Assert.Contains("7.8.0", list); + Assert.DoesNotContain("helm repo add", script); + Assert.Contains("\"oci://registry/chart\"", script); } [Fact] - public void BuildHelmInstallArgsOmitsRepoWhenNull() + public void BuildHelmScriptIncludesVersion() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, "7.8.0", "default")); - Assert.DoesNotContain("--repo", args); + Assert.Contains("--version \"7.8.0\"", script); } [Fact] - public void BuildHelmInstallArgsOmitsVersionWhenNull() + public void BuildHelmScriptOmitsVersionWhenNull() { - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", null, "/tmp/admin.yaml"); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default")); - Assert.DoesNotContain("--version", args); + Assert.DoesNotContain("--version", script); } [Fact] - public void BuildHelmInstallArgsIncludesSetValues() + public void BuildHelmScriptIncludesSetValues() { - var values = new Dictionary - { - ["service.type"] = "NodePort", - ["replicaCount"] = "2", - }; - var args = K3sHelmBuilderExtensions.BuildHelmInstallArgs( - "r", "chart", null, null, "default", values, "/tmp/admin.yaml"); - - var list = args.ToArray(); - Assert.Contains("--set", list); - Assert.Contains("service.type=NodePort", list); - Assert.Contains("replicaCount=2", list); + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default", new() + { + ["service.type"] = "NodePort", + ["replicaCount"] = "2", + })); + + Assert.Contains("--set \"service.type=NodePort\"", script); + Assert.Contains("--set \"replicaCount=2\"", script); } - // ── WaitFor support ─────────────────────────────────────────────────────── + // ── WaitForCompletion support ───────────────────────────────────────────── [Fact] - public void HelmReleaseHasHealthCheckForWaitForSupport() + public void HelmReleaseHasNoHealthCheckAnnotation() { - // WaitFor(helmRelease) is satisfied by the HelmReleaseHealthCheck, - // which flips IsReady once RunReleaseAsync completes. + // HelmReleaseResource is a run-to-completion container β€” consumers use + // WaitForCompletion(helmRelease) rather than WaitFor. No health check needed. var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); cluster.AddHelmRelease("argocd", "argo-cd"); @@ -347,17 +303,7 @@ public void HelmReleaseHasHealthCheckForWaitForSupport() var model = app.Services.GetRequiredService(); var resource = Assert.Single(model.Resources.OfType()); - Assert.Contains(resource.Annotations.OfType(), a => - a.Key == "helm_argocd_ready"); - } - - [Fact] - public void HelmReleaseIsReadyFlagStartsFalse() - { - var resource = new HelmReleaseResource( - "argocd", "argocd", "default", new K3sClusterResource("k8s")); - - Assert.False(resource.IsReady); + Assert.Empty(resource.Annotations.OfType()); } // ── Public API null-guard tests ─────────────────────────────────────────── @@ -397,10 +343,10 @@ public void WithHelmValueShouldThrowWhenBuilderIsNull() } [Fact] - public void WithEndpointShouldThrowWhenBuilderIsNull() + public void AddServiceEndpointShouldThrowWhenBuilderIsNull() { - IResourceBuilder builder = null!; - var action = () => builder.WithEndpoint("svc", 443, "ui"); + IResourceBuilder builder = null!; + var action = () => builder.AddServiceEndpoint("ui", "svc", 443); Assert.Throws(action); } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 9d1811255..0ca9b8bf9 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -211,13 +211,14 @@ public void WithReferenceSetsKubeconfigDataEnvForContainer() } [Fact] - public void K3sClusterResourceHasNoKubeconfigDirectoryByDefault() + public void AddK3sClusterSetsKubeconfigDirectory() { - // Kubeconfig is now stored in-memory (K8SConfiguration objects) and - // never written to disk by the resource itself β€” docker exec reads it. - var resource = new K3sClusterResource("k8s"); - Assert.Null(resource.AdminKubeconfig); - Assert.Null(resource.ContainerKubeconfig); + var appBuilder = DistributedApplication.CreateBuilder(); + var clusterBuilder = appBuilder.AddK3sCluster("k8s"); + + // KubeconfigDirectory is set by AddK3sCluster under AppHostDirectory/.k3s/{name}/ + Assert.NotNull(clusterBuilder.Resource.KubeconfigDirectory); + Assert.EndsWith(Path.Combine(".k3s", "k8s"), clusterBuilder.Resource.KubeconfigDirectory); } [Fact] diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs index 906d590fd..c657655fc 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -23,7 +23,7 @@ public void AddK8sManifestAddsResourceWithCorrectName() } [Fact] - public void AddK8sManifestStoresPath() + public void AddK8sManifestStoresAbsolutePath() { var appBuilder = DistributedApplication.CreateBuilder(); var cluster = appBuilder.AddK3sCluster("k8s"); @@ -34,7 +34,11 @@ public void AddK8sManifestStoresPath() var model = app.Services.GetRequiredService(); var resource = Assert.Single(model.Resources.OfType()); - Assert.Equal("./k8s/crds/", resource.Path); + // Path is resolved to absolute at registration time. + Assert.True(System.IO.Path.IsPathRooted(resource.Path)); + Assert.EndsWith(System.IO.Path.Combine("k8s", "crds"), + resource.Path.TrimEnd(System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase); } [Fact] @@ -50,26 +54,18 @@ public void AddK8sManifestParentIsCluster() var resource = Assert.Single(model.Resources.OfType()); Assert.Same(cluster.Resource, resource.Parent); + Assert.IsAssignableFrom>(resource); } [Fact] - public void AddK8sManifestImplementsIResourceWithParent() + public void K8sManifestResourceIsContainerResource() { + // K8sManifestResource extends ContainerResource β€” it runs bitnami/kubectl in Docker. + // No host-side kubectl binary required. WaitForCompletion waits for exit code 0. var resource = new K8sManifestResource( "crd", "./crd.yaml", new K3sClusterResource("k8s")); - var nonGeneric = resource as IResourceWithParent; - Assert.NotNull(nonGeneric); - Assert.Same(resource.Parent, nonGeneric.Parent); - } - - [Fact] - public void AddK8sManifestImplementsIResourceWithWaitSupport() - { - var resource = new K8sManifestResource( - "crd", "./crd.yaml", new K3sClusterResource("k8s")); - - Assert.IsAssignableFrom(resource); + Assert.IsAssignableFrom(resource); } [Fact] From a2ce723c786a23c93a6e05ec5c5e77af0d82b4eb Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Thu, 14 May 2026 01:45:39 +0200 Subject: [PATCH 03/13] feat: support files for helm/manifest --- .../HelmReleaseResource.cs | 7 ++ .../K3sBuilderExtensions.Helm.cs | 43 +++++++ .../K3sBuilderExtensions.Manifest.cs | 105 +++++++++++------- .../HelmReleaseResourceTests.cs | 52 +++++++++ .../K8sManifestResourceTests.cs | 60 ++++++++++ 5 files changed, 226 insertions(+), 41 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index b5de3d52b..65740bea8 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -33,4 +33,11 @@ public sealed class HelmReleaseResource( internal string? RepoUrl { get; set; } internal string? Version { get; set; } internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); + + /// + /// Absolute host paths of values files to inject into the helm container via + /// --values /helm-values/{filename}. + /// Populated by WithHelmValuesFile. + /// + internal List ValuesFiles { get; } = []; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index bd8dbb780..6799c8f8e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -82,6 +82,18 @@ public static IResourceBuilder AddHelmRelease( }]; }) .WithArgs("/helm-install.sh") + // Inject host-side values files declared via WithHelmValuesFile using + // Aspire's built-in WithContainerFiles(destinationPath, hostSourcePath). + // One call per file; each copies the file into /helm-values/ in the container. + // The callback wraps all files so it fires after all WithHelmValuesFile() calls. + .WithContainerFiles("/helm-values", async (ctx, ct) => + release.ValuesFiles + .Select(hostPath => (ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(hostPath), + SourcePath = hostPath, + }) + .ToList()) .WithBindMount(containerKubeconfigDir, "/root/.kube") .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Rocket") @@ -100,6 +112,33 @@ public static IResourceBuilder AddHelmRelease( }); } + /// + /// Injects a host-side YAML values file into the Helm installer container and + /// passes it as --values /helm-values/{filename} to helm upgrade --install. + /// Multiple files are applied in the order they are declared (last wins for overlapping keys). + /// + /// The Helm release resource builder. + /// + /// Path to the values YAML file on the host. Relative paths are resolved against + /// AppHostDirectory. + /// + public static IResourceBuilder WithHelmValuesFile( + this IResourceBuilder builder, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine( + builder.ApplicationBuilder.AppHostDirectory, path)); + + builder.Resource.ValuesFiles.Add(absolutePath); + return builder; + } + /// /// Adds a Helm --set key=value argument to this release. /// @@ -149,6 +188,10 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.Version is not null) sb.Append($" --version \"{release.Version}\""); + // Values files are injected into /helm-values/ by WithContainerFiles. + foreach (var hostPath in release.ValuesFiles) + sb.Append($" --values \"/helm-values/{System.IO.Path.GetFileName(hostPath)}\""); + foreach (var (key, value) in release.HelmValues) sb.Append($" --set \"{key}={value}\""); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 40a911e55..6cb79b838 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -12,18 +12,23 @@ namespace Aspire.Hosting; public static class K3sManifestBuilderExtensions { /// - /// Applies one or more Kubernetes YAML files to the cluster via - /// kubectl apply --server-side running inside a bitnami/kubectl container. - /// No host-side kubectl binary is required. + /// Applies one or more Kubernetes YAML files β€” or a Kustomize overlay β€” to the cluster + /// via a bitnami/kubectl container. No host-side kubectl binary is required. /// - /// After applying the manifests the container waits for any CRDs to reach the - /// Established condition, then exits with code 0. Use - /// WaitForCompletion(manifest) on dependent resources. + /// The container exits with code 0 after manifests are applied and any CRDs reach the + /// Established condition. Use WaitForCompletion(manifest) on dependent resources. /// + /// + /// Three modes, selected automatically based on : /// - /// A single file: cluster.AddK8sManifest("crd", "./k8s/widget-crd.yaml") - /// A directory: all .yaml/.yml files applied (kubectl handles ordering). + /// Single file β€” injected via WithContainerFiles, applied with kubectl apply -f. + /// Directory without kustomization.yaml β€” all .yaml/.yml files + /// injected via WithContainerFiles, applied with kubectl apply -f. + /// Kustomize overlay (directory contains kustomization.yaml or + /// kustomization.yml) β€” directory bind-mounted (preserving relative base references), + /// applied with kubectl apply -k. /// + /// /// [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] public static IResourceBuilder AddK8sManifest( @@ -37,25 +42,13 @@ public static IResourceBuilder AddK8sManifest( var cluster = builder.Resource; - // Resolve to an absolute path so the bind-mount and container path are stable. var absolutePath = System.IO.Path.IsPathRooted(path) ? path : System.IO.Path.GetFullPath( System.IO.Path.Combine(builder.ApplicationBuilder.AppHostDirectory, path)); - string hostBindDir; - string containerManifestPath; - - if (Directory.Exists(absolutePath)) - { - hostBindDir = absolutePath; - containerManifestPath = "/k8s-manifests"; - } - else - { - hostBindDir = System.IO.Path.GetDirectoryName(absolutePath)!; - containerManifestPath = $"/k8s-manifests/{System.IO.Path.GetFileName(absolutePath)}"; - } + bool isDirectory = Directory.Exists(absolutePath); + bool isKustomize = isDirectory && IsKustomizeDirectory(absolutePath); var manifest = new K8sManifestResource(name, absolutePath, cluster); cluster.AddManifest(manifest.Name); @@ -65,60 +58,92 @@ public static IResourceBuilder AddK8sManifest( var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; - return builder.ApplicationBuilder + var resourceBuilder = builder.ApplicationBuilder .AddResource(manifest) .WithImage(kubectlImage, kubectlTag) .WithImageRegistry(kubectlRegistry) .WithEntrypoint("/bin/sh") + // Script is injected at "/kubectl-apply.sh". The script auto-detects whether + // /k8s-manifests contains a kustomization.yaml and uses -k or -f accordingly. .WithContainerFiles("/", async (ctx, ct) => - { - var script = BuildManifestScript(containerManifestPath); - return [new ContainerFile + [new ContainerFile { Name = "kubectl-apply.sh", - Contents = script, + Contents = BuildManifestScript(), Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, - }]; - }) + }]) .WithArgs("/kubectl-apply.sh") - .WithBindMount(hostBindDir, "/k8s-manifests") - .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithBindMount(containerKubeconfigDir, "/root/.kube"); + + if (isKustomize) + { + // Bind-mount the overlay directory so kubectl kustomize can resolve relative + // references to base manifests (e.g. ../../base). WithContainerFiles copies + // files, not directory structure, so it would break cross-directory references. + resourceBuilder.WithBindMount(absolutePath, "/k8s-manifests"); + } + else + { + // Single file or regular directory β€” use Aspire's built-in + // WithContainerFiles(destinationPath, hostSourcePath) which copies the + // file or all files in the directory into the container without a bind-mount. + resourceBuilder.WithContainerFiles("/k8s-manifests", absolutePath); + } + + return resourceBuilder .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Code") .ExcludeFromManifest() .WithInitialState(new CustomResourceSnapshot { - ResourceType = "K8s Manifest", + ResourceType = isKustomize ? "K8s Kustomize" : "K8s Manifest", State = KnownResourceStates.NotStarted, - Properties = [new ResourcePropertySnapshot("Path", absolutePath)], + Properties = + [ + new ResourcePropertySnapshot("Path", absolutePath), + new ResourcePropertySnapshot("Mode", isKustomize ? "kustomize" : "apply"), + ], }); } // ── Script generation ───────────────────────────────────────────────────── - internal static string BuildManifestScript(string containerManifestPath) + internal static string BuildManifestScript() { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig β€” same pattern as the - // helm installer. Replaces WaitFor(cluster) for child resources. + // Poll until the k3s health check writes the kubeconfig. sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); - sb.AppendLine($"kubectl apply -f \"{containerManifestPath}\" --server-side --field-manager=aspire-k3s --force-conflicts"); + // Auto-detect kustomize: if a kustomization file is present, use -k. + // Otherwise use -f with server-side apply. + sb.AppendLine("if [ -f /k8s-manifests/kustomization.yaml ] || [ -f /k8s-manifests/kustomization.yml ]; then"); + sb.AppendLine(" echo 'Detected kustomization β€” using kubectl apply -k'"); + sb.AppendLine(" kubectl apply -k /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("else"); + sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("fi"); + // Wait for CRD Established condition if any CRDs are present. - // The check guard prevents failure when no CRDs were applied. sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); sb.AppendLine("fi"); + return sb.ToString(); } - // Keep for unit tests β€” file resolution logic is the same. + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static bool IsKustomizeDirectory(string directory) => + File.Exists(System.IO.Path.Combine(directory, "kustomization.yaml")) || + File.Exists(System.IO.Path.Combine(directory, "kustomization.yml")); + + // Exposed for unit tests. internal static IReadOnlyList ResolveFilesForTest(string path) { if (Directory.Exists(path)) @@ -135,9 +160,7 @@ internal static IReadOnlyList ResolveFilesForTest(string path) var pattern = System.IO.Path.GetFileName(path); if (pattern.Contains('*') || pattern.Contains('?')) - { return [..Directory.GetFiles(dir, pattern).Order(StringComparer.OrdinalIgnoreCase)]; - } return [path]; } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 02d3fc3c4..9d940fe58 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -349,4 +349,56 @@ public void AddServiceEndpointShouldThrowWhenBuilderIsNull() var action = () => builder.AddServiceEndpoint("ui", "svc", 443); Assert.Throws(action); } + + // ── WithHelmValuesFile tests ────────────────────────────────────────────── + + [Fact] + public void BuildHelmScriptIncludesValuesFiles() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) + { + Chart = "argo-cd", + }; + release.ValuesFiles.Add("/tmp/values.yaml"); + release.ValuesFiles.Add("/tmp/values-prod.yaml"); + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains("--values \"/helm-values/values.yaml\"", script); + Assert.Contains("--values \"/helm-values/values-prod.yaml\"", script); + // Values files are applied before --set overrides (last wins). + var valuesIndex = script.IndexOf("--values", StringComparison.Ordinal); + var setIndex = script.IndexOf("--set", StringComparison.Ordinal); + Assert.True(valuesIndex < setIndex || setIndex == -1, + "--values flags should appear before --set flags"); + } + + [Fact] + public void WithHelmValuesFileAccumulatesAbsolutePaths() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + // Use a temp file so the path resolution succeeds. + var tempFile = Path.Combine(appBuilder.AppHostDirectory, "values.yaml"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValuesFile("values.yaml"); // relative to AppHostDirectory + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Single(resource.ValuesFiles); + Assert.Equal(tempFile, resource.ValuesFiles[0]); + } + + [Fact] + public void WithHelmValuesFileShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValuesFile("values.yaml"); + Assert.Throws(action); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs index c657655fc..a286f79ea 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -184,4 +184,64 @@ public void ResolveFilesDirectory() Directory.Delete(dir, recursive: true); } } + + // ── Script generation ───────────────────────────────────────────────────── + + [Fact] + public void BuildManifestScriptIncludesKubectlApply() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kubectl apply -f /k8s-manifests", script); + Assert.Contains("--server-side", script); + } + + [Fact] + public void BuildManifestScriptAutoDetectsKustomize() + { + // The script checks for kustomization.yaml at runtime β€” no path argument needed. + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kustomization.yaml", script); + Assert.Contains("kubectl apply -k /k8s-manifests", script); + } + + [Fact] + public void BuildManifestScriptWaitsForKubeconfigBeforeApplying() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + var kubeconfigWaitIndex = script.IndexOf("/root/.kube/kubeconfig.yaml", StringComparison.Ordinal); + var applyIndex = script.IndexOf("kubectl apply", StringComparison.Ordinal); + Assert.True(kubeconfigWaitIndex < applyIndex, "Kubeconfig wait must precede kubectl apply"); + } + + // ── Kustomize detection ─────────────────────────────────────────────────── + + [Fact] + public void AddK8sManifestKustomizeDirectoryShowsKustomizeResourceType() + { + var dir = Path.Combine(Path.GetTempPath(), $"kustomize-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "kustomization.yaml"), "resources: []"); + + try + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddK8sManifest("kustom", dir); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var typeSnapshot = resource.Annotations + .OfType() + .Select(a => a.InitialSnapshot.ResourceType) + .FirstOrDefault(); + + Assert.Equal("K8s Kustomize", typeSnapshot); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } } From d343ee3a65321600ec7a851efd1c51884b4f79c7 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 14:50:19 +0200 Subject: [PATCH 04/13] compliting left work --- .../Program.cs | 7 +- .../ValidationAppHost/apphost.ts | 82 ++++++++ .../ValidationAppHost/aspire.config.json | 26 +++ .../ValidationAppHost/package.json | 19 ++ .../ValidationAppHost/tsconfig.json | 20 ++ .../Annotations/HelmReleaseAnnotation.cs | 35 ---- .../KubernetesDashboardAnnotation.cs | 11 - .../Annotations/KustomizeAnnotation.cs | 11 - ...CommunityToolkit.Aspire.Hosting.K3s.csproj | 2 +- .../HelmReleaseResource.cs | 3 + .../K3sBuilderExtensions.Helm.cs | 17 +- .../K3sBuilderExtensions.cs | 2 + .../K3sClusterResource.cs | 5 + .../K3sServiceEndpointResource.cs | 3 + .../K8sManifestResource.cs | 3 + .../README.md | 192 +++++++++++++++--- .../CommunityToolkit.Aspire.Hosting.K3s.cs | 150 +++++++++++--- .../K3sIntegrationTests.cs | 160 ++++++++++++++- .../HelmReleaseResourceTests.cs | 50 ++++- 19 files changed, 651 insertions(+), 147 deletions(-) create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json create mode 100644 playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs delete mode 100644 src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs index 527bf16d0..a25dce592 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -1,9 +1,10 @@ // K3s hosting example // ────────────────────────────────────────────────────────────────────────────── // Prerequisites (host machine): -// β€’ Docker with --privileged support (Linux or Docker Desktop on Mac/Windows) -// β€’ helm β†’ https://helm.sh/docs/intro/install/ -// β€’ kubectl β†’ https://kubernetes.io/docs/tasks/tools/ +// β€’ A container runtime that supports privileged Linux containers: +// - Linux: Docker Engine 20.10+ or rootful Podman 4.0+ +// - macOS / Windows: Docker Desktop (WSL2 / Hyper-V) +// No host-side helm or kubectl required β€” both run as containers. // // What this demonstrates: // 1. A k3s cluster starts inside a Docker container. diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts new file mode 100644 index 000000000..c9830b8f3 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts @@ -0,0 +1,82 @@ +import { createBuilder, ContainerLifetime } from './.modules/aspire.js'; + +const builder = await createBuilder(); + +// ── Runtime path (actually executed) ───────────────────────────────────────── +// Minimal cluster startup β€” validates that the core add/build/run path works. +const cluster = builder.addK3sCluster('k8s'); +const clusterResource = await cluster; +const _apiEndpoint = await clusterResource.apiEndpoint.get(); + +// ── Compile-time coverage ───────────────────────────────────────────────────── +// Guards with false so these are type-checked but never executed. +// Covers the full exported API surface without requiring Docker/k3s in CI. +const includeCompileOnlyScenarios = false; + +if (includeCompileOnlyScenarios) { + + // ── Cluster configuration ──────────────────────────────────────────────── + const configuredCluster = builder.addK3sCluster('k8s-configured') + .withK3sVersion('v1.32.3-k3s1') + .withPodSubnet('10.42.0.0/16') + .withServiceSubnet('10.43.0.0/16') + .withDisabledComponent('traefik') + .withExtraArg('--write-kubeconfig-mode=644') + .withDataVolume({ name: 'k8s-data' }) + .withLifetime(ContainerLifetime.Persistent); + + const configuredClusterResource = await configuredCluster; + const _configuredApiEndpoint = await configuredClusterResource.apiEndpoint.get(); + + // ── Helm release ───────────────────────────────────────────────────────── + const argocd = configuredCluster.addHelmRelease('argocd', 'argo-cd', { + repo: 'https://argoproj.github.io/argo-helm', + version: '7.8.0', + namespace: 'argocd', + }) + .withHelmValue('server.insecure', 'true') + .withHelmValuesFile('./deploy/argocd-values.yaml'); + + const argocdResource = await argocd; + const _argocdParent = await argocdResource.parent.get(); + const _argocdReleaseName = await argocdResource.releaseName.get(); + const _argocdNamespace = await argocdResource.namespace.get(); + + // ── K8s manifest / Kustomize overlay ───────────────────────────────────── + const widgetCrd = configuredCluster.addK8sManifest('widget-crd', './k8s/crds/'); + + const widgetCrdResource = await widgetCrd; + const _crdParent = await widgetCrdResource.parent.get(); + const _crdPath = await widgetCrdResource.path.get(); + + // ── Service endpoint ───────────────────────────────────────────────────── + const ui = configuredCluster.addServiceEndpoint('argocd-ui', 'argocd-server', 443, { + namespace: 'argocd', + }); + + const uiResource = await ui; + const _uiParent = await uiResource.parent.get(); + const _uiServiceName = await uiResource.serviceName.get(); + const _uiServicePort = await uiResource.servicePort.get(); + const _uiNamespace = await uiResource.namespace.get(); + const _uiHostPort = await uiResource.hostPort.get(); + + // ── WithReference β€” cluster kubeconfig injection ────────────────────────── + // Project: receives KUBECONFIG=.../.k3s/k8s/local/kubeconfig.yaml + const _projectRef = builder + .addProject('operator', { projectPath: '../WidgetOperator/WidgetOperator.csproj' }) + .withReference(configuredCluster); + + // Container: receives KUBECONFIG_DATA= + const _containerRef = builder + .addContainer('sidecar', 'myorg/sidecar') + .withReference(configuredCluster); + + // ── WithReference β€” service endpoint URL injection ──────────────────────── + // Host: receives services__argocd-ui__url=https://localhost:{port} + const _endpointRef = builder + .addProject('api', { projectPath: '../WidgetApi/WidgetApi.csproj' }) + .withReference(ui); +} + +await builder.build().run(); diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json new file mode 100644 index 000000000..80fbafe65 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/aspire.config.json @@ -0,0 +1,26 @@ +{ + "appHost": { + "path": "apphost.ts", + "language": "typescript/nodejs" + }, + "profiles": { + "https": { + "applicationUrl": "https://localhost:17149;http://localhost:15243", + "environmentVariables": { + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21065", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22170" + } + }, + "http": { + "applicationUrl": "http://localhost:15243", + "environmentVariables": { + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19027", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20141", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + }, + "packages": { + "CommunityToolkit.Aspire.Hosting.K3s": "" + } +} diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json new file mode 100644 index 000000000..54f1c98e3 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/package.json @@ -0,0 +1,19 @@ +{ + "name": "validationapphost", + "version": "1.0.0", + "type": "module", + "scripts": { + "start": "aspire run", + "build": "tsc", + "dev": "tsc --watch" + }, + "dependencies": { + "vscode-jsonrpc": "^8.2.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "nodemon": "^3.1.11", + "tsx": "^4.19.0", + "typescript": "^5.3.0" + } +} diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json new file mode 100644 index 000000000..05c7cf2f8 --- /dev/null +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "outDir": "./dist", + "rootDir": "." + }, + "include": [ + "apphost.ts", + ".modules/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs deleted file mode 100644 index b1d44bf9c..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that describes a Helm chart release to install into the cluster. -/// -/// The Helm release name. -/// The chart name or local path. -/// The Kubernetes namespace to install into. -/// Optional Helm repository URL containing the chart. -/// Optional chart version. -public sealed class HelmReleaseAnnotation( - string releaseName, - string chart, - string @namespace, - string? repoUrl, - string? version) : IResourceAnnotation -{ - /// Gets the Helm release name. - public string ReleaseName { get; } = releaseName; - - /// Gets the chart name or local path. - public string Chart { get; } = chart; - - /// Gets the target Kubernetes namespace. - public string Namespace { get; } = @namespace; - - /// Gets the optional Helm repository URL. - public string? RepoUrl { get; } = repoUrl; - - /// Gets the optional chart version. - public string? Version { get; } = version; - - /// Gets the extra --set values passed to helm install. - public IDictionary Values { get; } = new Dictionary(); -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs deleted file mode 100644 index 635eebd5e..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that signals the Kubernetes Dashboard should be installed alongside the cluster. -/// -/// The Kubernetes Dashboard chart version to install. -public sealed class KubernetesDashboardAnnotation(string? version = null) : IResourceAnnotation -{ - /// Gets the dashboard chart version, or to use the latest. - public string? Version { get; } = version; -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs deleted file mode 100644 index 4525fa5a1..000000000 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Aspire.Hosting.ApplicationModel; - -/// -/// Annotation that describes a Kustomize overlay to apply into the cluster. -/// -/// Path to the kustomization directory or remote URL. -public sealed class KustomizeAnnotation(string path) : IResourceAnnotation -{ - /// Gets the kustomization path. - public string Path { get; } = path; -} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj index 1800d1555..67a2f57a5 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -2,7 +2,7 @@ kubernetes k3s hosting cluster - An Aspire hosting integration for k3s β€” a lightweight Kubernetes distribution. + An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via alpine/k8s), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs index 65740bea8..e288b9ab4 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -13,6 +15,7 @@ namespace Aspire.Hosting.ApplicationModel; /// The Helm release name passed to helm upgrade --install. /// The Kubernetes namespace to install into. /// The parent k3s cluster resource. +[AspireExport(ExposeProperties = true)] public sealed class HelmReleaseResource( string name, string releaseName, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 6799c8f8e..9782bfd71 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -88,9 +88,12 @@ public static IResourceBuilder AddHelmRelease( // The callback wraps all files so it fires after all WithHelmValuesFile() calls. .WithContainerFiles("/helm-values", async (ctx, ct) => release.ValuesFiles - .Select(hostPath => (ContainerFileSystemItem)new ContainerFile + .Select((hostPath, i) => (ContainerFileSystemItem)new ContainerFile { - Name = System.IO.Path.GetFileName(hostPath), + // Prefix with index so files are unique even if basenames collide + // (e.g. prod/values.yaml + base/values.yaml β†’ 0-values.yaml, 1-values.yaml) + // and order is explicit on the filesystem as well as in the script. + Name = $"{i}-{System.IO.Path.GetFileName(hostPath)}", SourcePath = hostPath, }) .ToList()) @@ -122,6 +125,7 @@ public static IResourceBuilder AddHelmRelease( /// Path to the values YAML file on the host. Relative paths are resolved against /// AppHostDirectory. /// + [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] public static IResourceBuilder WithHelmValuesFile( this IResourceBuilder builder, string path) @@ -142,6 +146,7 @@ public static IResourceBuilder WithHelmValuesFile( /// /// Adds a Helm --set key=value argument to this release. /// + [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] public static IResourceBuilder WithHelmValue( this IResourceBuilder builder, string key, @@ -188,10 +193,12 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.Version is not null) sb.Append($" --version \"{release.Version}\""); - // Values files are injected into /helm-values/ by WithContainerFiles. - foreach (var hostPath in release.ValuesFiles) - sb.Append($" --values \"/helm-values/{System.IO.Path.GetFileName(hostPath)}\""); + // Values files: injected as {index}-{filename} to guarantee uniqueness and order. + // Applied first so --set flags below can override individual keys. + for (var i = 0; i < release.ValuesFiles.Count; i++) + sb.Append($" --values \"/helm-values/{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}\""); + // --set flags override everything above (highest Helm precedence). foreach (var (key, value) in release.HelmValues) sb.Append($" --set \"{key}={value}\""); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 6e87500dc..b630b613b 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -300,6 +300,7 @@ public static IResourceBuilder WithExtraArg( /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. /// + [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] public static IResourceBuilder WithDataVolume( this IResourceBuilder builder, string? name = null) @@ -371,6 +372,7 @@ public static IResourceBuilder WithReference( /// /// Sets the container lifetime for the k3s cluster and all its agent nodes. /// + [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] public static IResourceBuilder WithLifetime( this IResourceBuilder builder, ContainerLifetime lifetime) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 4e121d7f9..48baf3907 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -1,9 +1,12 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// /// Represents a k3s Kubernetes cluster running as a privileged container resource. /// /// The resource name. +[AspireExport(ExposeProperties = true)] public sealed class K3sClusterResource(string name) : ContainerResource(name) { internal const string ApiServerEndpointName = "api"; @@ -52,6 +55,7 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) new(StringComparer.OrdinalIgnoreCase); /// A dictionary of registered Helm releases keyed by resource name. + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public IReadOnlyDictionary HelmReleases => _helmReleases; internal void AddHelmRelease(string resourceName, string releaseName) => @@ -60,6 +64,7 @@ internal void AddHelmRelease(string resourceName, string releaseName) => private readonly List _manifests = []; /// Names of registered children. + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] public IReadOnlyList Manifests => _manifests; internal void AddManifest(string resourceName) => _manifests.Add(resourceName); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index 95d4c9f9a..0af117b1e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -12,6 +14,7 @@ namespace Aspire.Hosting.ApplicationModel; /// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. /// /// +[AspireExport(ExposeProperties = true)] public sealed class K3sServiceEndpointResource( string name, string serviceName, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index a87a687ee..2610f033c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -1,3 +1,5 @@ +#pragma warning disable ASPIREATS001 // AspireExport is experimental + namespace Aspire.Hosting.ApplicationModel; /// @@ -12,6 +14,7 @@ namespace Aspire.Hosting.ApplicationModel; /// The Aspire resource name. /// Absolute path to a single YAML file or a directory. /// The parent k3s cluster resource. +[AspireExport(ExposeProperties = true)] public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) : ContainerResource(name), IResourceWithParent { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index f6a5d8150..cc830bcbe 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -1,67 +1,197 @@ # CommunityToolkit.Aspire.Hosting.K3s -An Aspire hosting integration for [k3s](https://k3s.io/) β€” a lightweight, certified Kubernetes distribution by Rancher/SUSE. +Provides extension methods and resource definitions for the .NET Aspire AppHost to run a +[k3s](https://k3s.io/) lightweight Kubernetes cluster as part of the local development +inner loop. The cluster, Helm chart installs, manifest applies, and service endpoint +exposures all appear as first-class resources in the Aspire dashboard β€” no external +tooling beyond Docker is required. -## Getting started +## Getting Started ### Prerequisites -- Docker with support for `--privileged` containers (Linux host or Docker Desktop on macOS/Windows) +- A container runtime that supports privileged Linux containers: + - **Docker Engine 20.10+** (Linux) or **Docker Desktop** (macOS / Windows) + - **Podman 4.0+** (Linux) β€” works with rootful Podman; rootless requires cgroup v2 delegation -### Installation +> **Note:** k3s requires `--privileged`, `--cgroupns=host`, and a writable cgroup +> filesystem mount. These flags are passed automatically by the integration. Whether your +> runtime honours them depends on your system configuration. + +### Install the package + +In your AppHost project: ```sh dotnet add package CommunityToolkit.Aspire.Hosting.K3s ``` -## Usage +### Quick start ```csharp -var builder = DistributedApplication.CreateBuilder(args); +var cluster = builder.AddK3sCluster("k8s"); + +// Inject kubeconfig into a project β€” KUBECONFIG env var points to local/kubeconfig.yaml +builder.AddProject("operator") + .WaitFor(cluster) + .WithReference(cluster); +``` + +## Deploying Helm charts + +`AddHelmRelease` runs `helm upgrade --install` inside an `alpine/helm` container on the +DCP network. No host-side `helm` binary is required. The container exits with code 0 on +success, so use `WaitForCompletion` on any resource that depends on the chart being installed. -var cluster = builder.AddK3sCluster("k8s") - .WithPersistentState(); +```csharp +var argocd = cluster.AddHelmRelease("argocd", "argo-cd", + repo: "https://argoproj.github.io/argo-helm", + version: "7.8.0", + @namespace: "argocd") + .WithHelmValue("server.insecure", "true") + .WithHelmValuesFile("./deploy/argocd-values.yaml"); builder.AddProject("api") - .WithReference(cluster) - .WaitFor(cluster); + .WaitForCompletion(argocd) // wait for the chart install to finish + .WithReference(cluster); +``` + +### Helm override precedence + +Values are applied in this order (last wins): + +1. `WithHelmValuesFile` calls β€” in the order they are declared (`0-`, `1-`, … prefix) +2. `WithHelmValue` `--set` flags β€” always override values files + +## Applying Kubernetes manifests -builder.Build().Run(); +`AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/k8s` container. +No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. + +```csharp +// Plain YAML file or directory +var crd = cluster.AddK8sManifest("widget-crd", "./k8s/crds/"); + +// Kustomize overlay β€” detected automatically when kustomization.yaml is present +var overlay = cluster.AddK8sManifest("prod-overlay", "./k8s/overlays/local"); + +// Gate dependent resources until the CRD is Established +builder.AddProject("operator") + .WaitForCompletion(crd) + .WithReference(cluster); +``` + +For Kustomize overlays that reference base directories outside the overlay path, point +`AddK8sManifest` to the common root and specify the overlay path in `kustomization.yaml`. + +## Exposing k8s services to the Aspire network + +`AddServiceEndpoint` starts an in-process KubernetesClient WebSocket port-forward bound +to `0.0.0.0:{allocatedPort}`. No NodePort configuration is required. + +```csharp +var ui = cluster.AddServiceEndpoint("argocd-ui", + serviceName: "argocd-server", + servicePort: 443, + @namespace: "argocd") + .WaitForCompletion(argocd); // wait for chart install before port-forwarding + +// Host processes receive services__argocd-ui__url=https://localhost:{port} +builder.AddProject("consumer") + .WaitFor(ui) + .WithReference(ui); + +// DCP containers receive https://host.docker.internal:{port} +// --add-host=host.docker.internal:host-gateway is injected automatically on Linux +builder.AddContainer("sidecar", "myorg/sidecar") + .WaitFor(ui) + .WithReference(ui); ``` -### Kubeconfig injection +## Kubeconfig injection -`WithReference(cluster)` automatically selects the injection mode: +`WithReference(cluster)` selects the injection mode automatically: -| Resource type | Environment variable set | +| Consumer type | What is injected | |---|---| -| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=/tmp/aspire-k3s-k8s/admin.yaml` | -| `ContainerResource` | `KUBECONFIG_DATA=` | +| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml` | +| `ContainerResource` | `KUBECONFIG_DATA=` | -### Configuration options +Reading in .NET: +```csharp +// Project / executable β€” standard kubectl convention +var config = KubernetesClientConfiguration.BuildConfigFromConfigFile( + Environment.GetEnvironmentVariable("KUBECONFIG")); + +// Container β€” decode from env var +var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); +using var stream = new MemoryStream(bytes); +var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); +``` + +## Persistent cluster state + +```csharp +builder.AddK3sCluster("k8s") + .WithDataVolume() // persists /var/lib/rancher/k3s across AppHost restarts + .WithLifetime(ContainerLifetime.Persistent); +``` + +Without `WithDataVolume` the cluster state is ephemeral β€” each AppHost start produces a +fresh cluster. With it, subsequent starts reuse the existing cluster and skip +reinitialisation, making startup significantly faster. + +## Multi-node clusters ```csharp builder.AddK3sCluster("k8s", configure: opts => { - opts.ClusterCidr = "10.42.0.0/16"; - opts.ServiceCidr = "10.43.0.0/16"; - opts.DisabledComponents.Add("traefik"); + opts.AgentCount = 2; // 1 server + 2 agents }); ``` -Or use the fluent API: +The health check waits for all nodes to reach `Ready` before the cluster is marked healthy. + +## Image overrides + +The `alpine/helm` and `alpine/k8s` images are pinned but configurable: ```csharp -builder.AddK3sCluster("k8s") - .WithK3sVersion("v1.32.3-k3s1") - .WithPodSubnet("10.42.0.0/16") - .WithServiceSubnet("10.43.0.0/16") - .WithDisabledComponent("traefik") - .WithPersistentState(); +builder.AddK3sCluster("k8s", configure: opts => +{ + opts.HelmImage = "my-registry/helm"; + opts.HelmTag = "3.18.0"; + opts.KubectlImage = "my-registry/k8s"; + opts.KubectlTag = "1.33.0"; +}); ``` -## Known limitations +## Reaching Aspire services from k3s pods + +k3s pods run on the internal pod network (`10.42.0.0/16`). k3s's Flannel CNI masquerades +pod traffic through the k3s container's DCP network IP, so pods can reach DCP services by +their host-mapped port β€” but not by their Aspire DNS name. Use Helm values or a ConfigMap +to inject the host-accessible address: + +```csharp +var postgres = builder.AddPostgres("db"); + +// Resolve the host-mapped port at configuration time +var dbPort = postgres.GetEndpoint("tcp"); + +cluster.AddHelmRelease("my-operator", "my-operator-chart") + .WithHelmValue("database.host", "host.docker.internal") + .WithHelmValue("database.port", dbPort.Property(EndpointProperty.Port)); +``` + +Inside the pod, `host.docker.internal` resolves to the Docker host because k3s runs as a +privileged container on the DCP network and Flannel masquerades outbound pod traffic +through it. + +## Additional information + +https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s + +## Feedback & contributing -- Requires a privileged Docker runtime; `--privileged` is passed automatically. -- On Linux hosts the `/lib/modules` directory should be present for CNI networking. -- The first cluster start can take 30–60 seconds while container images and CNI plugins are initialised. +https://github.com/CommunityToolkit/Aspire diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs index 50701b0d7..770655906 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -8,47 +8,125 @@ //------------------------------------------------------------------------------ namespace Aspire.Hosting { + public static partial class K3sBuilderExtensions + { + [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + + [AspireExport("withDataVolume", Description = "Adds a named volume for the k3s cluster data directory so state survives AppHost restarts")] + public static ApplicationModel.IResourceBuilder WithDataVolume(this ApplicationModel.IResourceBuilder builder, string? name = null) { throw null; } + + [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } + + [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } + + [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } + + [AspireExport("withLifetime", Description = "Sets the container lifetime for the k3s cluster and all its agent nodes")] + public static ApplicationModel.IResourceBuilder WithLifetime(this ApplicationModel.IResourceBuilder builder, ApplicationModel.ContainerLifetime lifetime) { throw null; } + + [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + + [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) + where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } + + [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + } + public static partial class K3sHelmBuilderExtensions { + [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } + + [AspireExport("withHelmValue", Description = "Adds a --set key=value argument to the Helm release")] public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } - public static ApplicationModel.IResourceBuilder WithEndpoint(this ApplicationModel.IResourceBuilder builder, string serviceName, int servicePort, string name) { throw null; } + + [AspireExport("withHelmValuesFile", Description = "Injects a host-side YAML values file into the Helm installer container")] + public static ApplicationModel.IResourceBuilder WithHelmValuesFile(this ApplicationModel.IResourceBuilder builder, string path) { throw null; } } - public static partial class K3sBuilderExtensions + public static partial class K3sManifestBuilderExtensions { - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source, CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy strategy = CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy.Auto) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } - [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] - public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } - public static ApplicationModel.IResourceBuilder WithPersistentState(this ApplicationModel.IResourceBuilder builder, string? volumeName = null) { throw null; } + [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + public static ApplicationModel.IResourceBuilder AddK8sManifest(this ApplicationModel.IResourceBuilder builder, string name, string path) { throw null; } + } + + public static partial class K3sServiceEndpointExtensions + { + [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default") { throw null; } + + [AspireExport("withReference", Description = "Injects the k3s service URL into a dependent resource")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source) + where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } } } namespace Aspire.Hosting.ApplicationModel { - public sealed partial class HelmReleaseResource : Resource, IResourceWithParent, IResourceWithWaitSupport + [AspireExport(ExposeProperties = true)] + public sealed partial class HelmReleaseResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource { - public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!) { } + public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!, default) { } + + public string Namespace { get { throw null; } } + public K3sClusterResource Parent { get { throw null; } } + public string ReleaseName { get { throw null; } } - public string Namespace { get { throw null; } } } + public sealed partial class K3sAgentResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource + { + public K3sAgentResource(string name, K3sClusterResource cluster) : base(default!, default) { } + + public K3sClusterResource Parent { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] public sealed partial class K3sClusterResource : ContainerResource { - public K3sClusterResource(string name) : base(default!) { } + public K3sClusterResource(string name) : base(default!, default) { } + public EndpointReference ApiEndpoint { get { throw null; } } + + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] + public System.Collections.Generic.IReadOnlyDictionary HelmReleases { get { throw null; } } + + [AspireExportIgnore(Reason = "Internal tracking collection; not needed by guest SDK consumers.")] + public System.Collections.Generic.IReadOnlyList Manifests { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] + public sealed partial class K3sServiceEndpointResource : Resource, IResourceWithParent, IResourceWithParent, IResource, IResourceWithWaitSupport + { + public K3sServiceEndpointResource(string name, string serviceName, int servicePort, string @namespace, K3sClusterResource cluster) : base(default!) { } + + public int HostPort { get { throw null; } } + + public string Namespace { get { throw null; } } + + public K3sClusterResource Parent { get { throw null; } } + + public string ServiceName { get { throw null; } } + + public int ServicePort { get { throw null; } } + } + + [AspireExport(ExposeProperties = true)] + public sealed partial class K8sManifestResource : ContainerResource, IResourceWithParent, IResourceWithParent, IResource + { + public K8sManifestResource(string name, string path, K3sClusterResource cluster) : base(default!, default) { } + + public K3sClusterResource Parent { get { throw null; } } + + public string Path { get { throw null; } } } } @@ -56,18 +134,28 @@ namespace CommunityToolkit.Aspire.Hosting { public sealed partial class K3sClusterOptions { - public K3sClusterOptions() { } + public int AgentCount { get { throw null; } set { } } + public string? ClusterCidr { get { throw null; } set { } } - public string? ImageTag { get { throw null; } set { } } - public string? ServiceCidr { get { throw null; } set { } } + public System.Collections.Generic.IList DisabledComponents { get { throw null; } } + public System.Collections.Generic.IList ExtraArgs { get { throw null; } } - } - public enum KubeconfigInjectionStrategy - { - Auto = 0, - HostNetwork = 1, - ContainerNetwork = 2, + public string HelmImage { get { throw null; } set { } } + + public string HelmRegistry { get { throw null; } set { } } + + public string HelmTag { get { throw null; } set { } } + + public string? ImageTag { get { throw null; } set { } } + + public string KubectlImage { get { throw null; } set { } } + + public string KubectlRegistry { get { throw null; } set { } } + + public string KubectlTag { get { throw null; } set { } } + + public string? ServiceCidr { get { throw null; } set { } } } -} +} \ No newline at end of file diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index f08fa0588..226335163 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -3,6 +3,8 @@ using Aspire.Hosting.ApplicationModel; using Aspire.Hosting.Utils; using CommunityToolkit.Aspire.Testing; +using k8s; +using k8s.Models; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -13,12 +15,14 @@ namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; /// /// Requirements: /// -/// Linux with Docker (privileged containers required by k3s). -/// helm on PATH β€” used by AddHelmRelease. -/// kubectl on PATH β€” used by AddK8sManifest. +/// A container runtime that supports privileged Linux containers. +/// Linux: Docker Engine 20.10+ or rootful Podman 4.0+. +/// macOS / Windows: Docker Desktop (containers run inside WSL2 / Hyper-V VM). /// -/// The tests are gated by [RequiresDocker] and intended for the -/// ubuntu-latest-only CI job in tests.yaml. +/// No host-side helm or kubectl is needed β€” both run as Docker containers +/// (alpine/helm and alpine/k8s). +/// Tests are gated by [RequiresDocker] and run on both ubuntu-latest +/// and windows-latest CI jobs since privileged Linux containers work on Docker Desktop. /// /// [RequiresDocker] @@ -72,7 +76,7 @@ public async Task ClusterReachesRunningAndKubeconfigIsValid() } [Fact] - public async Task HelmReleaseReachesRunning() + public async Task HelmReleaseExitsSuccessfully() { var cluster = _builder!.AddK3sCluster("k8s"); @@ -90,8 +94,11 @@ public async Task HelmReleaseReachesRunning() using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // HelmReleaseResource is a run-to-completion container β€” it exits with code 0 + // when helm upgrade --install --wait completes successfully. await rns.WaitForResourceAsync("nginx", - s => s.Snapshot.State?.Text == KnownResourceStates.Running, cts.Token); + s => s.Snapshot.State?.Text == "Exited", cts.Token); } [Fact] @@ -116,12 +123,10 @@ public async Task ServiceEndpointExposesHttpPort() using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); await rns.WaitForResourceHealthyAsync("k8s", cts.Token); - // nginx is a run-to-completion container β€” wait for it to exit (Exited state) await rns.WaitForResourceAsync("nginx", s => s.Snapshot.State?.Text == "Exited", cts.Token); await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); - // Find the allocated port from the endpoint resource. var model = _app.Services.GetRequiredService(); var ep = model.Resources.OfType().Single(); @@ -138,8 +143,6 @@ public async Task WithReferenceInjectsKubeconfigForProject() { var cluster = _builder!.AddK3sCluster("k8s"); - // WithReference on a project injects KUBECONFIG pointing to local/kubeconfig.yaml. - // We verify the env var would be set by checking the cluster state. _app = _builder.Build(); await _app.StartAsync(); @@ -155,4 +158,139 @@ public async Task WithReferenceInjectsKubeconfigForProject() Assert.Contains("localhost", yaml); Assert.DoesNotContain("127.0.0.1:6443", yaml); } + + [Fact] + public async Task ManifestAppliesCrdAndReachesEstablished() + { + // Write a minimal CRD manifest to a temp file so AddK8sManifest can find it. + var crdYaml = """ + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: widgets.example.com + spec: + group: example.com + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + color: + type: string + """; + + var manifestDir = Path.Combine(_builder!.AppHostDirectory, "k8s-test-crds"); + Directory.CreateDirectory(manifestDir); + var crdPath = Path.Combine(manifestDir, "widget-crd.yaml"); + await File.WriteAllTextAsync(crdPath, crdYaml); + + try + { + var cluster = _builder.AddK3sCluster("k8s"); + var crd = cluster.AddK8sManifest("widget-crd", crdPath); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // K8sManifestResource is run-to-completion β€” exits 0 after CRD reaches Established. + await rns.WaitForResourceAsync("widget-crd", + s => s.Snapshot.State?.Text == "Exited", cts.Token); + + // Independently verify the CRD is Established via KubernetesClient. + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); + + var widgetCrd = await k8sClient.ApiextensionsV1 + .ReadCustomResourceDefinitionAsync("widgets.example.com", cancellationToken: cts.Token); + + var established = widgetCrd.Status?.Conditions?.Any(c => + c.Type == "Established" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true; + + Assert.True(established, "CRD 'widgets.example.com' should be Established"); + } + finally + { + Directory.Delete(manifestDir, recursive: true); + } + } + + [Fact] + public async Task WithDataVolumePreservesStateAcrossRestarts() + { + // Use an explicit volume name shared between both app instances. + var volumeName = $"aspire-k3s-persist-{Guid.NewGuid():N}"; + + // ── First run ───────────────────────────────────────────────────── + _builder!.AddK3sCluster("k8s").WithDataVolume(volumeName); + _app = _builder.Build(); + await _app.StartAsync(); + + var rns1 = _app.Services.GetRequiredService(); + using var cts1 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns1.WaitForResourceHealthyAsync("k8s", cts1.Token); + + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + using (var k8sClient = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath))) + { + await k8sClient.CoreV1.CreateNamespacedConfigMapAsync( + new V1ConfigMap + { + Metadata = new V1ObjectMeta { Name = "persist-check" }, + Data = new Dictionary { ["run"] = "first" }, + }, + "default", + cancellationToken: cts1.Token); + } + + // Stop the first app. The volume is retained; the container is removed by DCP. + await _app.StopAsync(); + await _app.DisposeAsync(); + _app = null; + + // ── Second run with the same named volume ───────────────────────── + var builder2 = TestDistributedApplicationBuilder.Create(); + builder2.AddK3sCluster("k8s").WithDataVolume(volumeName); + + await using var app2 = builder2.Build(); + await app2.StartAsync(); + + var rns2 = app2.Services.GetRequiredService(); + using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); + + var kubeconfigPath2 = Path.Combine( + builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + using var k8sClient2 = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); + + var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( + "persist-check", "default", cancellationToken: cts2.Token); + + Assert.Equal("first", cm.Data["run"]); + + await app2.StopAsync(); + } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 9d940fe58..7677ad7aa 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -353,7 +353,7 @@ public void AddServiceEndpointShouldThrowWhenBuilderIsNull() // ── WithHelmValuesFile tests ────────────────────────────────────────────── [Fact] - public void BuildHelmScriptIncludesValuesFiles() + public void BuildHelmScriptIncludesValuesFilesWithIndexPrefix() { var cluster = new K3sClusterResource("k8s"); var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) @@ -365,13 +365,47 @@ public void BuildHelmScriptIncludesValuesFiles() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - Assert.Contains("--values \"/helm-values/values.yaml\"", script); - Assert.Contains("--values \"/helm-values/values-prod.yaml\"", script); - // Values files are applied before --set overrides (last wins). - var valuesIndex = script.IndexOf("--values", StringComparison.Ordinal); - var setIndex = script.IndexOf("--set", StringComparison.Ordinal); - Assert.True(valuesIndex < setIndex || setIndex == -1, - "--values flags should appear before --set flags"); + // Index prefix guarantees uniqueness even when basenames collide. + Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); + Assert.Contains("--values \"/helm-values/1-values-prod.yaml\"", script); + } + + [Fact] + public void BuildHelmScriptValuesFilesOrderedBeforeSetFlags() + { + // Helm precedence: --values (ascending index) β†’ --set (highest, always wins). + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) + { + Chart = "argo-cd", + }; + release.ValuesFiles.Add("/tmp/base.yaml"); + release.ValuesFiles.Add("/tmp/prod.yaml"); + release.HelmValues["key"] = "override"; + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + var firstValuesIdx = script.IndexOf("--values \"/helm-values/0-", StringComparison.Ordinal); + var secondValuesIdx = script.IndexOf("--values \"/helm-values/1-", StringComparison.Ordinal); + var setIdx = script.IndexOf("--set", StringComparison.Ordinal); + + Assert.True(firstValuesIdx < secondValuesIdx, "0-base.yaml must precede 1-prod.yaml"); + Assert.True(secondValuesIdx < setIdx, "--values flags must precede --set flags"); + } + + [Fact] + public void BuildHelmScriptCollisionSafeWithSameBasename() + { + // Two files from different directories with the same name must not collide. + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("r", "r", "default", cluster) { Chart = "chart" }; + release.ValuesFiles.Add("/prod/values.yaml"); + release.ValuesFiles.Add("/dev/values.yaml"); + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); + Assert.Contains("--values \"/helm-values/1-values.yaml\"", script); } [Fact] From 0a4d13a3a07f66516205be4551e8bf31930d6533 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 14:59:29 +0200 Subject: [PATCH 05/13] upgrade example aspire version Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj index 65e9770a0..387601982 100644 --- a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe From f9a50d13942c7a37c46ebb75a7d458dcc9c6c45f Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 16:47:11 +0200 Subject: [PATCH 06/13] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 30 ++++++-- .../K3sBuilderExtensions.Manifest.cs | 32 +++++++- .../K3sBuilderExtensions.ServiceEndpoint.cs | 48 ++++++------ .../K3sBuilderExtensions.cs | 71 +++++++++++------ .../K3sContainerImageTags.cs | 4 +- .../K3sInProcessPortForwarder.cs | 77 ++++++++++++++++++- .../HelmReleaseResourceTests.cs | 26 +++---- .../K3sClusterResourceTests.cs | 34 ++++++-- 8 files changed, 239 insertions(+), 83 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index 9782bfd71..c5ade685f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -178,32 +178,48 @@ internal static string BuildHelmScript(HelmReleaseResource release) if (release.RepoUrl is not null) { var alias = $"aspire-k3s-{release.ReleaseName}"; - sb.AppendLine($"helm repo add --force-update \"{alias}\" \"{release.RepoUrl}\""); - sb.AppendLine($"helm repo update \"{alias}\""); + sb.AppendLine($"helm repo add --force-update {ShellEscape(alias)} {ShellEscape(release.RepoUrl)}"); + sb.AppendLine($"helm repo update {ShellEscape(alias)}"); } var chartRef = release.RepoUrl is not null ? $"aspire-k3s-{release.ReleaseName}/{release.Chart}" : release.Chart!; - sb.Append($"helm upgrade --install \"{release.ReleaseName}\" \"{chartRef}\""); - sb.Append($" --namespace \"{release.Namespace}\" --create-namespace"); + sb.Append($"helm upgrade --install {ShellEscape(release.ReleaseName)} {ShellEscape(chartRef)}"); + sb.Append($" --namespace {ShellEscape(release.Namespace)} --create-namespace"); sb.Append(" --wait --timeout 10m"); if (release.Version is not null) - sb.Append($" --version \"{release.Version}\""); + sb.Append($" --version {ShellEscape(release.Version)}"); // Values files: injected as {index}-{filename} to guarantee uniqueness and order. // Applied first so --set flags below can override individual keys. + // Paths are single-quoted so spaces or special characters in filenames are safe. for (var i = 0; i < release.ValuesFiles.Count; i++) - sb.Append($" --values \"/helm-values/{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}\""); + { + var filename = $"{i}-{System.IO.Path.GetFileName(release.ValuesFiles[i])}"; + sb.Append($" --values {ShellEscape($"/helm-values/{filename}")}"); + } // --set flags override everything above (highest Helm precedence). + // Both key and value are single-quoted so quotes, spaces, or $ in values are safe. foreach (var (key, value) in release.HelmValues) - sb.Append($" --set \"{key}={value}\""); + sb.Append($" --set {ShellEscape($"{key}={value}")}"); return sb.ToString(); } + + /// + /// Wraps in POSIX single quotes so that all shell + /// metacharacters ($, `, \, ", ;, + /// &, |, space, etc.) are treated as literals in Dockerfile + /// RUN shell-form commands and Delve's --build-flags parser. + /// Embedded single quotes are escaped with the standard POSIX technique: + /// ' β†’ '\''. + /// + private static string ShellEscape(string value) => + $"'{value.Replace("'", "'\\''")}'"; } #pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 6cb79b838..33c7ac128 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -86,10 +86,34 @@ [new ContainerFile } else { - // Single file or regular directory β€” use Aspire's built-in - // WithContainerFiles(destinationPath, hostSourcePath) which copies the - // file or all files in the directory into the container without a bind-mount. - resourceBuilder.WithContainerFiles("/k8s-manifests", absolutePath); + // Single file or regular directory β€” copy via async callback so the file(s) + // need not exist when the AppHost is built (only when the container starts). + // This mirrors WithContainerFiles(path, hostPath) semantics but without the + // build-time path validation that Aspire's string overload performs. + resourceBuilder.WithContainerFiles("/k8s-manifests", async (ctx, ct) => + { + if (Directory.Exists(absolutePath)) + { + var files = Directory + .GetFiles(absolutePath, "*.yaml", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(absolutePath, "*.yml", SearchOption.TopDirectoryOnly)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase); + + return [.. files + .Select(f => (ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(f), + SourcePath = f, + })]; + } + + return [(ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(absolutePath), + SourcePath = absolutePath, + }]; + }); } return resourceBuilder diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index dfbe5ecb8..0a2a12eb1 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -3,6 +3,7 @@ using System.Net.Sockets; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; +using k8s; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -20,20 +21,12 @@ public static class K3sServiceEndpointExtensions /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. /// /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, - /// binding to 0.0.0.0:{hostPort}. Use WaitFor to sequence after a - /// or that deploys the service. + /// binding to 0.0.0.0:{hostPort}. The endpoint only becomes healthy after the + /// Kubernetes service has a ready pod β€” use WaitForCompletion on a + /// or to sequence the + /// install before starting the port-forward. /// /// - /// - /// - /// var nginx = cluster.AddHelmRelease("nginx", "nginx", repo: "https://charts.bitnami.com/bitnami"); - /// var ui = cluster.AddServiceEndpoint("nginx-ui", "nginx", servicePort: 80) - /// .WaitFor(nginx); - /// builder.AddProject<Projects.Api>("api") - /// .WaitFor(ui) - /// .WithReference(ui); - /// - /// [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] public static IResourceBuilder AddServiceEndpoint( @@ -47,6 +40,10 @@ public static IResourceBuilder AddServiceEndpoint( ArgumentNullException.ThrowIfNull(name); ArgumentNullException.ThrowIfNull(serviceName); + if (servicePort is < 1 or > 65535) + throw new ArgumentOutOfRangeException(nameof(servicePort), + servicePort, "Service port must be in the range 1–65535."); + var cluster = builder.Resource; var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); @@ -79,10 +76,10 @@ public static IResourceBuilder AddServiceEndpoint( /// Injects the service URL exposed by into /// using the Aspire services__{name}__url convention. /// - /// Host processes receive https://localhost:{port}. - /// Container resources receive https://host.docker.internal:{port}. - /// On Linux without Docker Desktop, add - /// --add-host=host.docker.internal:host-gateway to the container runtime args. + /// Host processes receive http(s)://localhost:{port}. + /// Container resources receive http(s)://host.docker.internal:{port}. + /// The --add-host=host.docker.internal:host-gateway runtime arg is injected + /// automatically so the hostname resolves on Linux Docker Engine. /// /// [AspireExport("withReference", @@ -101,6 +98,14 @@ public static IResourceBuilder WithReference( if (destination.Resource is ContainerResource) { + // Inject --add-host so host.docker.internal resolves inside Linux containers. + // DCP does not inject this automatically; Docker Desktop on Mac/Windows resolves + // it natively, but Docker Engine on Linux requires the explicit mapping. + // ContainerRuntimeArgsCallbackAnnotation receives IList directly. + destination.Resource.Annotations.Add( + new ContainerRuntimeArgsCallbackAnnotation( + args => args.Add("--add-host=host.docker.internal:host-gateway"))); + return destination.WithEnvironment(ctx => { if (ep.IsReady) @@ -137,9 +142,6 @@ await notifications.PublishUpdateAsync(endpoint, "k3s local kubeconfig is not yet available for service endpoint."); } - // Allocate a host port by opening a listener, reading the OS-assigned port, - // then closing it before the forwarder binds β€” the port stays reserved in the - // kernel TIME_WAIT for long enough that the forwarder wins the race. var hostPort = AllocatePort(); endpoint.HostPort = hostPort; @@ -154,7 +156,6 @@ await notifications.PublishUpdateAsync(endpoint, isReady => { endpoint.IsReady = isReady; - var state = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy; var urls = isReady ? BuildUrls(scheme, endpoint.Name, hostPort, cluster.Name) : ImmutableArray.Empty; @@ -167,10 +168,6 @@ await notifications.PublishUpdateAsync(endpoint, }); _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); - - // Wait for the forwarder to signal ready (IsReady set via callback above). - // The health check also reads IsReady, so WaitFor on dependent resources - // naturally blocks until the port-forward is accepting connections. } catch (Exception ex) when (!ct.IsCancellationRequested) { @@ -203,7 +200,8 @@ private static int AllocatePort() /// /// Health check that satisfies WaitFor(serviceEndpoint). -/// Returns once the port-forward is active. +/// Returns once the port-forward has a confirmed +/// connection to a ready pod. /// internal sealed class K3sServiceEndpointHealthCheck(K3sServiceEndpointResource endpoint) : IHealthCheck { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index b630b613b..3f6a11865 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -1,4 +1,3 @@ -using System.Text; using Aspire.Hosting.ApplicationModel; using CommunityToolkit.Aspire.Hosting; using Microsoft.Extensions.DependencyInjection; @@ -245,7 +244,27 @@ public static IResourceBuilder WithK3sVersion( ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(tag); - return builder.WithImageTag(tag); + // Update the server image tag. + builder.WithImageTag(tag); + + // Sync all agent nodes to the same tag β€” mismatched server/agent versions can + // break node joins and exceed Kubernetes' supported Β±1 minor version skew. + foreach (var agent in builder.Resource.AgentResources) + { + var existing = agent.Annotations.OfType().FirstOrDefault(); + if (existing is not null) + { + agent.Annotations.Remove(existing); + agent.Annotations.Add(new ContainerImageAnnotation + { + Image = existing.Image, + Tag = tag, + Registry = existing.Registry, + }); + } + } + + return builder; } /// Sets the pod subnet CIDR (--cluster-cidr). @@ -317,17 +336,20 @@ public static IResourceBuilder WithDataVolume( /// to the cluster. The injection method is selected automatically based on the resource type: /// /// - /// s receive KUBECONFIG_DATA (base-64-encoded YAML - /// of the container-network kubeconfig) and KUBECONFIG=/var/k3s/kubeconfig.yaml - /// once the file is injected at container-start time. + /// s receive a physical kubeconfig file copied to + /// /var/k3s/kubeconfig.yaml (container-network variant, + /// server: https://{resourceName}:6443). KUBECONFIG=/var/k3s/kubeconfig.yaml + /// is set automatically so all standard Kubernetes tooling (kubectl, helm, + /// KubernetesClient SDK) works without any custom bootstrap code. /// /// /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml - /// pointing to a file that is accessible directly on the host filesystem. + /// pointing directly to a file on the host filesystem. /// /// - /// The variable is populated only after the cluster health check passes; use - /// WaitFor(cluster) on the dependent resource to guarantee ordering. + /// Both files are written by the health check after all nodes reach Ready state. + /// Use WaitFor(cluster) on the dependent resource to guarantee the files exist + /// before the resource starts. /// [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] public static IResourceBuilder WithReference( @@ -342,21 +364,24 @@ public static IResourceBuilder WithReference( if (destination.Resource is ContainerResource) { - // Containers receive KUBECONFIG_DATA (base64-encoded container-network kubeconfig). - // KubernetesClient reads this via: - // var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); - // using var stream = new MemoryStream(bytes); - // var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); - // For standard kubectl, use WithContainerFiles to inject a real file instead. - return destination.WithEnvironment(ctx => - { - if (cluster.KubeconfigDirectory is null) return; - var path = Path.Combine(cluster.KubeconfigDirectory, "container", "kubeconfig.yaml"); - if (!File.Exists(path)) return; - var yaml = File.ReadAllText(path); - ctx.EnvironmentVariables["KUBECONFIG_DATA"] = - Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); - }); + // Containers get a bind-mount of the container/ kubeconfig directory at /var/k3s. + // ContainerMountAnnotation is added directly to bypass the T : ContainerResource + // constraint on WithBindMount β€” the annotation is equivalent. + // + // Bind-mount (not file copy) is used for the same reason as in the helm and + // kubectl installer containers: if the cluster is recreated while a container is + // running, the new kubeconfig appears automatically without restarting the container. + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + destination.Resource.Annotations.Add( + new ContainerMountAnnotation( + containerKubeconfigDir, + "/var/k3s", + ContainerMountType.BindMount, + isReadOnly: true)); + + return destination.WithEnvironment("KUBECONFIG", "/var/k3s/kubeconfig.yaml"); } // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs index 605e5c65c..a312b2e25 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -8,6 +8,6 @@ internal static class K3sContainerImageTags /// rancher/k3s public const string Image = "rancher/k3s"; - /// v1.31.4-k3s1 - public const string Tag = "v1.31.4-k3s1"; + /// v1.32.3-k3s1 + public const string Tag = "v1.32.3-k3s1"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 1d704399b..64ceceacb 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -14,6 +14,12 @@ namespace CommunityToolkit.Aspire.Hosting; /// (localhost:{port}) and DCP-network containers /// (host.docker.internal:{port}) can reach the service. /// +/// +/// The callback is invoked with +/// only after a ready pod is confirmed via ListNamespacedPodAsync β€” not when the +/// TCP listener starts. This ensures WaitFor(endpoint) on dependent resources +/// correctly waits until the k8s service has a reachable pod. +/// /// internal sealed class K3sInProcessPortForwarder( string kubeconfigPath, @@ -33,12 +39,17 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) try { listener.Start(); - onReadyChanged(true); logger.LogInformation( "Port-forward: 0.0.0.0:{Local} β†’ svc/{Service}.{Ns}:{Port}", localPort, serviceName, @namespace, servicePort); + // Probe the service before signalling ready β€” the Kubernetes service and + // a ready pod must exist before any connection can succeed. + // This makes the ready signal meaningful for WaitFor(endpoint) consumers. + await WaitForServiceReadyAsync(logger, ct).ConfigureAwait(false); + onReadyChanged(true); + while (!ct.IsCancellationRequested) { var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); @@ -70,6 +81,57 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) } } + /// + /// Polls until the named service has at least one fully-ready pod. + /// This ensures the ready signal is only emitted when connections can actually succeed. + /// + private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try + { + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); + + var svc = await k8sClient.CoreV1 + .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) + .ConfigureAwait(false); + + var selector = string.Join(",", + (svc.Spec.Selector ?? new Dictionary()) + .Select(kv => $"{kv.Key}={kv.Value}")); + + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ConfigureAwait(false); + + var hasReadyPod = pods.Items.Any(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (hasReadyPod) + { + logger.LogDebug( + "Service {Service}/{Ns} has a ready pod β€” port-forward is ready.", + serviceName, @namespace); + return; + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return; + } + catch (Exception ex) + { + logger.LogDebug(ex, + "Service {Service}/{Ns} not yet ready; retrying…", serviceName, @namespace); + } + + await Task.Delay(TimeSpan.FromSeconds(3), ct).ConfigureAwait(false); + } + } + private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, CancellationToken ct) { using var _ = tcp; @@ -102,9 +164,20 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell return; } + // Resolve the pod container port from the service's targetPort. + // WebSocketNamespacedPodPortForwardAsync requires the pod/container port, + // NOT the service port. For services where port != targetPort the wrong + // port would be forwarded otherwise. + var svcPort = svc.Spec.Ports.FirstOrDefault(p => p.Port == servicePort); + // TargetPort is IntOrString β€” its Value property is always a string. + // Parse it as an integer; if it's a named port or unset, fall back to the service port. + var podPort = svcPort?.TargetPort?.Value is { } tp && int.TryParse(tp, out var tpInt) + ? tpInt + : servicePort; + // Open WebSocket port-forward to the pod. using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( - pod.Metadata.Name, @namespace, [servicePort], + pod.Metadata.Name, @namespace, [podPort], cancellationToken: ct).ConfigureAwait(false); using var demuxer = new StreamDemuxer(ws, StreamType.PortForward); diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs index 7677ad7aa..d5f6feb5b 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -220,8 +220,8 @@ public void BuildHelmScriptIncludesUpgradeInstall() MakeRelease("argocd", "argo-cd", null, null, "argocd")); Assert.Contains("helm upgrade --install", script); - Assert.Contains("\"argocd\"", script); - Assert.Contains("\"argo-cd\"", script); + Assert.Contains("'argocd'", script); + Assert.Contains("'argo-cd'", script); } [Fact] @@ -231,7 +231,7 @@ public void BuildHelmScriptIncludesWaitAndNamespace() MakeRelease("r", "chart", null, null, "my-ns")); Assert.Contains("--wait", script); - Assert.Contains("--namespace \"my-ns\"", script); + Assert.Contains("--namespace 'my-ns'", script); Assert.Contains("--create-namespace", script); } @@ -253,7 +253,7 @@ public void BuildHelmScriptWithoutRepoSkipsRepoSteps() MakeRelease("r", "oci://registry/chart", null, null, "default")); Assert.DoesNotContain("helm repo add", script); - Assert.Contains("\"oci://registry/chart\"", script); + Assert.Contains("'oci://registry/chart'", script); } [Fact] @@ -262,7 +262,7 @@ public void BuildHelmScriptIncludesVersion() var script = K3sHelmBuilderExtensions.BuildHelmScript( MakeRelease("r", "chart", null, "7.8.0", "default")); - Assert.Contains("--version \"7.8.0\"", script); + Assert.Contains("--version '7.8.0'", script); } [Fact] @@ -284,8 +284,8 @@ public void BuildHelmScriptIncludesSetValues() ["replicaCount"] = "2", })); - Assert.Contains("--set \"service.type=NodePort\"", script); - Assert.Contains("--set \"replicaCount=2\"", script); + Assert.Contains("--set 'service.type=NodePort'", script); + Assert.Contains("--set 'replicaCount=2'", script); } // ── WaitForCompletion support ───────────────────────────────────────────── @@ -366,8 +366,8 @@ public void BuildHelmScriptIncludesValuesFilesWithIndexPrefix() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); // Index prefix guarantees uniqueness even when basenames collide. - Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); - Assert.Contains("--values \"/helm-values/1-values-prod.yaml\"", script); + Assert.Contains("--values '/helm-values/0-values.yaml'", script); + Assert.Contains("--values '/helm-values/1-values-prod.yaml'", script); } [Fact] @@ -385,8 +385,8 @@ public void BuildHelmScriptValuesFilesOrderedBeforeSetFlags() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - var firstValuesIdx = script.IndexOf("--values \"/helm-values/0-", StringComparison.Ordinal); - var secondValuesIdx = script.IndexOf("--values \"/helm-values/1-", StringComparison.Ordinal); + var firstValuesIdx = script.IndexOf("--values '/helm-values/0-", StringComparison.Ordinal); + var secondValuesIdx = script.IndexOf("--values '/helm-values/1-", StringComparison.Ordinal); var setIdx = script.IndexOf("--set", StringComparison.Ordinal); Assert.True(firstValuesIdx < secondValuesIdx, "0-base.yaml must precede 1-prod.yaml"); @@ -404,8 +404,8 @@ public void BuildHelmScriptCollisionSafeWithSameBasename() var script = K3sHelmBuilderExtensions.BuildHelmScript(release); - Assert.Contains("--values \"/helm-values/0-values.yaml\"", script); - Assert.Contains("--values \"/helm-values/1-values.yaml\"", script); + Assert.Contains("--values '/helm-values/0-values.yaml'", script); + Assert.Contains("--values '/helm-values/1-values.yaml'", script); } [Fact] diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 0ca9b8bf9..19fd52758 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -185,29 +185,49 @@ public void AddK3sClusterWithClusterCidrViaOptions() public void WithReferenceSetsKubeconfigEnvForProject() { var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); - // ProjectResource would need a project file; use ExecutableResource as a proxy + // ProjectResource would need a project file; use ExecutableResource as a proxy. var exe = appBuilder.AddExecutable("myapp", "myapp", "."); exe.WithReference(cluster); - // Verify the environment callback was added (no exception thrown) using var app = appBuilder.Build(); - Assert.NotNull(app); + var model = app.Services.GetRequiredService(); + + var exeResource = Assert.Single(model.Resources.OfType()); + + // Executables receive KUBECONFIG pointing to local/kubeconfig.yaml on the host. + Assert.Contains( + exeResource.Annotations.OfType(), + a => a.Callback is not null); } [Fact] - public void WithReferenceSetsKubeconfigDataEnvForContainer() + public void WithReferenceMountsKubeconfigDirForContainer() { var appBuilder = DistributedApplication.CreateBuilder(); - var cluster = appBuilder.AddK3sCluster("k8s"); var container = appBuilder.AddContainer("operator", "myorg/operator"); container.WithReference(cluster); using var app = appBuilder.Build(); - Assert.NotNull(app); + var model = app.Services.GetRequiredService(); + + var containerResource = model.Resources + .OfType() + .Single(r => r.Name == "operator"); + + // All containers (user containers, helm, and kubectl installers) receive a bind-mount + // of the container/ kubeconfig directory. Bind-mount is used so the kubeconfig + // updates automatically if the cluster is recreated without restarting the container. + var mount = containerResource.Annotations + .OfType() + .FirstOrDefault(m => m.Target == "/var/k3s"); + + Assert.NotNull(mount); + Assert.Equal(ContainerMountType.BindMount, mount.Type); + Assert.True(mount.IsReadOnly); + Assert.EndsWith(Path.Combine(".k3s", "k8s", "container"), mount.Source); } [Fact] From da4bf9dfdadb2728b97353a7751e9c637a5322ca Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 16:58:59 +0200 Subject: [PATCH 07/13] chore: exluding integration tests for win runners --- .github/workflows/tests.yaml | 5 +++++ .../K3sIntegrationTests.cs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c4cc48db9..a08ff774c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -83,6 +83,11 @@ jobs: Sftp.Tests, SurrealDb.Tests, ] + exclude: + # k3s integration tests require privileged Linux containers. + # GitHub-hosted Windows runners do not support this reliably. + - os: windows-latest + name: Hosting.K3s.IntegrationTests steps: - name: Checkout code diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index 226335163..62887833a 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -21,8 +21,8 @@ namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; /// /// No host-side helm or kubectl is needed β€” both run as Docker containers /// (alpine/helm and alpine/k8s). -/// Tests are gated by [RequiresDocker] and run on both ubuntu-latest -/// and windows-latest CI jobs since privileged Linux containers work on Docker Desktop. +/// Tests are gated by [RequiresDocker] and run on ubuntu-latest only β€” +/// GitHub-hosted Windows runners do not support privileged Linux containers reliably. /// /// [RequiresDocker] From ff66be4a4d9d032192d45f7cde9bd8dddf07bf64 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 17:10:09 +0200 Subject: [PATCH 08/13] chore: update example comment Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../ValidationAppHost/apphost.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts index c9830b8f3..1085187e0 100644 --- a/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts +++ b/playground/polyglot/TypeScript/CommunityToolkit.Aspire.Hosting.K3s/ValidationAppHost/apphost.ts @@ -67,7 +67,7 @@ if (includeCompileOnlyScenarios) { .addProject('operator', { projectPath: '../WidgetOperator/WidgetOperator.csproj' }) .withReference(configuredCluster); - // Container: receives KUBECONFIG_DATA= + // Container: receives a bind-mounted kubeconfig and KUBECONFIG=/var/k3s/kubeconfig.yaml const _containerRef = builder .addContainer('sidecar', 'myorg/sidecar') .withReference(configuredCluster); From 37c11e32d384dbf7a7e7e0c3dfa06bf945f80495 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 19:56:11 +0200 Subject: [PATCH 09/13] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 39 ++++++------ .../K3sBuilderExtensions.Manifest.cs | 49 +++++++++------ .../K3sClusterOptions.cs | 4 +- .../K3sClusterResource.cs | 2 +- .../K3sInProcessPortForwarder.cs | 59 ++++++++++++++----- .../K3sReadinessHealthCheck.cs | 12 ++-- .../K8sManifestResource.cs | 2 +- .../KubectlContainerImageTags.cs | 9 ++- .../README.md | 16 +++-- .../K3sIntegrationTests.cs | 45 ++++++++++---- .../K3sClusterResourceTests.cs | 26 ++++++-- 11 files changed, 171 insertions(+), 92 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index c5ade685f..cccf24087 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -69,10 +69,10 @@ public static IResourceBuilder AddHelmRelease( // The install script is injected as /helm-install.sh via WithContainerFiles. // The callback fires when the container is being started (after WaitFor(cluster) // is satisfied), so all WithHelmValue() calls have been made by then. - .WithContainerFiles("/", async (ctx, ct) => + .WithContainerFiles("/", (ctx, ct) => { var script = BuildHelmScript(release); - return [new ContainerFile + IEnumerable items = [new ContainerFile { Name = "helm-install.sh", Contents = script, @@ -80,23 +80,22 @@ public static IResourceBuilder AddHelmRelease( | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, }]; + return Task.FromResult(items); }) .WithArgs("/helm-install.sh") - // Inject host-side values files declared via WithHelmValuesFile using - // Aspire's built-in WithContainerFiles(destinationPath, hostSourcePath). - // One call per file; each copies the file into /helm-values/ in the container. - // The callback wraps all files so it fires after all WithHelmValuesFile() calls. - .WithContainerFiles("/helm-values", async (ctx, ct) => - release.ValuesFiles + // Inject host-side values files declared via WithHelmValuesFile. + // The callback fires at container-start time so all WithHelmValuesFile() calls + // have been made and ValuesFiles is fully populated. + .WithContainerFiles("/helm-values", (ctx, ct) => + { + IEnumerable items = [.. release.ValuesFiles .Select((hostPath, i) => (ContainerFileSystemItem)new ContainerFile { - // Prefix with index so files are unique even if basenames collide - // (e.g. prod/values.yaml + base/values.yaml β†’ 0-values.yaml, 1-values.yaml) - // and order is explicit on the filesystem as well as in the script. - Name = $"{i}-{System.IO.Path.GetFileName(hostPath)}", + Name = $"{i}-{Path.GetFileName(hostPath)}", SourcePath = hostPath, - }) - .ToList()) + })]; + return Task.FromResult(items); + }) .WithBindMount(containerKubeconfigDir, "/root/.kube") .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") .WithIconName("Rocket") @@ -167,11 +166,13 @@ internal static string BuildHelmScript(HelmReleaseResource release) { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig β€” the file appears only - // after all nodes are Ready. This replaces WaitFor(cluster) since a container - // cannot WaitFor its IResourceWithParent. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); - sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + // Poll until the kubeconfig exists AND the k3s API server is reachable. + // DCP sets up container network aliases asynchronously, so the kubeconfig file + // can appear in the bind-mount before the k8s hostname resolves in the helm + // container. Using `helm list` (which calls the k8s API) verifies both the + // file and the network path before proceeding. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && helm list --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 33c7ac128..997131dcb 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -13,7 +13,7 @@ public static class K3sManifestBuilderExtensions { /// /// Applies one or more Kubernetes YAML files β€” or a Kustomize overlay β€” to the cluster - /// via a bitnami/kubectl container. No host-side kubectl binary is required. + /// via a rancher/kubectl container. No host-side kubectl binary is required. /// /// The container exits with code 0 after manifests are applied and any CRDs reach the /// Established condition. Use WaitForCompletion(manifest) on dependent resources. @@ -65,15 +65,18 @@ public static IResourceBuilder AddK8sManifest( .WithEntrypoint("/bin/sh") // Script is injected at "/kubectl-apply.sh". The script auto-detects whether // /k8s-manifests contains a kustomization.yaml and uses -k or -f accordingly. - .WithContainerFiles("/", async (ctx, ct) => - [new ContainerFile + .WithContainerFiles("/", (ctx, ct) => + { + IEnumerable items = [new ContainerFile { Name = "kubectl-apply.sh", Contents = BuildManifestScript(), Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupExecute | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, - }]) + }]; + return Task.FromResult(items); + }) .WithArgs("/kubectl-apply.sh") .WithBindMount(containerKubeconfigDir, "/root/.kube"); @@ -90,8 +93,10 @@ [new ContainerFile // need not exist when the AppHost is built (only when the container starts). // This mirrors WithContainerFiles(path, hostPath) semantics but without the // build-time path validation that Aspire's string overload performs. - resourceBuilder.WithContainerFiles("/k8s-manifests", async (ctx, ct) => + resourceBuilder.WithContainerFiles("/k8s-manifests", (ctx, ct) => { + IEnumerable items; + if (Directory.Exists(absolutePath)) { var files = Directory @@ -100,19 +105,22 @@ [new ContainerFile .Distinct(StringComparer.OrdinalIgnoreCase) .Order(StringComparer.OrdinalIgnoreCase); - return [.. files - .Select(f => (ContainerFileSystemItem)new ContainerFile - { - Name = System.IO.Path.GetFileName(f), - SourcePath = f, - })]; + items = [.. files.Select(f => new ContainerFile + { + Name = Path.GetFileName(f), + SourcePath = f, + })]; } - - return [(ContainerFileSystemItem)new ContainerFile + else { - Name = System.IO.Path.GetFileName(absolutePath), - SourcePath = absolutePath, - }]; + items = [new ContainerFile + { + Name = Path.GetFileName(absolutePath), + SourcePath = absolutePath, + }]; + } + + return Task.FromResult(items); }); } @@ -138,9 +146,12 @@ internal static string BuildManifestScript() { var sb = new StringBuilder("#!/bin/sh\nset -e\n"); - // Poll until the k3s health check writes the kubeconfig. - sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); - sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + // Poll until the kubeconfig exists AND the k3s API server is reachable. + // DCP sets up container network aliases asynchronously, so the kubeconfig file + // can appear in the bind-mount before the k8s hostname resolves in the kubectl + // container. Using `kubectl cluster-info` verifies both the file and the network. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ] && kubectl cluster-info --kubeconfig /root/.kube/kubeconfig.yaml > /dev/null 2>&1; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready and reachable...'"); sb.AppendLine(" sleep 5"); sb.AppendLine("done"); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs index ab67ca836..bc161e80f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -68,13 +68,13 @@ public sealed class K3sClusterOptions /// /// Gets or sets the kubectl container image name used by manifest applies. - /// Defaults to alpine/k8s. + /// Defaults to rancher/kubectl β€” maintained by the same team as k3s. /// public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; /// /// Gets or sets the kubectl container image tag used by manifest applies. - /// Defaults to 1.32.3. + /// Defaults to v1.32.3, matching the default k3s server version. /// public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs index 48baf3907..9390de879 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -17,7 +17,7 @@ public sealed class K3sClusterResource(string name) : ContainerResource(name) /// Container image settings for the kubectl manifest applier, resolved from cluster options. internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } - = ("docker.io", "alpine/k8s", "1.32.3"); + = ("docker.io", "rancher/kubectl", "v1.32.3"); /// /// Host-side directory that holds all kubeconfig variants for this cluster. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 64ceceacb..82a285587 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -98,12 +98,19 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - var selector = string.Join(",", - (svc.Spec.Selector ?? new Dictionary()) - .Select(kv => $"{kv.Key}={kv.Value}")); + if (svc.Spec.Selector is null or { Count: 0 }) + { + logger.LogWarning( + "Service {Service}/{Ns} has no pod selector β€” cannot determine readiness.", + serviceName, @namespace); + return; + } + + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) .ConfigureAwait(false); var hasReadyPod = pods.Items.Any(p => @@ -145,11 +152,19 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - var selector = string.Join(",", - (svc.Spec.Selector ?? new Dictionary()).Select(kv => $"{kv.Key}={kv.Value}")); + if (svc.Spec.Selector is null or { Count: 0 }) + { + logger.LogWarning( + "Service {Service}/{Ns} has no pod selector β€” connection dropped.", + serviceName, @namespace); + return; + } + + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) .ConfigureAwait(false); var pod = pods.Items.FirstOrDefault(p => @@ -165,15 +180,29 @@ private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, Cancell } // Resolve the pod container port from the service's targetPort. - // WebSocketNamespacedPodPortForwardAsync requires the pod/container port, - // NOT the service port. For services where port != targetPort the wrong - // port would be forwarded otherwise. + // WebSocketNamespacedPodPortForwardAsync requires the container port, not the + // service port. targetPort can be a numeric string or a named port string. var svcPort = svc.Spec.Ports.FirstOrDefault(p => p.Port == servicePort); - // TargetPort is IntOrString β€” its Value property is always a string. - // Parse it as an integer; if it's a named port or unset, fall back to the service port. - var podPort = svcPort?.TargetPort?.Value is { } tp && int.TryParse(tp, out var tpInt) - ? tpInt - : servicePort; + int podPort; + if (svcPort?.TargetPort?.Value is { } tp) + { + if (int.TryParse(tp, out var numeric)) + { + podPort = numeric; + } + else + { + // Named targetPort β€” resolve against the selected pod's container ports. + podPort = pod.Spec.Containers + .SelectMany(c => c.Ports ?? []) + .FirstOrDefault(p => p.Name == tp) + ?.ContainerPort ?? servicePort; + } + } + else + { + podPort = servicePort; + } // Open WebSocket port-forward to the pod. using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs index a51a9e2e2..22851041f 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -69,14 +69,18 @@ public async Task CheckHealthAsync( } catch (Exception ex) when (IsTlsOrAuthFailure(ex)) { - // Stale kubeconfig β€” cluster was recreated with new certs (e.g. data volume wiped). - // Invalidate everything; k3s will overwrite cluster/kubeconfig.yaml on next start. + // Stale cached client β€” the cluster was recreated with new certs while the + // health check held an old IKubernetes instance. k3s has already written a fresh + // kubeconfig to rawPath (it writes once at startup, not continuously), so we must + // NOT delete rawPath β€” that would remove the fresh file and leave the health check + // waiting forever for a file that k3s will never rewrite. + // Instead, discard only the cached client and the derived variants so they are + // regenerated from the fresh raw file on the next check cycle. _cachedClient?.Dispose(); _cachedClient = null; - TryDelete(rawPath); TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); - return HealthCheckResult.Unhealthy("k3s kubeconfig is stale β€” waiting for cluster refresh"); + return HealthCheckResult.Unhealthy("k3s kubeconfig is stale β€” retrying with fresh credentials"); } catch (Exception ex) { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs index 2610f033c..9893def84 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -4,7 +4,7 @@ namespace Aspire.Hosting.ApplicationModel; /// /// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via -/// kubectl apply --server-side running inside a bitnami/kubectl container. +/// kubectl apply --server-side running inside a rancher/kubectl container. /// /// The container polls for the cluster kubeconfig (written when the cluster health check /// first passes), applies the manifests, waits for any CRDs to reach Established, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs index eb6eedb68..5ead02f04 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -3,9 +3,8 @@ namespace CommunityToolkit.Aspire.Hosting; internal static class KubectlContainerImageTags { internal const string Registry = "docker.io"; - // alpine/k8s: lightweight Alpine-based image that includes kubectl and other k8s tools. - // Same organisation as alpine/helm β€” consistent image family. - internal const string Image = "alpine/k8s"; - // Matches the Kubernetes version shipped by the default k3s tag. - internal const string Tag = "1.32.3"; + // rancher/kubectl: maintained by the same team as k3s. Version tags mirror + // the Kubernetes version, so v1.32.x pairs correctly with the default k3s tag. + internal const string Image = "rancher/kubectl"; + internal const string Tag = "v1.32.3"; } diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index cc830bcbe..cef56ce7a 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -65,7 +65,7 @@ Values are applied in this order (last wins): ## Applying Kubernetes manifests -`AddK8sManifest` runs `kubectl apply --server-side` inside an `alpine/k8s` container. +`AddK8sManifest` runs `kubectl apply --server-side` inside an `rancher/kubectl` container. No host-side `kubectl` binary is required. Kustomize overlays are auto-detected. ```csharp @@ -115,18 +115,16 @@ builder.AddContainer("sidecar", "myorg/sidecar") | Consumer type | What is injected | |---|---| | `ProjectResource` / `ExecutableResource` | `KUBECONFIG=…/.k3s/k8s/local/kubeconfig.yaml` | -| `ContainerResource` | `KUBECONFIG_DATA=` | +| `ContainerResource` | Bind-mount of `container/kubeconfig.yaml` at `/var/k3s/` + `KUBECONFIG=/var/k3s/kubeconfig.yaml` | + +All standard Kubernetes tooling (`kubectl`, `helm`, KubernetesClient SDK) reads `KUBECONFIG` automatically β€” no custom bootstrap code required. Reading in .NET: ```csharp -// Project / executable β€” standard kubectl convention +// Works identically for both projects and containers β€” the SDK reads KUBECONFIG automatically. var config = KubernetesClientConfiguration.BuildConfigFromConfigFile( Environment.GetEnvironmentVariable("KUBECONFIG")); - -// Container β€” decode from env var -var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); -using var stream = new MemoryStream(bytes); -var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); +using var client = new Kubernetes(config); ``` ## Persistent cluster state @@ -154,7 +152,7 @@ The health check waits for all nodes to reach `Ready` before the cluster is mark ## Image overrides -The `alpine/helm` and `alpine/k8s` images are pinned but configurable: +The `alpine/helm` and `rancher/kubectl` images are pinned but configurable: ```csharp builder.AddK3sCluster("k8s", configure: opts => diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs index 62887833a..3861c4c16 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -270,27 +270,46 @@ await k8sClient.CoreV1.CreateNamespacedConfigMapAsync( _app = null; // ── Second run with the same named volume ───────────────────────── - var builder2 = TestDistributedApplicationBuilder.Create(); + using var builder2 = TestDistributedApplicationBuilder.Create(); builder2.AddK3sCluster("k8s").WithDataVolume(volumeName); await using var app2 = builder2.Build(); - await app2.StartAsync(); + try + { + await app2.StartAsync(); - var rns2 = app2.Services.GetRequiredService(); - using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); + var rns2 = app2.Services.GetRequiredService(); + using var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await rns2.WaitForResourceHealthyAsync("k8s", cts2.Token); - var kubeconfigPath2 = Path.Combine( - builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + var kubeconfigPath2 = Path.Combine( + builder2.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); - using var k8sClient2 = new Kubernetes( - KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); + using var k8sClient2 = new Kubernetes( + KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath2)); - var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( - "persist-check", "default", cancellationToken: cts2.Token); + var cm = await k8sClient2.CoreV1.ReadNamespacedConfigMapAsync( + "persist-check", "default", cancellationToken: cts2.Token); - Assert.Equal("first", cm.Data["run"]); + Assert.Equal("first", cm.Data["run"]); + } + finally + { + await app2.StopAsync(); - await app2.StopAsync(); + // Remove the named volume so it does not accumulate on CI runners. + try + { + using var process = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("docker") + { + ArgumentList = { "volume", "rm", "--force", volumeName }, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }); + if (process is not null) await process.WaitForExitAsync(); + } + catch { /* best effort */ } + } } } diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs index 19fd52758..2e8b98d5b 100644 --- a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -251,7 +251,13 @@ public void WithPodSubnetAddsClusterCidrArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var args = resource.Annotations.OfType(); + Assert.Contains(args, a => a is not null); // arg callbacks registered + // Verify the actual arg value by evaluating the callbacks. + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in args) a.Callback(ctx); + Assert.Contains("--cluster-cidr=10.88.0.0/16", ctx.Args); } [Fact] @@ -264,7 +270,11 @@ public void WithServiceSubnetAddsServiceCidrArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--service-cidr=10.89.0.0/16", ctx.Args); } [Fact] @@ -277,7 +287,11 @@ public void WithDisabledComponentAddsDisableArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--disable=traefik", ctx.Args); } [Fact] @@ -290,7 +304,11 @@ public void WithExtraArgAddsRawArg() using var app = appBuilder.Build(); var appModel = app.Services.GetRequiredService(); - Assert.Single(appModel.Resources.OfType()); + var resource = Assert.Single(appModel.Resources.OfType()); + var ctx = new CommandLineArgsCallbackContext([]); + foreach (var a in resource.Annotations.OfType()) + a.Callback(ctx); + Assert.Contains("--write-kubeconfig-mode=644", ctx.Args); } [Fact] From 99297cdfd52f885dd89e7e6b8991417b6bc40411 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:15:10 +0200 Subject: [PATCH 10/13] fix: WaitForServiceReadyAsync service health check Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../K3sInProcessPortForwarder.cs | 45 +++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 82a285587..014d21a34 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -98,31 +98,40 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) .ConfigureAwait(false); - if (svc.Spec.Selector is null or { Count: 0 }) + var exposesRequestedPort = svc.Spec?.Ports?.Any(p => p.Port == servicePort) == true; + if (!exposesRequestedPort) + { + logger.LogDebug( + "Service {Service}/{Ns} does not expose requested port {ServicePort}; retrying…", + serviceName, @namespace, servicePort); + } + else if (svc.Spec?.Selector is null or { Count: 0 }) { logger.LogWarning( "Service {Service}/{Ns} has no pod selector β€” cannot determine readiness.", serviceName, @namespace); return; } - - var labelSelector = string.Join(",", - svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); - - var pods = await k8sClient.CoreV1 - .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) - .ConfigureAwait(false); - - var hasReadyPod = pods.Items.Any(p => - p.Status?.Phase == "Running" && - p.Status?.ContainerStatuses?.All(c => c.Ready) == true); - - if (hasReadyPod) + else { - logger.LogDebug( - "Service {Service}/{Ns} has a ready pod β€” port-forward is ready.", - serviceName, @namespace); - return; + var labelSelector = string.Join(",", + svc.Spec.Selector.Select(kv => $"{kv.Key}={kv.Value}")); + + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: labelSelector, cancellationToken: ct) + .ConfigureAwait(false); + + var hasReadyPod = pods.Items.Any(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (hasReadyPod) + { + logger.LogDebug( + "Service {Service}/{Ns} exposes requested port {ServicePort} and has a ready pod β€” port-forward is ready.", + serviceName, @namespace, servicePort); + return; + } } } catch (OperationCanceledException) when (ct.IsCancellationRequested) From 7d34ddc3724d9c080fcc69d63b0728634198e2cb Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:16:23 +0200 Subject: [PATCH 11/13] chore: addressing reviews --- .../CommunityToolkit.Aspire.Hosting.K3s.csproj | 2 +- .../K3sBuilderExtensions.cs | 3 +-- src/CommunityToolkit.Aspire.Hosting.K3s/README.md | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj index 67a2f57a5..1ef4e1af1 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -2,7 +2,7 @@ kubernetes k3s hosting cluster - An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via alpine/k8s), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. + An Aspire hosting integration for k3s. Provides AddK3sCluster, AddHelmRelease (via alpine/helm), AddK8sManifest with Kustomize support (via rancher/kubectl), and AddServiceEndpoint for in-process WebSocket port-forwarding. No host-side kubectl or helm required. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index 3f6a11865..c5908b7dc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -327,8 +327,7 @@ public static IResourceBuilder WithDataVolume( ArgumentNullException.ThrowIfNull(builder); return builder - .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") - .WithContainerRuntimeArgs("--restart=unless-stopped"); + .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s"); } /// diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md index cef56ce7a..23112e81e 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -4,7 +4,7 @@ Provides extension methods and resource definitions for the .NET Aspire AppHost [k3s](https://k3s.io/) lightweight Kubernetes cluster as part of the local development inner loop. The cluster, Helm chart installs, manifest applies, and service endpoint exposures all appear as first-class resources in the Aspire dashboard β€” no external -tooling beyond Docker is required. +tooling beyond a supported container runtime is required. ## Getting Started From 7473170af228b0a5d7f15ffb8adb3b2c9415ff31 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:24:56 +0200 Subject: [PATCH 12/13] fix: copilot reviews --- .../K3sBuilderExtensions.ServiceEndpoint.cs | 17 +++++++++++++---- .../K3sInProcessPortForwarder.cs | 15 +++++++++++---- .../K3sServiceEndpointResource.cs | 7 +++++++ 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs index 0a2a12eb1..f84c99e00 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -34,7 +34,8 @@ public static IResourceBuilder AddServiceEndpoint( [ResourceName] string name, string serviceName, int servicePort, - string @namespace = "default") + string @namespace = "default", + string? scheme = null) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(name); @@ -44,8 +45,16 @@ public static IResourceBuilder AddServiceEndpoint( throw new ArgumentOutOfRangeException(nameof(servicePort), servicePort, "Service port must be in the range 1–65535."); + // Infer scheme from the port when not explicitly provided. + // Callers should pass an explicit scheme whenever the Kubernetes service port + // does not reliably indicate the application protocol (e.g. HTTPS on port 80). + var resolvedScheme = scheme ?? (servicePort is 443 or 8443 ? "https" : "http"); + var cluster = builder.Resource; - var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); + var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster) + { + Scheme = resolvedScheme, + }; var healthCheckKey = $"k3s_endpoint_{name}_ready"; builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( @@ -93,7 +102,7 @@ public static IResourceBuilder WithReference( ArgumentNullException.ThrowIfNull(source); var ep = source.Resource; - var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + var scheme = ep.Scheme; var envKey = $"services__{ep.Name}__url"; if (destination.Resource is ContainerResource) @@ -145,7 +154,7 @@ await notifications.PublishUpdateAsync(endpoint, var hostPort = AllocatePort(); endpoint.HostPort = hostPort; - var scheme = endpoint.ServicePort is 443 or 8443 ? "https" : "http"; + var scheme = endpoint.Scheme; var forwarder = new K3sInProcessPortForwarder( kubeconfigPath, diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 014d21a34..96ce3b79a 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -62,6 +62,15 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) { break; } + catch (InvalidOperationException ioe) when (!ct.IsCancellationRequested) + { + // Non-retryable configuration error (e.g. service has no pod selector). + // Log and stop β€” retrying will never succeed. + logger.LogError(ioe, "Port-forward for svc/{Service}/{Ns} cannot be established.", + serviceName, @namespace); + onReadyChanged(false); + break; + } catch (Exception ex) { logger.LogWarning(ex, @@ -107,10 +116,8 @@ private async Task WaitForServiceReadyAsync(ILogger logger, CancellationToken ct } else if (svc.Spec?.Selector is null or { Count: 0 }) { - logger.LogWarning( - "Service {Service}/{Ns} has no pod selector β€” cannot determine readiness.", - serviceName, @namespace); - return; + throw new InvalidOperationException( + $"Service {serviceName}/{@namespace} has no pod selector and cannot be port-forwarded by {nameof(K3sInProcessPortForwarder)}."); } else { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs index 0af117b1e..d2a039de0 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -42,6 +42,13 @@ public sealed class K3sServiceEndpointResource( /// public int HostPort { get; internal set; } + /// + /// The URL scheme used for services__{name}__url injection and dashboard URLs. + /// Set by AddServiceEndpoint; callers can override the default port-based inference + /// via the scheme parameter. + /// + internal string Scheme { get; init; } = "http"; + /// /// when the port-forward is active and accepting connections. /// Set by K3sInProcessPortForwarder; read by the health check. From 98bf92164fc5c3b383ed8f025a166de5343ebce2 Mon Sep 17 00:00:00 2001 From: edmondshtogu Date: Mon, 18 May 2026 20:39:40 +0200 Subject: [PATCH 13/13] fix: copilot reviews --- .../K3sBuilderExtensions.Helm.cs | 27 ++++++++++++++----- .../K3sBuilderExtensions.Manifest.cs | 11 +++++--- .../K3sBuilderExtensions.cs | 6 ++++- .../K3sInProcessPortForwarder.cs | 5 ++++ .../CommunityToolkit.Aspire.Hosting.K3s.cs | 2 +- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs index cccf24087..d0d159bfc 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -204,20 +204,33 @@ internal static string BuildHelmScript(HelmReleaseResource release) } // --set flags override everything above (highest Helm precedence). - // Both key and value are single-quoted so quotes, spaces, or $ in values are safe. + // Values are Helm-escaped then shell-escaped: + // 1. HelmEscape: escapes Helm's --set parser metacharacters (`,`, `{`, `}`, `\`) + // so Helm treats them as literals rather than array/map/list syntax. + // 2. ShellEscape: wraps in POSIX single quotes so the shell passes the value + // to Helm without any shell interpretation. + // For values containing commas, braces, or backslashes that Helm --set cannot + // represent safely (e.g. multi-line strings), use WithHelmValuesFile instead. foreach (var (key, value) in release.HelmValues) - sb.Append($" --set {ShellEscape($"{key}={value}")}"); + sb.Append($" --set {ShellEscape($"{key}={HelmEscape(value)}")}"); return sb.ToString(); } + /// + /// Escapes Helm --set value metacharacters so that Helm's own parser treats + /// them as literals rather than as array/map/list syntax delimiters. + /// + private static string HelmEscape(string value) => + value.Replace("\\", "\\\\") // backslash first to avoid double-escaping + .Replace(",", "\\,") // comma separates multiple assignments + .Replace("{", "\\{") // brace opens a map/list literal + .Replace("}", "\\}"); + /// /// Wraps in POSIX single quotes so that all shell - /// metacharacters ($, `, \, ", ;, - /// &, |, space, etc.) are treated as literals in Dockerfile - /// RUN shell-form commands and Delve's --build-flags parser. - /// Embedded single quotes are escaped with the standard POSIX technique: - /// ' β†’ '\''. + /// metacharacters are treated as literals. Embedded single quotes are escaped + /// with the standard POSIX technique: ' β†’ '\''. /// private static string ShellEscape(string value) => $"'{value.Replace("'", "'\\''")}'"; diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs index 997131dcb..d79ef7344 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -164,9 +164,14 @@ internal static string BuildManifestScript() sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); sb.AppendLine("fi"); - // Wait for CRD Established condition if any CRDs are present. - sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); - sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); + // Wait only for CRDs applied by this manifest, identified via the aspire-k3s + // field-manager. Using --all would also wait for pre-existing or concurrently + // installed CRDs that are stuck, causing an unrelated manifest to hang. + sb.AppendLine("CRDS=$(kubectl get crd -o name --no-headers 2>/dev/null \\"); + sb.AppendLine(" | xargs -r -I{} sh -c 'kubectl get {} -o jsonpath=\"{.metadata.managedFields[*].manager}\" 2>/dev/null | grep -q aspire-k3s && echo {}' 2>/dev/null)"); + sb.AppendLine("if [ -n \"$CRDS\" ]; then"); + sb.AppendLine(" # shellcheck disable=SC2086"); + sb.AppendLine(" kubectl wait --for=condition=Established $CRDS --timeout=300s"); sb.AppendLine("fi"); return sb.ToString(); diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs index c5908b7dc..ff82744db 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -108,10 +108,14 @@ public static IResourceBuilder AddK3sCluster( .WithArgs("--kubelet-arg=v=0") // ── API server endpoint ─────────────────────────────────────────── + // Proxy support must be disabled: the kubeconfig embeds the k3s server CA cert + // for TLS validation. An Aspire HTTPS proxy would present its own certificate, + // causing Kubernetes client TLS validation to fail on every connection. .WithHttpsEndpoint( targetPort: 6443, port: apiServerPort, - name: K3sClusterResource.ApiServerEndpointName) + name: K3sClusterResource.ApiServerEndpointName, + isProxied: false) // ── Docker / container runtime flags (mirrors k3d) ──────────────── .WithContainerRuntimeArgs("--privileged") diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs index 96ce3b79a..b00dc9141 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -35,6 +35,11 @@ public async Task RunAsync(ILogger logger, CancellationToken ct) while (!ct.IsCancellationRequested) { + // IPAddress.Any (0.0.0.0) is required: DCP-network containers reach the + // forwarded service via host.docker.internal:{port}, which resolves to the + // Docker host IP β€” not 127.0.0.1. Binding to loopback would silently drop + // all container traffic. Users on shared networks should be aware that the + // forwarded service is reachable from other hosts on the same LAN. var listener = new TcpListener(IPAddress.Any, localPort); try { diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs index 770655906..4abe3607c 100644 --- a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -60,7 +60,7 @@ public static partial class K3sManifestBuilderExtensions public static partial class K3sServiceEndpointExtensions { [AspireExport("addServiceEndpoint", Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] - public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default") { throw null; } + public static ApplicationModel.IResourceBuilder AddServiceEndpoint(this ApplicationModel.IResourceBuilder builder, string name, string serviceName, int servicePort, string @namespace = "default", string? scheme = null) { throw null; } [AspireExport("withReference", Description = "Injects the k3s service URL into a dependent resource")] public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source)