diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b4902e507..c4cc48db9 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -37,6 +37,8 @@ jobs: Hosting.Golang.Tests, Hosting.JavaScript.Extensions.Tests, Hosting.Java.Tests, + Hosting.K3s.Tests, + Hosting.K3s.IntegrationTests, Hosting.k6.Tests, Hosting.Keycloak.Extensions.Tests, Hosting.KurrentDB.Tests, diff --git a/.gitignore b/.gitignore index 4c96eff2b..c914256dc 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,10 @@ examples/perl/**/local/* **cpanfile.snapshot **/.modules/ **/*.AppHost.TypeScript/nuget.config +<<<<<<< main + +**/.k3s/ +======= tsconfig.apphost.json .ngrok bun.lock @@ -31,3 +35,4 @@ yarn.lock solr-data *.lscache apphost.js +>>>>>>> main diff --git a/CommunityToolkit.Aspire.slnx b/CommunityToolkit.Aspire.slnx index 1233718eb..2d71685f9 100644 --- a/CommunityToolkit.Aspire.slnx +++ b/CommunityToolkit.Aspire.slnx @@ -59,6 +59,9 @@ + + + @@ -214,6 +217,7 @@ + @@ -276,6 +280,8 @@ + + diff --git a/Directory.Packages.props b/Directory.Packages.props index edee3bc3b..1e423dc61 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -93,6 +93,7 @@ + diff --git a/README.md b/README.md index ce0170510..e9d3b7336 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This repository contains the source code for the Aspire Community Toolkit, a col | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | - **Learn More**: [`Hosting.Golang`][golang-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields]][golang-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Golang][golang-shields-preview]][golang-nuget-preview] | A hosting integration Golang apps. | | - **Learn More**: [`Hosting.Java`][java-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields]][java-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Java][java-shields-preview]][java-nuget-preview] | An integration for running Java code in Aspire either using the local JDK or using a container. | +| - **Learn More**: [`Hosting.K3s`][k3s-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields]][k3s-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.K3s][k3s-shields-preview]][k3s-nuget-preview] | An Aspire hosting integration for [k3s](https://k3s.io/), a lightweight Kubernetes distribution by Rancher. | | - **Learn More**: [`Hosting.NodeJS.Extensions`][nodejs-ext-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.NodeJS.Extensions][nodejs-ext-shields]][nodejs-ext-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.JavaScript.Extensions][nodejs-ext-shields-preview]][nodejs-ext-nuget-preview] | An integration that contains some additional extensions for running Node.js applications | | - **Learn More**: [`Hosting.Ollama`][ollama-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields]][ollama-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.Hosting.Ollama][ollama-shields-preview]][ollama-nuget-preview] | An Aspire hosting integration leveraging the [Ollama](https://ollama.com) container with support for downloading a model on startup. | | - **Learn More**: [`OllamaSharp`][ollama-integration-docs]
- Stable 📦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields]][ollamasharp-nuget]
- Preview 📦: [![CommunityToolkit.Aspire.OllamaSharp][ollamasharp-shields-preview]][ollamasharp-nuget-preview] | An Aspire client integration for the [OllamaSharp](https://github.com/awaescher/OllamaSharp) package. | @@ -106,6 +107,11 @@ This project is supported by the [.NET Foundation](https://dotnetfoundation.org) [java-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/ [java-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.Java?label=nuget%20(preview) [java-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.Java/absoluteLatest +[k3s-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-k3s +[k3s-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.K3s +[k3s-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/ +[k3s-shields-preview]: https://img.shields.io/nuget/vpre/CommunityToolkit.Aspire.Hosting.K3s?label=nuget%20(preview) +[k3s-nuget-preview]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.K3s/absoluteLatest [nodejs-ext-integration-docs]: https://learn.microsoft.com/dotnet/aspire/community-toolkit/hosting-nodejs-extensions [nodejs-ext-shields]: https://img.shields.io/nuget/v/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions [nodejs-ext-nuget]: https://nuget.org/packages/CommunityToolkit.Aspire.Hosting.JavaScript.Extensions/ diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj new file mode 100644 index 000000000..65e9770a0 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/CommunityToolkit.Aspire.Hosting.K3s.AppHost.csproj @@ -0,0 +1,14 @@ + + + + Exe + enable + enable + true + + + + + + + diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs new file mode 100644 index 000000000..527bf16d0 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Program.cs @@ -0,0 +1,37 @@ +// K3s hosting example +// ────────────────────────────────────────────────────────────────────────────── +// Prerequisites (host machine): +// • Docker with --privileged support (Linux or Docker Desktop on Mac/Windows) +// • helm → https://helm.sh/docs/intro/install/ +// • kubectl → https://kubernetes.io/docs/tasks/tools/ +// +// What this demonstrates: +// 1. A k3s cluster starts inside a Docker container. +// 2. podinfo is installed via Helm — a lightweight demo app. +// 3. A K3sServiceEndpointResource exposes the podinfo service: +// • Host processes reach it at http://localhost:{port} +// • DCP-network containers reach it at http://host.docker.internal:{port} +// 4. WithDataVolume keeps the cluster state alive across AppHost restarts. +// ────────────────────────────────────────────────────────────────────────────── + +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder + .AddK3sCluster("k8s") + .WithDataVolume() + .WithLifetime(ContainerLifetime.Persistent); + +var podinfo = cluster.AddHelmRelease( + name: "podinfo", + chart: "podinfo", + repo: "https://stefanprodan.github.io/podinfo", + version: "6.7.1", + @namespace: "podinfo"); + +// Expose the podinfo service as an Aspire endpoint resource. +// WaitForCompletion waits for the helm install container to exit with code 0 +// before starting the port-forward — no NodePort required. +cluster.AddServiceEndpoint("podinfo-web", "podinfo", servicePort: 9898, @namespace: "podinfo") + .WaitForCompletion(podinfo); + +builder.Build().Run(); diff --git a/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json new file mode 100644 index 000000000..dcfbae059 --- /dev/null +++ b/examples/k3s/CommunityToolkit.Aspire.Hosting.K3s.AppHost/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:53201;http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:53203", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:53204" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:53202", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:53205", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:53206" + } + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs new file mode 100644 index 000000000..b1d44bf9c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/HelmReleaseAnnotation.cs @@ -0,0 +1,35 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Helm chart release to install into the cluster. +/// +/// The Helm release name. +/// The chart name or local path. +/// The Kubernetes namespace to install into. +/// Optional Helm repository URL containing the chart. +/// Optional chart version. +public sealed class HelmReleaseAnnotation( + string releaseName, + string chart, + string @namespace, + string? repoUrl, + string? version) : IResourceAnnotation +{ + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName; + + /// Gets the chart name or local path. + public string Chart { get; } = chart; + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace; + + /// Gets the optional Helm repository URL. + public string? RepoUrl { get; } = repoUrl; + + /// Gets the optional chart version. + public string? Version { get; } = version; + + /// Gets the extra --set values passed to helm install. + public IDictionary Values { get; } = new Dictionary(); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs new file mode 100644 index 000000000..635eebd5e --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KubernetesDashboardAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that signals the Kubernetes Dashboard should be installed alongside the cluster. +/// +/// The Kubernetes Dashboard chart version to install. +public sealed class KubernetesDashboardAnnotation(string? version = null) : IResourceAnnotation +{ + /// Gets the dashboard chart version, or to use the latest. + public string? Version { get; } = version; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs new file mode 100644 index 000000000..4525fa5a1 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/Annotations/KustomizeAnnotation.cs @@ -0,0 +1,11 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Annotation that describes a Kustomize overlay to apply into the cluster. +/// +/// Path to the kustomization directory or remote URL. +public sealed class KustomizeAnnotation(string path) : IResourceAnnotation +{ + /// Gets the kustomization path. + public string Path { get; } = path; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj new file mode 100644 index 000000000..1800d1555 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/CommunityToolkit.Aspire.Hosting.K3s.csproj @@ -0,0 +1,19 @@ + + + + kubernetes k3s hosting cluster + An Aspire hosting integration for k3s — a lightweight Kubernetes distribution. + + + + + + + + + + + + + + diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs new file mode 100644 index 000000000..885c77724 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmContainerImageTags.cs @@ -0,0 +1,8 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class HelmContainerImageTags +{ + internal const string Registry = "docker.io"; + internal const string Image = "alpine/helm"; + internal const string Tag = "3.17.3"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs new file mode 100644 index 000000000..65740bea8 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/HelmReleaseResource.cs @@ -0,0 +1,43 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a Helm chart release deployed to a k3s cluster. +/// +/// Runs as an alpine/helm container on the DCP network. The container polls for the +/// cluster kubeconfig (written when the cluster health check first passes), executes +/// helm upgrade --install --wait, and exits with code 0 on success. Use +/// WaitForCompletion(helmRelease) on resources that depend on the release being installed. +/// +/// +/// The Aspire resource name (also used as the Helm release name). +/// The Helm release name passed to helm upgrade --install. +/// The Kubernetes namespace to install into. +/// The parent k3s cluster resource. +public sealed class HelmReleaseResource( + string name, + string releaseName, + string @namespace, + K3sClusterResource cluster) + : ContainerResource(name), IResourceWithParent +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Helm release name. + public string ReleaseName { get; } = releaseName ?? throw new ArgumentNullException(nameof(releaseName)); + + /// Gets the target Kubernetes namespace. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + internal string? Chart { get; set; } + internal string? RepoUrl { get; set; } + internal string? Version { get; set; } + internal Dictionary HelmValues { get; } = new(StringComparer.Ordinal); + + /// + /// Absolute host paths of values files to inject into the helm container via + /// --values /helm-values/{filename}. + /// Populated by WithHelmValuesFile. + /// + internal List ValuesFiles { get; } = []; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs new file mode 100644 index 000000000..f6850faaa --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sAgentResource.cs @@ -0,0 +1,22 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s agent (worker) node that is a child of a . +/// +/// Agent nodes run k3s agent and join the cluster by connecting to the server's API +/// server at https://{serverName}:6443, resolved via DCP's built-in Docker DNS. +/// Agents start immediately alongside the server (no WaitFor dependency) and use +/// k3s's built-in retry loop to connect once the server becomes reachable. The cluster's +/// health check waits for all 1 + nodes +/// to reach Ready state before transitioning to Running. +/// +/// +/// The resource name (e.g. k8s-agent-0). +/// The parent k3s cluster resource. +public sealed class K3sAgentResource(string name, K3sClusterResource cluster) + : ContainerResource(name), IResourceWithParent +{ + /// + public K3sClusterResource Parent { get; } = cluster + ?? throw new ArgumentNullException(nameof(cluster)); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs new file mode 100644 index 000000000..6799c8f8e --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Helm.cs @@ -0,0 +1,202 @@ +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for adding Helm release resources to a k3s cluster. +/// +public static class K3sHelmBuilderExtensions +{ + /// + /// Adds a Helm release as a child resource of the k3s cluster. + /// + /// The release runs as a bitnami/helm container on the DCP network, executing + /// helm upgrade --install --wait then exiting. No host-side helm binary + /// is required. Use WaitForCompletion(helmRelease) on resources that depend on + /// the release being fully installed. + /// + /// + /// The k3s cluster resource builder. + /// Resource name — also used as the Helm release name. + /// Chart name. Add for remote charts. + /// Optional Helm repository URL. + /// Optional chart version. + /// Target namespace (created automatically). + /// A builder for the . + [AspireExport("addHelmRelease", Description = "Adds a Helm chart release to the k3s cluster")] + public static IResourceBuilder AddHelmRelease( + this IResourceBuilder builder, + [ResourceName] string name, + string chart, + string? repo = null, + string? version = null, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(chart); + + var cluster = builder.Resource; + + var release = new HelmReleaseResource(name, releaseName: name, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + + cluster.AddHelmRelease(release.Name, release.ReleaseName); + + // The helm installer container mounts container/kubeconfig.yaml so it can reach + // the k3s API via DCP DNS (https://{clusterName}:6443). The directory is created + // by AddK3sCluster; the kubeconfig file is written by K3sReadinessHealthCheck on + // first successful health check. WaitFor(cluster) guarantees the file exists. + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + var (helmRegistry, helmImage, helmTag) = cluster.HelmImageInfo; + + return builder.ApplicationBuilder + .AddResource(release) + .WithImage(helmImage, helmTag) + .WithImageRegistry(helmRegistry) + .WithEntrypoint("/bin/sh") + + // The install script is injected as /helm-install.sh via WithContainerFiles. + // The callback fires when the container is being started (after WaitFor(cluster) + // is satisfied), so all WithHelmValue() calls have been made by then. + .WithContainerFiles("/", async (ctx, ct) => + { + var script = BuildHelmScript(release); + return [new ContainerFile + { + Name = "helm-install.sh", + Contents = script, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]; + }) + .WithArgs("/helm-install.sh") + // Inject host-side values files declared via WithHelmValuesFile using + // Aspire's built-in WithContainerFiles(destinationPath, hostSourcePath). + // One call per file; each copies the file into /helm-values/ in the container. + // The callback wraps all files so it fires after all WithHelmValuesFile() calls. + .WithContainerFiles("/helm-values", async (ctx, ct) => + release.ValuesFiles + .Select(hostPath => (ContainerFileSystemItem)new ContainerFile + { + Name = System.IO.Path.GetFileName(hostPath), + SourcePath = hostPath, + }) + .ToList()) + .WithBindMount(containerKubeconfigDir, "/root/.kube") + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Rocket") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "Helm Release", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ReleaseName", name), + new ResourcePropertySnapshot("Chart", chart), + new ResourcePropertySnapshot("Namespace", @namespace), + new ResourcePropertySnapshot("Version", version ?? "latest"), + ], + }); + } + + /// + /// Injects a host-side YAML values file into the Helm installer container and + /// passes it as --values /helm-values/{filename} to helm upgrade --install. + /// Multiple files are applied in the order they are declared (last wins for overlapping keys). + /// + /// The Helm release resource builder. + /// + /// Path to the values YAML file on the host. Relative paths are resolved against + /// AppHostDirectory. + /// + public static IResourceBuilder WithHelmValuesFile( + this IResourceBuilder builder, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine( + builder.ApplicationBuilder.AppHostDirectory, path)); + + builder.Resource.ValuesFiles.Add(absolutePath); + return builder; + } + + /// + /// Adds a Helm --set key=value argument to this release. + /// + public static IResourceBuilder WithHelmValue( + this IResourceBuilder builder, + string key, + string value) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(value); + + builder.Resource.HelmValues[key] = value; + return builder; + } + + // ── Script generation ───────────────────────────────────────────────────── + + // Visible for testing. + internal static string BuildHelmScript(HelmReleaseResource release) + { + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); + + // Poll until the k3s health check writes the kubeconfig — the file appears only + // after all nodes are Ready. This replaces WaitFor(cluster) since a container + // cannot WaitFor its IResourceWithParent. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); + + if (release.RepoUrl is not null) + { + var alias = $"aspire-k3s-{release.ReleaseName}"; + sb.AppendLine($"helm repo add --force-update \"{alias}\" \"{release.RepoUrl}\""); + sb.AppendLine($"helm repo update \"{alias}\""); + } + + var chartRef = release.RepoUrl is not null + ? $"aspire-k3s-{release.ReleaseName}/{release.Chart}" + : release.Chart!; + + sb.Append($"helm upgrade --install \"{release.ReleaseName}\" \"{chartRef}\""); + sb.Append($" --namespace \"{release.Namespace}\" --create-namespace"); + sb.Append(" --wait --timeout 10m"); + + if (release.Version is not null) + sb.Append($" --version \"{release.Version}\""); + + // Values files are injected into /helm-values/ by WithContainerFiles. + foreach (var hostPath in release.ValuesFiles) + sb.Append($" --values \"/helm-values/{System.IO.Path.GetFileName(hostPath)}\""); + + foreach (var (key, value) in release.HelmValues) + sb.Append($" --set \"{key}={value}\""); + + return sb.ToString(); + } +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs new file mode 100644 index 000000000..6cb79b838 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.Manifest.cs @@ -0,0 +1,169 @@ +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for applying Kubernetes YAML manifests to a k3s cluster. +/// +public static class K3sManifestBuilderExtensions +{ + /// + /// Applies one or more Kubernetes YAML files — or a Kustomize overlay — to the cluster + /// via a bitnami/kubectl container. No host-side kubectl binary is required. + /// + /// The container exits with code 0 after manifests are applied and any CRDs reach the + /// Established condition. Use WaitForCompletion(manifest) on dependent resources. + /// + /// + /// Three modes, selected automatically based on : + /// + /// Single file — injected via WithContainerFiles, applied with kubectl apply -f. + /// Directory without kustomization.yaml — all .yaml/.yml files + /// injected via WithContainerFiles, applied with kubectl apply -f. + /// Kustomize overlay (directory contains kustomization.yaml or + /// kustomization.yml) — directory bind-mounted (preserving relative base references), + /// applied with kubectl apply -k. + /// + /// + /// + [AspireExport("addK8sManifest", Description = "Applies Kubernetes YAML manifests to the k3s cluster")] + public static IResourceBuilder AddK8sManifest( + this IResourceBuilder builder, + [ResourceName] string name, + string path) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(path); + + var cluster = builder.Resource; + + var absolutePath = System.IO.Path.IsPathRooted(path) + ? path + : System.IO.Path.GetFullPath( + System.IO.Path.Combine(builder.ApplicationBuilder.AppHostDirectory, path)); + + bool isDirectory = Directory.Exists(absolutePath); + bool isKustomize = isDirectory && IsKustomizeDirectory(absolutePath); + + var manifest = new K8sManifestResource(name, absolutePath, cluster); + cluster.AddManifest(manifest.Name); + + var containerKubeconfigDir = Path.Combine(cluster.KubeconfigDirectory!, "container"); + Directory.CreateDirectory(containerKubeconfigDir); + + var (kubectlRegistry, kubectlImage, kubectlTag) = cluster.KubectlImageInfo; + + var resourceBuilder = builder.ApplicationBuilder + .AddResource(manifest) + .WithImage(kubectlImage, kubectlTag) + .WithImageRegistry(kubectlRegistry) + .WithEntrypoint("/bin/sh") + // Script is injected at "/kubectl-apply.sh". The script auto-detects whether + // /k8s-manifests contains a kustomization.yaml and uses -k or -f accordingly. + .WithContainerFiles("/", async (ctx, ct) => + [new ContainerFile + { + Name = "kubectl-apply.sh", + Contents = BuildManifestScript(), + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithArgs("/kubectl-apply.sh") + .WithBindMount(containerKubeconfigDir, "/root/.kube"); + + if (isKustomize) + { + // Bind-mount the overlay directory so kubectl kustomize can resolve relative + // references to base manifests (e.g. ../../base). WithContainerFiles copies + // files, not directory structure, so it would break cross-directory references. + resourceBuilder.WithBindMount(absolutePath, "/k8s-manifests"); + } + else + { + // Single file or regular directory — use Aspire's built-in + // WithContainerFiles(destinationPath, hostSourcePath) which copies the + // file or all files in the directory into the container without a bind-mount. + resourceBuilder.WithContainerFiles("/k8s-manifests", absolutePath); + } + + return resourceBuilder + .WithEnvironment("KUBECONFIG", "/root/.kube/kubeconfig.yaml") + .WithIconName("Code") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = isKustomize ? "K8s Kustomize" : "K8s Manifest", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("Path", absolutePath), + new ResourcePropertySnapshot("Mode", isKustomize ? "kustomize" : "apply"), + ], + }); + } + + // ── Script generation ───────────────────────────────────────────────────── + + internal static string BuildManifestScript() + { + var sb = new StringBuilder("#!/bin/sh\nset -e\n"); + + // Poll until the k3s health check writes the kubeconfig. + sb.AppendLine("until [ -f /root/.kube/kubeconfig.yaml ]; do"); + sb.AppendLine(" echo 'Waiting for k3s cluster to be ready...'"); + sb.AppendLine(" sleep 5"); + sb.AppendLine("done"); + + // Auto-detect kustomize: if a kustomization file is present, use -k. + // Otherwise use -f with server-side apply. + sb.AppendLine("if [ -f /k8s-manifests/kustomization.yaml ] || [ -f /k8s-manifests/kustomization.yml ]; then"); + sb.AppendLine(" echo 'Detected kustomization — using kubectl apply -k'"); + sb.AppendLine(" kubectl apply -k /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("else"); + sb.AppendLine(" kubectl apply -f /k8s-manifests --server-side --field-manager=aspire-k3s --force-conflicts"); + sb.AppendLine("fi"); + + // Wait for CRD Established condition if any CRDs are present. + sb.AppendLine("if kubectl get crd --no-headers 2>/dev/null | grep -q .; then"); + sb.AppendLine(" kubectl wait --for=condition=Established crd --all --timeout=300s"); + sb.AppendLine("fi"); + + return sb.ToString(); + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static bool IsKustomizeDirectory(string directory) => + File.Exists(System.IO.Path.Combine(directory, "kustomization.yaml")) || + File.Exists(System.IO.Path.Combine(directory, "kustomization.yml")); + + // Exposed for unit tests. + internal static IReadOnlyList ResolveFilesForTest(string path) + { + if (Directory.Exists(path)) + { + return [ + ..Directory.GetFiles(path, "*.yaml", SearchOption.TopDirectoryOnly) + .Concat(Directory.GetFiles(path, "*.yml", SearchOption.TopDirectoryOnly)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Order(StringComparer.OrdinalIgnoreCase) + ]; + } + + var dir = System.IO.Path.GetDirectoryName(path) ?? "."; + var pattern = System.IO.Path.GetFileName(path); + + if (pattern.Contains('*') || pattern.Contains('?')) + return [..Directory.GetFiles(dir, pattern).Order(StringComparer.OrdinalIgnoreCase)]; + + return [path]; + } +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs new file mode 100644 index 000000000..dfbe5ecb8 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.ServiceEndpoint.cs @@ -0,0 +1,218 @@ +using System.Collections.Immutable; +using System.Net; +using System.Net.Sockets; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental + +namespace Aspire.Hosting; + +/// +/// Extension methods for exposing Kubernetes services from a k3s cluster into the Aspire network. +/// +public static class K3sServiceEndpointExtensions +{ + /// + /// Exposes a Kubernetes service as a first-class Aspire endpoint resource. + /// + /// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready, + /// binding to 0.0.0.0:{hostPort}. Use WaitFor to sequence after a + /// or that deploys the service. + /// + /// + /// + /// + /// var nginx = cluster.AddHelmRelease("nginx", "nginx", repo: "https://charts.bitnami.com/bitnami"); + /// var ui = cluster.AddServiceEndpoint("nginx-ui", "nginx", servicePort: 80) + /// .WaitFor(nginx); + /// builder.AddProject<Projects.Api>("api") + /// .WaitFor(ui) + /// .WithReference(ui); + /// + /// + [AspireExport("addServiceEndpoint", + Description = "Exposes a Kubernetes service as an Aspire endpoint resource")] + public static IResourceBuilder AddServiceEndpoint( + this IResourceBuilder builder, + [ResourceName] string name, + string serviceName, + int servicePort, + string @namespace = "default") + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + ArgumentNullException.ThrowIfNull(serviceName); + + var cluster = builder.Resource; + var endpoint = new K3sServiceEndpointResource(name, serviceName, servicePort, @namespace, cluster); + + var healthCheckKey = $"k3s_endpoint_{name}_ready"; + builder.ApplicationBuilder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + healthCheckKey, + sp => new K3sServiceEndpointHealthCheck(endpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + return builder.ApplicationBuilder + .AddResource(endpoint) + .ExcludeFromManifest() + .WithHealthCheck(healthCheckKey) + .WithIconName("ArrowRouting") + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Service Endpoint", + State = KnownResourceStates.NotStarted, + Properties = + [ + new ResourcePropertySnapshot("ServiceName", serviceName), + new ResourcePropertySnapshot("ServicePort", servicePort.ToString()), + new ResourcePropertySnapshot("Namespace", @namespace), + ], + }); + } + + /// + /// Injects the service URL exposed by into + /// using the Aspire services__{name}__url convention. + /// + /// Host processes receive https://localhost:{port}. + /// Container resources receive https://host.docker.internal:{port}. + /// On Linux without Docker Desktop, add + /// --add-host=host.docker.internal:host-gateway to the container runtime args. + /// + /// + [AspireExport("withReference", + Description = "Injects the k3s service URL into a dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var ep = source.Resource; + var scheme = ep.ServicePort is 443 or 8443 ? "https" : "http"; + var envKey = $"services__{ep.Name}__url"; + + if (destination.Resource is ContainerResource) + { + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://host.docker.internal:{ep.HostPort}"; + }); + } + + return destination.WithEnvironment(ctx => + { + if (ep.IsReady) + ctx.EnvironmentVariables[envKey] = $"{scheme}://localhost:{ep.HostPort}"; + }); + } + + // ── Lifecycle ───────────────────────────────────────────────────────────── + + internal static async Task RunEndpointAsync( + K3sServiceEndpointResource endpoint, + K3sClusterResource cluster, + ResourceNotificationService notifications, + ILogger logger, + CancellationToken ct) + { + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.Starting }) + .ConfigureAwait(false); + + try + { + var kubeconfigPath = K3sBuilderExtensions.GetLocalKubeconfigPath(cluster); + if (kubeconfigPath is null || !File.Exists(kubeconfigPath)) + { + throw new InvalidOperationException( + "k3s local kubeconfig is not yet available for service endpoint."); + } + + // Allocate a host port by opening a listener, reading the OS-assigned port, + // then closing it before the forwarder binds — the port stays reserved in the + // kernel TIME_WAIT for long enough that the forwarder wins the race. + var hostPort = AllocatePort(); + endpoint.HostPort = hostPort; + + var scheme = endpoint.ServicePort is 443 or 8443 ? "https" : "http"; + + var forwarder = new K3sInProcessPortForwarder( + kubeconfigPath, + endpoint.Namespace, + endpoint.ServiceName, + hostPort, + endpoint.ServicePort, + isReady => + { + endpoint.IsReady = isReady; + var state = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy; + var urls = isReady + ? BuildUrls(scheme, endpoint.Name, hostPort, cluster.Name) + : ImmutableArray.Empty; + + _ = notifications.PublishUpdateAsync(endpoint, s => s with + { + State = isReady ? KnownResourceStates.Running : KnownResourceStates.RuntimeUnhealthy, + Urls = urls, + }); + }); + + _ = Task.Run(() => forwarder.RunAsync(logger, ct), ct); + + // Wait for the forwarder to signal ready (IsReady set via callback above). + // The health check also reads IsReady, so WaitFor on dependent resources + // naturally blocks until the port-forward is accepting connections. + } + catch (Exception ex) when (!ct.IsCancellationRequested) + { + logger.LogError(ex, "Service endpoint '{Name}' failed to start.", endpoint.Name); + await notifications.PublishUpdateAsync(endpoint, + state => state with { State = KnownResourceStates.FailedToStart }) + .ConfigureAwait(false); + } + } + + private static ImmutableArray BuildUrls( + string scheme, string endpointName, int hostPort, string clusterName) + => [ + new UrlSnapshot(endpointName, $"{scheme}://localhost:{hostPort}", IsInternal: false), + new UrlSnapshot( + $"{endpointName} (container)", + $"{scheme}://host.docker.internal:{hostPort}", + IsInternal: true), + ]; + + private static int AllocatePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; + } +} + +/// +/// Health check that satisfies WaitFor(serviceEndpoint). +/// Returns once the port-forward is active. +/// +internal sealed class K3sServiceEndpointHealthCheck(K3sServiceEndpointResource endpoint) : IHealthCheck +{ + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + => Task.FromResult(endpoint.IsReady + ? HealthCheckResult.Healthy("Port-forward is active") + : HealthCheckResult.Unhealthy("Port-forward not yet active")); +} + +#pragma warning restore ASPIREATS001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs new file mode 100644 index 000000000..6e87500dc --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sBuilderExtensions.cs @@ -0,0 +1,437 @@ +using System.Text; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +#pragma warning disable ASPIREATS001 // AspireExport is experimental +#pragma warning disable ASPIRECERTIFICATES001 // WithHttpsDeveloperCertificate is experimental + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding k3s cluster resources to an . +/// +public static class K3sBuilderExtensions +{ + /// + /// Adds a k3s Kubernetes cluster resource to the distributed application. + /// + /// The distributed application builder. + /// The resource name used for DNS resolution within the DCP network. + /// + /// Optional host port to bind the Kubernetes API server (port 6443) to. + /// When a random available port is assigned. + /// + /// Optional callback to configure . + /// A builder for the . + [AspireExport("addK3sCluster", Description = "Adds a k3s Kubernetes cluster resource")] + public static IResourceBuilder AddK3sCluster( + this IDistributedApplicationBuilder builder, + [ResourceName] string name, + int? apiServerPort = null, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + + var options = new K3sClusterOptions(); + configure?.Invoke(options); + + var resource = new K3sClusterResource(name) + { + HelmImageInfo = (options.HelmRegistry, options.HelmImage, options.HelmTag), + KubectlImageInfo = (options.KubectlRegistry, options.KubectlImage, options.KubectlTag), + }; + var tag = options.ImageTag ?? K3sContainerImageTags.Tag; + + // ── Kubeconfig directory on the host ────────────────────────────────── + // AppHostDirectory/.k3s/{name}/ holds three sub-directories: + // cluster/ — bind-mounted into the k3s container; k3s writes kubeconfig.yaml here + // local/ — rewritten by the health check with server: https://localhost:{port} + // container/ — rewritten by the health check with server: https://{name}:6443 + var kubeconfigDir = Path.Combine(builder.AppHostDirectory, ".k3s", name); + var clusterDir = Path.Combine(kubeconfigDir, "cluster"); + Directory.CreateDirectory(clusterDir); + + resource.KubeconfigDirectory = kubeconfigDir; + + var resourceBuilder = builder.AddResource(resource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + + // ── k3d-style init entrypoint ───────────────────────────────────── + // Runs mount --make-rshared / and the cgroupsv2 evacuation fix before + // k3s starts — exactly what k3d does via /bin/k3d-entrypoint*.sh. + // WithContainerFiles injects the script via `docker cp` — no bind mounts, + // no host-side temp files, works on all platforms. + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + + // ── k3s server command ──────────────────────────────────────────── + .WithArgs("server") + + // NOTE: --cluster-init (embedded etcd) is intentionally NOT used. + // etcd stores the container's IP in its peer URL. When Docker assigns a new IP + // on container restart, etcd refuses to start ("not a member of the etcd cluster"). + // k3s uses SQLite by default for single-server clusters — it handles restarts and + // IP changes gracefully. k3d only adds --cluster-init for HA multi-server setups. + + // Add TLS SANs so the API server certificate is valid for all addresses + // that clients use to reach it: host-side (localhost), DCP-network containers + // ({resourceName}), and any forwarded address (0.0.0.0). + .WithArgs($"--tls-san=127.0.0.1") + .WithArgs($"--tls-san=localhost") + .WithArgs($"--tls-san={name}") + .WithArgs("--tls-san=0.0.0.0") + + // Disable components not needed for local development. + // servicelb and metrics-server are the biggest resource consumers and slowest + // to start; disabling them speeds up cluster readiness significantly. + .WithArgs("--disable=servicelb") + .WithArgs("--disable=metrics-server") + + // Reduce log verbosity — k3s writes everything to stderr which DCP surfaces + // as "error" level entries in the dashboard log viewer. + .WithArgs("-v", "0") + .WithArgs("--kube-apiserver-arg=v=0") + .WithArgs("--kube-controller-manager-arg=v=0") + .WithArgs("--kube-scheduler-arg=v=0") + // Suppress kubelet INFO-level noise including the harmless cgroupsv2 race warning. + .WithArgs("--kubelet-arg=v=0") + + // ── API server endpoint ─────────────────────────────────────────── + .WithHttpsEndpoint( + targetPort: 6443, + port: apiServerPort, + name: K3sClusterResource.ApiServerEndpointName) + + // ── Docker / container runtime flags (mirrors k3d) ──────────────── + .WithContainerRuntimeArgs("--privileged") + .WithContainerRuntimeArgs("--init") + .WithContainerRuntimeArgs("--userns=host") + .WithContainerRuntimeArgs("--cgroupns=host") + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + + // ── Kubeconfig bind-mount ───────────────────────────────────────── + // Mounts AppHostDirectory/.k3s/{name}/cluster/ into the container so k3s + // writes its kubeconfig into a host-accessible directory. + // K3S_KUBECONFIG_OUTPUT tells k3s where to write the kubeconfig file. + // The health check polls File.Exists on the host side — no docker exec needed. + .WithBindMount(clusterDir, "/tmp/k3s-kubeconfig") + + // ── Environment ─────────────────────────────────────────────────── + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + .WithEnvironment("K3S_KUBECONFIG_MODE", "644") + .WithEnvironment("K3S_KUBECONFIG_OUTPUT", "/tmp/k3s-kubeconfig/kubeconfig.yaml") + + .WithIconName("Kubernetes") + .WithHttpsDeveloperCertificate(); + + if (options.ClusterCidr is not null) + { + resourceBuilder.WithArgs($"--cluster-cidr={options.ClusterCidr}"); + } + + if (options.ServiceCidr is not null) + { + resourceBuilder.WithArgs($"--service-cidr={options.ServiceCidr}"); + } + + foreach (var component in options.DisabledComponents) + { + resourceBuilder.WithArgs($"--disable={component}"); + } + + foreach (var arg in options.ExtraArgs) + { + resourceBuilder.WithArgs(arg); + } + + // Create agent nodes specified via options.AgentCount. + // Agents use DCP DNS: K3S_URL=https://{name}:6443 resolves to the server container. + // NO WaitFor — k3s agent retries indefinitely until the server is reachable. + // This avoids a deadlock where the cluster health check waits for nodes to be Ready + // while nodes wait for the cluster to be healthy. + for (var i = 0; i < options.AgentCount; i++) + { + resource.AgentCount++; + var agentName = $"{name}-agent-{i}"; + var agentResource = new K3sAgentResource(agentName, resource); + resource.AddAgentResource(agentResource); + + builder.AddResource(agentResource) + .WithImage(K3sContainerImageTags.Image, tag) + .WithImageRegistry(K3sContainerImageTags.Registry) + .WithContainerFiles("/", [new ContainerFile + { + Name = "aspire-k3s-entrypoint.sh", + Contents = K3sInitEntrypointScript, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute + | UnixFileMode.GroupRead | UnixFileMode.GroupExecute + | UnixFileMode.OtherRead | UnixFileMode.OtherExecute, + }]) + .WithEntrypoint("/bin/sh") + .WithArgs("/aspire-k3s-entrypoint.sh") + .WithArgs("agent") + .WithArgs("-v", "0") + .WithArgs("--kubelet-arg=v=0") + .WithEnvironment("K3S_URL", $"https://{name}:6443") + .WithEnvironment("K3S_TOKEN", $"aspire-k3s-{name}-token") + .WithEnvironment("K3S_NODE_NAME", agentName) + .WithContainerRuntimeArgs("--privileged") + .WithContainerRuntimeArgs("--init") + .WithContainerRuntimeArgs("--userns=host") + .WithContainerRuntimeArgs("--cgroupns=host") + .WithContainerRuntimeArgs("--volume=/sys/fs/cgroup:/sys/fs/cgroup:rw") + .WithContainerRuntimeArgs("--tmpfs=/run", "--tmpfs=/var/run") + .ExcludeFromManifest() + .WithInitialState(new CustomResourceSnapshot + { + ResourceType = "K3s Agent", + State = KnownResourceStates.Starting, + Properties = [new ResourcePropertySnapshot("Cluster", name)], + }); + } + + resourceBuilder.WithHealthCheck($"k3s_{name}_ready"); + + builder.Services.AddHealthChecks().Add(new HealthCheckRegistration( + $"k3s_{name}_ready", + sp => new K3sReadinessHealthCheck(resource, resource.ApiEndpoint), + failureStatus: HealthStatus.Unhealthy, + tags: null)); + + // The cluster's ResourceReadyEvent drives service endpoint port-forwards. + // HelmReleaseResource and K8sManifestResource containers are managed directly + // by DCP — they WaitFor the cluster and exit when their work completes. + builder.Eventing.Subscribe(resource, (@event, ct) => + { + var appModel = @event.Services.GetRequiredService(); + var notifications = @event.Services.GetRequiredService(); + var loggerService = @event.Services.GetRequiredService(); + + // Start all service endpoint forwarders concurrently. + foreach (var ep in appModel.Resources + .OfType() + .Where(e => ReferenceEquals(e.Parent, resource))) + { + var logger = loggerService.GetLogger(ep); + _ = Task.Run(() => K3sServiceEndpointExtensions.RunEndpointAsync( + ep, resource, notifications, logger, ct), ct); + } + + return Task.CompletedTask; + }); + + return resourceBuilder; + } + + /// Overrides the k3s server image version. + [AspireExport("withK3sVersion", Description = "Overrides the k3s server image version")] + public static IResourceBuilder WithK3sVersion( + this IResourceBuilder builder, + string tag) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(tag); + + return builder.WithImageTag(tag); + } + + /// Sets the pod subnet CIDR (--cluster-cidr). + [AspireExport("withPodSubnet", Description = "Sets the pod subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithPodSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--cluster-cidr={cidr}"); + } + + /// Sets the service subnet CIDR (--service-cidr). + [AspireExport("withServiceSubnet", Description = "Sets the service subnet CIDR for the k3s cluster")] + public static IResourceBuilder WithServiceSubnet( + this IResourceBuilder builder, + string cidr) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(cidr); + + return builder.WithArgs($"--service-cidr={cidr}"); + } + + /// Disables a built-in k3s component (e.g. traefik). + [AspireExport("withDisabledComponent", Description = "Disables a built-in k3s component")] + public static IResourceBuilder WithDisabledComponent( + this IResourceBuilder builder, + string component) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(component); + + return builder.WithArgs($"--disable={component}"); + } + + /// Appends a raw argument to the k3s server command. + [AspireExport("withExtraArg", Description = "Appends a raw argument to the k3s server command")] + public static IResourceBuilder WithExtraArg( + this IResourceBuilder builder, + string arg) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(arg); + + return builder.WithArgs(arg); + } + + /// + /// Adds a named volume for the k3s cluster data directory (/var/lib/rancher/k3s) + /// so the cluster state (SQLite database, certificates, kubeconfig) survives AppHost restarts. + /// + public static IResourceBuilder WithDataVolume( + this IResourceBuilder builder, + string? name = null) + { + ArgumentNullException.ThrowIfNull(builder); + + return builder + .WithVolume(name ?? VolumeNameGenerator.Generate(builder, "data"), "/var/lib/rancher/k3s") + .WithContainerRuntimeArgs("--restart=unless-stopped"); + } + + /// + /// Injects the k3s kubeconfig into so it can authenticate + /// to the cluster. The injection method is selected automatically based on the resource type: + /// + /// + /// s receive KUBECONFIG_DATA (base-64-encoded YAML + /// of the container-network kubeconfig) and KUBECONFIG=/var/k3s/kubeconfig.yaml + /// once the file is injected at container-start time. + /// + /// + /// Projects and executables receive KUBECONFIG=<host path>/local/kubeconfig.yaml + /// pointing to a file that is accessible directly on the host filesystem. + /// + /// + /// The variable is populated only after the cluster health check passes; use + /// WaitFor(cluster) on the dependent resource to guarantee ordering. + /// + [AspireExport("withReference", Description = "Injects kubeconfig credentials into the dependent resource")] + public static IResourceBuilder WithReference( + this IResourceBuilder destination, + IResourceBuilder source) + where TDestination : IResourceWithEnvironment + { + ArgumentNullException.ThrowIfNull(destination); + ArgumentNullException.ThrowIfNull(source); + + var cluster = source.Resource; + + if (destination.Resource is ContainerResource) + { + // Containers receive KUBECONFIG_DATA (base64-encoded container-network kubeconfig). + // KubernetesClient reads this via: + // var bytes = Convert.FromBase64String(Environment.GetEnvironmentVariable("KUBECONFIG_DATA")!); + // using var stream = new MemoryStream(bytes); + // var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(stream); + // For standard kubectl, use WithContainerFiles to inject a real file instead. + return destination.WithEnvironment(ctx => + { + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "container", "kubeconfig.yaml"); + if (!File.Exists(path)) return; + var yaml = File.ReadAllText(path); + ctx.EnvironmentVariables["KUBECONFIG_DATA"] = + Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); + }); + } + + // Projects and executables: KUBECONFIG points to the host-accessible local kubeconfig. + // This file is regenerated on every AppHost start (port may change). + return destination.WithEnvironment(ctx => + { + if (cluster.KubeconfigDirectory is null) return; + var path = Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + ctx.EnvironmentVariables["KUBECONFIG"] = path; + }); + } + + /// + /// Sets the container lifetime for the k3s cluster and all its agent nodes. + /// + public static IResourceBuilder WithLifetime( + this IResourceBuilder builder, + ContainerLifetime lifetime) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.WithAnnotation( + new ContainerLifetimeAnnotation { Lifetime = lifetime }, + ResourceAnnotationMutationBehavior.Replace); + + foreach (var agent in builder.Resource.AgentResources) + { + var existing = agent.Annotations.OfType().ToList(); + foreach (var ann in existing) + { + agent.Annotations.Remove(ann); + } + + agent.Annotations.Add(new ContainerLifetimeAnnotation { Lifetime = lifetime }); + } + + return builder; + } + + /// + /// Returns the path to the local/kubeconfig.yaml file for this cluster, + /// used by helm and manifest runners that invoke host-side tools. + /// Returns if the cluster directory is not yet configured. + /// + internal static string? GetLocalKubeconfigPath(K3sClusterResource cluster) => + cluster.KubeconfigDirectory is null + ? null + : Path.Combine(cluster.KubeconfigDirectory, "local", "kubeconfig.yaml"); + + // cgroupsv2 fix adapted from moby/moby (Apache-2.0, used with permission by k3d). + // See: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh + private const string K3sInitEntrypointScript = """ + #!/bin/sh + # Aspire k3s init entrypoint — adapted from k3d (https://github.com/k3d-io/k3d) + # cgroupsv2 fix adapted from moby/moby (Apache-2.0), used with permission. + + # Make mountpoints recursively shared — required for volume propagation in Docker-in-Docker. + mount --make-rshared / 2>/dev/null || true + + # cgroupsv2: evacuate root cgroup so k3s kubelet can create pod sub-cgroups. + # Without this, writing to cgroup.subtree_control fails with EBUSY because + # init processes still live in the root cgroup. + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + if command -v xargs >/dev/null 2>&1; then + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + else + busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs 2>/dev/null || true + fi + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true + fi + + exec k3s "$@" + """; +} + +#pragma warning restore ASPIREATS001 +#pragma warning restore ASPIRECERTIFICATES001 diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs new file mode 100644 index 000000000..ab67ca836 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterOptions.cs @@ -0,0 +1,80 @@ +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Configuration options for a k3s cluster resource. +/// +public sealed class K3sClusterOptions +{ + /// + /// Gets or sets the CIDR range for pod IPs (passed as --cluster-cidr). + /// + public string? ClusterCidr { get; set; } + + /// + /// Gets or sets the CIDR range for service IPs (passed as --service-cidr). + /// + public string? ServiceCidr { get; set; } + + /// + /// Gets the list of k3s components to disable (each passed as --disable=<component>). + /// + public IList DisabledComponents { get; } = new List(); + + /// + /// Gets the list of raw extra arguments appended to the k3s server command. + /// + public IList ExtraArgs { get; } = new List(); + + /// + /// Gets or sets the number of agent (worker) nodes to add to the cluster. + /// Equivalent to k3d's --agents N flag. + /// Defaults to 0 (single-node cluster). + /// + public int AgentCount { get; set; } + + /// + /// Gets or sets the k3s image tag (e.g. v1.31.4-k3s1). + /// When , the default tag embedded in the package is used. + /// + public string? ImageTag { get; set; } + + // ── Helm installer image ────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the Helm installer container image. + /// Defaults to docker.io. + /// + public string HelmRegistry { get; set; } = HelmContainerImageTags.Registry; + + /// + /// Gets or sets the Helm installer container image name. + /// Defaults to alpine/helm. + /// + public string HelmImage { get; set; } = HelmContainerImageTags.Image; + + /// + /// Gets or sets the Helm installer container image tag. + /// Defaults to 3.17.3. + /// + public string HelmTag { get; set; } = HelmContainerImageTags.Tag; + + // ── kubectl image ───────────────────────────────────────────────────────── + + /// + /// Gets or sets the registry for the kubectl container image used by manifest applies. + /// Defaults to docker.io. + /// + public string KubectlRegistry { get; set; } = KubectlContainerImageTags.Registry; + + /// + /// Gets or sets the kubectl container image name used by manifest applies. + /// Defaults to alpine/k8s. + /// + public string KubectlImage { get; set; } = KubectlContainerImageTags.Image; + + /// + /// Gets or sets the kubectl container image tag used by manifest applies. + /// Defaults to 1.32.3. + /// + public string KubectlTag { get; set; } = KubectlContainerImageTags.Tag; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs new file mode 100644 index 000000000..4e121d7f9 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sClusterResource.cs @@ -0,0 +1,66 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents a k3s Kubernetes cluster running as a privileged container resource. +/// +/// The resource name. +public sealed class K3sClusterResource(string name) : ContainerResource(name) +{ + internal const string ApiServerEndpointName = "api"; + + /// Container image settings for the Helm installer, resolved from cluster options. + internal (string Registry, string Image, string Tag) HelmImageInfo { get; set; } + = ("docker.io", "alpine/helm", "3.17.3"); + + /// Container image settings for the kubectl manifest applier, resolved from cluster options. + internal (string Registry, string Image, string Tag) KubectlImageInfo { get; set; } + = ("docker.io", "alpine/k8s", "1.32.3"); + + /// + /// Host-side directory that holds all kubeconfig variants for this cluster. + /// Set by AddK3sCluster to AppHostDirectory/.k3s/{name}/. + /// Sub-directories: + /// + /// cluster/kubeconfig.yaml — raw file written by k3s (bind-mounted) + /// local/kubeconfig.yaml — server: https://localhost:{port} (host processes) + /// container/kubeconfig.yaml — server: https://{name}:6443 (DCP-network containers) + /// + /// + internal string? KubeconfigDirectory { get; set; } + + private EndpointReference? _apiEndpoint; + + /// Gets the endpoint reference for the k3s API server (port 6443). + public EndpointReference ApiEndpoint => _apiEndpoint ??= new(this, ApiServerEndpointName); + + // ── Child resource tracking (Postgres pattern) ──────────────────────────── + + /// + /// Number of agent (worker) nodes. The health check waits for all + /// 1 + AgentCount nodes to reach Ready state before marking the cluster healthy. + /// + internal int AgentCount { get; set; } + + private readonly List _agentResources = []; + + /// Agent resource instances, used to propagate annotations (e.g. lifetime) to all nodes. + internal IReadOnlyList AgentResources => _agentResources; + + internal void AddAgentResource(K3sAgentResource agent) => _agentResources.Add(agent); + + private readonly Dictionary _helmReleases = + new(StringComparer.OrdinalIgnoreCase); + + /// A dictionary of registered Helm releases keyed by resource name. + public IReadOnlyDictionary HelmReleases => _helmReleases; + + internal void AddHelmRelease(string resourceName, string releaseName) => + _helmReleases.TryAdd(resourceName, releaseName); + + private readonly List _manifests = []; + + /// Names of registered children. + public IReadOnlyList Manifests => _manifests; + + internal void AddManifest(string resourceName) => _manifests.Add(resourceName); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs new file mode 100644 index 000000000..605e5c65c --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sContainerImageTags.cs @@ -0,0 +1,13 @@ +namespace Aspire.Hosting.ApplicationModel; + +internal static class K3sContainerImageTags +{ + /// docker.io + public const string Registry = "docker.io"; + + /// rancher/k3s + public const string Image = "rancher/k3s"; + + /// v1.31.4-k3s1 + public const string Tag = "v1.31.4-k3s1"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs new file mode 100644 index 000000000..1d704399b --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sInProcessPortForwarder.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.Net; +using System.Net.Sockets; +using k8s; +using Microsoft.Extensions.Logging; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Forwards a local TCP port to a Kubernetes service using the KubernetesClient +/// WebSocket port-forward API — no kubectl binary required. +/// +/// The listener binds to 0.0.0.0:{localPort} so both host processes +/// (localhost:{port}) and DCP-network containers +/// (host.docker.internal:{port}) can reach the service. +/// +/// +internal sealed class K3sInProcessPortForwarder( + string kubeconfigPath, + string @namespace, + string serviceName, + int localPort, + int servicePort, + Action onReadyChanged) +{ + public async Task RunAsync(ILogger logger, CancellationToken ct) + { + var backoff = TimeSpan.FromSeconds(2); + + while (!ct.IsCancellationRequested) + { + var listener = new TcpListener(IPAddress.Any, localPort); + try + { + listener.Start(); + onReadyChanged(true); + + logger.LogInformation( + "Port-forward: 0.0.0.0:{Local} → svc/{Service}.{Ns}:{Port}", + localPort, serviceName, @namespace, servicePort); + + while (!ct.IsCancellationRequested) + { + var tcp = await listener.AcceptTcpClientAsync(ct).ConfigureAwait(false); + _ = Task.Run( + () => ForwardConnectionAsync(tcp, logger, ct), + CancellationToken.None); + } + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogWarning(ex, + "Port-forward for svc/{Service} failed; retrying in {Delay}s…", + serviceName, backoff.TotalSeconds); + onReadyChanged(false); + } + finally + { + listener.Stop(); + } + + if (ct.IsCancellationRequested) break; + + try { await Task.Delay(backoff, ct).ConfigureAwait(false); } catch (OperationCanceledException) { break; } + backoff = TimeSpan.FromSeconds(Math.Min(backoff.TotalSeconds * 2, 30)); + } + } + + private async Task ForwardConnectionAsync(TcpClient tcp, ILogger logger, CancellationToken ct) + { + using var _ = tcp; + try + { + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeconfigPath); + using var k8sClient = new Kubernetes(config); + + // Resolve the service to a running pod. + var svc = await k8sClient.CoreV1 + .ReadNamespacedServiceAsync(serviceName, @namespace, cancellationToken: ct) + .ConfigureAwait(false); + + var selector = string.Join(",", + (svc.Spec.Selector ?? new Dictionary()).Select(kv => $"{kv.Key}={kv.Value}")); + + var pods = await k8sClient.CoreV1 + .ListNamespacedPodAsync(@namespace, labelSelector: selector, cancellationToken: ct) + .ConfigureAwait(false); + + var pod = pods.Items.FirstOrDefault(p => + p.Status?.Phase == "Running" && + p.Status?.ContainerStatuses?.All(c => c.Ready) == true); + + if (pod is null) + { + logger.LogWarning( + "No ready pod found for service {Service}/{Ns} — connection dropped.", + serviceName, @namespace); + return; + } + + // Open WebSocket port-forward to the pod. + using var ws = await k8sClient.WebSocketNamespacedPodPortForwardAsync( + pod.Metadata.Name, @namespace, [servicePort], + cancellationToken: ct).ConfigureAwait(false); + + using var demuxer = new StreamDemuxer(ws, StreamType.PortForward); + demuxer.Start(); + + using var k8sStream = demuxer.GetStream((byte?)0, (byte?)0); + using var tcpStream = tcp.GetStream(); + + // Bidirectional byte pump until either side closes. + await Task.WhenAny( + tcpStream.CopyToAsync(k8sStream, ct), + k8sStream.CopyToAsync(tcpStream, ct)).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { } + catch (Exception ex) + { + logger.LogDebug(ex, "Port-forward connection for svc/{Service} closed.", serviceName); + } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs new file mode 100644 index 000000000..a51a9e2e2 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sReadinessHealthCheck.cs @@ -0,0 +1,139 @@ +using Aspire.Hosting.ApplicationModel; +using k8s; +using k8s.KubeConfigModels; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace CommunityToolkit.Aspire.Hosting; + +/// +/// Health check for . +/// +/// Polls for the kubeconfig file written by k3s into the bind-mounted +/// AppHostDirectory/.k3s/{name}/cluster/kubeconfig.yaml. On first appearance +/// it rewrites the server URL for two variants: +/// +/// local/kubeconfig.yaml — server: https://localhost:{allocatedPort} (host processes) +/// container/kubeconfig.yaml — server: https://{name}:6443 (DCP-network containers) +/// +/// Then uses a cached client to call ListNodeAsync, +/// confirming that all expected nodes (server + agents) are in Ready state. +/// No docker exec is involved — works with any container runtime. +/// +/// +internal sealed class K3sReadinessHealthCheck : IHealthCheck +{ + private readonly K3sClusterResource _resource; + private readonly EndpointReference _endpoint; + private Kubernetes? _cachedClient; + + internal K3sReadinessHealthCheck(K3sClusterResource resource, EndpointReference endpoint) + { + _resource = resource; + _endpoint = endpoint; + } + + /// + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + if (!_endpoint.IsAllocated) + return HealthCheckResult.Unhealthy("k3s API server endpoint not yet allocated"); + + var dir = _resource.KubeconfigDirectory; + if (dir is null) + return HealthCheckResult.Unhealthy("Kubeconfig directory not configured on resource"); + + var rawPath = Path.Combine(dir, "cluster", "kubeconfig.yaml"); + if (!File.Exists(rawPath)) + return HealthCheckResult.Unhealthy("Waiting for k3s to write kubeconfig"); + + try + { + var client = await EnsureClientAsync(rawPath, cancellationToken).ConfigureAwait(false); + + var nodes = await client.CoreV1 + .ListNodeAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var readyCount = nodes.Items.Count(n => + n.Status?.Conditions?.Any(c => + c.Type == "Ready" && + string.Equals(c.Status, "True", StringComparison.OrdinalIgnoreCase)) == true); + + var expected = 1 + _resource.AgentCount; + if (readyCount < expected) + return HealthCheckResult.Unhealthy($"k3s cluster: {readyCount}/{expected} nodes Ready"); + + return HealthCheckResult.Healthy("k3s cluster is ready"); + } + catch (Exception ex) when (IsTlsOrAuthFailure(ex)) + { + // Stale kubeconfig — cluster was recreated with new certs (e.g. data volume wiped). + // Invalidate everything; k3s will overwrite cluster/kubeconfig.yaml on next start. + _cachedClient?.Dispose(); + _cachedClient = null; + TryDelete(rawPath); + TryDelete(Path.Combine(dir, "local", "kubeconfig.yaml")); + TryDelete(Path.Combine(dir, "container", "kubeconfig.yaml")); + return HealthCheckResult.Unhealthy("k3s kubeconfig is stale — waiting for cluster refresh"); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy(ex.Message, ex); + } + } + + private async Task EnsureClientAsync(string rawPath, CancellationToken ct) + { + if (_cachedClient is not null) + return _cachedClient; + + var port = _endpoint.Port; + var dir = _resource.KubeconfigDirectory!; + + var rawYaml = await File.ReadAllTextAsync(rawPath, ct).ConfigureAwait(false); + var parsed = KubernetesYaml.Deserialize(rawYaml); + + var localDir = Path.Combine(dir, "local"); + Directory.CreateDirectory(localDir); + var localPath = Path.Combine(localDir, "kubeconfig.yaml"); + await File.WriteAllTextAsync(localPath, BuildConfigYaml(parsed, $"https://localhost:{port}"), ct) + .ConfigureAwait(false); + + var containerDir = Path.Combine(dir, "container"); + Directory.CreateDirectory(containerDir); + await File.WriteAllTextAsync( + Path.Combine(containerDir, "kubeconfig.yaml"), + BuildConfigYaml(parsed, $"https://{_resource.Name}:6443"), + ct).ConfigureAwait(false); + + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(localPath); + _cachedClient = new Kubernetes(config); + return _cachedClient; + } + + private static string BuildConfigYaml(K8SConfiguration source, string serverUrl) + { + var yaml = KubernetesYaml.Serialize(source); + var copy = KubernetesYaml.Deserialize(yaml); + foreach (var cluster in copy.Clusters ?? []) + { + if (cluster.ClusterEndpoint is not null) + cluster.ClusterEndpoint.Server = serverUrl; + } + + return KubernetesYaml.Serialize(copy); + } + + private static bool IsTlsOrAuthFailure(Exception ex) => + ex is System.Security.Authentication.AuthenticationException + || ex.InnerException is System.Security.Authentication.AuthenticationException + || (ex is k8s.Autorest.HttpOperationException op && + op.Response?.StatusCode == System.Net.HttpStatusCode.Unauthorized); + + private static void TryDelete(string path) + { + try { File.Delete(path); } catch { /* best effort */ } + } +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs new file mode 100644 index 000000000..95d4c9f9a --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K3sServiceEndpointResource.cs @@ -0,0 +1,47 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Exposes a Kubernetes service running inside a k3s cluster as an Aspire endpoint resource. +/// +/// An in-process KubernetesClient WebSocket port-forward is started when the cluster is ready. +/// The forwarder binds to 0.0.0.0:{hostPort} so both host processes and DCP-network +/// containers can reach the service. +/// +/// +/// Host consumers receive services__{name}__url=https://localhost:{port}. +/// Container consumers receive services__{name}__url=https://host.docker.internal:{port}. +/// +/// +public sealed class K3sServiceEndpointResource( + string name, + string serviceName, + int servicePort, + string @namespace, + K3sClusterResource cluster) + : Resource(name), IResourceWithParent, IResourceWithWaitSupport +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the Kubernetes service name. + public string ServiceName { get; } = serviceName ?? throw new ArgumentNullException(nameof(serviceName)); + + /// Gets the service port number. + public int ServicePort { get; } = servicePort; + + /// Gets the Kubernetes namespace containing the service. + public string Namespace { get; } = @namespace ?? throw new ArgumentNullException(nameof(@namespace)); + + /// + /// The host port allocated for the port-forward listener. + /// Set by RunEndpointAsync before the resource transitions to Running. + /// Consumers can use this to construct the service URL directly when needed. + /// + public int HostPort { get; internal set; } + + /// + /// when the port-forward is active and accepting connections. + /// Set by K3sInProcessPortForwarder; read by the health check. + /// + internal volatile bool IsReady; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs new file mode 100644 index 000000000..a87a687ee --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/K8sManifestResource.cs @@ -0,0 +1,23 @@ +namespace Aspire.Hosting.ApplicationModel; + +/// +/// Represents one or more Kubernetes YAML manifests applied to the parent k3s cluster via +/// kubectl apply --server-side running inside a bitnami/kubectl container. +/// +/// The container polls for the cluster kubeconfig (written when the cluster health check +/// first passes), applies the manifests, waits for any CRDs to reach Established, +/// then exits with code 0. Use WaitForCompletion(manifest) on dependent resources. +/// +/// +/// The Aspire resource name. +/// Absolute path to a single YAML file or a directory. +/// The parent k3s cluster resource. +public sealed class K8sManifestResource(string name, string path, K3sClusterResource cluster) + : ContainerResource(name), IResourceWithParent +{ + /// + public K3sClusterResource Parent { get; } = cluster ?? throw new ArgumentNullException(nameof(cluster)); + + /// Gets the manifest path or directory on the host. + public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs new file mode 100644 index 000000000..eb6eedb68 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/KubectlContainerImageTags.cs @@ -0,0 +1,11 @@ +namespace CommunityToolkit.Aspire.Hosting; + +internal static class KubectlContainerImageTags +{ + internal const string Registry = "docker.io"; + // alpine/k8s: lightweight Alpine-based image that includes kubectl and other k8s tools. + // Same organisation as alpine/helm — consistent image family. + internal const string Image = "alpine/k8s"; + // Matches the Kubernetes version shipped by the default k3s tag. + internal const string Tag = "1.32.3"; +} diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/README.md b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md new file mode 100644 index 000000000..f6a5d8150 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/README.md @@ -0,0 +1,67 @@ +# CommunityToolkit.Aspire.Hosting.K3s + +An Aspire hosting integration for [k3s](https://k3s.io/) — a lightweight, certified Kubernetes distribution by Rancher/SUSE. + +## Getting started + +### Prerequisites + +- Docker with support for `--privileged` containers (Linux host or Docker Desktop on macOS/Windows) + +### Installation + +```sh +dotnet add package CommunityToolkit.Aspire.Hosting.K3s +``` + +## Usage + +```csharp +var builder = DistributedApplication.CreateBuilder(args); + +var cluster = builder.AddK3sCluster("k8s") + .WithPersistentState(); + +builder.AddProject("api") + .WithReference(cluster) + .WaitFor(cluster); + +builder.Build().Run(); +``` + +### Kubeconfig injection + +`WithReference(cluster)` automatically selects the injection mode: + +| Resource type | Environment variable set | +|---|---| +| `ProjectResource` / `ExecutableResource` | `KUBECONFIG=/tmp/aspire-k3s-k8s/admin.yaml` | +| `ContainerResource` | `KUBECONFIG_DATA=` | + +### Configuration options + +```csharp +builder.AddK3sCluster("k8s", configure: opts => +{ + opts.ClusterCidr = "10.42.0.0/16"; + opts.ServiceCidr = "10.43.0.0/16"; + opts.DisabledComponents.Add("traefik"); +}); +``` + +Or use the fluent API: + +```csharp +builder.AddK3sCluster("k8s") + .WithK3sVersion("v1.32.3-k3s1") + .WithPodSubnet("10.42.0.0/16") + .WithServiceSubnet("10.43.0.0/16") + .WithDisabledComponent("traefik") + .WithPersistentState(); +``` + +## Known limitations + +- Requires a privileged Docker runtime; `--privileged` is passed automatically. +- On Linux hosts the `/lib/modules` directory should be present for CNI networking. +- The first cluster start can take 30–60 seconds while container images and CNI plugins are initialised. diff --git a/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs new file mode 100644 index 000000000..50701b0d7 --- /dev/null +++ b/src/CommunityToolkit.Aspire.Hosting.K3s/api/CommunityToolkit.Aspire.Hosting.K3s.cs @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ +namespace Aspire.Hosting +{ + public static partial class K3sHelmBuilderExtensions + { + public static ApplicationModel.IResourceBuilder AddHelmRelease(this ApplicationModel.IResourceBuilder builder, string name, string chart, string? repo = null, string? version = null, string @namespace = "default") { throw null; } + public static ApplicationModel.IResourceBuilder WithHelmValue(this ApplicationModel.IResourceBuilder builder, string key, string value) { throw null; } + public static ApplicationModel.IResourceBuilder WithEndpoint(this ApplicationModel.IResourceBuilder builder, string serviceName, int servicePort, string name) { throw null; } + } + + public static partial class K3sBuilderExtensions + { + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder AddK3sCluster(this IDistributedApplicationBuilder builder, string name, int? apiServerPort = null, System.Action? configure = null) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithReference(this ApplicationModel.IResourceBuilder destination, ApplicationModel.IResourceBuilder source, CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy strategy = CommunityToolkit.Aspire.Hosting.KubeconfigInjectionStrategy.Auto) where TDestination : ApplicationModel.IResourceWithEnvironment { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithK3sVersion(this ApplicationModel.IResourceBuilder builder, string tag) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithPodSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithServiceSubnet(this ApplicationModel.IResourceBuilder builder, string cidr) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithDisabledComponent(this ApplicationModel.IResourceBuilder builder, string component) { throw null; } + [System.Diagnostics.CodeAnalysis.Experimental("ASPIREATS001")] + public static ApplicationModel.IResourceBuilder WithExtraArg(this ApplicationModel.IResourceBuilder builder, string arg) { throw null; } + public static ApplicationModel.IResourceBuilder WithPersistentState(this ApplicationModel.IResourceBuilder builder, string? volumeName = null) { throw null; } + } +} + +namespace Aspire.Hosting.ApplicationModel +{ + public sealed partial class HelmReleaseResource : Resource, IResourceWithParent, IResourceWithWaitSupport + { + public HelmReleaseResource(string name, string releaseName, string @namespace, K3sClusterResource cluster) : base(default!) { } + public K3sClusterResource Parent { get { throw null; } } + public string ReleaseName { get { throw null; } } + public string Namespace { get { throw null; } } + } + + public sealed partial class K3sClusterResource : ContainerResource + { + public K3sClusterResource(string name) : base(default!) { } + public EndpointReference ApiEndpoint { get { throw null; } } + } +} + +namespace CommunityToolkit.Aspire.Hosting +{ + public sealed partial class K3sClusterOptions + { + public K3sClusterOptions() { } + public string? ClusterCidr { get { throw null; } set { } } + public string? ImageTag { get { throw null; } set { } } + public string? ServiceCidr { get { throw null; } set { } } + public System.Collections.Generic.IList DisabledComponents { get { throw null; } } + public System.Collections.Generic.IList ExtraArgs { get { throw null; } } + } + + public enum KubeconfigInjectionStrategy + { + Auto = 0, + HostNetwork = 1, + ContainerNetwork = 2, + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj new file mode 100644 index 000000000..95df55b8f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests.csproj @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs new file mode 100644 index 000000000..f08fa0588 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests/K3sIntegrationTests.cs @@ -0,0 +1,158 @@ +using Aspire.Components.Common.Tests; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Utils; +using CommunityToolkit.Aspire.Testing; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace CommunityToolkit.Aspire.Hosting.K3s.IntegrationTests; + +/// +/// End-to-end integration tests that spin up a real k3s cluster inside Docker. +/// +/// Requirements: +/// +/// Linux with Docker (privileged containers required by k3s). +/// helm on PATH — used by AddHelmRelease. +/// kubectl on PATH — used by AddK8sManifest. +/// +/// The tests are gated by [RequiresDocker] and intended for the +/// ubuntu-latest-only CI job in tests.yaml. +/// +/// +[RequiresDocker] +public class K3sIntegrationTests : IAsyncLifetime +{ + private DistributedApplication? _app; + private IDistributedApplicationTestingBuilder? _builder; + + public async ValueTask InitializeAsync() + { + _builder = TestDistributedApplicationBuilder.Create(); + await ValueTask.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + if (_app is not null) + { + await _app.StopAsync(); + await _app.DisposeAsync(); + } + + _builder?.Dispose(); + } + + [Fact] + public async Task ClusterReachesRunningAndKubeconfigIsValid() + { + var cluster = _builder!.AddK3sCluster("k8s"); + _app = _builder.Build(); + + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + // local/kubeconfig.yaml must exist on the host. + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + Assert.True(File.Exists(kubeconfigPath), + $"Expected local kubeconfig at {kubeconfigPath}"); + + // container/kubeconfig.yaml must also exist. + var containerKubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "container", "kubeconfig.yaml"); + Assert.True(File.Exists(containerKubeconfigPath), + $"Expected container kubeconfig at {containerKubeconfigPath}"); + } + + [Fact] + public async Task HelmReleaseReachesRunning() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == KnownResourceStates.Running, cts.Token); + } + + [Fact] + public async Task ServiceEndpointExposesHttpPort() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + var nginx = cluster.AddHelmRelease( + name: "nginx", + chart: "nginx", + repo: "https://charts.bitnami.com/bitnami", + version: "18.3.6", + @namespace: "nginx"); + + cluster.AddServiceEndpoint("nginx-web", "nginx", servicePort: 80, @namespace: "nginx") + .WaitForCompletion(nginx); + + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(8)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + // nginx is a run-to-completion container — wait for it to exit (Exited state) + await rns.WaitForResourceAsync("nginx", + s => s.Snapshot.State?.Text == "Exited", cts.Token); + await rns.WaitForResourceHealthyAsync("nginx-web", cts.Token); + + // Find the allocated port from the endpoint resource. + var model = _app.Services.GetRequiredService(); + var ep = model.Resources.OfType().Single(); + + Assert.True(ep.HostPort > 0, "HostPort should be allocated"); + + using var http = new HttpClient(); + var response = await http.GetAsync($"http://localhost:{ep.HostPort}", cts.Token); + Assert.True(response.IsSuccessStatusCode, + $"Expected HTTP 200 from nginx at localhost:{ep.HostPort}, got {response.StatusCode}"); + } + + [Fact] + public async Task WithReferenceInjectsKubeconfigForProject() + { + var cluster = _builder!.AddK3sCluster("k8s"); + + // WithReference on a project injects KUBECONFIG pointing to local/kubeconfig.yaml. + // We verify the env var would be set by checking the cluster state. + _app = _builder.Build(); + await _app.StartAsync(); + + var rns = _app.Services.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + await rns.WaitForResourceHealthyAsync("k8s", cts.Token); + + var kubeconfigPath = Path.Combine( + _builder.AppHostDirectory, ".k3s", "k8s", "local", "kubeconfig.yaml"); + + var yaml = await File.ReadAllTextAsync(kubeconfigPath, cts.Token); + Assert.Contains("localhost", yaml); + Assert.DoesNotContain("127.0.0.1:6443", yaml); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj new file mode 100644 index 000000000..bad157c30 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/CommunityToolkit.Aspire.Hosting.K3s.Tests.csproj @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs new file mode 100644 index 000000000..9d940fe58 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/HelmReleaseResourceTests.cs @@ -0,0 +1,404 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class HelmReleaseResourceTests +{ + [Fact] + public void AddHelmReleaseAddsHelmReleaseResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Name); + } + + [Fact] + public void HelmReleaseResourceIsContainerResource() + { + // HelmReleaseResource extends ContainerResource — it runs bitnami/helm in Docker. + // No host-side helm binary required. WaitForCompletion waits for exit code 0. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.IsAssignableFrom(resource); + } + + [Fact] + public void AddHelmReleaseDefaultsReleaseName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("my-release", "my-chart"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("my-release", resource.ReleaseName); + } + + [Fact] + public void AddHelmReleaseDefaultsNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("default", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseWithExplicitNamespace() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", @namespace: "argocd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd", resource.Namespace); + } + + [Fact] + public void AddHelmReleaseStoresRepoUrl() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease( + "argocd", "argo-cd", + repo: "https://argoproj.github.io/argo-helm"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("https://argoproj.github.io/argo-helm", resource.RepoUrl); + } + + [Fact] + public void AddHelmReleaseStoresVersion() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd", version: "7.8.0"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("7.8.0", resource.Version); + } + + [Fact] + public void HelmReleaseParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + Assert.IsAssignableFrom>(resource); + } + + [Fact] + public void WithHelmValueAccumulatesValues() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValue("server.service.type", "NodePort") + .WithHelmValue("server.insecure", "true"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("NodePort", resource.HelmValues["server.service.type"]); + Assert.Equal("true", resource.HelmValues["server.insecure"]); + } + + [Fact] + public void AddServiceEndpointAddsEndpointResource() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + cluster.AddServiceEndpoint("argocd-ui", "argocd-server", servicePort: 443, @namespace: "argocd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var ep = Assert.Single(model.Resources.OfType()); + Assert.Equal("argocd-server", ep.ServiceName); + Assert.Equal(443, ep.ServicePort); + Assert.Equal("argocd", ep.Namespace); + } + + [Fact] + public void AddServiceEndpointMultipleEndpointsAllRegistered() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddServiceEndpoint("ui", "argocd-server", 443); + cluster.AddServiceEndpoint("http", "argocd-server", 80); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Equal(2, model.Resources.OfType().Count()); + } + + [Fact] + public void HelmReleaseIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + // ── BuildHelmScript tests (pure logic, no DI needed) ────────────────────── + + private static HelmReleaseResource MakeRelease( + string releaseName, string chart, string? repo, string? version, + string @namespace, Dictionary? values = null) + { + var cluster = new K3sClusterResource("k8s"); + var r = new HelmReleaseResource(releaseName, releaseName, @namespace, cluster) + { + Chart = chart, + RepoUrl = repo, + Version = version, + }; + foreach (var kv in values ?? []) + r.HelmValues[kv.Key] = kv.Value; + return r; + } + + [Fact] + public void BuildHelmScriptIncludesUpgradeInstall() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("argocd", "argo-cd", null, null, "argocd")); + + Assert.Contains("helm upgrade --install", script); + Assert.Contains("\"argocd\"", script); + Assert.Contains("\"argo-cd\"", script); + } + + [Fact] + public void BuildHelmScriptIncludesWaitAndNamespace() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "my-ns")); + + Assert.Contains("--wait", script); + Assert.Contains("--namespace \"my-ns\"", script); + Assert.Contains("--create-namespace", script); + } + + [Fact] + public void BuildHelmScriptWithRepoAddsRepoSteps() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", "https://my-repo.example.com", null, "default")); + + Assert.Contains("helm repo add", script); + Assert.Contains("helm repo update", script); + Assert.Contains("aspire-k3s-r/chart", script); + } + + [Fact] + public void BuildHelmScriptWithoutRepoSkipsRepoSteps() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "oci://registry/chart", null, null, "default")); + + Assert.DoesNotContain("helm repo add", script); + Assert.Contains("\"oci://registry/chart\"", script); + } + + [Fact] + public void BuildHelmScriptIncludesVersion() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, "7.8.0", "default")); + + Assert.Contains("--version \"7.8.0\"", script); + } + + [Fact] + public void BuildHelmScriptOmitsVersionWhenNull() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default")); + + Assert.DoesNotContain("--version", script); + } + + [Fact] + public void BuildHelmScriptIncludesSetValues() + { + var script = K3sHelmBuilderExtensions.BuildHelmScript( + MakeRelease("r", "chart", null, null, "default", new() + { + ["service.type"] = "NodePort", + ["replicaCount"] = "2", + })); + + Assert.Contains("--set \"service.type=NodePort\"", script); + Assert.Contains("--set \"replicaCount=2\"", script); + } + + // ── WaitForCompletion support ───────────────────────────────────────────── + + [Fact] + public void HelmReleaseHasNoHealthCheckAnnotation() + { + // HelmReleaseResource is a run-to-completion container — consumers use + // WaitForCompletion(helmRelease) rather than WaitFor. No health check needed. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddHelmRelease("argocd", "argo-cd"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Empty(resource.Annotations.OfType()); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddHelmReleaseShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddHelmRelease("argocd", "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease(null!, "argo-cd"); + Assert.Throws(action); + } + + [Fact] + public void AddHelmReleaseShouldThrowWhenChartIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddHelmRelease("argocd", null!); + Assert.Throws(action); + } + + [Fact] + public void WithHelmValueShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValue("key", "value"); + Assert.Throws(action); + } + + [Fact] + public void AddServiceEndpointShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddServiceEndpoint("ui", "svc", 443); + Assert.Throws(action); + } + + // ── WithHelmValuesFile tests ────────────────────────────────────────────── + + [Fact] + public void BuildHelmScriptIncludesValuesFiles() + { + var cluster = new K3sClusterResource("k8s"); + var release = new HelmReleaseResource("argocd", "argocd", "argocd", cluster) + { + Chart = "argo-cd", + }; + release.ValuesFiles.Add("/tmp/values.yaml"); + release.ValuesFiles.Add("/tmp/values-prod.yaml"); + + var script = K3sHelmBuilderExtensions.BuildHelmScript(release); + + Assert.Contains("--values \"/helm-values/values.yaml\"", script); + Assert.Contains("--values \"/helm-values/values-prod.yaml\"", script); + // Values files are applied before --set overrides (last wins). + var valuesIndex = script.IndexOf("--values", StringComparison.Ordinal); + var setIndex = script.IndexOf("--set", StringComparison.Ordinal); + Assert.True(valuesIndex < setIndex || setIndex == -1, + "--values flags should appear before --set flags"); + } + + [Fact] + public void WithHelmValuesFileAccumulatesAbsolutePaths() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + // Use a temp file so the path resolution succeeds. + var tempFile = Path.Combine(appBuilder.AppHostDirectory, "values.yaml"); + + cluster.AddHelmRelease("argocd", "argo-cd") + .WithHelmValuesFile("values.yaml"); // relative to AppHostDirectory + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Single(resource.ValuesFiles); + Assert.Equal(tempFile, resource.ValuesFiles[0]); + } + + [Fact] + public void WithHelmValuesFileShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.WithHelmValuesFile("values.yaml"); + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs new file mode 100644 index 000000000..b9dcf1f0c --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sAgentNodeTests.cs @@ -0,0 +1,201 @@ +using Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sAgentNodeTests +{ + // ── Agent creation via K3sClusterOptions ───────────────────────────────── + + [Fact] + public void AgentCountInOptionsCreatesK3sAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + + Assert.Equal(2, agents.Count); + Assert.Contains(agents, a => a.Name == "k8s-agent-0"); + Assert.Contains(agents, a => a.Name == "k8s-agent-1"); + } + + [Fact] + public void AgentNodesAreChildrenOfCluster() + { + // Implements IResourceWithParent so they appear nested under k8s in the dashboard. + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, agent.Parent); + + // Non-generic IResourceWithParent — used by the dashboard for grouping. + var nonGeneric = agent as IResourceWithParent; + Assert.NotNull(nonGeneric); + Assert.Same(cluster.Resource, nonGeneric.Parent); + } + + [Fact] + public void AgentCountZeroProducesNoAgentResources() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 0); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentCountUpdatesClusterAgentCount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 3); + + Assert.Equal(3, cluster.Resource.AgentCount); + } + + [Fact] + public void AgentNodesDoNotHaveWaitForDependencyOnCluster() + { + // Agents must NOT WaitFor the cluster — that would create a deadlock because the + // cluster health check waits for all nodes (including agents) to be Ready. + // Instead, k3s agent retries connecting to the server independently. + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var waitAnnotations = agent.Annotations.OfType().ToList(); + + Assert.DoesNotContain(waitAnnotations, w => w.Resource is K3sClusterResource); + } + + [Fact] + public void AgentNodesUseSameImageAsServer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var img = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(K3sContainerImageTags.Image, img.Image); + Assert.Equal(K3sContainerImageTags.Registry, img.Registry); + } + + [Fact] + public void AgentNodesAreExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, agent.Annotations); + } + + [Fact] + public void AgentNodesHaveEnvironmentAnnotations() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var envCallbacks = agent.Annotations.OfType().ToList(); + Assert.True(envCallbacks.Count >= 3, + $"Expected at least 3 env annotations (K3S_URL, K3S_TOKEN, K3S_NODE_NAME), got {envCallbacks.Count}"); + } + + [Fact] + public void DefaultClusterHasNoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void AgentNodeNamesFollowConvention() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("mycluster", configure: opts => opts.AgentCount = 3); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-0"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-1"); + Assert.Contains(model.Resources, r => r.Name == "mycluster-agent-2"); + } + + [Fact] + public void NegativeAgentCountIsIgnored() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder.AddK3sCluster("k8s", configure: opts => opts.AgentCount = -1); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Empty(model.Resources.OfType()); + } + + [Fact] + public void WithLifetimePersistentPropagatestoAgentNodes() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 2) + .WithLifetime(ContainerLifetime.Persistent); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agents = model.Resources.OfType().ToList(); + Assert.Equal(2, agents.Count); + + foreach (var agent in agents) + { + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Persistent, annotation.Lifetime); + } + } + + [Fact] + public void WithLifetimeSessionDoesNotAddPersistentAnnotationToAgents() + { + var appBuilder = DistributedApplication.CreateBuilder(); + appBuilder + .AddK3sCluster("k8s", configure: opts => opts.AgentCount = 1) + .WithLifetime(ContainerLifetime.Session); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var agent = Assert.Single(model.Resources.OfType()); + var annotation = Assert.Single(agent.Annotations.OfType()); + Assert.Equal(ContainerLifetime.Session, annotation.Lifetime); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs new file mode 100644 index 000000000..0ca9b8bf9 --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sClusterResourceTests.cs @@ -0,0 +1,296 @@ +using System.Net.Sockets; +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using CommunityToolkit.Aspire.Hosting; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sClusterResourceTests +{ + [Fact] + public void AddK3sClusterAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.Equal("k8s", resource.Name); + } + + [Fact] + public void AddK3sClusterAddsCorrectContainerImage() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal(K3sContainerImageTags.Image, annotation.Image); + Assert.Equal(K3sContainerImageTags.Tag, annotation.Tag); + Assert.Equal(K3sContainerImageTags.Registry, annotation.Registry); + } + + [Fact] + public void AddK3sClusterAddsApiServerEndpointOnPort6443() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(ProtocolType.Tcp, endpoint.Protocol); + Assert.Equal("https", endpoint.UriScheme); + } + + [Fact] + public void AddK3sClusterWithExplicitPortBindsToThatPort() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", apiServerPort: 16443); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var endpoint = Assert.Single( + resource.Annotations.OfType(), + e => e.Name == K3sClusterResource.ApiServerEndpointName); + + Assert.Equal(6443, endpoint.TargetPort); + Assert.Equal(16443, endpoint.Port); + } + + [Fact] + public void AddK3sClusterAddsServerArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var commandLineArgs = resource.Annotations.OfType(); + Assert.NotEmpty(commandLineArgs); + } + + [Fact] + public void WithK3sVersionOverridesImageTag() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithK3sVersion("v1.32.3-k3s1"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var annotation = Assert.Single(resource.Annotations.OfType()); + + Assert.Equal("v1.32.3-k3s1", annotation.Tag); + } + + [Fact] + public void AddK3sClusterHasNoVolumeByDefault() + { + // Persistence is opt-in via WithPersistentState(). + // No volume is mounted by default so the cluster is ephemeral. + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.DoesNotContain( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + } + + [Fact] + public void WithDataVolumeAddsSingleVolumeMount() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume(); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + // VolumeNameGenerator format: {appName}-{sha256}-{resourceName}-data + Assert.EndsWith("-k8s-data", volume.Source); + } + + [Fact] + public void WithDataVolumeUsesCustomName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDataVolume("my-k3s-data"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + var volume = Assert.Single( + resource.Annotations.OfType(), + v => v.Target == "/var/lib/rancher/k3s" && v.Type == ContainerMountType.Volume); + + Assert.Equal("my-k3s-data", volume.Source); + } + + [Fact] + public void AddK3sClusterWithClusterCidrViaOptions() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s", configure: opts => + { + opts.ClusterCidr = "10.99.0.0/16"; + }); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + var resource = Assert.Single(appModel.Resources.OfType()); + Assert.NotNull(resource); + } + + [Fact] + public void WithReferenceSetsKubeconfigEnvForProject() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + + // ProjectResource would need a project file; use ExecutableResource as a proxy + var exe = appBuilder.AddExecutable("myapp", "myapp", "."); + exe.WithReference(cluster); + + // Verify the environment callback was added (no exception thrown) + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void WithReferenceSetsKubeconfigDataEnvForContainer() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + var cluster = appBuilder.AddK3sCluster("k8s"); + var container = appBuilder.AddContainer("operator", "myorg/operator"); + container.WithReference(cluster); + + using var app = appBuilder.Build(); + Assert.NotNull(app); + } + + [Fact] + public void AddK3sClusterSetsKubeconfigDirectory() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var clusterBuilder = appBuilder.AddK3sCluster("k8s"); + + // KubeconfigDirectory is set by AddK3sCluster under AppHostDirectory/.k3s/{name}/ + Assert.NotNull(clusterBuilder.Resource.KubeconfigDirectory); + Assert.EndsWith(Path.Combine(".k3s", "k8s"), clusterBuilder.Resource.KubeconfigDirectory); + } + + [Fact] + public void WithPodSubnetAddsClusterCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithPodSubnet("10.88.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithServiceSubnetAddsServiceCidrArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithServiceSubnet("10.89.0.0/16"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithDisabledComponentAddsDisableArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithDisabledComponent("traefik"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void WithExtraArgAddsRawArg() + { + var appBuilder = DistributedApplication.CreateBuilder(); + + appBuilder.AddK3sCluster("k8s").WithExtraArg("--write-kubeconfig-mode=644"); + + using var app = appBuilder.Build(); + var appModel = app.Services.GetRequiredService(); + + Assert.Single(appModel.Resources.OfType()); + } + + [Fact] + public void ApiEndpointReturnsEndpointReferenceWithCorrectName() + { + var resource = new K3sClusterResource("k8s"); + var endpoint = resource.ApiEndpoint; + + Assert.NotNull(endpoint); + Assert.Equal(K3sClusterResource.ApiServerEndpointName, endpoint.EndpointName); + } + + [Fact] + public void ApiEndpointIsCached() + { + var resource = new K3sClusterResource("k8s"); + + var first = resource.ApiEndpoint; + var second = resource.ApiEndpoint; + + Assert.Same(first, second); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs new file mode 100644 index 000000000..f803d976f --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K3sPublicApiTests.cs @@ -0,0 +1,109 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K3sPublicApiTests +{ + [Fact] + public void AddK3sClusterShouldThrowWhenBuilderIsNull() + { + IDistributedApplicationBuilder builder = null!; + + var action = () => builder.AddK3sCluster("k8s"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void AddK3sClusterShouldThrowWhenNameIsNull() + { + IDistributedApplicationBuilder builder = new DistributedApplicationBuilder([]); + string name = null!; + + var action = () => builder.AddK3sCluster(name); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(name), exception.ParamName); + } + + [Fact] + public void WithK3sVersionShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithK3sVersion("v1.32.3-k3s1"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithPodSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithPodSubnet("10.42.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithServiceSubnetShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithServiceSubnet("10.43.0.0/16"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithDisabledComponentShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithDisabledComponent("traefik"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithExtraArgShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + + var action = () => builder.WithExtraArg("--write-kubeconfig-mode=644"); + + var exception = Assert.Throws(action); + Assert.Equal(nameof(builder), exception.ParamName); + } + + [Fact] + public void WithReferenceShouldThrowWhenDestinationIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var cluster = appBuilder.AddK3sCluster("k8s"); + IResourceBuilder destination = null!; + + var action = () => destination.WithReference(cluster); + + Assert.Throws(action); + } + + [Fact] + public void WithReferenceShouldThrowWhenSourceIsNull() + { + var appBuilder = new DistributedApplicationBuilder([]); + var container = appBuilder.AddContainer("app", "myimage"); + IResourceBuilder source = null!; + + var action = () => container.WithReference(source); + + Assert.Throws(action); + } +} diff --git a/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs new file mode 100644 index 000000000..a286f79ea --- /dev/null +++ b/tests/CommunityToolkit.Aspire.Hosting.K3s.Tests/K8sManifestResourceTests.cs @@ -0,0 +1,247 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; + +namespace CommunityToolkit.Aspire.Hosting.K3s.Tests; + +public class K8sManifestResourceTests +{ + // ── AddK8sManifest registration ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestAddsResourceWithCorrectName() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./k8s/widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Equal("widget-crd", resource.Name); + } + + [Fact] + public void AddK8sManifestStoresAbsolutePath() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crds", "./k8s/crds/"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + // Path is resolved to absolute at registration time. + Assert.True(System.IO.Path.IsPathRooted(resource.Path)); + Assert.EndsWith(System.IO.Path.Combine("k8s", "crds"), + resource.Path.TrimEnd(System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void AddK8sManifestParentIsCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Same(cluster.Resource, resource.Parent); + Assert.IsAssignableFrom>(resource); + } + + [Fact] + public void K8sManifestResourceIsContainerResource() + { + // K8sManifestResource extends ContainerResource — it runs bitnami/kubectl in Docker. + // No host-side kubectl binary required. WaitForCompletion waits for exit code 0. + var resource = new K8sManifestResource( + "crd", "./crd.yaml", new K3sClusterResource("k8s")); + + Assert.IsAssignableFrom(resource); + } + + [Fact] + public void AddK8sManifestIsExcludedFromManifest() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + Assert.Contains(ManifestPublishingCallbackAnnotation.Ignore, resource.Annotations); + } + + [Fact] + public void ClusterTracksRegisteredManifests() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("widget-crd", "./widget-crd.yaml"); + cluster.AddK8sManifest("rbac", "./rbac.yaml"); + + Assert.Contains("widget-crd", cluster.Resource.Manifests); + Assert.Contains("rbac", cluster.Resource.Manifests); + Assert.Equal(2, cluster.Resource.Manifests.Count); + } + + [Fact] + public void MultipleManifestsCanBeRegisteredOnSameCluster() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + + cluster.AddK8sManifest("crd1", "./crd1.yaml"); + cluster.AddK8sManifest("crd2", "./crd2.yaml"); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + Assert.Equal(2, model.Resources.OfType().Count()); + } + + // ── Public API null-guard tests ─────────────────────────────────────────── + + [Fact] + public void AddK8sManifestShouldThrowWhenBuilderIsNull() + { + IResourceBuilder builder = null!; + var action = () => builder.AddK8sManifest("crd", "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenNameIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest(null!, "./crd.yaml"); + Assert.Throws(action); + } + + [Fact] + public void AddK8sManifestShouldThrowWhenPathIsNull() + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + var action = () => cluster.AddK8sManifest("crd", null!); + Assert.Throws(action); + } + + // ── File resolution ─────────────────────────────────────────────────────── + + [Fact] + public void ResolveFilesSingleFile() + { + // Create a temp file to test with + var file = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}.yaml"); + File.WriteAllText(file, "apiVersion: v1\nkind: ConfigMap"); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(file); + Assert.Single(files, f => f == file); + } + finally + { + File.Delete(file); + } + } + + [Fact] + public void ResolveFilesDirectory() + { + var dir = Path.Combine(Path.GetTempPath(), $"test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "b.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "a.yaml"), ""); + File.WriteAllText(Path.Combine(dir, "c.yml"), ""); + + try + { + var files = K3sManifestBuilderExtensions.ResolveFilesForTest(dir); + + // Lexicographic order, all YAML extensions + Assert.Equal(3, files.Count); + Assert.Equal("a.yaml", Path.GetFileName(files[0])); + Assert.Equal("b.yaml", Path.GetFileName(files[1])); + Assert.Equal("c.yml", Path.GetFileName(files[2])); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } + + // ── Script generation ───────────────────────────────────────────────────── + + [Fact] + public void BuildManifestScriptIncludesKubectlApply() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kubectl apply -f /k8s-manifests", script); + Assert.Contains("--server-side", script); + } + + [Fact] + public void BuildManifestScriptAutoDetectsKustomize() + { + // The script checks for kustomization.yaml at runtime — no path argument needed. + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + Assert.Contains("kustomization.yaml", script); + Assert.Contains("kubectl apply -k /k8s-manifests", script); + } + + [Fact] + public void BuildManifestScriptWaitsForKubeconfigBeforeApplying() + { + var script = K3sManifestBuilderExtensions.BuildManifestScript(); + var kubeconfigWaitIndex = script.IndexOf("/root/.kube/kubeconfig.yaml", StringComparison.Ordinal); + var applyIndex = script.IndexOf("kubectl apply", StringComparison.Ordinal); + Assert.True(kubeconfigWaitIndex < applyIndex, "Kubeconfig wait must precede kubectl apply"); + } + + // ── Kustomize detection ─────────────────────────────────────────────────── + + [Fact] + public void AddK8sManifestKustomizeDirectoryShowsKustomizeResourceType() + { + var dir = Path.Combine(Path.GetTempPath(), $"kustomize-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "kustomization.yaml"), "resources: []"); + + try + { + var appBuilder = DistributedApplication.CreateBuilder(); + var cluster = appBuilder.AddK3sCluster("k8s"); + cluster.AddK8sManifest("kustom", dir); + + using var app = appBuilder.Build(); + var model = app.Services.GetRequiredService(); + + var resource = Assert.Single(model.Resources.OfType()); + var typeSnapshot = resource.Annotations + .OfType() + .Select(a => a.InitialSnapshot.ResourceType) + .FirstOrDefault(); + + Assert.Equal("K8s Kustomize", typeSnapshot); + } + finally + { + Directory.Delete(dir, recursive: true); + } + } +}