diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs index 3f8b40771..392be3ef3 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs @@ -9,7 +9,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; -public abstract class KubernetesAgentIntegrationTest +public abstract class KubernetesAgentIntegrationTest(int agentMajorVersion) { KubernetesAgentInstaller? kubernetesAgentInstaller; KubeCtlTool? kubeCtl; @@ -27,10 +27,12 @@ public abstract class KubernetesAgentIntegrationTest protected readonly IDictionary CustomHelmValues = new Dictionary(); - HalibutRuntime serverHalibutRuntime; + HalibutRuntime serverHalibutRuntime = null!; string? agentThumbprint; + protected int AgentMajorVersion { get; } = agentMajorVersion; + [OneTimeSetUp] public async Task OneTimeSetUp() { @@ -39,6 +41,7 @@ public async Task OneTimeSetUp() KubernetesTestsGlobalContext.Instance.HelmExePath, KubernetesTestsGlobalContext.Instance.KubeCtlExePath, KubernetesTestsGlobalContext.Instance.KubeConfigPath, + AgentMajorVersion, KubernetesTestsGlobalContext.Instance.Logger); kubeCtl = new KubeCtlTool( diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs index e47ef6df0..34af82625 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs @@ -8,7 +8,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; -public class KubernetesAgentMetricsIntegrationTest : KubernetesAgentIntegrationTest +public class KubernetesAgentMetricsIntegrationTest() : KubernetesAgentIntegrationTest(KubernetesAgentMajorVersion.Latest) { readonly ISystemLog systemLog = new SystemLog(); diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs index db24e8e1d..ebc1c6026 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs @@ -13,8 +13,9 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; -[TestFixture] -public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrationTest +[TestFixture(KubernetesAgentMajorVersion.V2)] +[TestFixture(KubernetesAgentMajorVersion.V3)] +public class KubernetesScriptServiceV1IntegrationTest(int agentMajorVersion) : KubernetesAgentIntegrationTest(agentMajorVersion) { IRecordedMethodUsages recordedMethodUsages = null!; @@ -196,7 +197,7 @@ public async Task TentaclePodIsTerminatedDuringScriptExecution_ShouldRestartAndP // Arrange var logs = new List(); var scriptCompleted = false; - const int count = 100; + const int count = 60; var semaphoreSlim = new SemaphoreSlim(0, 1); var builder = new ExecuteKubernetesScriptCommandBuilder(LoggingUtils.CurrentTestHash()) @@ -341,6 +342,11 @@ Task ScriptCompleted(CancellationToken ct) [Test] public async Task NfsPodIsTerminatedDuringNormalScriptExecution_ScriptFails() { + if (AgentMajorVersion != KubernetesAgentMajorVersion.V2) + { + Assert.Ignore("NFS is only tested against V2"); + } + // Arrange var logs = new List(); var scriptCompleted = false; @@ -395,6 +401,11 @@ Task ScriptCompleted(CancellationToken ct) [Test] public async Task NfsPodIsTerminatedDuringRawScriptExecution_ShouldRestartAndPickUpPodStatus() { + if (AgentMajorVersion != KubernetesAgentMajorVersion.V2) + { + Assert.Ignore("NFS is only tested against V2"); + } + // Arrange var logs = new List(); var scriptCompleted = false; diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMajorVersion.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMajorVersion.cs new file mode 100644 index 000000000..11bdc8b96 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMajorVersion.cs @@ -0,0 +1,9 @@ +namespace Octopus.Tentacle.Kubernetes.Tests.Integration; + +public static class KubernetesAgentMajorVersion +{ + public const int V2 = 2; + public const int V3 = 3; + + public static int Latest => V3; +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs index a0c6ae878..52ddeaf4a 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs @@ -21,9 +21,9 @@ public class KubernetesClientCompatibilityTests { static readonly object[] TestClusterVersions = [ - new object[] {new ClusterVersion(1, 35)}, - new object[] {new ClusterVersion(1, 34)}, - new object[] {new ClusterVersion(1, 33)} + new object[] { new ClusterVersion(1, 35) }, + new object[] { new ClusterVersion(1, 34) }, + new object[] { new ClusterVersion(1, 33) } ]; KubernetesTestsGlobalContext? testContext; @@ -66,6 +66,7 @@ public async Task TearDown() await cancellationTokenSource.CancelAsync(); cancellationTokenSource.Dispose(); } + clusterInstaller?.Dispose(); testContext?.Dispose(); @@ -76,11 +77,11 @@ public async Task TearDown() } [Test] - [TestCaseSource(nameof(TestClusterVersions))] - public async Task RunSimpleScript(ClusterVersion clusterVersion) + [Combinatorial] + public async Task RunSimpleScript([ValueSource(nameof(TestClusterVersions))] ClusterVersion clusterVersion, [Values(KubernetesAgentMajorVersion.V2, KubernetesAgentMajorVersion.V3)] int agentVersion) { - await SetUp(clusterVersion); - + await SetUp(clusterVersion, agentVersion); + // Arrange var logs = new List(); var scriptCompleted = false; @@ -120,11 +121,11 @@ Task ScriptCompleted(CancellationToken ct) return Task.CompletedTask; } } - - async Task SetUp(ClusterVersion clusterVersion) + + async Task SetUp(ClusterVersion clusterVersion, int agentMajorVersion) { testContext = new KubernetesTestsGlobalContext(logger); - + await SetupCluster(clusterVersion); kubernetesAgentInstaller = new KubernetesAgentInstaller( @@ -132,6 +133,7 @@ async Task SetUp(ClusterVersion clusterVersion) testContext.HelmExePath, testContext.KubeCtlExePath, testContext.KubeConfigPath, + agentMajorVersion, testContext.Logger); //create a new server halibut runtime @@ -159,7 +161,7 @@ async Task SetUp(ClusterVersion clusterVersion) recordedMethodUsages = recordedUsages; }); } - + async Task SetupCluster(ClusterVersion clusterVersion) { clusterInstaller = new KubernetesClusterInstaller(testContext.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, testContext.Logger); diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 92656fcb8..f40f1593a 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -47,10 +47,12 @@ - - + PreserveNewest + + PreserveNewest + diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs index 79de31e5f..eae908f4a 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesAgentInstaller.cs @@ -20,15 +20,17 @@ public class KubernetesAgentInstaller readonly TemporaryDirectory temporaryDirectory; readonly ILogger logger; readonly string kubeConfigPath; + readonly int agentMajorVersion; bool isAgentInstalled; - public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, ILogger logger) + public KubernetesAgentInstaller(TemporaryDirectory temporaryDirectory, string helmExePath, string kubeCtlExePath, string kubeConfigPath, int agentMajorVersion, ILogger logger) { this.temporaryDirectory = temporaryDirectory; this.helmExePath = helmExePath; this.kubeCtlExePath = kubeCtlExePath; this.kubeConfigPath = kubeConfigPath; + this.agentMajorVersion = agentMajorVersion; this.logger = logger; AgentName = Guid.NewGuid().ToString("N"); @@ -88,7 +90,7 @@ string ToHelmCommandArgs(IDictionary customValues) async Task WriteValuesFile(int listeningPort) { - using var reader = new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStreamFromPartialName("agent-values.yaml")); + using var reader = new StreamReader(Assembly.GetExecutingAssembly().GetManifestResourceStreamFromPartialName($"agent-values-v{agentMajorVersion}.yaml")); var valuesFile = await reader.ReadToEndAsync(); @@ -135,11 +137,11 @@ string BuildAgentInstallArguments(string valuesFilePath, string? tentacleImageAn return string.Join(" ", args.WhereNotNull()); } - static string GetChartVersion() + string GetChartVersion() { var customHelmChartVersion = Environment.GetEnvironmentVariable("KubernetesIntegrationTests_HelmChartVersion"); - return !string.IsNullOrWhiteSpace(customHelmChartVersion) ? customHelmChartVersion : "2.*.*"; + return !string.IsNullOrWhiteSpace(customHelmChartVersion) ? customHelmChartVersion : $"{agentMajorVersion}.*.*"; } static string? GetImageAndRepository(string? tentacleImageAndTag) diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values-v2.yaml similarity index 100% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values.yaml rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values-v2.yaml diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values-v3.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values-v3.yaml new file mode 100644 index 000000000..75d5b0dec --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/agent-values-v3.yaml @@ -0,0 +1,22 @@ +agent: + acceptEula: "Y" + name: "#{TargetName}" + serverCommsAddress: "#{ServerCommsAddress}" + serverUrl: "https://this.is.not.required.com/" + bearerToken: "this-is-a-fake-bearer-token" + space: "Default" + + deploymentTarget: + enabled: true + initial: + environments: ["development"] + tags: ["Testing Cluster", "another-testing-cluster"] + enabled: "true" + + image: + repository: docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle + +testing: + tentacle: + configMap: + data: #{ConfigMapData} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesPodTemplateService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesPodTemplateService.cs index 3f5d147f3..6bad3de14 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesPodTemplateService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesPodTemplateService.cs @@ -65,7 +65,7 @@ public KubernetesPodTemplateService(IKubernetesClientConfigProvider configProvid return scriptPodTemplate; } - public async Task GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken) + async Task GetOldestScriptPodTemplateCustomResource(CancellationToken cancellationToken) { return await RetryPolicy.ExecuteAsync(async () => { diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs index 09ec0ccb0..894a7d7ab 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesScriptPodCreator.cs @@ -230,7 +230,7 @@ async Task CreatePod(StartKubernetesScriptCommandV1 command, IScriptWorkspace wo pod.Spec.InitContainers = await CreateInitContainers(command, podName, homeDir, workspacePath, tentacleScriptLog, scriptPodTemplate?.ScriptInitContainerSpec); pod.Spec.Containers = await CreateScriptContainers(command, podName, scriptName, homeDir, workspacePath, workspace.ScriptArguments, tentacleScriptLog, scriptPodTemplate); - pod.Spec.ImagePullSecrets = imagePullSecretNames; + pod.Spec.ImagePullSecrets = Merge(imagePullSecretNames, scriptPodTemplate?.PodSpec?.ImagePullSecrets); pod.Spec.ServiceAccountName = serviceAccountName; pod.Spec.Volumes = Merge(pod.Spec.Volumes, CreateVolumes(command)); diff --git a/source/Octopus.Tentacle/Kubernetes/ScriptPodTemplate.cs b/source/Octopus.Tentacle/Kubernetes/ScriptPodTemplate.cs index 8abe007e5..2d832e11d 100644 --- a/source/Octopus.Tentacle/Kubernetes/ScriptPodTemplate.cs +++ b/source/Octopus.Tentacle/Kubernetes/ScriptPodTemplate.cs @@ -21,19 +21,19 @@ public class ScriptPodTemplate [JsonPropertyName("watchdogContainerSpec")] public V1Container? WatchdogContainerSpec { get; set; } - public static ScriptPodTemplate? GetScriptPodTemplateFromDeployment(V1Deployment deployment) + public static ScriptPodTemplate GetScriptPodTemplateFromDeployment(V1Deployment deployment) { var template = new ScriptPodTemplate { PodMetadata = new PodMetadata { - Labels = deployment.Spec.Template.Metadata.Labels.Clone(), - Annotations = deployment.Spec.Template.Metadata.Annotations.Clone(), + Labels = deployment.Spec.Template.Metadata.Labels?.Clone(), + Annotations = deployment.Spec.Template.Metadata.Annotations?.Clone(), }, PodSpec = deployment.Spec.Template.Spec.Clone(), - ScriptContainerSpec = deployment.Spec.Template.Spec.Containers.First(c => c.Name == ContainerNames.PodTemplateScriptContainerName).Clone(), - ScriptInitContainerSpec = deployment.Spec.Template.Spec.Containers.First(c => c.Name == ContainerNames.PodTemplateScriptContainerName).Clone(), - WatchdogContainerSpec = deployment.Spec.Template.Spec.Containers.First(c => c.Name == ContainerNames.PodTemplateWatchdogContainerName).Clone() + ScriptContainerSpec = deployment.Spec.Template.Spec.Containers.FirstOrDefault(c => c.Name == ContainerNames.PodTemplateScriptContainerName)?.Clone(), + ScriptInitContainerSpec = deployment.Spec.Template.Spec.Containers.FirstOrDefault(c => c.Name == ContainerNames.PodTemplateScriptContainerName)?.Clone(), + WatchdogContainerSpec = deployment.Spec.Template.Spec.Containers.FirstOrDefault(c => c.Name == ContainerNames.PodTemplateWatchdogContainerName)?.Clone() }; // The deployment will have the containers, we should not pull them in here though - we overwrite them and programatically create them later @@ -43,15 +43,15 @@ public class ScriptPodTemplate return template; } - public static ScriptPodTemplate? GetScriptPodTemplateFromCustomResource(ScriptPodTemplateCustomResource scriptPodTemplateCustomResource) + public static ScriptPodTemplate GetScriptPodTemplateFromCustomResource(ScriptPodTemplateCustomResource scriptPodTemplateCustomResource) { var template = new ScriptPodTemplate { - PodMetadata = scriptPodTemplateCustomResource.Spec.PodMetadata.Clone(), - PodSpec = scriptPodTemplateCustomResource.Spec.PodSpec.Clone(), - ScriptContainerSpec = scriptPodTemplateCustomResource.Spec.ScriptContainerSpec.Clone(), - ScriptInitContainerSpec = scriptPodTemplateCustomResource.Spec.ScriptContainerSpec.Clone(), - WatchdogContainerSpec = scriptPodTemplateCustomResource.Spec.WatchdogContainerSpec.Clone() + PodMetadata = scriptPodTemplateCustomResource.Spec.PodMetadata?.Clone(), + PodSpec = scriptPodTemplateCustomResource.Spec.PodSpec?.Clone(), + ScriptContainerSpec = scriptPodTemplateCustomResource.Spec.ScriptContainerSpec?.Clone(), + ScriptInitContainerSpec = scriptPodTemplateCustomResource.Spec.ScriptContainerSpec?.Clone(), + WatchdogContainerSpec = scriptPodTemplateCustomResource.Spec.WatchdogContainerSpec?.Clone() }; return template; }