diff --git a/Justfile b/Justfile index 37cb4e689..a18d9cdf7 100644 --- a/Justfile +++ b/Justfile @@ -292,6 +292,19 @@ run-homelab-restore: argo submit --from workflowtemplate/homelab-restore-drill \ -n {{ argo_ns }} --wait --log +# ── Service-catalog workload validation (#51) ──────────────────────────────── + +# Run service-catalog pipeline for a given lane (default: media) +# Usage: just run-service-catalog-smoke +# Usage: just run-service-catalog-smoke lane=non-media +# Usage: just run-service-catalog-smoke lane=media image-tag=lts branch=feat/my-branch +run-service-catalog-smoke lane="media" image-tag="latest" branch="main": + argo submit --from workflowtemplate/service-catalog-pipeline \ + -p lane={{ lane }} \ + -p image-tag={{ image-tag }} \ + -p branch={{ branch }} \ + -n {{ argo_ns }} --wait --log + # ── Ghost maintenance ───────────────────────────────────────────────────────── # Patch ghost OTel collector config to remove noisy process scraper (#117) diff --git a/RUNBOOK.md b/RUNBOOK.md index 4f4f34a85..7c3c42ff8 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -84,6 +84,45 @@ Golden disks can be patched by workflow after key rotation; titan disk key refre - `findChild(..., requireResult=...)` is not a supported dogtail pattern in this repo's stack. - `findChildren(...)` and `findChild(..., retry=False)` are the canonical presence-check APIs. +## Service-catalog pipeline + +A second execution path validates homelab service workloads directly in +Kubernetes, without VMs or GNOME infrastructure: + +``` +Argo Workflow (argo namespace) + │ + ├─ create-namespace ─► ephemeral namespace svc-- + ├─ deploy-workload ─► clone repo, kubectl apply lane manifests + ├─ run-service-tests ─► pytest against live k8s workload + └─ cleanup (onExit) ─► delete namespace +``` + +| Property | Value | +|---|---| +| Entry point | `just run-service-catalog-smoke` or `argo submit --from workflowtemplate/service-catalog-pipeline` | +| Parameters | `lane` (default: media), `image-tag` (default: latest), `branch` (default: main) | +| Wall-clock | ~3–5 min | +| Evidence | pytest JUnit XML + per-check artifacts in pod logs (Loki) | + +### Adding a new lane + +1. Create `tests/service_catalog//manifests.yaml` with the workload's + Kubernetes manifests (Deployment, Service, PVC, Secrets, ConfigMaps). +2. Create `tests/service_catalog//test_.py` importing helpers + from `tests/service_catalog/shared/` (deploy, persistence, reachability, + redeploy, teardown). +3. Run: `just run-service-catalog-smoke lane=` +4. The pipeline creates a namespace, applies manifests, runs tests, cleans up. + +### Inspecting results + +- `argo logs @latest` — test output including summary and artifact list. +- Loki query: `{namespace="argo"} |= "svc-catalog"` — all service-catalog + workflow logs. +- JUnit XML is emitted to the pod stdout and captured by Argo's log + archival. No separate artifact upload path is needed. + ## Common failure modes | Symptom | Root cause | Durable fix | @@ -96,6 +135,9 @@ Golden disks can be patched by workflow after key rotation; titan disk key refre | VM stuck `Terminating` | KubeVirt controller race with launcher cleanup | Delete the `virt-launcher-*` pod and let reconciliation finish | | `run-gnome-tests` pod fails at startup | Workflow template structure error, often misplaced `volumes:` | Fix the template in git and let ArgoCD reconcile it | | WorkflowTemplate change appears ignored | Workflow was submitted before the new template was reconciled | Verify ArgoCD revision, wait or sync, then submit a new workflow | +| Service-catalog deploy step fails with "No manifests found" | Lane directory missing `manifests.yaml` | Create `tests/service_catalog//manifests.yaml` per the contract | +| Service-catalog test step fails with "No test suite" | Lane directory missing under `tests/service_catalog/` | Create the lane test directory with at least one `test_*.py` file | +| Service-catalog namespace stuck terminating | Finalizer or PVC not released | Check for stuck PVCs or pods with `kubectl get all -n `, delete manually if needed | ## Historical notes diff --git a/WORKFLOWS.md b/WORKFLOWS.md index a0074da42..5f2bfec4c 100644 --- a/WORKFLOWS.md +++ b/WORKFLOWS.md @@ -99,6 +99,40 @@ selinux=0, sudoers) on an existing golden disk without rebuilding it. |---|---|---| | `image-tag` | `latest` | Disk dir under `/var/tmp/bluefin-golden/`. | +### `service-catalog-pipeline` + +K8s-native service-catalog validation pipeline. Deploys a lane's workload +manifests into an ephemeral namespace, runs the lane's pytest suite, and +tears down on exit. Does not use VMs or GNOME infrastructure — runs +directly against k8s-hosted workloads. + +| Parameter | Default | Notes | +|---|---|---| +| `lane` | `media` | Lane name — must match a directory under `tests/service_catalog//` and `tests/service_catalog//manifests.yaml` | +| `image-tag` | `latest` | Passed to lane manifests (available for future per-lane image selection) | +| `branch` | `main` | testing-lab branch to clone for manifests and tests | + +Wall-clock: ~3–5 min (depends on image pull and test count). + +``` +just run-service-catalog-smoke # media lane, latest +just run-service-catalog-smoke lane=non-media # non-media lane +just run-service-catalog-smoke lane=media branch=feat/my-branch +``` + +Pipeline structure: +``` +create-namespace → deploy-workload → run-tests → cleanup (onExit) +``` + +The deploy step reads `tests/service_catalog//manifests.yaml` from +the cloned repo and applies it to the ephemeral namespace. Each lane owns +its own manifests — the pipeline is lane-agnostic. + +Test runner: `run-service-tests` (see supporting templates below). Uses +the shared helpers in `tests/service_catalog/shared/` for deployment, +persistence, reachability, redeploy, and teardown assertions. + --- ## Supporting templates (called via `templateRef`) @@ -106,6 +140,18 @@ selinux=0, sudoers) on an existing golden disk without rebuilding it. These are exposed only because they are referenced by the entry points; submit them directly only for diagnosis. +### `run-service-tests` (template: `run-pytest`) + +Non-GNOME test runner for service-catalog lanes. Clones testing-lab, +discovers the test suite under `tests/service_catalog//`, and runs +pytest with JUnit XML output. Emits a summary line (`N/M pytest checks +passed`) to stdout for Argo/Loki consumption. + +Env vars passed to the test container: `TEST_NAMESPACE`, `TEST_LANE`, +`TEST_RESULTS_DIR`. Lane-specific tests import shared helpers from +`tests/service_catalog/shared/` (deploy, persistence, reachability, +redeploy, teardown). + ### `bib-build-and-push` (template: `ensure-disk`) Builds the golden raw disk via `bootc-image-builder` if missing or stale. diff --git a/argo/service-catalog-smoke.yaml b/argo/service-catalog-smoke.yaml new file mode 100644 index 000000000..fc4cb36e1 --- /dev/null +++ b/argo/service-catalog-smoke.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: svc-catalog- + namespace: argo +spec: + serviceAccountName: argo + workflowTemplateRef: + name: service-catalog-pipeline + arguments: + parameters: + - name: lane + value: "media" + - name: image-tag + value: "latest" + - name: branch + value: "main" diff --git a/argo/workflow-templates/run-service-tests.yaml b/argo/workflow-templates/run-service-tests.yaml new file mode 100644 index 000000000..4df9dd445 --- /dev/null +++ b/argo/workflow-templates/run-service-tests.yaml @@ -0,0 +1,126 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: run-service-tests + namespace: argo + annotations: + bluefin.io/stack: "k8s workload + pytest" +spec: + templates: + - name: run-pytest + metadata: + labels: + app.kubernetes.io/part-of: bluefin-test-suite + bluefin.io/step: run-service-tests + bluefin.io/lane: "svc-{{inputs.parameters.lane}}" + inputs: + parameters: + - name: namespace + - name: lane + - name: branch + value: "main" + activeDeadlineSeconds: 1800 + initContainers: + - name: git-sync + image: quay.io/fedora/fedora:latest + env: + - name: BRANCH + value: "{{inputs.parameters.branch}}" + command: [bash, -c] + args: + - | + set -euo pipefail + dnf install -y --quiet git 2>&1 | tail -2 + for attempt in 1 2 3; do + rm -rf /workspace/testing-lab + if git clone --depth 1 --branch "${BRANCH}" \ + https://github.com/projectbluefin/testing-lab \ + /workspace/testing-lab 2>&1; then + echo "Synced testing-lab @ ${BRANCH} (attempt ${attempt})" + exit 0 + fi + rm -rf /workspace/testing-lab + if git clone https://github.com/projectbluefin/testing-lab /workspace/testing-lab 2>&1 && \ + git -C /workspace/testing-lab checkout "${BRANCH}" 2>&1; then + echo "Synced testing-lab @ ${BRANCH} via SHA fallback (attempt ${attempt})" + exit 0 + fi + echo "Attempt ${attempt} failed, retrying in 15s..." + sleep 15 + done + exit 1 + volumeMounts: + - name: workspace + mountPath: /workspace + container: + image: quay.io/fedora/fedora:latest + imagePullPolicy: Always + resources: + requests: + cpu: "500m" + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + env: + - name: TEST_NAMESPACE + value: "{{inputs.parameters.namespace}}" + - name: TEST_LANE + value: "{{inputs.parameters.lane}}" + - name: TEST_RESULTS_DIR + value: /tmp/results + volumeMounts: + - name: workspace + mountPath: /workspace + readOnly: true + command: [bash, -c] + args: + - | + set -euo pipefail + mkdir -p /tmp/results + LOG=/tmp/results/runner.log + exec > >(tee -a "${LOG}") 2>&1 + + dnf install -y --quiet python3 python3-pytest kubernetes-client jq curl openssl 2>&1 | tail -4 + + LANE="{{inputs.parameters.lane}}" + SUITE="/workspace/testing-lab/tests/service_catalog/${LANE}" + + if [[ ! -d "${SUITE}" ]]; then + echo "ERROR: No test suite at ${SUITE}" >&2 + exit 1 + fi + + export PYTHONPATH="/workspace/testing-lab:${PYTHONPATH:-}" + + TEST_RC=0 + python3 -m pytest -q "${SUITE}" \ + --junitxml /tmp/results/pytest-results.xml || TEST_RC=$? + + echo "Collected result files:" >&2 + find /tmp/results -maxdepth 1 -type f -printf '%f\n' | sort >&2 + + if [[ -f /tmp/results/pytest-results.xml ]]; then + SUMMARY="$(python3 -c " + import xml.etree.ElementTree as ET + root = ET.parse('/tmp/results/pytest-results.xml').getroot() + suites = [root] if root.tag == 'testsuite' else root.findall('testsuite') + tests = sum(int(s.attrib.get('tests', 0)) for s in suites) + skipped = sum(int(s.attrib.get('skipped', 0)) for s in suites) + failures = sum(int(s.attrib.get('failures', 0)) + int(s.attrib.get('errors', 0)) for s in suites) + passed = tests - skipped - failures + print(f'{passed}/{tests} pytest checks passed') + " 2>/dev/null || true)" + if [[ -n "${SUMMARY}" ]]; then + printf '%s\n' "${SUMMARY}" + else + echo "pytest results present" + fi + else + echo "pytest results missing" + fi + + exit "${TEST_RC}" + volumes: + - name: workspace + emptyDir: {} diff --git a/argo/workflow-templates/service-catalog-pipeline.yaml b/argo/workflow-templates/service-catalog-pipeline.yaml new file mode 100644 index 000000000..db079907d --- /dev/null +++ b/argo/workflow-templates/service-catalog-pipeline.yaml @@ -0,0 +1,136 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: service-catalog-pipeline + namespace: argo + labels: + app.kubernetes.io/component: service-catalog + app.kubernetes.io/part-of: bluefin-test-suite +spec: + entrypoint: pipeline + onExit: cleanup + serviceAccountName: homelab-runner + arguments: + parameters: + - name: lane + value: "media" + - name: image-tag + value: "latest" + - name: branch + value: "main" + templates: + - name: pipeline + dag: + tasks: + - name: create-namespace + template: create-namespace + arguments: + parameters: + - name: namespace + value: "svc-{{workflow.parameters.lane}}-{{workflow.uid}}" + - name: lane + value: "{{workflow.parameters.lane}}" + - name: deploy-workload + depends: "create-namespace.Succeeded" + template: deploy-workload + arguments: + parameters: + - name: namespace + value: "svc-{{workflow.parameters.lane}}-{{workflow.uid}}" + - name: lane + value: "{{workflow.parameters.lane}}" + - name: image-tag + value: "{{workflow.parameters.image-tag}}" + - name: branch + value: "{{workflow.parameters.branch}}" + - name: run-tests + depends: "deploy-workload.Succeeded" + templateRef: + name: run-service-tests + template: run-pytest + arguments: + parameters: + - name: namespace + value: "svc-{{workflow.parameters.lane}}-{{workflow.uid}}" + - name: lane + value: "{{workflow.parameters.lane}}" + - name: branch + value: "{{workflow.parameters.branch}}" + + - name: create-namespace + inputs: + parameters: + - name: namespace + - name: lane + resource: + action: create + manifest: | + apiVersion: v1 + kind: Namespace + metadata: + name: "{{inputs.parameters.namespace}}" + labels: + app.kubernetes.io/part-of: bluefin-test-suite + bluefin.io/lane: "svc-{{inputs.parameters.lane}}" + + - name: deploy-workload + inputs: + parameters: + - name: namespace + - name: lane + - name: image-tag + - name: branch + script: + image: quay.io/fedora/fedora:latest + command: [bash, -c] + source: | + set -euo pipefail + NS="{{inputs.parameters.namespace}}" + LANE="{{inputs.parameters.lane}}" + BRANCH="{{inputs.parameters.branch}}" + + dnf install -y --quiet git kubernetes-client 2>&1 | tail -4 + + REPO_DIR=$(mktemp -d) + git clone --depth 1 --branch "${BRANCH}" \ + https://github.com/projectbluefin/testing-lab "${REPO_DIR}" 2>&1 + + MANIFEST="${REPO_DIR}/tests/service_catalog/${LANE}/manifests.yaml" + if [[ ! -f "${MANIFEST}" ]]; then + echo "ERROR: No manifests found at ${MANIFEST}" >&2 + echo "Each lane must provide tests/service_catalog//manifests.yaml" >&2 + exit 1 + fi + + echo "Deploying lane=${LANE} into namespace=${NS}" >&2 + kubectl apply -n "${NS}" -f "${MANIFEST}" >&2 + + DEPLOYMENT=$(kubectl get deployments -n "${NS}" \ + -l "bluefin.io/lane=svc-${LANE}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + + if [[ -z "${DEPLOYMENT}" ]]; then + DEPLOYMENT=$(kubectl get deployments -n "${NS}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + fi + + if [[ -n "${DEPLOYMENT}" ]]; then + echo "Waiting for deployment/${DEPLOYMENT} in ${NS}..." >&2 + kubectl rollout status "deployment/${DEPLOYMENT}" \ + -n "${NS}" --timeout=300s >&2 + else + echo "WARNING: No deployment found; lane may use a different workload type" >&2 + fi + + echo "deploy-complete" + + - name: cleanup + script: + image: cgr.dev/chainguard/kubectl:latest-dev + command: [bash, -c] + source: | + set -euo pipefail + NS="svc-{{workflow.parameters.lane}}-{{workflow.uid}}" + kubectl delete namespace "${NS}" --ignore-not-found=true >&2 || true + timeout 180 bash -c "until ! kubectl get namespace \"${NS}\" >/dev/null 2>&1; do sleep 5; done" >&2 || true + echo "cleanup-complete" diff --git a/tests/service_catalog/shared/deploy.py b/tests/service_catalog/shared/deploy.py new file mode 100644 index 000000000..833698f0b --- /dev/null +++ b/tests/service_catalog/shared/deploy.py @@ -0,0 +1,86 @@ +""" +Deployment assertion helpers for service-catalog lanes. + +Validates that a workload deployed into the test namespace reaches +a ready state and has the expected structure (Deployment, Service, +endpoints). Corresponds to §2 of the service-catalog contract (#66). +""" + +from __future__ import annotations + +import json + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + TEST_LANE, + require_kubectl, + run_kubectl, + write_artifact, +) + + +def get_deployment(name: str) -> dict: + raw = require_kubectl("get", "deployment", name, "-n", NAMESPACE, "-o", "json") + write_artifact(f"{TEST_LANE}-deployment.json", raw) + return json.loads(raw) + + +def assert_deployment_ready(name: str) -> dict: + data = get_deployment(name) + status = data.get("status", {}) + available = status.get("availableReplicas", 0) + ready = status.get("readyReplicas", 0) + assert available >= 1, ( + f"Deployment {name} has {available} available replicas, expected >= 1" + ) + assert ready >= 1, ( + f"Deployment {name} has {ready} ready replicas, expected >= 1" + ) + return data + + +def get_pods(app_label: str) -> dict: + raw = require_kubectl("get", "pods", "-n", NAMESPACE, "-l", app_label, "-o", "json") + write_artifact(f"{TEST_LANE}-pods.json", raw) + return json.loads(raw) + + +def assert_pod_running(app_label: str) -> dict: + data = get_pods(app_label) + items = data.get("items", []) + assert items, f"No pods found with label {app_label} in {NAMESPACE}" + running = [ + p for p in items + if p.get("status", {}).get("phase") == "Running" + ] + assert running, ( + f"No pods in Running phase for {app_label}; " + f"phases: {[p.get('status', {}).get('phase') for p in items]}" + ) + return data + + +def get_endpoints(service_name: str) -> dict: + raw = require_kubectl("get", "endpoints", service_name, "-n", NAMESPACE, "-o", "json") + write_artifact(f"{TEST_LANE}-endpoints.json", raw) + return json.loads(raw) + + +def assert_service_has_endpoints(service_name: str) -> dict: + data = get_endpoints(service_name) + subsets = data.get("subsets") or [] + addresses = [addr for s in subsets for addr in s.get("addresses", [])] + assert addresses, ( + f"Service {service_name} has no endpoint addresses in {NAMESPACE}" + ) + return data + + +def capture_events() -> str: + result = run_kubectl( + "get", "events", "-n", NAMESPACE, + "--sort-by=.lastTimestamp", + ) + output = result.stdout + result.stderr + write_artifact(f"{TEST_LANE}-events.txt", output) + return output diff --git a/tests/service_catalog/shared/persistence.py b/tests/service_catalog/shared/persistence.py new file mode 100644 index 000000000..48d1a9b2a --- /dev/null +++ b/tests/service_catalog/shared/persistence.py @@ -0,0 +1,104 @@ +""" +Persistence assertion helpers for service-catalog lanes. + +Validates PVC lifecycle and data survival across rollout restarts. +Corresponds to §3 of the service-catalog contract (#66). +""" + +from __future__ import annotations + +import json +import uuid + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + TEST_LANE, + require_kubectl, + run_kubectl, + write_artifact, +) + + +def assert_pvc_bound(pvc_name: str) -> dict: + raw = require_kubectl("get", "pvc", pvc_name, "-n", NAMESPACE, "-o", "json") + write_artifact(f"{TEST_LANE}-pvc.json", raw) + data = json.loads(raw) + phase = data.get("status", {}).get("phase", "") + assert phase == "Bound", f"PVC {pvc_name} phase is {phase}, expected Bound" + return data + + +def write_sentinel(pod_name: str, mount_path: str) -> str: + sentinel_value = f"sentinel-{uuid.uuid4().hex[:12]}" + sentinel_path = f"{mount_path}/.sentinel" + result = run_kubectl( + "exec", pod_name, "-n", NAMESPACE, "--", + "sh", "-c", f'echo "{sentinel_value}" > "{sentinel_path}"', + ) + assert result.returncode == 0, ( + f"Failed to write sentinel to {pod_name}:{sentinel_path}: {result.stderr}" + ) + write_artifact(f"{TEST_LANE}-sentinel-write.txt", f"{sentinel_value}\n") + return sentinel_value + + +def read_sentinel(pod_name: str, mount_path: str) -> str: + sentinel_path = f"{mount_path}/.sentinel" + result = run_kubectl( + "exec", pod_name, "-n", NAMESPACE, "--", + "cat", sentinel_path, + ) + assert result.returncode == 0, ( + f"Failed to read sentinel from {pod_name}:{sentinel_path}: {result.stderr}" + ) + value = result.stdout.strip() + write_artifact(f"{TEST_LANE}-sentinel-read.txt", f"{value}\n") + return value + + +def get_pod_uids(app_label: str) -> set[str]: + raw = require_kubectl("get", "pods", "-n", NAMESPACE, "-l", app_label, "-o", "json") + data = json.loads(raw) + return {item["metadata"]["uid"] for item in data.get("items", [])} + + +def rollout_restart(deployment_name: str) -> None: + pods_before = require_kubectl( + "get", "pods", "-n", NAMESPACE, "-o", "json", + ) + write_artifact(f"{TEST_LANE}-pods-before.json", pods_before) + + restart = run_kubectl( + "rollout", "restart", f"deployment/{deployment_name}", "-n", NAMESPACE, + ) + assert restart.returncode == 0, ( + f"rollout restart failed: {restart.stdout}{restart.stderr}" + ) + write_artifact(f"{TEST_LANE}-restart.txt", restart.stdout + restart.stderr) + + status = run_kubectl( + "rollout", "status", f"deployment/{deployment_name}", + "-n", NAMESPACE, "--timeout=300s", + ) + assert status.returncode == 0, ( + f"rollout status failed: {status.stdout}{status.stderr}" + ) + write_artifact(f"{TEST_LANE}-rollout-status.txt", status.stdout + status.stderr) + + pods_after = require_kubectl( + "get", "pods", "-n", NAMESPACE, "-o", "json", + ) + write_artifact(f"{TEST_LANE}-pods-after.json", pods_after) + + +def assert_restart_changes_pods(deployment_name: str, app_label: str) -> None: + before_uids = get_pod_uids(app_label) + assert before_uids, f"No pods found before restart for {app_label}" + + rollout_restart(deployment_name) + + after_uids = get_pod_uids(app_label) + assert after_uids, f"No pods found after restart for {app_label}" + assert before_uids != after_uids, ( + f"Pod UIDs did not change across restart: {sorted(before_uids)}" + ) diff --git a/tests/service_catalog/shared/reachability.py b/tests/service_catalog/shared/reachability.py new file mode 100644 index 000000000..393421c6a --- /dev/null +++ b/tests/service_catalog/shared/reachability.py @@ -0,0 +1,82 @@ +""" +Reachability assertion helpers for service-catalog lanes. + +Validates cluster DNS resolution, endpoint presence, and HTTP(S) +responses. Corresponds to §4 of the service-catalog contract (#66). +""" + +from __future__ import annotations + +import subprocess +import urllib.request + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + TEST_LANE, + write_artifact, +) + +TIMEOUT_SECONDS = 30 +HTTP_TIMEOUT_SECONDS = 15 + + +def run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(args, capture_output=True, text=True, timeout=TIMEOUT_SECONDS) + + +def service_fqdn(service_name: str) -> str: + return f"{service_name}.{NAMESPACE}.svc.cluster.local" + + +def assert_dns_resolves(service_name: str) -> str: + fqdn = service_fqdn(service_name) + result = run("getent", "hosts", fqdn) + write_artifact(f"{TEST_LANE}-dns.txt", result.stdout + result.stderr) + assert result.returncode == 0, f"DNS resolution failed for {fqdn}: {result.stderr}" + return result.stdout.strip() + + +def assert_http_reachable(service_name: str, port: int = 80, path: str = "/") -> str: + fqdn = service_fqdn(service_name) + url = f"http://{fqdn}:{port}{path}" + try: + with urllib.request.urlopen(url, timeout=HTTP_TIMEOUT_SECONDS) as resp: + body = resp.read().decode() + write_artifact(f"{TEST_LANE}-http.txt", body) + return body + except Exception as exc: + write_artifact(f"{TEST_LANE}-http.txt", f"ERROR: {exc}") + raise AssertionError(f"HTTP request to {url} failed: {exc}") from exc + + +def assert_https_reachable( + service_name: str, + hostname: str, + port: int = 8443, + path: str = "/healthz", +) -> str: + fqdn = service_fqdn(service_name) + result = run( + "curl", "-sk", + "-H", f"Host: {hostname}", + f"https://{fqdn}:{port}{path}", + ) + combined = result.stdout + result.stderr + write_artifact(f"{TEST_LANE}-https.txt", combined) + assert result.returncode == 0, f"HTTPS request failed: {result.stderr}" + return result.stdout.strip() + + +def assert_tls_handshake(service_name: str, hostname: str, port: int = 8443) -> str: + fqdn = service_fqdn(service_name) + result = run( + "openssl", "s_client", + "-connect", f"{fqdn}:{port}", + "-servername", hostname, + "-brief", + ) + combined = result.stdout + result.stderr + write_artifact(f"{TEST_LANE}-tls-handshake.txt", combined) + assert result.returncode == 0, f"TLS handshake failed: {combined}" + assert "Protocol version" in combined, f"No protocol version in TLS output: {combined}" + return combined diff --git a/tests/service_catalog/shared/redeploy.py b/tests/service_catalog/shared/redeploy.py new file mode 100644 index 000000000..0e43cd5c0 --- /dev/null +++ b/tests/service_catalog/shared/redeploy.py @@ -0,0 +1,76 @@ +""" +Redeploy / upgrade assertion helpers for service-catalog lanes. + +Validates that a workload can be upgraded by changing image tag or +config and re-applying manifests. Corresponds to §5 of the +service-catalog contract (#66). +""" + +from __future__ import annotations + +import json + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + TEST_LANE, + require_kubectl, + run_kubectl, + write_artifact, +) + + +def capture_deployment_state(deployment_name: str, suffix: str) -> dict: + raw = require_kubectl( + "get", "deployment", deployment_name, "-n", NAMESPACE, "-o", "json", + ) + write_artifact(f"{TEST_LANE}-upgrade-{suffix}.json", raw) + return json.loads(raw) + + +def set_image(deployment_name: str, container_name: str, new_image: str) -> None: + result = run_kubectl( + "set", "image", + f"deployment/{deployment_name}", + f"{container_name}={new_image}", + "-n", NAMESPACE, + ) + assert result.returncode == 0, ( + f"set image failed: {result.stdout}{result.stderr}" + ) + write_artifact(f"{TEST_LANE}-upgrade-apply.txt", result.stdout + result.stderr) + + +def wait_for_rollout(deployment_name: str) -> None: + result = run_kubectl( + "rollout", "status", f"deployment/{deployment_name}", + "-n", NAMESPACE, "--timeout=300s", + ) + assert result.returncode == 0, ( + f"rollout status failed: {result.stdout}{result.stderr}" + ) + write_artifact(f"{TEST_LANE}-upgrade-rollout.txt", result.stdout + result.stderr) + + +def get_running_image(deployment_name: str, app_label: str) -> str: + raw = require_kubectl( + "get", "pods", "-n", NAMESPACE, "-l", app_label, + "-o", "jsonpath={.items[0].spec.containers[0].image}", + ) + write_artifact(f"{TEST_LANE}-upgrade-image.txt", raw) + return raw.strip() + + +def assert_image_upgrade( + deployment_name: str, + container_name: str, + new_image: str, + app_label: str, +) -> None: + capture_deployment_state(deployment_name, "before") + set_image(deployment_name, container_name, new_image) + wait_for_rollout(deployment_name) + capture_deployment_state(deployment_name, "after") + running = get_running_image(deployment_name, app_label) + assert running == new_image, ( + f"Running image {running} does not match expected {new_image}" + ) diff --git a/tests/service_catalog/shared/teardown.py b/tests/service_catalog/shared/teardown.py new file mode 100644 index 000000000..363c7fc35 --- /dev/null +++ b/tests/service_catalog/shared/teardown.py @@ -0,0 +1,44 @@ +""" +Teardown assertion helpers for service-catalog lanes. + +Provides namespace cleanup verification. The actual teardown is +performed by the Argo Workflow onExit handler, not by test code. +These helpers allow tests to verify pre-conditions about namespace +state. Corresponds to §6 of the service-catalog contract (#66). +""" + +from __future__ import annotations + +import json + +from tests.service_catalog.shared.kube import ( + NAMESPACE, + TEST_LANE, + run_kubectl, + write_artifact, +) + + +def assert_namespace_exists() -> dict: + result = run_kubectl("get", "namespace", NAMESPACE, "-o", "json") + assert result.returncode == 0, ( + f"Namespace {NAMESPACE} does not exist: {result.stderr}" + ) + data = json.loads(result.stdout) + write_artifact(f"{TEST_LANE}-namespace.json", result.stdout) + return data + + +def assert_no_cluster_scoped_leaks(label_selector: str) -> None: + for kind in ("clusterrole", "clusterrolebinding"): + result = run_kubectl("get", kind, "-l", label_selector, "-o", "json") + if result.returncode != 0: + continue + data = json.loads(result.stdout) + items = data.get("items", []) + names = [item["metadata"]["name"] for item in items] + write_artifact(f"{TEST_LANE}-{kind}-leak-check.json", result.stdout) + assert not names, ( + f"Found cluster-scoped {kind} resources with label {label_selector}: {names}. " + f"These must be cleaned up in the onExit handler." + )