diff --git a/.github/workflows/lxc-test.yml b/.github/workflows/lxc-test.yml index c0ddc1e..e22eaf1 100644 --- a/.github/workflows/lxc-test.yml +++ b/.github/workflows/lxc-test.yml @@ -45,8 +45,9 @@ jobs: run: | i=1 while IFS= read -r line || [ -n "$line" ]; do - [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue - echo "cmd${i}=${line}" >> "$GITHUB_OUTPUT" + trimmed=$(echo "$line" | xargs) + [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue + echo "cmd${i}=${trimmed}" >> "$GITHUB_OUTPUT" i=$((i+1)) done <<< "$CMLXC_COMMANDS" echo "count=$((i-1))" >> "$GITHUB_OUTPUT" @@ -69,7 +70,6 @@ jobs: repository: chatmail/cmlxc ref: ${{ inputs.cmlxc_version }} path: cmlxc - fetch-depth: 0 persist-credentials: false - name: Install Incus (Zabbly) @@ -78,7 +78,7 @@ jobs: sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list sudo apt-get update - sudo apt-get install -y incus=1:7.2* incus-client=1:7.2* incus-base=1:7.2* + sudo apt-get install -y incus-client=1:7.2* incus-base=1:7.2* incus --version - name: Initialise Incus @@ -87,6 +87,10 @@ jobs: sudo iptables -P FORWARD ACCEPT sudo sysctl -w fs.inotify.max_user_instances=65535 sudo sysctl -w fs.inotify.max_user_watches=65535 + # Disable AppArmor restrictions so Docker-in-LXC containers + # can run systemd (needs cgroup notification socket access). + sudo systemctl stop apparmor || true + sudo apparmor_parser -R /etc/apparmor.d/* 2>/dev/null || true sudo incus admin init --auto sudo chmod 666 /var/lib/incus/unix.socket sudo chown -R $USER:$USER ~/.config/incus @@ -104,17 +108,19 @@ jobs: uv tool install ./cmlxc uv --color never tool dir --bin >> "$GITHUB_PATH" - - name: Cache Incus images + - name: Restore Incus image cache id: cache-images - uses: actions/cache@v6 + uses: actions/cache/restore@v5 with: path: /tmp/incus-cache key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }} + restore-keys: | + incus-v5-${{ runner.os }}- - name: Import cached images run: | mkdir -p /tmp/incus-cache - for alias in localchat-base localchat-builder; do + for alias in localchat-base localchat-builder localchat-cmdeploy localchat-docker; do if [ -f /tmp/incus-cache/$alias.tar.gz ]; then echo "Importing: $alias" incus image import /tmp/incus-cache/$alias.tar.gz --alias $alias || true @@ -177,12 +183,12 @@ jobs: set -eu i=0 while IFS= read -r cmd || [ -n "$cmd" ]; do - [[ "$cmd" =~ ^[[:space:]]*(#|$) ]] && continue + trimmed=$(echo "$cmd" | xargs) + [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue i=$((i+1)) if [ $i -le 12 ]; then continue; fi - - echo "::group::Run: $cmd" - eval "$cmd" || { echo "::endgroup::"; exit 1; } + echo "::group::Run: $trimmed" + eval "$trimmed" || { echo "::endgroup::"; exit 1; } echo "::endgroup::" done <<< "$CMLXC_COMMANDS" @@ -191,7 +197,23 @@ jobs: run: | for c in $(incus list -c n --format csv); do echo "::group::Logs for $c" - incus exec "$c" -- journalctl -p warning --no-pager -n 100 || true + incus exec "$c" -- journalctl --no-pager -n 200 || true + # Dump Docker container logs if present + svc=chatmail + if incus exec "$c" -- docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "$svc"; then + echo "--- docker logs $svc ---" + incus exec "$c" -- docker logs "$svc" --tail 200 2>&1 || true + echo "--- dovecot journal ---" + incus exec "$c" -- docker exec "$svc" journalctl -u dovecot --no-pager -n 50 2>&1 || true + echo "--- postfix journal ---" + incus exec "$c" -- docker exec "$svc" journalctl -u postfix --no-pager -n 50 2>&1 || true + echo "--- failed units ---" + incus exec "$c" -- docker exec "$svc" systemctl --failed --no-pager 2>&1 || true + echo "--- dovecot -n (effective config) ---" + incus exec "$c" -- docker exec "$svc" dovecot -n 2>&1 | tail -40 || true + echo "--- ssl cert check ---" + incus exec "$c" -- docker exec "$svc" ls -la /etc/ssl/certs/mailserver.pem /etc/ssl/private/mailserver.key 2>&1 || true + fi echo "::endgroup::" done @@ -199,18 +221,49 @@ jobs: if: always() && steps.cache-images.outputs.cache-hit != 'true' run: | mkdir -p /tmp/incus-cache + # Publish the builder LXC container as a cached image (the Docker + # container inside gets recreated on compose up, so the LXC is clean). + # Only skip localchat-cmdeploy on failure -- it bakes deploy state + # directly into the LXC and would carry broken config into the next run. if incus list -c n --format csv | grep -q builder-localchat; then - incus exec builder-localchat -- rm -rf /root/relays /root/minitest-venv /root/.ssh/config* + echo "Cleaning up builder container before publishing ..." + incus exec builder-localchat -- bash -c 'rm -rf /root/relays/* /root/.cache/* /root/.npm /root/.bun' echo "Publishing builder container as image ..." incus publish builder-localchat --alias localchat-builder --force || true fi - for alias in localchat-base localchat-builder; do + # Publish Docker relay container with engine only (strip images to keep cache small) + for ct in $(incus list -c n --format csv | grep -v builder); do + if incus exec "$ct" -- docker info >/dev/null 2>&1; then + echo "Stripping Docker images from $ct ..." + incus exec "$ct" -- docker system prune -af --volumes 2>/dev/null || true + echo "Publishing $ct as localchat-docker ..." + incus publish "$ct" --alias localchat-docker --force || true + break + fi + done + exported=0 + if [ "${{ job.status }}" = "success" ]; then + aliases="localchat-base localchat-builder localchat-cmdeploy localchat-docker" + else + aliases="localchat-base localchat-builder localchat-docker" + fi + for alias in $aliases; do if incus image list --format csv -c l | grep -q "^$alias$"; then echo "Exporting: $alias" incus image export $alias /tmp/incus-cache/$alias || true if [ -f /tmp/incus-cache/$alias ] && [ ! -f /tmp/incus-cache/$alias.tar.gz ]; then mv /tmp/incus-cache/$alias /tmp/incus-cache/$alias.tar.gz fi + exported=$((exported+1)) fi done + echo "exported=$exported" >> "$GITHUB_OUTPUT" + id: export-images + + - name: Save Incus image cache + if: always() && steps.export-images.outputs.exported > 0 + uses: actions/cache/save@v5 + with: + path: /tmp/incus-cache + key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a6c2dcf..c25c5b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list sudo apt-get update - sudo apt-get install -y incus=1:7.2* incus-client=1:7.2* incus-base=1:7.2* + sudo apt-get install -y incus-client=1:7.2* incus-base=1:7.2* incus --version - name: Initialise Incus diff --git a/README.md b/README.md index 5586000..3ec98da 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,34 @@ Each `deploy-*` invocation initialises the driver's source in the builder (wipe-and-reclone). +**Deploy via Docker Compose** (runs chatmail inside Docker-in-LXC): + + # Pull a pre-built image directly from GHCR + cmlxc docker deploy --source ghcr:main dk0 + cmlxc docker deploy --source ghcr:sha-ce05b26 dk0 + + # Load a local image tarball + cmlxc docker deploy --image ./chatmail.tar dk0 + + # Inject a locally-built image from the host Docker daemon + cmlxc docker deploy --source docker:chatmail-relay:latest dk0 + +Pull a newer image into an already-deployed relay: + + cmlxc docker pull dk0 + cmlxc docker pull dk0 --tag sha-ce05b26 + +Inspect running services and logs: + + cmlxc docker ps dk0 + cmlxc docker logs dk0 + cmlxc docker logs dk0 -f + +SSH into a Docker service (auto-configured by ``cmlxc``): + + ssh chatmail@dk0.localchat + + **Run integration tests** inside the builder: cmlxc test-mini cm0 @@ -159,13 +187,14 @@ the host only needs `cmlxc` itself. **Relay containers** (e.g. `cm0-localchat`, `mad1-localchat`) -- ephemeral containers that receive a deployed chatmail service. -Each relay is locked to a single deployment driver (`cmdeploy` or -`madmail`); switching requires destroying and re-creating the container. +Each relay is locked to a single deployment driver (`cmdeploy`, +`madmail`, or `docker`); switching requires destroying and re-creating +the container. ### Deployment drivers -Drivers live in `driver_cmdeploy.py` and `driver_madmail.py`. +Drivers live in `driver_cmdeploy.py`, `driver_madmail.py`, and `driver_docker.py`. Each driver module exports its CLI subcommand metadata, builder init, and deploy orchestration. `cli.py` generates the `deploy-*` subcommands from a `DRIVER_BY_NAME` mapping. @@ -179,6 +208,45 @@ builder init, and deploy orchestration. pushes it via SCP and runs `madmail install --simple --ip `. No DNS entries are needed. +- **docker** -- deploys chatmail via Docker Compose inside a Docker-in-LXC +relay container (`security.nesting=true`), either directly pulled from GHCR or +injected from a host docker instance. Docker is installed inside the relay +automatically; no host Docker installation is required. + +#### Docker subcommands + +- `docker deploy RELAY` -- deploy chatmail into a relay container via + Docker Compose. Three image sources are supported: + - `--source ghcr:TAG` -- pull a pre-built image from GHCR directly + into the relay. No builder container is involved. + - `--source docker:TAG` -- pipe a locally-built image from the host + Docker daemon into the relay via `docker save | docker load`. + - `--image PATH` -- load a pre-exported image tarball. + A docker-compose.yaml is fetched from + [chatmail/docker](https://github.com/chatmail/docker) unless + `--compose URL` overrides the source. + +- `docker pull RELAY` -- pull a newer image from GHCR into an already + deployed relay without a full redeploy. Use `--tag` to specify the + image tag (default: `main`). + +- `docker ps RELAY` -- list running Docker Compose services in a relay. + +- `docker logs RELAY` -- show Docker Compose logs (last 100 lines). + Pass `-f` to follow in real time. + +- `docker shell RELAY [SERVICE]` -- open an interactive shell inside + the named Compose service (default: `chatmail`). + +#### SSH into Docker services + +For Docker-deployed relays, `cmlxc` auto-generates SSH config entries for +each running Compose service. After any deploy or `cmlxc status`, you can: + + ssh chatmail@dk0.localchat + +This uses `ProxyCommand` to run `docker exec` inside the LXC container. + ## Releasing diff --git a/src/cmlxc/cli.py b/src/cmlxc/cli.py index 02dff4c..a724d85 100644 --- a/src/cmlxc/cli.py +++ b/src/cmlxc/cli.py @@ -1,10 +1,11 @@ """cmlxc -- Manage local chatmail relay containers via Incus. Standard workflow: -init -> deploy-cmdeploy/deploy-madmail -> test-cmdeploy/test-madmail/test-mini. +init -> deploy-cmdeploy/deploy-madmail/docker deploy -> test-*/test-mini. """ import argparse +import os import subprocess from pathlib import Path @@ -21,6 +22,7 @@ ) from cmlxc.driver_base import __version__ from cmlxc.driver_cmdeploy import CmdeployDriver +from cmlxc.driver_docker import DockerDriver from cmlxc.driver_madmail import MadmailDriver, print_admin_info from cmlxc.incus import Incus, _is_ip_address, check_cgroup_compat from cmlxc.output import Out @@ -41,15 +43,7 @@ def _container_completer(prefix, **kwargs): def _check_init(ix, out): - managed = ix.list_managed() - dns_running = any( - c["name"] == DNS_CONTAINER_NAME and c["status"] == "Running" for c in managed - ) - if not dns_running or not ix.find_image([BASE_IMAGE_ALIAS]): - out.red("Error: cmlxc environment not initialized.") - out.red("Please run 'cmlxc init' first to set up the base image and DNS.") - return False - return True + return ix.check_init() def _destroy_all(ix, out): @@ -266,13 +260,25 @@ def _add_test_relay_args(parser): def test_cmdeploy_cmd_options(parser): _add_test_relay_args(parser) + parser.add_argument( + "--relay-ref", + default=None, + help="Override relay git ref for tests (default: SHA from deployed image label).", + ) def test_cmdeploy_cmd(args, out): """Run cmdeploy integration tests inside the builder container.""" ix = Incus(out) ct = ix.get_running_relay(args.relay) - driver = CmdeployDriver(ct, out) + drv_cls = DRIVER_BY_NAME.get(ct.driver_name) + if drv_cls is None: + out.red( + f"Warning: unknown driver {ct.driver_name!r} for" + f" {ct.shortname}, falling back to cmdeploy." + ) + drv_cls = CmdeployDriver + driver = drv_cls(ct, out) if not driver.check_init(): return 1 @@ -302,6 +308,8 @@ def test_cmdeploy_cmd(args, out): drv2 = DRIVER_BY_NAME[ct2.driver_name](ct2, out) second_domain = drv2.get_test_domain_or_ip() + if args.relay_ref is not None: + os.environ["RELAY_REF"] = args.relay_ref return driver.run_tests(second_domain=second_domain) @@ -527,11 +535,16 @@ def _print_container_status(out, c, ix): def _print_builder_repos(out, ct): try: - for name in DRIVER_BY_NAME: - path = f"/root/{name}-git-main" + seen = set() + for name, drv_cls in DRIVER_BY_NAME.items(): + repo = drv_cls.REPO_NAME + if repo in seen: + continue + seen.add(repo) + path = f"/root/{repo}-git-main" status = ct.get_repo_status(path) if status: - out.print(f"{name}: {status}") + out.print(f"{repo}: {status}") except Exception: out.print("repos: (unavailable)") @@ -612,7 +625,11 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False): ("destroy", destroy_cmd, destroy_cmd_options), ] -DRIVER_BY_NAME = {"cmdeploy": CmdeployDriver, "madmail": MadmailDriver} +DRIVER_BY_NAME = { + "cmdeploy": CmdeployDriver, + "docker": DockerDriver, + "madmail": MadmailDriver, +} def _add_subcommand(subparsers, name, func, addopts, shared): @@ -677,6 +694,10 @@ def main(args=None): if args.func is None: return parser.parse_args(["-h"]) + # GitHub Actions: auto-enable max verbosity when debug logging is on + if not args.verbose and os.environ.get("RUNNER_DEBUG") == "1": + args.verbose = 3 + out = Out(verbosity=args.verbose) try: res = args.func(args, out) diff --git a/src/cmlxc/container.py b/src/cmlxc/container.py index 9301d18..c54c4ed 100644 --- a/src/cmlxc/container.py +++ b/src/cmlxc/container.py @@ -5,6 +5,7 @@ All interaction with Incus containers goes through these types. """ +import ipaddress import shlex import socket import subprocess @@ -49,12 +50,15 @@ class SetupError(Exception): """User-facing error raised when a pre-condition is not met.""" -def _extract_ip(net_data, family="inet"): +def _extract_ip(net_data, family="inet", subnet=None): for iface_name, iface in net_data.items(): if iface_name == "lo": continue for addr in iface.get("addresses", []): if addr["family"] == family and addr["scope"] == "global": + if subnet is not None: + if ipaddress.ip_address(addr["address"]) not in subnet: + continue return addr["address"] return None @@ -116,6 +120,23 @@ def bash(self, script, check=True): cmd = ["exec", self.name, "--", "bash", "-ec", script] return self.incus.run_output(cmd, check=check) + def bash_get(self, script): + """Run script, return stdout or None on failure without printing errors. + + Use for existence checks and polls where None is the expected "absent" signal. + """ + return self.bash(script, check=False) + + def bash_do(self, script): + """Run script, print errors on failure but return None instead of raising. + + Use when failure should be visible and the caller handles the None return. + """ + try: + return self.bash(script) + except subprocess.CalledProcessError: + return None + def run_cmd(self, *args, check=True): """Run command in container and return stdout.""" return self.incus.run_output( @@ -132,7 +153,7 @@ def stop(self, force=False): cmd.append("--force") self.incus.run(cmd, check=False) - def launch(self, image_candidates=None): + def launch(self, image_candidates=None, extra_config=None): """Launch from the base image or a provided candidate.""" if image_candidates is None: image_candidates = [BASE_IMAGE_ALIAS] @@ -147,6 +168,9 @@ def launch(self, image_candidates=None): cfg = [] cfg += ("-c", f"{LABEL_KEY}=true") cfg += ("-c", f"{LABEL_DOMAIN}={self.domain}") + if extra_config: + for k, v in extra_config.items(): + cfg += ("-c", f"{k}={v}") self.incus.run(["launch", image, self.name, *cfg]) return image @@ -164,7 +188,7 @@ def is_ipv6_disabled(self): ) return result == "1" - def ensure(self, ipv4_only=False, image_candidates=None): + def ensure(self, ipv4_only=False, image_candidates=None, extra_config=None): data = self.incus.run_json(["list", self.name], check=False) or [] existing = [c for c in data if c["name"] == self.name] @@ -174,7 +198,7 @@ def ensure(self, ipv4_only=False, image_candidates=None): if not ipv4_only: self.enable_ipv6() else: - self.launch(image_candidates=image_candidates) + self.launch(image_candidates=image_candidates, extra_config=extra_config) self.wait_ready(expect_ipv6=not ipv4_only) if ipv4_only: self.disable_ipv6() @@ -283,7 +307,8 @@ def wait_ready(self, timeout=60, expect_ipv6=False): ) if data and data[0].get("status") == "Running": net = data[0].get("state", {}).get("network", {}) - self.ipv4 = _extract_ip(net, "inet") + # Docker containers expose docker0 as well, filter + self.ipv4 = _extract_ip(net, "inet", subnet=self.incus.bridge_subnet) self.ipv6 = _extract_ip(net, "inet6") if self.ipv4 and (not expect_ipv6 or self.ipv6): return @@ -356,8 +381,10 @@ def destroy(self): ) super().destroy() - def launch(self, image_candidates=None): - image = super().launch(image_candidates=image_candidates) + def launch(self, image_candidates=None, extra_config=None): + image = super().launch( + image_candidates=image_candidates, extra_config=extra_config + ) # Re-inject the current SSH key; cached images may have a stale one. pub_key = self.incus.ssh_key_path.with_suffix(".pub").read_text().strip() self.bash(f""" @@ -371,12 +398,15 @@ def launch(self, image_candidates=None): """) return image - def ensure(self, ipv4_only=False, image_candidates=None): + def ensure(self, ipv4_only=False, image_candidates=None, extra_config=None): out = self.out out.green(f"Ensuring container {self.name!r} ({self.domain}) ...") - super().ensure(ipv4_only=ipv4_only, image_candidates=image_candidates) - + super().ensure( + ipv4_only=ipv4_only, + image_candidates=image_candidates, + extra_config=extra_config, + ) if self.get_deploy_state(): self.wait_services() @@ -573,7 +603,12 @@ def init_ssh(self): key = self.incus.ssh_key_path self.bash("mkdir -p /root/.ssh/config.d && chmod 700 /root/.ssh") self.incus.run( - ["file", "push", str(key), f"{self.name}/root/.ssh/id_localchat"] + [ + "file", + "push", + str(key), + f"{self.name}/root/.ssh/id_localchat", + ] ) self.bash("chown root:root /root/.ssh/id_localchat") self.bash("chmod 600 /root/.ssh/id_localchat") diff --git a/src/cmlxc/driver_base.py b/src/cmlxc/driver_base.py index c6f7ea2..a70a463 100644 --- a/src/cmlxc/driver_base.py +++ b/src/cmlxc/driver_base.py @@ -5,7 +5,8 @@ """ import re -from dataclasses import dataclass +import subprocess +from dataclasses import dataclass, replace from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import Literal @@ -20,6 +21,7 @@ DNS_CONTAINER_NAME, BuilderContainer, DNSContainer, + SetupError, ) from cmlxc.incus import Incus @@ -46,6 +48,7 @@ def parse_source(value: str, default_url: str) -> SourceSpec: Accepted forms: @ref -- branch/tag on the default remote + @latest -- newest released tag on the default remote /path or ./path -- local directory URL@ref -- custom remote at a given ref """ @@ -64,6 +67,35 @@ def parse_source(value: str, default_url: str) -> SourceSpec: raise ValueError(f"Invalid SOURCE: {value!r}. Use @ref, /path, ./path, or URL@ref.") +def get_latest_tag(url: str) -> str: + """Return the newest release tag on url via git ls-remote.""" + try: + result = subprocess.run( + # version-sort tags (newest first) + ["git", "ls-remote", "--tags", "--refs", "--sort=-v:refname", url], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + except (subprocess.SubprocessError, OSError) as e: + raise SetupError(f"Failed to resolve @latest for {url}: {e}") + + # Lines are "\trefs/tags/", newest first. Extract first match of X.Y.Z, + # optional leading "v", tags, skips pre-release (X.Y.Z-rc1) and non-semver tags. + tags = re.findall(r"\trefs/tags/(v?\d+\.\d+\.\d+)$", result.stdout, re.M) + if tags: + return tags[0] + raise SetupError(f"No tags found at {url} to resolve @latest") + + +def resolve_source(source: SourceSpec) -> SourceSpec: + """Resolve a ref="latest" remote spec to a concrete tag; pass others through.""" + if source.kind == "remote" and source.ref == "latest": + return replace(source, ref=get_latest_tag(source.url)) + return source + + _RELAY_NAME_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9-]*$") @@ -146,7 +178,10 @@ def add_cli_options(cls, parser, completer=None): "--source", default=f"@{cls.DEFAULT_REF}", metavar="SOURCE", - help=f"Driver source: @ref, /path, ./path, or URL@ref (default: @{cls.DEFAULT_REF}).", + help=( + "Driver source: @ref (branch, tag), @latest (release tag), /path, ./path," + f" or URL@ref (default: @{cls.DEFAULT_REF})." + ), ) action = parser.add_argument( "name", @@ -192,7 +227,7 @@ def on_prep_builder(cls, out, bld_ct, tmp_dest): """Hook called by ``prep_builder`` after the git-main checkout is ready.""" pass - def on_init_relay(self, repo_path): + def on_init_relay(self, repo_path, source): """Hook called by ``init_builder`` after a relay checkout is ready.""" pass @@ -202,7 +237,10 @@ def get_git_main_path(self): @classmethod def prep_builder(cls, ix, out, bld_ct): - """Hook called by ``cmlxc init`` to prepare toolchains and main checkout.""" + """Prepare builder toolchains and the persistent ``-git-main`` cache clone. + + Called by ``cmlxc init`` and, via ``init_builder``, by every deploy. + """ # Trust all repo paths inside the builder (ownership differs from host). bld_ct.bash("mkdir -p /root/relays") bld_ct.bash("git config --global --add safe.directory '*'", check=False) @@ -221,11 +259,13 @@ def prep_builder(cls, ix, out, bld_ct): tmp_dest = f"/root/{cls.REPO_NAME}-git-main" if bld_ct.bash(f"test -d {tmp_dest}", check=False) is None: - source = parse_source(f"@{cls.DEFAULT_REF}", cls.DEFAULT_SOURCE_URL) + source = resolve_source( + parse_source(f"@{cls.DEFAULT_REF}", cls.DEFAULT_SOURCE_URL) + ) bld_ct.setup_repo(tmp_dest, out, source) else: out.print(f" Fetching {cls.REPO_NAME}-git-main from upstream ...") - bld_ct.bash(f"cd {tmp_dest} && git fetch origin") + bld_ct.bash(f"cd {tmp_dest} && git fetch origin --tags") # Driver-specific toolchain setup cls.on_prep_builder(out, bld_ct, tmp_dest) @@ -241,12 +281,24 @@ def init_builder(self, source): f" Copying {self.REPO_NAME}-git-main to {repo_path} on builder" ) self.bld_ct.bash(f"rm -rf {repo_path} && cp -a {tmp_dest} {repo_path}") - if source.ref != "main": + is_sha = bool(re.fullmatch(r"[0-9a-f]{40}", source.ref or "")) + if is_sha: + # Shallow clone won't have arbitrary commits; fetch just this one. + self.out.print(f" Fetching {source.ref[:12]} ...") + self.bld_ct.bash( + f"cd {repo_path} && git fetch --depth 1 origin {source.ref}" + ) + elif source.ref != "main": self.out.print(f" Checking out {source.ref!r} ...") + reset_cmd = "" + if not is_sha: + reset_cmd = ( + f"git reset --hard -q origin/{source.ref} 2>/dev/null || true" + ) self.bld_ct.bash(f""" cd {repo_path} git checkout -q {source.ref} - git reset --hard -q origin/{source.ref} 2>/dev/null || true + {reset_cmd} git clean -fdx if [ -f .gitmodules ]; then git submodule update --init --recursive @@ -260,7 +312,7 @@ def init_builder(self, source): self.bld_ct.sync_to(source.path, repo_path) # Relay-specific preparation (e.g. build binary, init venv) - self.on_init_relay(repo_path) + self.on_init_relay(repo_path, source) def run_deploy(self, *, source, ipv4_only): """Perform the driver-specific deployment. @@ -312,6 +364,7 @@ def cmd(args, out): out.print(f"cmlxc {__version__}") with out.section(f"Preparing {cls.CLI_NAME} source in builder"): + source = resolve_source(source) out.print(f" Source: {source.description}") driver.init_builder(source=source) diff --git a/src/cmlxc/driver_cmdeploy.py b/src/cmlxc/driver_cmdeploy.py index 696a676..e9f441e 100644 --- a/src/cmlxc/driver_cmdeploy.py +++ b/src/cmlxc/driver_cmdeploy.py @@ -13,6 +13,36 @@ CMDEPLOY = "cmdeploy" +def run_test_cmdeploy(driver, second_domain=None): + """Run the cmdeploy pytest suite via incus exec on the builder. + + Shared by CmdeployDriver and DockerDriver. + """ + env = {} + if second_domain: + env["CHATMAIL_DOMAIN2"] = second_domain + + test_addr = driver.get_test_domain_or_ip() + driver.out.print(f"Running cmdeploy tests against {test_addr} ...") + + ini_path = f"{driver.repo_path}/chatmail.ini" + env_exports = f"export CHATMAIL_INI={ini_path}" + for k, v in env.items(): + env_exports += f" && export {k}={v}" + cmd = ( + f"incus exec {driver.bld_ct.name} --" + f" bash -c '" + f"{env_exports} &&" + f" source {driver.venv_path}/bin/activate &&" + f" cd {driver.repo_path} &&" + f" pytest cmdeploy/src/ -n4 -rs -x -v --durations=5'" + ) + ret = driver.out.shell(cmd) + if ret: + driver.out.red(f"test-cmdeploy failed (exit {ret})") + return ret + + class CmdeployDriver(Driver): """Deploys chatmail relays via the ``cmdeploy`` tool.""" @@ -60,8 +90,8 @@ def get_test_domain_or_ip(self): case _: return self.ct.domain - def on_init_relay(self, repo_path): - """Hook called by ``init_builder`` to run initenv.sh for the relay.""" + def on_init_relay(self, repo_path, source): + """Run scripts/initenv.sh on the builder to create the relay venv.""" self.out.print(f" Running scripts/initenv.sh for {self.ct.shortname} ...") self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh") @@ -75,7 +105,12 @@ def init_builder(self, source): remote_path = f"{self.repo_path}/filtermail" self.out.print(f" Syncing {local_path.name} to builder ...") self.ix.run( - ["file", "push", str(local_path), f"{self.bld_ct.name}{remote_path}"] + [ + "file", + "push", + str(local_path), + f"{self.bld_ct.name}{remote_path}", + ] ) self.custom_env["CHATMAIL_FILTERMAIL_BINARY"] = remote_path @@ -96,26 +131,7 @@ def run_tests(self, second_domain=None): write_ini( self.bld_ct, self.ct, domain, disable_ipv6=self.ct.is_ipv6_disabled ) - - ini_path = f"{self.repo_path}/chatmail.ini" - env = {"CHATMAIL_INI": ini_path} - if second_domain: - env["CHATMAIL_DOMAIN2"] = second_domain - - self.out.print(f"Running cmdeploy tests against {domain} ...") - - env_args = "".join(f" --env {k}={v}" for k, v in env.items()) - cmd = ( - f"incus exec {self.bld_ct.name}{env_args} --" - f" bash -c '" - f" source {self.venv_path}/bin/activate &&" - f" cd {self.repo_path} &&" - f" pytest cmdeploy/src/ -n4 -rs -x -v --durations=5'" - ) - ret = self.out.shell(cmd) - if ret: - self.out.red(f"test-cmdeploy failed (exit {ret})") - return ret + return run_test_cmdeploy(self, second_domain) def deploy(self, source=None): """Deploy chatmail services to a single relay via cmdeploy.""" @@ -191,40 +207,54 @@ def _run_cmdeploy(self, subcmd, *extra): # ------------------------------------------------------------------ +TEST_INI_OVERRIDES = { + "max_user_send_per_minute": 600, + "max_user_send_burst_size": 100, + # Relays reject new address creation while the machine looks + # busy, which a CI runner hosting several containers always + # does. Test relays are throwaway, so lift the gates instead + # of losing accounts to unrelated load. + "max_load_1m": 1000, + "min_available_memory": "1M", + "min_free_disk_space": "1M", + "mtail_address": "127.0.0.1", +} + + def get_ini_overrides(domain, disable_ipv6=False): """Return chatmail.ini settings suited for throwaway test relays.""" - overrides = { - "max_user_send_per_minute": 600, - "max_user_send_burst_size": 100, - # Relays reject new address creation while the machine looks - # busy, which a CI runner hosting several containers always - # does. Test relays are throwaway, so lift the gates instead - # of losing accounts to unrelated load. - "max_load_1m": 1000, - "min_available_memory": "1M", - "min_free_disk_space": "1M", - "mtail_address": "127.0.0.1", - "ssh_host": domain, - } + overrides = dict(TEST_INI_OVERRIDES) + overrides["ssh_host"] = domain if disable_ipv6: overrides["disable_ipv6"] = "True" return overrides -def write_ini(builder_ct, ct, domain, disable_ipv6=False): - """Write a chatmail.ini for *ct* using the builder container.""" - overrides = get_ini_overrides(domain, disable_ipv6=disable_ipv6) +def make_ini_script(domain, ini_path, overrides): + """Return a Python -c snippet that calls write_initial_config.""" overrides_str = ", ".join( f"'{k}': '{v}'" if isinstance(v, str) else f"'{k}': {v}" for k, v in overrides.items() ) + return "\n".join( + [ + "from chatmaild.config import write_initial_config", + "from pathlib import Path", + f"write_initial_config(Path('{ini_path}'), '{domain}', {{{overrides_str}}})", + ] + ) + + +def write_ini(builder_ct, ct, domain, disable_ipv6=False): + """Write a chatmail.ini for *ct* using the builder container.""" + overrides = get_ini_overrides(domain, disable_ipv6=disable_ipv6) repo_path = ct.get_repo_path(CMDEPLOY) ini_path = f"{repo_path}/chatmail.ini" + venv_path = f"{repo_path}/venv" + script = make_ini_script(domain, ini_path, overrides) builder_ct.bash(f""" - source {repo_path}/venv/bin/activate + source {venv_path}/bin/activate python3 -c " -from chatmaild.config import write_initial_config -from pathlib import Path -write_initial_config(Path('{ini_path}'), '{domain}', {{{overrides_str}}}) +{script} " """) diff --git a/src/cmlxc/driver_docker.py b/src/cmlxc/driver_docker.py new file mode 100644 index 0000000..771a9c2 --- /dev/null +++ b/src/cmlxc/driver_docker.py @@ -0,0 +1,753 @@ +"""Docker driver and management commands for cmlxc. + +Contains the DockerDriver (``cmlxc docker deploy``) and the +``docker logs / ps / shell / pull`` CLI subcommands. +""" + +import os +import re +import shlex +import subprocess +import time +from datetime import datetime, timezone +from types import SimpleNamespace + +from cmlxc.container import SetupError +from cmlxc.driver_base import Driver, __version__, parse_source, validate_relay_name +from cmlxc.driver_cmdeploy import ( + TEST_INI_OVERRIDES, + make_ini_script, + run_test_cmdeploy, + write_ini, +) +from cmlxc.incus import Incus + +DOCKER = "docker" +DOCKER_COMPOSE_SERVICE = "chatmail" +DOCKER_IMAGE_TAG = "chatmail-relay" +GHCR_IMAGE = "ghcr.io/chatmail/docker" + +# Validates the TAG portion of ``docker:TAG``. Allows colons and slashes +# so users can pass ``image:version`` or ``registry/repo:tag`` forms. +_DOCKER_TAG_RE = re.compile(r"^[a-zA-Z0-9._:/-]+$") + + +def _add_relay_arg(parser, completer=None, *, help="Relay container name."): + """Add the positional RELAY argument with optional tab-completion.""" + arg = parser.add_argument("relay", metavar="RELAY", help=help) + if completer: + arg.completer = completer + + +def _parse_inject_tag(source_arg): + """Extract and validate tag from a ``docker:TAG`` source argument. + + Returns the tag string if *source_arg* starts with ``docker:`` and the + tag passes validation, ``None`` if the prefix is absent, or raises + ``ValueError`` if the prefix is present but the tag is empty/malformed. + """ + if not source_arg.startswith("docker:"): + return None + tag = source_arg[len("docker:") :] + if not tag or not _DOCKER_TAG_RE.match(tag): + raise ValueError(f"Invalid Docker tag: {tag!r}") + return tag + + +# ------------------------------------------------------------------- +# Image helpers +# ------------------------------------------------------------------- + + +def image_tag(sha): + """Docker image tag for a given git SHA.""" + return f"{DOCKER_IMAGE_TAG}:{sha[:12]}" + + +def ensure_docker(ct): + """Install Docker engine in container if not present.""" + if ct.bash_get("docker info >/dev/null 2>&1") is not None: + return + ct.bash(""" + mkdir -p /etc/apt/keyrings + /usr/lib/apt/apt-helper download-file \ + https://download.docker.com/linux/debian/gpg \ + /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) \ + signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -qq + apt-get install -y -qq \ + docker-ce docker-ce-cli containerd.io docker-compose-plugin + mkdir -p /etc/docker + # Docker iptables rules conflict with LXC-managed networking. + printf '{"iptables": false}\\n' > /etc/docker/daemon.json + systemctl enable --now docker + """) + + +def inject_image_from_host(ct, tag, out): + """Pipe a locally-built image from the host Docker daemon into the relay.""" + out.print(f" Injecting {tag} into {ct.shortname} ...") + cmd = f"docker save {shlex.quote(tag)} | incus exec {ct.name} -- docker load" + ret = out.shell(cmd) + if ret: + raise SetupError(f"Failed to inject {tag} into {ct.name}") + ct.bash(f"docker tag {shlex.quote(tag)} {DOCKER_IMAGE_TAG}:latest") + + +def pull_image(ct, tag, out): + """Pull a Docker image from GHCR into a container and tag locally. + + Returns the relay git SHA extracted from image labels, or None. + """ + ref = f"{GHCR_IMAGE}:{tag}" + ensure_docker(ct) + out.print(f" Pulling {ref} ...") + try: + ct.bash(f"docker pull {ref}") + except subprocess.CalledProcessError: + out.red(f" Failed to pull {ref}") + return None + ct.bash(f"docker tag {ref} {DOCKER_IMAGE_TAG}:latest") + sha = get_image_label_sha(ct, ref) + if sha: + local_tag = image_tag(sha) + ct.bash(f"docker tag {ref} {local_tag}") + out.print(f" Tagged as {local_tag}") + return sha + out.print(f" Pulled {ref} (no SHA label found)") + return None + + +def get_image_label_sha(ct, tag): + """Read the relay commit SHA from a Docker image's OCI labels.""" + sha = ct.bash_get( + f"docker inspect {tag}" + " --format '{{index .Config.Labels \"org.opencontainers.image.revision\"}}'" + ) + return sha.strip() if sha and sha.strip() else None + + +def logs_docker_cmd(args, out): + """Show Docker Compose logs from a deployed relay container.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + state = ct.get_deploy_state() + if state is None or state.get("driver") != DOCKER: + out.red(f"Container {ct.shortname!r} is not a Docker deployment.") + return 1 + + follow = "-f " if args.follow else "" + cmd = f"incus exec {ct.name} -- docker compose -f /opt/chatmail-docker/docker-compose.yaml logs {follow}--tail=100" + return out.shell(cmd) + + +def _get_docker_services(ix, name): + """Query running Docker Compose service names from a relay container.""" + raw = ix.run_output( + [ + "exec", + name, + "--", + "docker", + "compose", + "-f", + "/opt/chatmail-docker/docker-compose.yaml", + "ps", + "--services", + "--status", + "running", + ], + check=False, + ) + if not raw: + return [] + return [s.strip() for s in raw.splitlines() if s.strip()] + + +def ps_docker_cmd(args, out): + """Show running Docker Compose services in a deployed relay.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + for svc in _get_docker_services(ix, ct.name): + out.print(svc) + + +def shell_docker_cmd(args, out): + """Open an interactive shell (or run a command) in a Docker container.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + svc = args.service + if args.command: + cmd_str = " ".join(shlex.quote(c) for c in args.command) + cmd = [ + "incus", + "exec", + ct.name, + "--", + "docker", + "exec", + "-i", + svc, + "bash", + "-c", + cmd_str, + ] + else: + cmd = [ + "incus", + "exec", + ct.name, + "--", + "docker", + "exec", + "-it", + svc, + "bash", + "-l", + ] + return subprocess.call(cmd) + + +def pull_docker_cmd_options(parser, completer=None): + _add_relay_arg( + parser, completer, help="Relay container name to pull the image into." + ) + parser.add_argument( + "--tag", + default="main", + metavar="TAG", + help="GHCR image tag to pull (default: main).", + ) + + +def pull_docker_cmd(args, out): + """Pull a chatmail Docker image from GHCR into a relay container.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + with out.section(f"Pulling {GHCR_IMAGE}:{args.tag}"): + sha = pull_image(ct, args.tag, out) + if sha is None: + out.red(f"Pull failed for {GHCR_IMAGE}:{args.tag}") + return 1 + out.green(f"Done. Image: {DOCKER_IMAGE_TAG}:{sha[:12] if sha else 'latest'}") + return 0 + + +# ------------------------------------------------------------------- +# Deployment driver +# ------------------------------------------------------------------- + + +class DockerDriver(Driver): + """Deploys chatmail relays via Docker Compose in LXC containers.""" + + CLI_NAME = "docker" + CLI_DOC = "Docker relay management (deploy, pull, logs, ps, shell)." + DEFAULT_SOURCE_URL = "https://github.com/chatmail/relay.git" + REPO_NAME = "cmdeploy" + REQUIRED_SOURCE_PATHS = ["cmdeploy"] + + NESTING_CONFIG = { + "security.nesting": "true", + "security.syscalls.intercept.mknod": "true", + "security.syscalls.intercept.setxattr": "true", + } + # CI runners have AppArmor enforcing, which blocks systemd inside + # Docker-in-LXC. These overrides must not be used outside disposable CI. + _CI_NESTING_EXTRA = { + "security.privileged": "true", + "raw.lxc": "lxc.apparmor.profile=unconfined", + } + + @classmethod + def get_nesting_config(cls): + cfg = dict(cls.NESTING_CONFIG) + if os.environ.get("CI"): + cfg.update(cls._CI_NESTING_EXTRA) + return cfg + + @classmethod + def add_cli_options(cls, parser, completer=None): + super().add_cli_options(parser, completer=completer) + parser.add_argument( + "--image", + metavar="PATH", + help="Load a pre-exported image tarball into the relay.", + ) + parser.add_argument( + "--compose", + default="https://raw.githubusercontent.com/chatmail/docker/main/docker-compose.yaml", + help="docker-compose.yaml URL to fetch (inject path only; default: chatmail/docker main).", + ) + + @classmethod + def logs_add_cli_options(cls, parser, completer=None): + _add_relay_arg(parser, completer) + parser.add_argument( + "-f", + "--follow", + action="store_true", + help="Follow log output (like tail -f).", + ) + + @classmethod + def ps_add_cli_options(cls, parser, completer=None): + _add_relay_arg(parser, completer) + + @classmethod + def shell_add_cli_options(cls, parser, completer=None): + _add_relay_arg(parser, completer) + parser.add_argument( + "service", + nargs="?", + default=DOCKER_COMPOSE_SERVICE, + help=f"Docker Compose service (default: {DOCKER_COMPOSE_SERVICE}).", + ) + parser.add_argument( + "command", + nargs="*", + default=[], + metavar="CMD", + help="Command to run (default: interactive bash).", + ) + + # (name, help, func, options_func) -- options_func may accept completer kwarg; + # logs/ps/shell use {name}_add_cli_options classmethods instead (options_func=None) + _DOCKER_SUBCOMMANDS = [ + ( + "logs", + "Show Docker Compose logs from a deployed relay", + logs_docker_cmd, + None, + ), + ("ps", "Show running Docker Compose services", ps_docker_cmd, None), + ("shell", "Open a shell in a Docker container", shell_docker_cmd, None), + ( + "pull", + "Pull a Docker image from GHCR into a relay", + pull_docker_cmd, + pull_docker_cmd_options, + ), + ] + + @classmethod + def add_subcommand(cls, subparsers, shared, *, completer=None): + """Register 'docker' with deploy/build/list/prune sub-subcommands.""" + docker_parser = subparsers.add_parser( + cls.CLI_NAME, + description=cls.CLI_DOC, + help=cls.CLI_DOC.split(".")[0], + parents=[shared], + ) + docker_parser.set_defaults(func=lambda args, out: docker_parser.print_help()) + docker_subs = docker_parser.add_subparsers(title="docker subcommands") + + # docker deploy (special: uses driver make_cmd + add_cli_options) + deploy_p = docker_subs.add_parser( + "deploy", + description="Deploy a chatmail relay via Docker Compose.", + help="Deploy a chatmail relay via Docker Compose", + parents=[shared], + ) + deploy_p.set_defaults(func=cls.make_cmd()) + cls.add_cli_options(deploy_p, completer=completer) + + for name, help_text, func, addopts in cls._DOCKER_SUBCOMMANDS: + p = docker_subs.add_parser( + name, + description=func.__doc__, + help=help_text, + parents=[shared], + ) + p.set_defaults(func=func) + classmethod_opts = getattr(cls, f"{name}_add_cli_options", None) + if classmethod_opts is not None: + classmethod_opts(p, completer=completer) + elif addopts is not None: + addopts(p, completer=completer) + + @classmethod + def make_cmd(cls): + """Build the deploy CLI command, routing by source/image type.""" + + def cmd(args, out): + source_str = getattr(args, "source", "") or "" + if source_str.startswith("ghcr:") or getattr(args, "image", None): + return cls._deploy_cmd(args, out) + try: + inject_tag = _parse_inject_tag(source_str) + except ValueError as exc: + out.red(str(exc)) + return 1 + if inject_tag: + return cls._deploy_cmd(args, out) + out.red( + "Specify an image: --source docker:TAG, --source ghcr:TAG," + " or --image PATH." + ) + return 1 + + cmd.__doc__ = cls.CLI_DOC + return cmd + + @classmethod + def _deploy_cmd(cls, args, out): + """Unified deploy handler for all Docker image sources.""" + try: + validate_relay_name(args.name) + except ValueError as exc: + out.red(str(exc)) + return 1 + + source_str = getattr(args, "source", "") or "" + if source_str.startswith("docker:") and getattr(args, "image", None): + out.red("Error: --image and --source docker:TAG are mutually exclusive.") + return 1 + + ix = Incus(out) + ct = ix.get_relay_container(args.name) + driver = cls(ct, out) + if not driver.check_init(): + return 1 + + driver.image_path = getattr(args, "image", None) + driver.compose_url = getattr( + args, + "compose", + "https://raw.githubusercontent.com/chatmail/docker/main/docker-compose.yaml", + ) + driver.ghcr_tag = None + driver.inject_tag = None + if source_str.startswith("ghcr:"): + driver.ghcr_tag = source_str[5:] or "main" + else: + try: + driver.inject_tag = _parse_inject_tag(source_str) + except ValueError: + pass # already validated in make_cmd + + out.print(f"cmlxc {__version__}") + driver.run_deploy(ipv4_only=args.ipv4_only) + return 0 + + def run_deploy(self, *, ipv4_only=False): + """Deploy Docker Compose relay into an LXC container.""" + with self.out.section(f"Preparing container: {self.ct.shortname}"): + self.ct.ensure( + ipv4_only=ipv4_only, + image_candidates=["localchat-docker", "localchat-base"], + extra_config=self.get_nesting_config(), + ) + + t_total = time.time() + self.deploy() + elapsed = time.time() - t_total + self.out.section_line(f"deploy docker complete ({elapsed:.1f}s)") + + def deploy(self): + """Deploy chatmail via Docker Compose.""" + self.ct.check_deploy_lock(DOCKER) + if not re.fullmatch(r"[a-zA-Z0-9._-]+", self.ct.domain): + raise SetupError(f"Unsafe domain value: {self.ct.domain!r}") + self.ix.write_ssh_config() + + dns_ct = self.get_dns_container() + # Cached images (localchat-docker) have the previous run's DNS IP baked in, + # update resolv.conf to current ip + self.ct.bash(f"echo 'nameserver {dns_ct.ipv4}' > /etc/resolv.conf") + + with self.out.section("Installing Docker in relay"): + ensure_docker(self.ct) + + if self.image_path: + self._load_local_image() + elif self.ghcr_tag: + with self.out.section(f"Pulling image from GHCR ({self.ghcr_tag})"): + sha = pull_image(self.ct, self.ghcr_tag, self.out) + if sha is None: + raise SetupError(f"Failed to pull {GHCR_IMAGE}:{self.ghcr_tag}") + elif self.inject_tag: + with self.out.section(f"Injecting image ({self.inject_tag})"): + inject_image_from_host(self.ct, self.inject_tag, self.out) + + with self.out.section("Fetching compose file"): + self._fetch_compose_file() + + # Register domain after the pull so set_dns_records()'s recursor cache + # wipe doesn't break public DNS resolution during the image pull. + dns_ct.set_dns_records( + self.ct.domain, + f"{self.ct.domain}. 3600 IN A {self.ct.ipv4}", + ) + + with self.out.section("Preparing chatmail.ini"): + self._write_host_ini() + + with self.out.section("Starting Docker Compose"): + self._start_compose() + + with self.out.section("Waiting for healthcheck"): + self._wait_healthy() + + with self.out.section("Loading DNS zone"): + self._load_dns(dns_ct) + + desc = "" + if self.ghcr_tag: + desc = f"ghcr:{self.ghcr_tag}" + elif self.inject_tag: + desc = f"docker:{self.inject_tag}" + elif self.image_path: + desc = f"image:{self.image_path}" + sha = get_image_label_sha(self.ct, f"{DOCKER_IMAGE_TAG}:latest") + if sha: + desc += f" (relay {sha[:12]})" + source = SimpleNamespace(description=desc) if desc else None + self.ct.write_deploy_state(DOCKER, source=source, deploy_type="ipv4") + + def _load_local_image(self): + """Load a pre-exported image tarball into the relay.""" + with self.out.section(f"Loading image from {self.image_path}"): + path = shlex.quote(str(self.image_path)) + cmd = f"cat {path} | incus exec {self.ct.name} -- docker load" + ret = self.out.shell(cmd) + if ret: + raise SetupError(f"Failed to load image from {self.image_path}") + loaded = self.ct.bash( + f"docker images {DOCKER_IMAGE_TAG} --format '{{{{.Tag}}}}' | head -1" + ) + if loaded and loaded.strip() != "latest": + self.ct.bash( + f"docker tag {DOCKER_IMAGE_TAG}:{loaded.strip()}" + f" {DOCKER_IMAGE_TAG}:latest" + ) + + def _fetch_compose_file(self): + """Download docker-compose.yaml into the relay; raise SetupError on failure.""" + dest = "/opt/chatmail-docker/docker-compose.yaml" + self.ct.bash("mkdir -p /opt/chatmail-docker") + # apt-helper is always present (part of apt); avoids installing curl/wget. + result = self.ct.bash_get( + f"/usr/lib/apt/apt-helper download-file" + f" {shlex.quote(self.compose_url)} {dest} 2>&1" + ) + if result is None: + raise SetupError(f"Failed to fetch compose file from {self.compose_url}") + + def _write_host_ini(self): + """Write chatmail.ini into the relay via the Docker image's Python.""" + ini_path = "/srv/chatmail/chatmail.ini" + overrides = dict(TEST_INI_OVERRIDES) + if self.ct.is_ipv6_disabled: + overrides["disable_ipv6"] = "True" + script = make_ini_script(self.ct.domain, ini_path, overrides) + self.ct.bash(f""" + mkdir -p /srv/chatmail + docker run --rm \\ + --entrypoint /opt/cmdeploy/bin/python3 \\ + -v /srv/chatmail:/srv/chatmail \\ + {DOCKER_IMAGE_TAG}:latest \\ + -c " +{script} +" + """) + + def _start_compose(self): + """Write .env, compose override, copy compose file, and start.""" + self.ct.bash(f""" + mkdir -p /opt/chatmail-docker + cd /opt/chatmail-docker + cat > .env <<'DOTENV' +MAIL_DOMAIN={self.ct.domain} +CHATMAIL_IMAGE=chatmail-relay:latest +DOTENV + """) + # Always write the override so the chatmail.ini volume mount reflects the + # current deploy. A stale cached override would silently let the container + # start with default rate limits. + # NOTE: do NOT add `privileged: true` here as it causes Docker to mount a + # fresh devtmpfs and request `a *:* rwm` in the sub-cgroup, which cgroup v2's + # hierarchical eBPF filter on the parent LXC container denies, breaking + # /dev/null access for Dovecot. + self.ct.bash(""" + cat > /opt/chatmail-docker/docker-compose.override.yaml <<'OVERRIDE' +services: + chatmail: + volumes: + - /srv/chatmail/chatmail.ini:/etc/chatmail/chatmail.ini +OVERRIDE + """) + + self.ct.bash(""" + cd /opt/chatmail-docker + docker compose down -v 2>/dev/null || true + docker compose up -d --no-build + """) + + def _wait_healthy(self, timeout=180, interval=5): + """Poll Docker healthcheck until healthy or timeout.""" + since = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deadline = time.time() + timeout + while time.time() < deadline: + status = self.ct.bash_get( + f"docker inspect {DOCKER_COMPOSE_SERVICE}" + " --format '{{.State.Health.Status}}' 2>/dev/null" + ) + s = status.strip() if status else "" + if s == "healthy": + self.out.print(" Container healthy.") + return + if self.out.verbosity >= 1: + new_logs = self.ct.bash_get( + f"docker logs {DOCKER_COMPOSE_SERVICE} --since {since} 2>&1" + ) + if new_logs: + lines = new_logs.splitlines() + if len(lines) > 20: + self.out.print( + f" [docker] ... ({len(lines) - 20} lines skipped)" + ) + lines = lines[-20:] + for line in lines: + self.out.print(f" [docker] {line}") + since = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + time.sleep(interval) + self._dump_docker_logs(tail=80) + raise SetupError(f"Docker container not healthy after {timeout}s") + + def _dump_docker_logs(self, tail=80): + """Print recent Docker container logs for debugging.""" + svc = DOCKER_COMPOSE_SERVICE + sections = [ + ( + f"docker logs {svc} (last {tail})", + f"docker logs {svc} --tail {tail} 2>&1", + ), + ( + "healthcheck state", + f"docker inspect {svc} --format '{{{{json .State.Health}}}}' 2>/dev/null", + ), + ( + "dovecot journal", + f"docker exec {svc} journalctl -u dovecot --no-pager -n 30 2>&1", + ), + ( + "postfix journal", + f"docker exec {svc} journalctl -u postfix --no-pager -n 30 2>&1", + ), + ( + "failed systemd units", + f"docker exec {svc} systemctl --failed --no-pager 2>&1", + ), + ] + for label, cmd in sections: + self.out.red(f" --- {label} ---") + output = self.ct.bash_get(cmd) + if output: + for line in output.strip().splitlines(): + self.out.print(f" {line}") + + def _load_dns(self, dns_ct): + """Extract DNS zone from Docker container and load into PowerDNS.""" + tmp = "/tmp/localchat-forward.conf" + self.ct.push_file_content( + tmp, + f""" + server: + domain-insecure: "localchat" + + forward-zone: + name: "localchat" + forward-addr: {dns_ct.ipv4} + """, + ) + svc = DOCKER_COMPOSE_SERVICE + self.ct.bash( + f"docker cp {tmp} {svc}:/etc/unbound/unbound.conf.d/localchat-forward.conf" + f" && docker exec {svc} systemctl restart unbound" + ) + zone_content = self.ct.bash_do( + f"docker exec {svc} cmdeploy dns --ssh-host @local --zonefile /dev/stdout" + ) + if zone_content: + dns_ct.set_dns_records(self.ct.domain, zone_content) + else: + # Minimal A record fallback + dns_ct.set_dns_records( + self.ct.domain, + f"{self.ct.domain}. 3600 IN A {self.ct.ipv4}", + ) + + def on_init_relay(self, repo_path): + """Initialize the venv in the builder checkout for this relay.""" + self.out.print(f" Running scripts/initenv.sh for {self.ct.shortname} ...") + self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh") + + def _setup_docker_ssh_forwarding(self): + """Rewrite authorized_keys on the LXC host to forward SSH into Docker. + + Tests use SSHExec (execnet over SSH) which lands on the LXC host. + Services (dovecot, opendkim, postfix) run inside the Docker container. + By wrapping the builder key with command="docker exec ...", every SSH + session transparently enters the container. The LXC host itself is + managed via incus exec, so losing direct SSH access is fine. + + A wrapper script is needed because $SSH_ORIGINAL_COMMAND contains + shell metacharacters (quotes, parens) from execnet's python bootstrap. + Bare $SSH_ORIGINAL_COMMAND expansion would mangle them; bash -c with + double-quoted expansion preserves the command correctly. + """ + self.ct.push_file_content( + "/usr/local/bin/docker-ssh-forward", + f'#!/bin/bash\nexec docker exec -i {DOCKER_COMPOSE_SERVICE} bash -c "$SSH_ORIGINAL_COMMAND"', + mode="755", + ) + pub_key = self.ct.incus.ssh_key_path.with_suffix(".pub").read_text().strip() + self.ct.bash("mkdir -p /root/.ssh && chmod 700 /root/.ssh") + self.ct.push_file_content( + "/root/.ssh/authorized_keys", + f'command="/usr/local/bin/docker-ssh-forward" {pub_key}', + mode="600", + ) + + def _get_image_relay_sha(self): + """Read the relay commit SHA from the running Docker image's OCI labels.""" + return get_image_label_sha(self.ct, f"{DOCKER_IMAGE_TAG}:latest") + + def run_tests(self, second_domain=None): + """Execute the cmdeploy test suite against the Docker relay. + + The builder checkout must match the relay image so that + ``test_deployed_state`` (which compares local ``git rev-parse HEAD`` + against ``/etc/chatmail-version``) passes. When the venv already + exists from a prior deploy, re-checkout if the current SHA differs + from the image SHA. + + Set ``RELAY_REF`` in the environment to override the relay git ref + used for the test checkout (default: SHA from the running image). + """ + with self.out.section("cmdeploytest"): + self._setup_docker_ssh_forwarding() + self.bld_ct.write_relay_ssh_config(self.ct) + + ref = os.environ.get("RELAY_REF") or self._get_image_relay_sha() or "main" + venv_exists = self.bld_ct.bash_get(f"test -d {self.venv_path}") is not None + if not venv_exists: + self.out.print( + f" Venv missing, initializing builder for {self.ct.shortname} ..." + ) + source = parse_source(f"@{ref}", self.DEFAULT_SOURCE_URL) + self.init_builder(source) + + self.out.print("Preparing chatmail.ini on builder ...") + write_ini( + self.bld_ct, + self.ct, + self.ct.domain, + disable_ipv6=self.ct.is_ipv6_disabled, + ) + return run_test_cmdeploy(self, second_domain) diff --git a/src/cmlxc/driver_madmail.py b/src/cmlxc/driver_madmail.py index 6327f5b..e452361 100644 --- a/src/cmlxc/driver_madmail.py +++ b/src/cmlxc/driver_madmail.py @@ -1,17 +1,19 @@ """madmail-based deployment driver for cmlxc. -The maddy binary is built inside the builder container +The madmail binary is built inside the builder container and transferred to relay containers via SCP. Madmail relays run on IP addresses and do not require DNS entries. """ +import re import time from cmlxc.container import SetupError from cmlxc.driver_base import Driver MADMAIL = "madmail" +RELEASED_TAG_RE = re.compile(r"^v\d+\.\d+\.\d+$") class MadmailDriver(Driver): @@ -21,8 +23,8 @@ class MadmailDriver(Driver): CLI_DOC = "Deploy a madmail relay service into a container." DEFAULT_SOURCE_URL = "https://github.com/themadorg/madmail.git" REPO_NAME = MADMAIL - REQUIRED_SOURCE_PATHS = ["go.mod", "Makefile"] - DEFAULT_REF = "v0.47.1" + REQUIRED_SOURCE_PATHS = ["Cargo.toml", "Makefile"] + DEFAULT_REF = "latest" type = "ipv4" @classmethod @@ -43,57 +45,75 @@ def configure_from_args(self, args): @classmethod def on_prep_builder(cls, out, bld_ct, tmp_dest): - """Hook called by ``prep_builder`` to ensure the Go toolchain is ready.""" - out.print(" Ensuring build environment (Go) ...") + """Hook called by ``prep_builder`` to ensure the Rust toolchain is ready.""" + out.print(" Ensuring build environment (Rust) ...") prepare_build_container(bld_ct, tmp_dest) - def on_init_relay(self, repo_path): - """Hook called by ``init_builder`` to build the maddy binary.""" + def on_init_relay(self, repo_path, source): + """Hook called by ``init_builder`` to obtain the madmail binary. + + Tries downloading a prebuilt release asset first (only possible for a + resolved release tag); falls back to building from source otherwise. + """ + if source.kind == "remote" and RELEASED_TAG_RE.match(source.ref or ""): + with self.out.section( + f"Fetching madmail {source.ref} release binary for {self.ct.shortname}" + ): + if self._download_release_binary(repo_path, source.ref): + return + self.out.print(" Download failed, falling back to source build ...") + + self._build_from_source(repo_path) + + def _download_release_binary(self, repo_path, ref): + """Try to fetch the musl release asset for *ref*. Return True on success.""" + arch = self.bld_ct.bash("dpkg --print-architecture").strip() + if arch not in ("amd64", "arm64"): + self.out.print(f" No release asset for arch {arch!r}") + return False + + asset = f"madmail-linux-{arch}-musl" + url = f"https://github.com/themadorg/madmail/releases/download/{ref}/{asset}" + self.bld_ct.bash(f"mkdir -p {repo_path}/target/release") + ret = self.bld_ct.bash( + f"curl -fsSL -o {repo_path}/target/release/madmail '{url}'", + check=False, + ) + if ret is None: + return False + self.bld_ct.bash(f"chmod +x {repo_path}/target/release/madmail") + self.out.print(f" Downloaded {asset}") + return True + + def _build_from_source(self, repo_path): mode = "with admin web UI" if self.with_admin else "without admin web UI" with self.out.section( - f"Building maddy binary for {self.ct.shortname} ({mode})" + f"Building madmail binary for {self.ct.shortname} ({mode})" ): + self.bld_ct.bash(f"cd '{repo_path}' && make init") + if self.with_admin: - # Ensure admin-web submodule is populated and dependencies installed; - # build.sh copy_admin_web() handles the actual SPA build. - self.bld_ct.bash(f""" - if [ ! -f '{repo_path}/admin-web/package.json' ]; then - cd '{repo_path}' && git submodule update --init admin-web - fi - cd '{repo_path}/admin-web' - if command -v bun >/dev/null 2>&1; then - bun install - elif command -v npm >/dev/null 2>&1; then - npm install - fi - """) + # v2's `build-admin-web` Makefile target runs + # `git submodule update --init` itself and no separate + # submodule/bun-install step is needed + ret = self.out.shell( + f"incus exec {self.bld_ct.name} -- bash -c " + f"'cd {repo_path} && make build-release'" + ) else: - # Hide package.json so build.sh creates a placeholder instead. - self.bld_ct.bash(f""" - PKG='{repo_path}/admin-web/package.json' - BAK='{repo_path}/admin-web/package.json.cmlxc-disabled' - if [ -f "$PKG" ]; then mv "$PKG" "$BAK"; fi - """) - - try: + # Clear any stale embed + self.bld_ct.bash(f"rm -rf {repo_path}/crates/chatmail-admin-web/embed") ret = self.out.shell( f"incus exec {self.bld_ct.name} -- bash -c " - f"'cd {repo_path} && make clean build'" + f"'cd {repo_path} && cargo build -p chatmail --release'" ) - finally: - # Restore package.json if we hid it. - self.bld_ct.bash(f""" - BAK='{repo_path}/admin-web/package.json.cmlxc-disabled' - PKG='{repo_path}/admin-web/package.json' - if [ -f "$BAK" ] && [ ! -f "$PKG" ]; then mv "$BAK" "$PKG"; fi - """) if ret: - raise SetupError(f"maddy build failed in {repo_path} (exit {ret})") + raise SetupError(f"madmail build failed in {repo_path} (exit {ret})") if self.with_admin: check = self.bld_ct.bash( - f"test -f {repo_path}/internal/adminweb/build/index.html", + f"test -f {repo_path}/crates/chatmail-admin-web/embed/index.html", check=False, ) if check is None: @@ -104,11 +124,12 @@ def run_tests(self, second_domain=None, cool=False, simple=False): test_src = f"{self.get_git_main_path()}/tests/deltachat-test" with self.out.section("test-madmail"): - # Symlink the built maddy binary into the test directory so + # Symlink the built madmail binary into the test directory so # tests that spawn a local server can find it at build/maddy. self.bld_ct.bash( f"mkdir -p {test_src}/build" - f" && ln -sf {self.repo_path}/build/maddy {test_src}/build/maddy" + f" && ln -sf {self.repo_path}/target/release/madmail" + f" {test_src}/build/maddy" ) relay1 = self.get_test_domain_or_ip() @@ -172,17 +193,15 @@ def deploy(self, source=None): self.out.print("Pushing madmail binary via SCP ...") self.ct.bash("rm -f /tmp/madmail") self.bld_ct.scp_to_relay( - f"{self.repo_path}/build/maddy", + f"{self.repo_path}/target/release/madmail", ip, "/tmp/madmail", ) self.ct.bash("chmod +x /tmp/madmail") install_flags = ( - f"--simple --ip {ip}" - " --tls-mode self_signed" - " --enable-chatmail" - " --non-interactive" + f"--simple --ip {ip} --tls-mode self_signed" + " --enable-chatmail --enable-iroh --non-interactive" ) self.out.print(f"Running madmail install {install_flags} ...") self.ct.bash("systemctl stop madmail || true") @@ -247,8 +266,8 @@ def _parse_admin_web_status(status): return enabled, path -def prepare_build_container(bld_ct, go_mod_path): - """Install or update Go inside the builder according to go.mod.""" +def prepare_build_container(bld_ct, cargo_toml_path): + """Install or update Rust inside the builder according to Cargo.toml.""" bld_ct.bash(""" if ! command -v node >/dev/null 2>&1 || [ "$(node -v | cut -d. -f1 | tr -d v)" -lt 22 ]; then apt-get -o DPkg::Lock::Timeout=60 update @@ -262,21 +281,24 @@ def prepare_build_container(bld_ct, go_mod_path): ln -sf /root/.bun/bin/bun /usr/local/bin/bun fi """) + bld_ct.bash(""" + if ! command -v rustc >/dev/null 2>&1; then + apt-get -o DPkg::Lock::Timeout=60 update + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl build-essential + apt-get clean && rm -rf /var/lib/apt/lists/* + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \\ + | bash -s -- -y --default-toolchain stable --profile minimal + fi + ln -sf /root/.cargo/bin/cargo /usr/local/bin/cargo + ln -sf /root/.cargo/bin/rustc /usr/local/bin/rustc + ln -sf /root/.cargo/bin/rustup /usr/local/bin/rustup + """) bld_ct.bash(f""" - NEED=$(awk '/^go / {{print $2}}' {go_mod_path}/go.mod) - ARCH=$(dpkg --print-architecture) - case "$ARCH" in - amd64) GOARCH=amd64 ;; - arm64) GOARCH=arm64 ;; - *) echo "unsupported arch: $ARCH" >&2; exit 1 ;; - esac - if [ -x /usr/local/go/bin/go ]; then - HAVE=$(/usr/local/go/bin/go version | awk '{{print $3}}' | sed 's/^go//') - [ "$HAVE" = "$NEED" ] && exit 0 + NEED=$(awk -F'"' '/^rust-version/ {{print $2}}' {cargo_toml_path}/Cargo.toml) + HAVE=$(rustc --version | awk '{{print $2}}') + OLDEST=$(printf '%s\\n%s\\n' "$HAVE" "$NEED" | sort -V | head -n1) + if [ "$OLDEST" = "$HAVE" ] && [ "$HAVE" != "$NEED" ]; then + echo "Updating Rust: $HAVE < required $NEED ..." + rustup update stable fi - URL="https://go.dev/dl/go${{NEED}}.linux-${{GOARCH}}.tar.gz" - echo "Installing Go ${{NEED}} from ${{URL}} ..." - curl -fsSL "$URL" | tar -C /usr/local -xzf - - ln -sf /usr/local/go/bin/go /usr/local/bin/go - ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt """) diff --git a/src/cmlxc/incus.py b/src/cmlxc/incus.py index 18fb3c9..6092a36 100644 --- a/src/cmlxc/incus.py +++ b/src/cmlxc/incus.py @@ -17,6 +17,7 @@ from cmlxc.container import ( BASE_IMAGE_ALIAS, + DNS_CONTAINER_NAME, DOMAIN_SUFFIX, LABEL_DEPLOY_DRIVER, LABEL_DEPLOY_SOURCE, @@ -82,6 +83,37 @@ def __init__(self, out): check=True, ) self.ssh_config_path = self.config_dir / "ssh-config" + self._bridge_subnet = NotImplemented + + @property + def bridge_subnet(self): + """Return the IPv4 subnet of incusbr0, used to filter container IPs.""" + if self._bridge_subnet is NotImplemented: + self._bridge_subnet = None + result = self.run( + ["network", "get", "incusbr0", "ipv4.address"], check=False + ) + if result.returncode == 0 and result.stdout.strip(): + try: + self._bridge_subnet = ipaddress.ip_network( + result.stdout.strip(), strict=False + ) + except ValueError: + pass + return self._bridge_subnet + + def check_init(self): + """Return True if the cmlxc environment is initialized.""" + managed = self.list_managed() + dns_running = any( + c["name"] == DNS_CONTAINER_NAME and c["status"] == "Running" + for c in managed + ) + if not dns_running or not self.find_image([BASE_IMAGE_ALIAS]): + self.out.red("Error: cmlxc environment not initialized.") + self.out.red("Please run 'cmlxc init' first.") + return False + return True def write_ssh_config(self): """Write ``ssh-config`` mapping all containers to their IPs.""" @@ -210,7 +242,7 @@ def list_managed(self): containers.append( { "name": name, - "ip": _extract_ip(net, "inet"), + "ip": _extract_ip(net, "inet", subnet=self.bridge_subnet), "ipv6": _extract_ip(net, "inet6"), "domain": config.get(LABEL_DOMAIN, f"{name}{DOMAIN_SUFFIX}"), "status": ct.get("status", "Unknown"), diff --git a/src/relay_minitest/test_relay.py b/src/relay_minitest/test_relay.py index 8ac12e4..62522de 100644 --- a/src/relay_minitest/test_relay.py +++ b/src/relay_minitest/test_relay.py @@ -1,10 +1,10 @@ +import email import imaplib import ipaddress import smtplib import ssl from email.mime.text import MIMEText -import imap_tools import pytest import requests @@ -75,8 +75,11 @@ def test_delivery_port_blocked( relayadmin2.block_port(443) relayadmin2.block_port(80) chat.send_text("should not arrive") - # Postfix logs "status=deferred", madmail logs "delivery attempt failed". - lines = relayadmin.wait_for_journal_match("deferred|delivery attempt failed") + # Postfix logs "status=deferred"; madmail logs one of three phrasings + # depending on retry state (see chatmail-delivery/src/queue/worker.rs). + lines = relayadmin.wait_for_journal_match( + "deferred|outbound delivery (failed, requeued|permanent failure|exceeded max_tries)" + ) lp.indent(lines.splitlines()[-1]) def test_one_on_one_http_only(self, cmfactory, cmfactory2, relayadmin2, lp): @@ -115,11 +118,19 @@ def test_hide_senders_ip_address(cmfactory, ssl_context): addr = user2.get_config("addr") host = addr.split("@")[1].strip("[]") pw = user2.get_config("mail_pw") - mailbox = imap_tools.MailBox(host, ssl_context=ssl_context) - mailbox.login(addr, pw) - msgs = list(mailbox.fetch(mark_seen=False)) - assert msgs, "expected at least one message" - assert public_ip not in msgs[0].obj.as_string() + + # madmail's IMAP server doesn't implement SEARCH/UID SEARCH (only FETCH, + # UID FETCH/STORE/MOVE/COPY, see chatmail-imap/src/session.rs), so + # fetch by sequence range instead. + imap = imaplib.IMAP4_SSL(host, ssl_context=ssl_context) + imap.login(addr, pw) + imap.select("INBOX") + typ, data = imap.fetch("1:*", "(BODY.PEEK[])") + assert typ == "OK" + raw_messages = [item[1] for item in data if isinstance(item, tuple)] + assert raw_messages, "expected at least one message" + msg = email.message_from_bytes(raw_messages[-1]) + assert public_ip not in msg.as_string() def test_unencrypted_rejection(cmsetup, lp): diff --git a/tests/test_cli.py b/tests/test_cli.py index d245c14..e306125 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,61 @@ """Unit tests for CLI helpers.""" +import shutil +import subprocess from pathlib import Path import pytest -from cmlxc.driver_base import SourceSpec, parse_source, validate_relay_name +from cmlxc.container import SetupError +from cmlxc.driver_base import ( + SourceSpec, + parse_source, + resolve_source, + validate_relay_name, +) from cmlxc.driver_cmdeploy import get_ini_overrides URL = "https://github.com/chatmail/relay.git" +# @latest lookup uses git ls-remote, skip according tests if absent. +requires_git = pytest.mark.skipif( + shutil.which("git") is None, reason="git binary not on PATH" +) + + +def _git(cwd, *args): + """Run a git subcommand in *cwd*, quietly.""" + subprocess.run( + ["git", "-C", str(cwd), *args], + check=True, + capture_output=True, + text=True, + ) + + +@pytest.fixture +def tagged_repo(tmp_path): + """A real local git repo modelling the relay tag scheme.""" + repo = tmp_path / "repo" + subprocess.run( + ["git", "init", "-q", str(repo)], check=True, capture_output=True, text=True + ) + _git( + repo, + "-c", + "user.email=t@t", + "-c", + "user.name=t", + "commit", + "-q", + "--allow-empty", + "-m", + "init", + ) + for tag in ("1.8.0", "1.9.0", "1.10.0-rc1", "withlmtp"): + _git(repo, "tag", tag) + return str(repo) + @pytest.mark.parametrize( "value, expected", @@ -16,6 +63,7 @@ ("@main", SourceSpec("remote", url=URL, ref="main")), ("@fix-dovecot", SourceSpec("remote", url=URL, ref="fix-dovecot")), ("@v2.1", SourceSpec("remote", url=URL, ref="v2.1")), + ("@latest", SourceSpec("remote", url=URL, ref="latest")), ("/home/me/relay", SourceSpec("local", path=Path("/home/me/relay"))), ("./relay", SourceSpec("local", path=Path("./relay"))), ("../relay", SourceSpec("local", path=Path("../relay"))), @@ -61,3 +109,37 @@ def test_ini_overrides_disable_ipv6(): assert "disable_ipv6" not in get_ini_overrides("cm0.localchat") overrides = get_ini_overrides("cm0.localchat", disable_ipv6=True) assert overrides["disable_ipv6"] == "True" + + +@requires_git +def test_source_latest(tagged_repo): + """@latest resolves to the newest stable tag via real git ls-remote.""" + assert resolve_source(parse_source("@latest", tagged_repo)) == SourceSpec( + "remote", url=tagged_repo, ref="1.9.0" + ) + + +@requires_git +def test_source_main(tagged_repo): + """@main is a branch, not a tag, passed through""" + assert resolve_source(parse_source("@main", tagged_repo)) == SourceSpec( + "remote", url=tagged_repo, ref="main" + ) + + +@requires_git +def test_source_latest_no_tags(tmp_path): + """@latest against a real repo with no tags fails with SetupError.""" + empty = tmp_path / "empty" + subprocess.run( + ["git", "init", "-q", str(empty)], check=True, capture_output=True, text=True + ) + with pytest.raises(SetupError, match="No tags found"): + resolve_source(parse_source("@latest", str(empty))) + + +@requires_git +def test_source_latest_bad_repo(tmp_path): + """@latest against a nonexistent repo fails with SetupError.""" + with pytest.raises(SetupError, match="Failed to resolve @latest"): + resolve_source(parse_source("@latest", str(tmp_path / "does-not-exist"))) diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 0000000..da410c3 --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,35 @@ +"""Unit tests for Docker driver helpers.""" + +import pytest + +from cmlxc.driver_docker import _parse_inject_tag + + +@pytest.mark.parametrize( + "value, expected", + [ + ("docker:main", "main"), + ("docker:image:version", "image:version"), + ("docker:registry/repo:tag", "registry/repo:tag"), + ("docker:sha256:abc123", "sha256:abc123"), + ], +) +def test_parse_inject_tag_accepts_valid(value, expected): + assert _parse_inject_tag(value) == expected + + +@pytest.mark.parametrize( + "value", + ["ghcr:main", "@main", "", "main"], +) +def test_parse_inject_tag_returns_none_for_non_docker(value): + assert _parse_inject_tag(value) is None + + +@pytest.mark.parametrize( + "value", + ["docker:", "docker: ", "docker:tag with spaces", "docker:tag!bad"], +) +def test_parse_inject_tag_rejects_invalid_tag(value): + with pytest.raises(ValueError, match="Invalid Docker tag"): + _parse_inject_tag(value)