Automate nightly compatibility verification on a standalone VM

ops/vm/foundry-verify.service

@@ -0,0 +1,52 @@
+# Install: /etc/systemd/system/foundry-verify.service
+# Then: systemctl daemon-reload && systemctl enable --now foundry-verify.timer
+#
+# Credentials (FOUNDRY_USERNAME/PASSWORD/ADMIN_KEY) live in
+# /opt/foundry-playwright/.env and the push/issue token lives in `gh`'s own
+# config (`gh auth login`) — nothing is passed via this unit.
+#
+# Runs as a dedicated non-root user rather than root, with NO docker-group
+# membership - containers run rootless via Podman (see below), since this
+# box fetches and executes third-party FoundryVTT/system/module content on
+# every run, and docker-group access is root-equivalent. Before enabling:
+#   sudo useradd --system --create-home --shell /usr/sbin/nologin foundry-verify
+#   sudo apt install podman podman-docker uidmap slirp4netns
+#   sudo usermod --add-subuids 200000-265535 --add-subgids 200000-265535 foundry-verify
+#   sudo loginctl enable-linger foundry-verify
+#   sudo chown -R foundry-verify:foundry-verify /opt/foundry-playwright
+# (the .env file - which must set FOUNDRY_PLAYWRIGHT_ROOTLESS=1 - and
+# gh/git credentials need to be set up as that user too)
+#
+# podman-docker makes the `docker` binary a Podman wrapper, so nothing in
+# this repo needs to know it isn't talking to real Docker.
+#
+# Commits are SSH-signed. One-time setup (as foundry-verify):
+#   ssh-keygen -t ed25519 -N "" -C "foundry-verify commit signing" -f ~/.ssh/foundry-verify-signing
+#   git config --global gpg.format ssh
+#   git config --global user.signingkey ~/.ssh/foundry-verify-signing.pub
+#   git config --global commit.gpgsign true
+#   git config --global user.email "<id>+<github-username>@users.noreply.github.com"
+#   echo "<that email> $(cat ~/.ssh/foundry-verify-signing.pub)" > ~/.ssh/allowed_signers
+#   git config --global gpg.ssh.allowedSignersFile ~/.ssh/allowed_signers
+# Then register the public key on GitHub as a *Signing key* (not an
+# authentication key - it's never used for repo access, only for the
+# cryptographic signature) under the account matching that noreply email:
+# `gh ssh-key add ~/.ssh/foundry-verify-signing.pub --type signing --title "foundry-verify"`.
+# No passphrase, since this runs unattended with no agent/terminal - the
+# key's blast radius if leaked is "forge signed commits as this identity",
+# not repo or account access, so this is an acceptable tradeoff here.
+
+[Unit]
+Description=Foundry Playwright nightly compatibility verification
+After=network-online.target
+Wants=network-online.target
+# No Requires=docker.service: containers run via rootless Podman, which is
+# daemonless - there's no system-wide service for this unit to depend on.
+
+[Service]
+Type=oneshot
+User=foundry-verify
+Group=foundry-verify
+WorkingDirectory=/opt/foundry-playwright
+ExecStart=/opt/foundry-playwright/ops/vm/verify-nightly.sh
+TimeoutStartSec=6h

ops/vm/foundry-verify.timer

@@ -0,0 +1,20 @@
+# Install: /etc/systemd/system/foundry-verify.timer
+
+[Unit]
+Description=Run Foundry Playwright compatibility verification nightly
+
+[Timer]
+# 20:00 CEST: ~5h clear of the host's ~01:00-01:30/04:30-06:00 CEST backup-job
+# CPU peaks (from a week of RRD history on the PBS VM sharing this host), and
+# ~11.5h clear of the next weekday's 07:30 CEST push blackout - both wider
+# margins than a 02:00 slot would give.
+OnCalendar=*-*-* 20:00:00 Europe/Berlin
+RandomizedDelaySec=600
+# Deliberately no Persistent=true: a missed run should be skipped, not caught
+# up. Persistent=true fires as soon as the timer becomes active again (e.g.
+# right at boot) regardless of the time of day, which could land the git push
+# inside the Mon/Tue/Thu/Fri 07:30-16:30 CEST blackout this schedule exists to
+# avoid. Waiting for the next 20:00 keeps that guarantee unconditional.
+
+[Install]
+WantedBy=timers.target

ops/vm/verify-nightly.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Nightly compatibility verification, run by foundry-verify.timer on the VM.
+#
+# Consumes "pending" rows written by the cloud Monitor Releases workflow,
+# runs the Docker-based verification suite against each, and pushes the
+# result back. Requires FOUNDRY_USERNAME/PASSWORD/ADMIN_KEY in .env and a
+# `gh auth login`'d token with repo scope, both local to this VM only.
+
+REPO_DIR="${FP_REPO_DIR:-/opt/foundry-playwright}"
+# Repo-owned rather than /tmp: a world-writable, predictable path there is
+# open to a symlink pre-creation attack from another local user; this path is
+# only writable by foundry-verify itself.
+LOCK_FILE="$REPO_DIR/.verify-nightly.lock"
+MAX_DISK_USED_PCT=85
+
+cd "$REPO_DIR"
+
+exec 200>"$LOCK_FILE"
+if ! flock -n 200; then
+  echo "[verify-nightly] Another run is still in progress, exiting."
+  exit 0
+fi
+
+# Foundry-version-tagged images are reused across runs and left alone; this
+# only clears dangling layers/containers. Runs via EXIT trap rather than as a
+# last step so it still fires if an earlier command (disk guard, git push,
+# reconciliation) aborts the script under `set -e`.
+trap 'docker system prune -f >/dev/null 2>&1 || true' EXIT
+
+used_pct=$(df --output=pcent "$REPO_DIR" | tail -1 | tr -dc '0-9')
+if [ "$used_pct" -gt "$MAX_DISK_USED_PCT" ]; then
+  echo "[verify-nightly] Disk usage at ${used_pct}%, aborting run." >&2
+  exit 1
+fi
+
+git pull --rebase --autostash
+
+# Don't let a genuine test failure abort the script — --record-failures already
+# writes it to the registry as "failed" so it stops being retried; we still want
+# to push whatever did succeed and reconcile issues either way. Capture the
+# real exit status instead of masking it, so it still surfaces at the end
+# (e.g. to systemd/monitoring) rather than always reporting success.
+verify_status=0
+npm run verify:local -- --all-pending --docker --update-registry --record-failures --git-commit ||
+  verify_status=$?
+
+git pull --rebase --autostash
+
+# The 20:00 CEST schedule is chosen to sit well clear of the Mon/Tue/Thu/Fri
+# 07:30-16:30 CEST push blackout, but this system user's git isn't wired to
+# the global pre-push hook that enforces it interactively - so check again
+# here, right before the one action (the push) that actually leaves the VM.
+tz_day=$(TZ="Europe/Berlin" date +%u)
+tz_hm=$((10#$(TZ="Europe/Berlin" date +%H%M)))
+if [[ "$tz_day" =~ ^[1245]$ ]] && [ "$tz_hm" -ge 730 ] && [ "$tz_hm" -lt 1630 ]; then
+  echo "[verify-nightly] Within the Mon/Tue/Thu/Fri 07:30-16:30 Europe/Berlin push blackout; leaving results committed locally and skipping push/reconciliation for tonight." >&2
+  exit "$verify_status"
+fi
+
+git push
+
+npm run close-resolved-issues
+
+exit "$verify_status"

package.json

@@ -36,6 +36,7 @@
     "format": "oxfmt . --check",
     "format:fix": "oxfmt .",
     "verify:local": "tsx scripts/verify-local.ts",
+    "close-resolved-issues": "tsx scripts/close-resolved-issues.ts",
     "release": "release-it",
     "prepublishOnly": "npm run build",
     "prepare": "husky"

scripts/close-resolved-issues.ts

@@ -0,0 +1,148 @@
+import "dotenv/config";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+
+/**
+ * Reconciles verification-required GitHub issues against verified-versions.json.
+ *
+ * Run after a verification pass: any entry that has left "pending" gets its
+ * matching issue commented on and closed (or, for "failed", relabeled
+ * needs-investigation so a human looks at the real regression instead of it
+ * being silently retried forever).
+ */
+
+interface RegistryEntry {
+  fvtt: string;
+  system: string;
+  systemVersion: string;
+  status: "stable" | "pending" | "incompatible" | "failed";
+  notes: string;
+}
+
+interface GhIssue {
+  number: number;
+  title: string;
+}
+
+function getGithubToken(): string {
+  try {
+    const token = execSync("gh auth token", {
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "pipe"],
+    }).trim();
+    if (token) return token;
+  } catch {
+    console.warn("[close-resolved-issues] gh not available or not logged in.");
+  }
+  const envToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN;
+  if (envToken) return envToken;
+  throw new Error(
+    "No GitHub token available (`gh auth token` failed and GITHUB_TOKEN/GH_TOKEN are unset).",
+  );
+}
+
+function repoSlug(): string {
+  const pkg = JSON.parse(fs.readFileSync(path.join(process.cwd(), "package.json"), "utf8"));
+  const url: string = pkg.repository?.url ?? "";
+  const m = url.match(/github\.com[/:]([^/]+)\/([^/.]+?)(\.git)?$/);
+  if (!m)
+    throw new Error(`Could not determine owner/repo from package.json repository.url: "${url}"`);
+  return `${m[1]}/${m[2]}`;
+}
+
+async function githubRequest<T>(
+  token: string,
+  method: string,
+  urlPath: string,
+  body?: unknown,
+): Promise<T> {
+  const res = await fetch(`https://api.github.com${urlPath}`, {
+    method,
+    headers: {
+      Authorization: `Bearer ${token}`,
+      Accept: "application/vnd.github.v3+json",
+      "User-Agent": "foundry-playwright/close-resolved-issues",
+      ...(body ? { "Content-Type": "application/json" } : {}),
+    },
+    body: body ? JSON.stringify(body) : undefined,
+  });
+  if (!res.ok) {
+    throw new Error(`GitHub API ${method} ${urlPath} failed: ${res.status} ${await res.text()}`);
+  }
+  return (await res.json()) as T;
+}
+
+async function run() {
+  const registryPath = path.join(process.cwd(), "verified-versions.json");
+  const registry: RegistryEntry[] = JSON.parse(fs.readFileSync(registryPath, "utf8"));
+  const resolved = registry.filter((e) => e.status !== "pending");
+
+  if (resolved.length === 0) {
+    console.log("[close-resolved-issues] No resolved entries to reconcile.");
+    return;
+  }
+
+  const token = getGithubToken();
+  const repo = repoSlug();
+
+  // Paginate - a single page could silently miss issues once there are more
+  // than 100 open verification-required issues at once.
+  const openIssues: GhIssue[] = [];
+  for (let page = 1; ; page++) {
+    const batch = await githubRequest<GhIssue[]>(
+      token,
+      "GET",
+      `/repos/${repo}/issues?labels=verification-required&state=open&per_page=100&page=${page}`,
+    );
+    openIssues.push(...batch);
+    if (batch.length < 100) break;
+  }
+
+  for (const entry of resolved) {
+    const title = `Verification Required: FVTT ${entry.fvtt} + ${entry.system} v${entry.systemVersion}`;
+    try {
+      const issue = openIssues.find((i) => i.title === title);
+      if (!issue) continue;
+
+      const outcome =
+        entry.status === "stable"
+          ? `✅ Verified stable.\n\n${entry.notes}`
+          : entry.status === "incompatible"
+            ? `❌ Confirmed incompatible.\n\n${entry.notes}`
+            : `⚠️ Automated verification failed and needs investigation.\n\n${entry.notes}`;
+
+      console.log(`[close-resolved-issues] #${issue.number}: ${title} -> ${entry.status}`);
+      await githubRequest(token, "POST", `/repos/${repo}/issues/${issue.number}/comments`, {
+        body: outcome,
+      });
+
+      if (entry.status === "failed") {
+        await githubRequest(token, "POST", `/repos/${repo}/issues/${issue.number}/labels`, {
+          labels: ["needs-investigation"],
+        });
+        // Drop the label this query selects on, or a "failed" entry (which
+        // stays "failed" forever - it's not re-verified by --all-pending)
+        // would get re-commented and re-labeled every single night.
+        await githubRequest(
+          token,
+          "DELETE",
+          `/repos/${repo}/issues/${issue.number}/labels/verification-required`,
+        );
+      } else {
+        await githubRequest(token, "PATCH", `/repos/${repo}/issues/${issue.number}`, {
+          state: "closed",
+        });
+      }
+    } catch (e) {
+      // One entry's GitHub API call failing (rate limit, deleted issue,
+      // transient network error) shouldn't abort reconciliation for every
+      // other independent entry in this run.
+      console.error(
+        `[close-resolved-issues] Failed to reconcile "${title}": ${(e as Error).message}`,
+      );
+    }
+  }
+}
+
+run();