ci(e2e): CMT scheduling + macOS E2E stabilization + CI/security hardening

.github/workflows/cmt-provisioner.yml

@@ -1,10 +1,20 @@
 name: CMT Provisioner
 
-# This is a stub workflow. Its only purpose is to fire a workflow_run webhook
-# to Matterwick, which provisions cloud servers for each server version and
-# then dispatches compatibility-matrix-testing.yml with the CMT_MATRIX input.
+# Trigger workflow for Compatibility Matrix Testing (CMT).
 #
-# Usage: Go to Actions → CMT Provisioner → Run workflow
+# Previous design: this workflow just echoed the inputs and relied on GitHub
+# firing a workflow_run webhook to matterwick, which would then read the
+# server_versions input from the webhook payload. That path is broken --
+# GitHub's workflow_run payload does not include workflow_dispatch inputs,
+# so matterwick never received server_versions and never dispatched
+# compatibility-matrix-testing.yml.
+#
+# New design: POST directly to matterwick's /cmt_dispatch endpoint with
+# everything it needs (server_versions, run_id, sha, ref, repo, owner).
+# Matterwick returns 202 immediately and provisions instances + dispatches
+# compatibility-matrix-testing.yml asynchronously.
+#
+# Usage: Go to Actions -> CMT Provisioner -> Run workflow
 #   server_versions: comma-separated Mattermost server versions
 #   Example: "v11.1.0, v11.2.0, v12.0.0"
 
@@ -16,14 +26,57 @@ on:
         required: true
         type: string
 
+# This workflow does not use GITHUB_TOKEN. It does not check out the repo,
+# does not call the GitHub API, and does not interact with workflow files;
+# its single step is an outbound curl to matterwick authenticated by a
+# separate shared secret. Drop all GITHUB_TOKEN scopes so the runner-issued
+# token cannot accidentally be used to modify the repo if a step is added
+# carelessly later.
+permissions: {}
+
 jobs:
-  notify:
+  trigger-matterwick:
     runs-on: ubuntu-22.04
     steps:
-      - name: Log CMT request
+      - name: Request CMT provisioning from matterwick
         env:
+          MATTERWICK_URL: ${{ vars.MATTERWICK_URL }}
+          CMT_TRIGGER_TOKEN: ${{ secrets.MATTERWICK_CMT_TRIGGER_SECRET }}
           SERVER_VERSIONS: ${{ inputs.server_versions }}
+          REPO: ${{ github.event.repository.name }}
+          OWNER: ${{ github.repository_owner }}
+          SHA: ${{ github.sha }}
+          REF: ${{ github.ref_name }}
+          RUN_ID: ${{ github.run_id }}
         run: |
-          echo "CMT Provisioner triggered"
-          echo "Server versions: ${SERVER_VERSIONS}"
-          echo "Matterwick will provision cloud instances and dispatch compatibility-matrix-testing.yml"
+          if [ -z "${MATTERWICK_URL}" ]; then
+            echo "ERROR: vars.MATTERWICK_URL is not set on this repository" >&2
+            exit 1
+          fi
+          if [ -z "${CMT_TRIGGER_TOKEN}" ]; then
+            echo "ERROR: secrets.MATTERWICK_CMT_TRIGGER_SECRET is not set on this repository" >&2
+            exit 1
+          fi
+
+          payload="$(jq -nc \
+            --arg owner    "${OWNER}" \
+            --arg repo     "${REPO}" \
+            --arg sha      "${SHA}" \
+            --arg ref      "${REF}" \
+            --argjson run_id "${RUN_ID}" \
+            --arg versions "${SERVER_VERSIONS}" \
+            '{owner:$owner, repo:$repo, sha:$sha, ref:$ref, run_id:$run_id, server_versions:$versions}')"
+
+          echo "Requesting CMT provisioning for server versions: ${SERVER_VERSIONS}"
+
+          curl -sS -X POST "${MATTERWICK_URL}/cmt_dispatch" \
+            -H "Content-Type: application/json" \
+            -H "X-Trigger-Token: ${CMT_TRIGGER_TOKEN}" \
+            --data-binary "${payload}" \
+            --fail-with-body \
+            --retry 3 --retry-delay 5 --retry-connrefused \
+            --connect-timeout 10 --max-time 30
+
+          echo "matterwick accepted the CMT dispatch request."
+          echo "Provisioning will run asynchronously (~30 min) and then"
+          echo "compatibility-matrix-testing.yml will be dispatched automatically."

.github/workflows/e2e-functional-template.yml

@@ -229,6 +229,18 @@ jobs:
           npm ci
           cd e2e && npm ci
 
+      - name: e2e/suppress-macos-dialogs
+        if: runner.os == 'macOS'
+        run: |
+          # Suppress the "Reopen windows" dialog that blocks Electron startup
+          # when a previous test instance was killed by SIGTERM/SIGKILL.
+          # The global-setup.ts also does this, but running it here as well ensures
+          # the setting is applied even if the Playwright globalSetup step is skipped.
+          defaults write com.github.Electron NSQuitAlwaysKeepsWindows -bool false || true
+          defaults write com.github.Electron ApplePersistenceIgnoreState -bool YES || true
+          defaults write com.apple.LaunchServices LSQuarantine -bool false || true
+          defaults write com.apple.CrashReporter DialogType none || true
+
       - name: e2e/run-playwright-tests-linux
         if: runner.os == 'Linux'
         run: |

.github/workflows/e2e-functional.yml

@@ -334,6 +334,16 @@ jobs:
           npm ci
           cd e2e && npm ci
 
+      - name: e2e/suppress-macos-dialogs
+        if: runner.os == 'macOS'
+        run: |
+          # Suppress the "Reopen windows" dialog that blocks Electron startup
+          # when a previous test instance was killed by SIGTERM/SIGKILL.
+          defaults write com.github.Electron NSQuitAlwaysKeepsWindows -bool false || true
+          defaults write com.github.Electron ApplePersistenceIgnoreState -bool YES || true
+          defaults write com.apple.LaunchServices LSQuarantine -bool false || true
+          defaults write com.apple.CrashReporter DialogType none || true
+
       - name: e2e/build-test-bundle
         run: npm run build-test
 

.github/workflows/e2e-nightly-trigger.yml

@@ -4,11 +4,14 @@ name: E2E Nightly Trigger
 # Matterwick receives the workflow_run:requested event, provisions 3 Mattermost cloud
 # instances (linux/macos/windows), and dispatches e2e-functional.yml with instance details.
 # When e2e-functional.yml completes, Matterwick destroys the provisioned instances.
+#
+# Schedule-only. Pushes to master and release-* are handled by Matterwick's push-event
+# handler directly (E2EAutoTriggerOnMaster / E2EAutoTriggerOnRelease in the deployed
+# config). Previously this workflow also fired on push to those branches, which caused
+# both Matterwick paths to fire concurrently for every push -- producing 2x cloud
+# instances and 2x e2e-functional.yml runs per commit. Dropping the push trigger here
+# leaves Matterwick's native push handler as the single canonical driver for push runs.
 on:
-  push:
-    branches:
-      - master
-      - 'release-*'
   schedule:
     - cron: "0 0 * * 4,5" # Thursday and Friday midnight UTC
 

e2e/fixtures/index.ts

@@ -92,6 +92,12 @@ export const test = base.extend<Fixtures>({
                 '--disable-features=CrossOriginOpenerPolicy',
                 '--disable-renderer-backgrounding',
 
+                // Dialogs & first-run
+                '--no-first-run',
+                '--no-default-browser-check',
+                '--disable-default-apps',
+                '--disable-crash-reporter',
+
                 // Consistency
                 '--force-color-profile=srgb',
                 '--mute-audio',

e2e/global-setup.ts

@@ -28,11 +28,38 @@ export default async function globalSetup() {
     }
 
     if (process.platform === 'darwin') {
+        // Multiple bundle IDs may be involved: com.github.Electron (Electron binary
+        // launched directly) and the app's own bundle ID (when running signed builds).
+        const bundleIDs = ['com.github.Electron'];
+
+        for (const bundleID of bundleIDs) {
+            try {
+                execSync(`defaults write ${bundleID} NSQuitAlwaysKeepsWindows -bool false`, {stdio: 'pipe'});
+            } catch {
+                // Non-fatal — tests still run, just potentially with the Resume dialog
+            }
+            try {
+                execSync(`defaults write ${bundleID} ApplePersistenceIgnoreState -bool YES`, {stdio: 'pipe'});
+            } catch {
+                // Non-fatal
+            }
+        }
+
+        // Verify at least one bundle ID got the settings applied.
+        // Also suppress the "verification of developer" dialog that can appear
+        // on first launch of unsigned Electron builds.
+        try {
+            execSync('defaults write com.apple.LaunchServices LSQuarantine -bool false', {stdio: 'pipe'});
+        } catch {
+            // Non-fatal
+        }
+
+        // Suppress the macOS crash dialog ("Electron quit unexpectedly") that
+        // appears when a process is killed by SIGKILL in global-teardown.
         try {
-            execSync('defaults write com.github.Electron NSQuitAlwaysKeepsWindows -bool false', {stdio: 'ignore'});
-            execSync('defaults write com.github.Electron ApplePersistenceIgnoreState -bool YES', {stdio: 'ignore'});
+            execSync('defaults write com.apple.CrashReporter DialogType none', {stdio: 'pipe'});
         } catch {
-            // Non-fatal — tests still run, just potentially with the Resume dialog
+            // Non-fatal
         }
     }
 }

e2e/global-teardown.ts

@@ -51,7 +51,10 @@ export default async function globalTeardown() {
             continue;
         }
 
-        const deadline = Date.now() + 5_000;
+        // Give the process time to exit gracefully.
+        // On macOS, use a longer wait since Electron shutdown can be slow.
+        const waitMs = process.platform === 'darwin' ? 10_000 : 5_000;
+        const deadline = Date.now() + waitMs;
         while (Date.now() < deadline) {
             if (!isProcessAlive(pid)) {
                 break;
@@ -60,10 +63,16 @@ export default async function globalTeardown() {
         }
 
         if (isProcessAlive(pid)) {
-            try {
-                process.kill(pid, 'SIGKILL');
-            } catch {
-                // already exited
+            // On macOS, SIGKILL triggers the "quit unexpectedly" crash dialog
+            // which blocks subsequent Electron launches. Skip SIGKILL and let
+            // the process linger — global-setup clears the registry, and each
+            // test uses a unique userDataDir so orphans never block new tests.
+            if (process.platform !== 'darwin') {
+                try {
+                    process.kill(pid, 'SIGKILL');
+                } catch {
+                    // already exited
+                }
             }
         }
     }

e2e/helpers/appReadiness.ts

@@ -16,6 +16,10 @@ import type {ElectronApplication} from 'playwright';
  * We read the global directly, not via IPC.
  */
 export async function waitForAppReady(app: ElectronApplication): Promise<void> {
+    // macOS CI runners are slower and may show Resume dialogs that delay startup.
+    // Use a longer timeout to accommodate this.
+    const timeout = process.platform === 'darwin' ? 60_000 : 30_000;
+
     await expect.poll(
         async () => {
             try {
@@ -32,8 +36,14 @@ export async function waitForAppReady(app: ElectronApplication): Promise<void> {
             }
         },
         {
-            message: 'Timed out waiting for __e2eAppReady. Check that initialize.ts sets it after handleMainWindowIsShown().',
-            timeout: 30_000,
+            message: [
+                'Timed out waiting for __e2eAppReady.',
+                `Timeout: ${timeout}ms.`,
+                'Check that initialize.ts sets __e2eAppReady after handleMainWindowIsShown().',
+                'On macOS, verify that global-setup.ts successfully wrote NSQuitAlwaysKeepsWindows=false',
+                'to prevent the "Reopen windows" dialog from blocking startup.',
+            ].join(' '),
+            timeout,
             intervals: [200, 500, 1000, 2000],
         },
     ).toBe(true);

e2e/utils/analyze-flaky-test.js

@@ -27,10 +27,39 @@
     }
 
     if (suite.failures !== undefined || suite.errors !== undefined) {
+        // These are aggregated counts. Playwright includes retried failures in them.
+        // If the exit code is 0, all tests ultimately passed — only count definitive failures.
+        const exitCode = toNumber(process.env.PLAYWRIGHT_EXIT_CODE || '0');
+        if (exitCode === 0) {
+            return 0;
+        }
         return toNumber(suite.failures) + toNumber(suite.errors);
     }
 
-    return asArray(suite.testcase).filter((testcase) => testcase.failure || testcase.error).length;
+    // When no aggregated counts exist, inspect individual testcases.
+    // Filter out failures that were later retried and passed.
+    const cases = asArray(suite.testcase);
+    const definitiveFailures = cases.filter((testcase) => {
+        if (!testcase.failure && !testcase.error) {
+            return false;
+        }
+        // If this test name ends with a retry suffix like " (retry #1)",
+        // and the base test (without suffix) also appears as a passing case,
+        // this failure was retried and resolved — don't count it.
+        const name = testcase.name || '';
+        const retryMatch = name.match(/^(.*) \(retry #\d+\)$/);
+        if (retryMatch) {
+            const baseName = retryMatch[1];
+            const hasPassingRetry = cases.some(
+                (c) => c.name === baseName && !c.failure && !c.error,
+            );
+            if (hasPassingRetry) {
+                return false;
+            }
+        }
+        return true;
+    });
+    return definitiveFailures.length;
 }
 
 function getFailureCountFromReport(report) {