test(e2e): split @lifecycle into @ci + @real tiers (#621)

.github/workflows/ci-real.yml

@@ -0,0 +1,61 @@
+name: CI (real lifecycle)
+
+# Real-user lifecycle suite. Runs on a single OS (windows) because each
+# test downloads ~500 MB of standalone payload and the wall-clock for
+# the full chain is ~45-60 min. Scheduled nightly + manual dispatch.
+#
+# See e2e/AGENTS.md for the @real contract.
+on:
+  workflow_dispatch:
+  schedule:
+    # 09:00 UTC daily (02:00 PT) — after the typical US workday wraps,
+    # so a failure surfaces by morning standup.
+    - cron: '0 9 * * *'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-real-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  real:
+    name: Real lifecycle (windows)
+    runs-on: windows-latest
+    timeout-minutes: 90
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: pnpm
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build app
+        run: pnpm run build
+
+      - name: Run real lifecycle suite
+        run: pnpm run test:e2e:real
+
+      - name: Upload Playwright failure artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-real-failure
+          path: |
+            test-results/
+            playwright-report/
+          retention-days: 14
+          if-no-files-found: ignore

e2e/AGENTS.md

@@ -0,0 +1,106 @@
+# e2e — test tier contract
+
+Every Playwright test in this directory MUST carry exactly one tier tag.
+No exceptions, no untagged tests.
+
+## Tier tags
+
+### `@ci` — every-PR matrix
+
+Runs on Windows + macOS + Linux on every PR. Allowed to:
+
+- Seed installations via `launchApp({ installations, settings, ... })`.
+- Stage fixtures on disk (`writeFileSync`, `mkdirSync`, `execFileSync git`, etc.).
+- Use the `__e2e` backdoor (`seedDownloads`, `setInstallUpdate`, `setAppUpdateState`,
+  `seedRunningSession`, `ageReleaseCache`, IPC invocation spies, …).
+- Invoke `window.api.*` directly via the eval bridge.
+- Send synthetic IPC into the renderer (`wc.send(...)`).
+
+In short: `@ci` is the seeded-harness tier. Fast, loud, deterministic,
+covers renderer + IPC contracts.
+
+### `@real` — real lifecycle
+
+Run only via `pnpm run test:e2e:real` (single project, single OS, nightly /
+manual). The contract:
+
+> A `@real` test must behave as if a human had booted the app and clicked
+> through the UI. Nothing outside the start or end of the test may change
+> state except via DOM clicks and keyboard input dispatched through the
+> visible webContents.
+
+Concretely, **forbidden** in `@real`:
+
+- Passing `installations`, `settings`, or any other seed to `launchApp(...)`.
+- Calls into `__e2e.*` that **mutate** state (`seedDownloads`,
+  `setInstallUpdate`, `setAppUpdateState`, `seedRunningSession`,
+  `ageReleaseCache`, …).
+- Calls into `window.api.*` from test code (use button clicks / typing).
+- Sending synthetic IPC via `webContents.send(...)`.
+- Pre-staging fake install directories with `writeFile` / `mkdir` to skip
+  a real install.
+
+**Allowed** in `@real` (read-only observation only):
+
+- `__e2e.getIpcInvocations`, `__e2e.getRunningSessionSnapshot`,
+  `__e2e.getReleaseCacheCheckedAt` — observing live state never affects it.
+- `app.evaluate(({ BrowserWindow }) => ...)` for inspecting window /
+  webContents state.
+- `execFileSync('git', ['rev-parse', 'HEAD'], …)` for asserting against
+  on-disk state the app actually produced.
+- DOM-driven flows through `panel.click(...)`, `panel.fill(...)`,
+  `panel.pressKey(...)`.
+
+Prerequisite paths (consent → pick local → install → launch, etc.) that a
+test isn't itself asserting belong in `e2e/support/realPrereqs.ts`. Drive
+them with real clicks; never short-circuit with `window.api`.
+
+## OS axis (independent of tier)
+
+`@ci` tests default to running on **all three** OSes. Genuinely
+platform-specific behavior gets an opt-out tag:
+
+| Tag | Effect |
+|---|---|
+| `@windows-only` | excluded from `macos` + `linux` projects |
+| `@macos-only`   | excluded from `windows` + `linux` projects |
+| `@linux-only`   | excluded from `windows` + `macos` projects |
+
+Example: `quit-flow.spec.ts` is `@ci @macos-only` because tray-close mode
+only exists on macOS.
+
+`@real` runs on a single OS by configuration; OS opt-out tags don't apply.
+
+## Playwright projects
+
+```ts
+{ name: 'macos',   grep: /@ci/,   grepInvert: /@(windows|linux)-only/ }
+{ name: 'windows', grep: /@ci/,   grepInvert: /@(macos|linux)-only/   }
+{ name: 'linux',   grep: /@ci/,   grepInvert: /@(macos|windows)-only/ }
+{ name: 'real',    grep: /@real/, timeout: 600_000 }
+```
+
+## Commands
+
+| Command | Runs |
+|---|---|
+| `pnpm run test:e2e:windows` | `@ci` minus `@macos-only` / `@linux-only` |
+| `pnpm run test:e2e:macos`   | `@ci` minus `@windows-only` / `@linux-only` |
+| `pnpm run test:e2e:linux`   | `@ci` minus `@macos-only` / `@windows-only` |
+| `pnpm run test:e2e:real`    | `@real` only (real lifecycle suite) |
+
+When the user says "run the lifecycle tests" or "validate the lifecycle
+tests", they mean `pnpm run test:e2e:real`. That is the only suite that
+proves the app still works for a real user.
+
+## Skipping a broken test
+
+Don't silently delete a failing test. Choose one:
+
+- `test.skip(...)` with a `// TODO(#NNN): …` comment naming the issue.
+- `test.fail(...)` when the test correctly asserts a contract the product
+  is currently violating (known product bug). `test.fail` flips green
+  when the product is fixed, alerting the team to remove the marker.
+
+Every skip / fail must link to an issue. Stale skips rot — they're
+not "passing" tests.

e2e/chooser.test.ts

@@ -39,17 +39,17 @@ test.afterAll(async () => {
   await ctx.cleanup()
 })
 
-test('chooser body renders on cold start @windows @macos @linux', async () => {
+test('chooser body renders on cold start @ci', async () => {
   await expectChooserVisible(ctx.panel)
   expect(await ctx.panel.exists('.chooser-tile-new')).toBe(true)
 })
 
-test('title bar shows install-less pill on chooser host @windows @macos @linux', async () => {
+test('title bar shows install-less pill on chooser host @ci', async () => {
   expect(await ctx.titleBar.exists('.title-install-pill.is-install-less')).toBe(true)
   expect(await ctx.titleBar.textOf('.title-install-name')).toMatch(/Desktop 2\.0/i)
 })
 
-test('clicking New Install tile opens the new-install takeover @windows @macos @linux', async () => {
+test('clicking New Install tile opens the new-install takeover @ci', async () => {
   await clickNewInstallTile(ctx.panel)
   await expectTakeoverOpen(ctx.panel)
   await dismissOverlay(ctx.panel)
@@ -64,7 +64,7 @@ test('clicking New Install tile opens the new-install takeover @windows @macos @
 // after the dedup path runs.
 // ---------------------------------------------------------------------------
 
-test('activate hook focuses the existing chooser host instead of spawning a duplicate @windows @macos @linux', async () => {
+test('activate hook focuses the existing chooser host instead of spawning a duplicate @ci', async () => {
   // Baseline: exactly one host BrowserWindow is open from `beforeAll`.
   const before = await ctx.app.evaluate(({ BrowserWindow }) =>
     BrowserWindow.getAllWindows().filter((w) => !w.isDestroyed()).length,
@@ -92,14 +92,14 @@ test('activate hook focuses the existing chooser host instead of spawning a dupl
 // preserve.
 // ---------------------------------------------------------------------------
 
-test('title popup + system modal webContents are pre-warmed on the chooser host @windows @macos @linux', async () => {
+test('title popup + system modal webContents are pre-warmed on the chooser host @ci', async () => {
   // Both popups are pre-warmed in `comfy-window:title-bar-ready` so the
   // first user trigger doesn't pay the load cost.
   await waitForWebContents(ctx.app, 'comfyTitlePopup.html', 10_000)
   await waitForWebContents(ctx.app, 'comfySystemModal.html', 10_000)
 })
 
-test('title popup opens, renders menu items, and closes via bridge @windows @macos @linux', async () => {
+test('title popup opens, renders menu items, and closes via bridge @ci', async () => {
   // Click the waffle menu — main pushes a config to the cached title popup
   // and flips it visible. We assert the popup is no longer marked hidden
   // by the EmbeddedPopupView contract (the WebContentsView's bounds become
@@ -128,7 +128,7 @@ test('title popup opens, renders menu items, and closes via bridge @windows @mac
   ).toBe(false)
 })
 
-test('title popup reopens after a blur dismiss (menu-closed IPC clears the reopen guard) @windows @macos @linux', async () => {
+test('title popup reopens after a blur dismiss (menu-closed IPC clears the reopen guard) @ci', async () => {
   // The previous test dismissed the popup via the close bridge — that
   // stamps the title-bar's `menuClosedAt.menu` so a click within 100ms
   // is suppressed by the time-based reopen guard. Wait past that
@@ -173,7 +173,7 @@ test('title popup reopens after a blur dismiss (menu-closed IPC clears the reope
   ).toBe(true)
 })
 
-test('title-bar tooltip popup is created on demand and hides cleanly @windows @macos @linux', async () => {
+test('title-bar tooltip popup is created on demand and hides cleanly @ci', async () => {
   // No webContents for the tooltip popup exists before the first show —
   // unlike titlePopup / systemModal, the tooltip is NOT pre-warmed.
   expect(await findWebContentsId(ctx.app, 'comfyTitleTooltip.html')).toBeNull()

e2e/copy-update-destination.test.ts

@@ -68,7 +68,7 @@
   await resetIpcInvocations(ctx.app, 'open-install-window')
 })
 
-test('Copy success opens the destination install in a new window @lifecycle', async () => {
+test('Copy success opens the destination install in a new window @ci', async () => {
   // Drive ProgressModal with a synthetic copy result carrying the
   // destination install id. The renderer's `handleDone` consumes
   // `op.result.newInstallationId` and calls `openInstallWindow`.
@@ -82,7 +82,7 @@
 
   // The brand loader auto-closes 700ms after `finished`, then
   // handleDone fires. Poll the recorded IPC invocations.
   await expect
     .poll(async () => (await getIpcInvocations(ctx.app, 'open-install-window')).length, {
       timeout: 5_000,
       intervals: [100, 250],
@@ -95,7 +95,7 @@
   expect(calls[0]?.installationId).toBe(DEST_ID)
 })
 
-test('No newInstallationId → no open-install-window call @lifecycle', async () => {
+test('No newInstallationId → no open-install-window call @ci', async () => {
   // Same shape but no newInstallationId — should NOT trigger
   // openInstallWindow. Guards the conditional branch in handleDone
   // against accidentally firing for non-copy success paths.

e2e/dashboard-delete-flow.test.ts

@@ -73,12 +73,12 @@ test.afterAll(async () => {
   if (installPath) await rm(installPath, { recursive: true, force: true })
 })
 
-test('chooser shows the seeded tile @lifecycle', async () => {
+test('chooser shows the seeded tile @ci', async () => {
   await ctx.panel.waitForSelector(byTestId(TID.dashboardTile(INSTALL_ID)), { timeout: 10_000 })
   expect(await ctx.panel.textOf(`${byTestId(TID.dashboardTile(INSTALL_ID))} .chooser-tile-name`)).toBe(INSTALL_NAME)
 })
 
-test('Delete from kebab opens confirm without invoking get-detail-sections @lifecycle', async () => {
+test('Delete from kebab opens confirm without invoking get-detail-sections @ci', async () => {
   // Clear any cumulative invocation history so the assertion measures
   // only what fires during the Delete dispatch itself.
   await resetIpcInvocations(ctx.app, 'get-detail-sections')
@@ -104,7 +104,7 @@ test('Delete from kebab opens confirm without invoking get-detail-sections @life
   expect(invocations, 'Delete dispatch must not call get-detail-sections').toEqual([])
 })
 
-test('Confirm removes the install directory and tile @lifecycle', async () => {
+test('Confirm removes the install directory and tile @ci', async () => {
   // Confirm the delete via the BaseAlert primary button.
   const confirmClicked = await ctx.panel.click(byTestId(TID.baseAlertAction))
   expect(confirmClicked, 'confirm button click dispatched').toBe(true)

e2e/devhooks-smoke.test.ts

@@ -30,7 +30,7 @@ test.afterAll(async () => {
   await ctx.cleanup()
 })
 
-test('dev hooks bridge: seedDownloads, setInstallUpdate, setAppUpdateState all run @windows @macos @linux', async () => {
+test('dev hooks bridge: seedDownloads, setInstallUpdate, setAppUpdateState all run @ci', async () => {
   await seedDownloads(ctx.app, {
     active: [
       {
@@ -57,7 +57,7 @@ test('dev hooks bridge: seedDownloads, setInstallUpdate, setAppUpdateState all r
   await setAppUpdateState(ctx.app, { kind: null, version: null, autoUpdate: true })
 })
 
-test('dev hooks bridge: getTitlePopupBounds returns null when no popup is open @windows @macos @linux', async () => {
+test('dev hooks bridge: getTitlePopupBounds returns null when no popup is open @ci', async () => {
   const bounds = await getTitlePopupBounds(ctx.app)
   expect(bounds).toBeNull()
 })

e2e/downloads-shelf.test.ts

@@ -59,7 +59,7 @@ test.beforeEach(async () => {
 // flex-allocated-height regression without manual smoke testing.
 // ---------------------------------------------------------------------------
 
-test('empty drawer fits content (popup height < 260px) @windows @macos @linux', async () => {
+test('empty drawer fits content (popup height < 260px) @ci', async () => {
   await openDownloadsTray(ctx.titleBar)
   await waitForPopupVisible(ctx.app)
   await waitForStableBounds(ctx.app)
@@ -74,7 +74,7 @@ test('empty drawer fits content (popup height < 260px) @windows @macos @linux',
   expect(bounds!.bounds.height).toBeGreaterThan(40)
 })
 
-test('one downloading entry fits content (NOT clipped at 396 ceiling) @windows @macos @linux', async () => {
+test('one downloading entry fits content (NOT clipped at 396 ceiling) @ci', async () => {
   await seedDownloads(ctx.app, {
     active: [makeEntry({ url: 'https://example.test/m1.safetensors', filename: 'm1.safetensors', status: 'downloading', progress: 0.5 })],
     recent: [],
@@ -91,7 +91,7 @@ test('one downloading entry fits content (NOT clipped at 396 ceiling) @windows @
   expect(bounds!.bounds.height).toBeLessThan(260)
 })
 
-test('many entries cap at the ceiling and the list scrolls @windows @macos @linux', async () => {
+test('many entries cap at the ceiling and the list scrolls @ci', async () => {
   const active: DownloadProgressLike[] = []
   for (let i = 0; i < 12; i++) {
     active.push(
@@ -130,7 +130,7 @@ test('many entries cap at the ceiling and the list scrolls @windows @macos @linu
 // active rows, remove for terminal rows).
 // ---------------------------------------------------------------------------
 
-test('per-status row close button maps to the right action @windows @macos @linux', async () => {
+test('per-status row close button maps to the right action @ci', async () => {
   await seedDownloads(ctx.app, {
     active: [
       makeEntry({ url: 'u-dl', filename: 'dl.safetensors', status: 'downloading', progress: 0.5 }),
@@ -180,7 +180,7 @@ test('per-status row close button maps to the right action @windows @macos @linu
 // owes the user when a download starts mid-shelf-open).
 // ---------------------------------------------------------------------------
 
-test('the open popup repaints live when tray state changes @windows @macos @linux', async () => {
+test('the open popup repaints live when tray state changes @ci', async () => {
   await openDownloadsTray(ctx.titleBar)
   await waitForPopupVisible(ctx.app)
   await waitForStableBounds(ctx.app)

e2e/dropdowns.test.ts

@@ -47,7 +47,7 @@ test.beforeEach(async () => {
 // `Reset Zoom` menu-item gating.
 // ---------------------------------------------------------------------------
 
-test('Reset Zoom menu item is absent at zoom level 0 @windows @macos @linux', async () => {
+test('Reset Zoom menu item is absent at zoom level 0 @ci', async () => {
   await setComfyViewZoomLevel(ctx.app, 0)
   await openTitleMenu(ctx.titleBar)
   await popup.waitForSelector('[role="menuitem"]', { timeout: 5_000 })
@@ -56,7 +56,7 @@ test('Reset Zoom menu item is absent at zoom level 0 @windows @macos @linux', as
   expect(labels.some((l) => /reset zoom/i.test(l))).toBe(false)
 })
 
-test('Reset Zoom menu item appears with the current percent label when zoom is non-zero @windows @macos @linux', async () => {
+test('Reset Zoom menu item appears with the current percent label when zoom is non-zero @ci', async () => {
   await setComfyViewZoomLevel(ctx.app, 1)
   await openTitleMenu(ctx.titleBar)
   await popup.waitForSelector('[role="menuitem"]', { timeout: 5_000 })
@@ -77,7 +77,7 @@ test('Reset Zoom menu item appears with the current percent label when zoom is n
 // would leak event handlers (and `tray-state-changed` callbacks etc).
 // ---------------------------------------------------------------------------
 
-test('title-popup webContents listener counts are stable across repeated opens @windows @macos @linux', async () => {
+test('title-popup webContents listener counts are stable across repeated opens @ci', async () => {
   // Prime the popup once so the renderer has loaded and any first-run
   // wiring is in place.
   await openTitleMenu(ctx.titleBar)
@@ -107,7 +107,7 @@ test('title-popup webContents listener counts are stable across repeated opens @
 // tooltip; otherwise both popups overlap and the user reads garbage.
 // ---------------------------------------------------------------------------
 
-test('opening the title menu hides the title-bar tooltip @windows @macos @linux', async () => {
+test('opening the title menu hides the title-bar tooltip @ci', async () => {
   // Drive the tooltip directly via the title-bar bridge, mirroring the
   // existing tooltip-on-demand test in chooser.test.ts.
   await ctx.app.evaluate(({ webContents }) => {

e2e/lifecycle-add-existing.test.ts

@@ -103,7 +103,7 @@ test.afterAll(async () => {
   if (stagedPath) await rm(stagedPath, { recursive: true, force: true })
 })
 
-test('probe detects the staged standalone-shaped directory @lifecycle', async () => {
+test('probe detects the staged standalone-shaped directory @ci', async () => {
   const results = await ctx.panel.evaluate<ProbeResult[]>(
     `window.api.probeInstallation(${JSON.stringify(stagedPath)})`,
   )
@@ -115,7 +115,7 @@ test('probe detects the staged standalone-shaped directory @lifecycle', async ()
   expect(standalone!.comfyVersion?.commit).toMatch(/^[0-9a-f]{40}$/)
 })
 
-test('track-installation registers the directory and chooser shows the tile @lifecycle', async () => {
+test('track-installation registers the directory and chooser shows the tile @ci', async () => {
   const probeResults = await ctx.panel.evaluate<ProbeResult[]>(
     `window.api.probeInstallation(${JSON.stringify(stagedPath)})`,
   )