fix(tests): isolate feature user defaults #1916
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: UI Tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| ref: | |
| required: true | |
| type: string | |
| platform: | |
| default: all | |
| type: string | |
| configuration: | |
| default: default | |
| type: string | |
| update_reference: | |
| default: false | |
| type: boolean | |
| outputs: | |
| verified-sha: | |
| description: Commit verified on both platforms | |
| value: ${{ jobs.ios_uitest.outputs.sha == jobs.macos_uitest.outputs.sha && jobs.ios_uitest.outputs.sha || '' }} | |
| workflow_dispatch: | |
| inputs: | |
| platform: | |
| description: Platform to test | |
| required: true | |
| default: all | |
| type: choice | |
| options: | |
| - all | |
| - ios | |
| - macos | |
| update_reference: | |
| description: Update reference images before running UI tests | |
| required: false | |
| default: false | |
| type: boolean | |
| only-testing: | |
| description: 'Optional test identifier (e.g. OpenSwiftUIUITests/TextUITests/dateFormatStyleExample())' | |
| required: false | |
| default: '' | |
| type: string | |
| configuration: | |
| description: UI test configuration set | |
| required: false | |
| default: default | |
| type: choice | |
| options: | |
| - default | |
| - all | |
| - swiftui-renderer-ag | |
| - swiftui-renderer-iag | |
| - openswiftui-renderer-ag | |
| - openswiftui-renderer-iag | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| statuses: write | |
| jobs: | |
| prepare_uitests: | |
| name: Prepare UI tests | |
| if: >- | |
| (inputs.ref != '' && github.event_name != 'issue_comment') || | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'issue_comment' && | |
| github.event.issue.pull_request && | |
| contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) && | |
| contains(github.event.comment.body, '/uitest')) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| repository: ${{ steps.prepare.outputs.repository }} | |
| ref: ${{ steps.prepare.outputs.ref }} | |
| ios-requested: ${{ steps.prepare.outputs.ios-requested }} | |
| macos-requested: ${{ steps.prepare.outputs.macos-requested }} | |
| update-reference: ${{ steps.prepare.outputs.update-reference }} | |
| only-testing: ${{ steps.prepare.outputs.only-testing }} | |
| status-enabled: ${{ steps.prepare.outputs.status-enabled }} | |
| configuration-matrix: ${{ steps.prepare.outputs.configuration-matrix }} | |
| steps: | |
| - name: Prepare requested UI tests | |
| id: prepare | |
| uses: actions/github-script@v7 | |
| env: | |
| REQUESTED_REF: ${{ inputs.ref || '' }} | |
| PLATFORM: ${{ inputs.platform || 'all' }} | |
| CONFIGURATION: ${{ inputs.configuration || 'default' }} | |
| UPDATE_REFERENCE: ${{ inputs.update_reference && 'true' || 'false' }} | |
| UITEST_ONLY_TESTING: ${{ inputs.only-testing || '' }} | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const allowedAssociations = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); | |
| const configurations = { | |
| "swiftui-renderer-ag": { | |
| id: "swiftui-renderer-ag", | |
| swiftui_renderer: 1, | |
| attributegraph: 1, | |
| compute: 0, | |
| }, | |
| "swiftui-renderer-iag": { | |
| id: "swiftui-renderer-iag", | |
| swiftui_renderer: 1, | |
| attributegraph: 0, | |
| compute: 1, | |
| }, | |
| "openswiftui-renderer-ag": { | |
| id: "openswiftui-renderer-ag", | |
| swiftui_renderer: 0, | |
| attributegraph: 1, | |
| compute: 0, | |
| }, | |
| "openswiftui-renderer-iag": { | |
| id: "openswiftui-renderer-iag", | |
| swiftui_renderer: 0, | |
| attributegraph: 0, | |
| compute: 1, | |
| }, | |
| }; | |
| const configurationAliases = { | |
| "all-configs": "all", | |
| "sui-ag": "swiftui-renderer-ag", | |
| "sui-iag": "swiftui-renderer-iag", | |
| "osui-ag": "openswiftui-renderer-ag", | |
| "osui-iag": "openswiftui-renderer-iag", | |
| }; | |
| const defaultConfigurationIds = [ | |
| "swiftui-renderer-ag", | |
| "openswiftui-renderer-iag", | |
| ]; | |
| const allConfigurationIds = [ | |
| "swiftui-renderer-ag", | |
| "swiftui-renderer-iag", | |
| "openswiftui-renderer-ag", | |
| "openswiftui-renderer-iag", | |
| ]; | |
| const outputs = { | |
| repository: "", | |
| ref: "", | |
| "ios-requested": "false", | |
| "macos-requested": "false", | |
| "update-reference": "false", | |
| "only-testing": "", | |
| "status-enabled": "false", | |
| "configuration-matrix": JSON.stringify(defaultConfigurationIds.map((id) => configurations[id])), | |
| }; | |
| const setOutputs = () => { | |
| for (const [name, value] of Object.entries(outputs)) { | |
| core.setOutput(name, value); | |
| } | |
| }; | |
| const requestPlatforms = (requested) => { | |
| if (requested === "all") { | |
| return ["ios", "macos"]; | |
| } | |
| if (requested === "ios" || requested === "macos") { | |
| return [requested]; | |
| } | |
| return []; | |
| }; | |
| const requestConfigurations = (requested) => { | |
| const normalized = configurationAliases[requested] ?? requested; | |
| if (normalized === "default") { | |
| return defaultConfigurationIds.map((id) => configurations[id]); | |
| } | |
| if (normalized === "all") { | |
| return allConfigurationIds.map((id) => configurations[id]); | |
| } | |
| if (configurations[normalized]) { | |
| return [configurations[normalized]]; | |
| } | |
| return []; | |
| }; | |
| const applyRequest = (platforms, configurationMatrix, updateRequested, onlyTesting = "") => { | |
| outputs["ios-requested"] = String(platforms.includes("ios")); | |
| outputs["macos-requested"] = String(platforms.includes("macos")); | |
| outputs["update-reference"] = String(updateRequested); | |
| outputs["only-testing"] = onlyTesting; | |
| outputs["configuration-matrix"] = JSON.stringify(configurationMatrix); | |
| }; | |
| const requestedRef = process.env.REQUESTED_REF || ""; | |
| if (context.eventName !== "issue_comment" && (requestedRef || context.eventName === "workflow_dispatch")) { | |
| if (requestedRef && (requestedRef.length !== 40 || !/^[0-9a-f]+$/.test(requestedRef))) { | |
| throw new Error("Reusable CI requires a full commit SHA."); | |
| } | |
| const platforms = requestPlatforms(process.env.PLATFORM); | |
| const configurationMatrix = requestConfigurations(process.env.CONFIGURATION); | |
| if (platforms.length === 0 || configurationMatrix.length === 0) { | |
| setOutputs(); | |
| return; | |
| } | |
| outputs.repository = `${owner}/${repo}`; | |
| outputs.ref = requestedRef || context.sha; | |
| applyRequest(platforms, configurationMatrix, process.env.UPDATE_REFERENCE === "true", (process.env.UITEST_ONLY_TESTING ?? "").trim()); | |
| setOutputs(); | |
| return; | |
| } | |
| const issue = context.payload.issue; | |
| const comment = context.payload.comment; | |
| if ( | |
| context.eventName !== "issue_comment" || | |
| !issue?.pull_request || | |
| !comment || | |
| !allowedAssociations.has(comment.author_association) | |
| ) { | |
| setOutputs(); | |
| return; | |
| } | |
| const body = comment.body.trim(); | |
| const args = body.split(/\s+/); | |
| if (args[0] !== "/uitest") { | |
| setOutputs(); | |
| return; | |
| } | |
| let updateRequested = false; | |
| let requestedPlatform = "all"; | |
| let requestedConfiguration = "default"; | |
| let requestedOnlyTesting = ""; | |
| for (const rawArg of args.slice(1)) { | |
| const arg = rawArg.toLowerCase(); | |
| if (arg === "update") { | |
| updateRequested = true; | |
| } else if (arg === "ios" || arg === "macos" || arg === "all") { | |
| requestedPlatform = arg; | |
| } else if (arg.startsWith("config=") || arg.startsWith("configuration=")) { | |
| requestedConfiguration = arg.split("=", 2)[1] || "default"; | |
| } else if (arg.startsWith("only-testing=")) { | |
| requestedOnlyTesting = rawArg.slice("only-testing=".length); | |
| if (!requestedOnlyTesting) { | |
| core.warning("Ignoring /uitest command with an empty only-testing identifier."); | |
| setOutputs(); | |
| return; | |
| } | |
| } else if ( | |
| arg === "default" || | |
| arg === "all-configs" || | |
| configurations[arg] || | |
| configurationAliases[arg] | |
| ) { | |
| requestedConfiguration = arg; | |
| } else { | |
| core.warning(`Ignoring unsupported /uitest argument: ${rawArg}`); | |
| setOutputs(); | |
| return; | |
| } | |
| } | |
| const platforms = requestPlatforms(requestedPlatform); | |
| const configurationMatrix = requestConfigurations(requestedConfiguration); | |
| if (platforms.length === 0 || configurationMatrix.length === 0) { | |
| setOutputs(); | |
| return; | |
| } | |
| const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: issue.number }); | |
| outputs.repository = pull.head.repo.full_name; | |
| outputs.ref = pull.head.sha; | |
| applyRequest(platforms, configurationMatrix, updateRequested, requestedOnlyTesting); | |
| const sameRepository = pull.head.repo.full_name === `${owner}/${repo}`; | |
| if (!sameRepository) { | |
| core.warning(`Skipping PR commit status creation for fork PR ${pull.head.repo.full_name}@${pull.head.sha}.`); | |
| setOutputs(); | |
| return; | |
| } | |
| outputs["status-enabled"] = "true"; | |
| const details_url = `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`; | |
| for (const platform of platforms) { | |
| const label = platform === "ios" ? "iOS" : "macOS"; | |
| for (const configuration of configurationMatrix) { | |
| const description = `Queued by @${comment.user.login} with ${body}.`.slice(0, 140); | |
| await github.rest.repos.createCommitStatus({ | |
| owner, | |
| repo, | |
| sha: pull.head.sha, | |
| state: "pending", | |
| target_url: details_url, | |
| description, | |
| context: `UI Tests / ${label} / ${configuration.id}${requestedOnlyTesting ? " / Selected" : ""}`, | |
| }); | |
| } | |
| } | |
| setOutputs(); | |
| ios_uitest: | |
| name: Execute UI tests on iOS (${{ matrix.configuration.id }}) | |
| outputs: | |
| sha: ${{ steps.checkout.outputs.commit }} | |
| needs: prepare_uitests | |
| permissions: | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| statuses: write | |
| id-token: write | |
| if: >- | |
| needs.prepare_uitests.outputs.ios-requested == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| xcode-version: ["26.6"] | |
| release: [2024] | |
| ios-version: ["18.5"] | |
| configuration: ${{ fromJSON(needs.prepare_uitests.outputs.configuration-matrix) }} | |
| include: | |
| - ios-version: "18.5" | |
| ios-simulator-name: "iPhone 16 Pro" | |
| runs-on: | |
| - xcode-26.6 | |
| - ios-simulator-18.5 | |
| env: | |
| STATUS_CONTEXT: UI Tests / iOS / ${{ matrix.configuration.id }}${{ needs.prepare_uitests.outputs.only-testing != '' && ' / Selected' || '' }} | |
| STATUS_ENABLED: ${{ needs.prepare_uitests.outputs.status-enabled }} | |
| STATUS_SHA: ${{ needs.prepare_uitests.outputs.ref }} | |
| CONFIGURATION_NAME: ${{ matrix.configuration.id }} | |
| OPENSWIFTUI_WERROR: 0 # Disable it to avoid enable OAG's werror and hit conflicts | |
| OPENSWIFTUI_OPENATTRIBUTESHIMS_ATTRIBUTEGRAPH: ${{ matrix.configuration.attributegraph }} | |
| OPENSWIFTUI_OPENATTRIBUTESHIMS_COMPUTE: ${{ matrix.configuration.compute }} | |
| OPENSWIFTUI_COMPATIBILITY_TEST: 0 | |
| OPENSWIFTUI_SWIFT_LOG: 0 | |
| OPENSWIFTUI_SWIFT_CRYPTO: 0 | |
| OPENSWIFTUI_TARGET_RELEASE: ${{ matrix.release }} | |
| OPENSWIFTUI_USE_LOCAL_DEPS: 1 | |
| OPENSWIFTUI_LINK_TESTING: 0 | |
| OPENSWIFTUI_SWIFTUI_RENDERER: ${{ matrix.configuration.swiftui_renderer }} | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Mark PR status running | |
| if: github.event_name == 'issue_comment' && env.STATUS_ENABLED == 'true' && env.STATUS_SHA != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| await github.rest.repos.createCommitStatus({ | |
| owner, | |
| repo, | |
| sha: process.env.STATUS_SHA, | |
| state: "pending", | |
| target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`, | |
| description: `iOS UI tests (${process.env.CONFIGURATION_NAME}) are running.`, | |
| context: process.env.STATUS_CONTEXT, | |
| }); | |
| - uses: actions/checkout@v4 | |
| id: checkout | |
| with: | |
| repository: ${{ needs.prepare_uitests.outputs.repository || github.repository }} | |
| ref: ${{ needs.prepare_uitests.outputs.ref || github.sha }} | |
| - name: Run UI Tests | |
| id: run-tests | |
| uses: ./.github/actions/uitests | |
| with: | |
| xcode-version: ${{ matrix.xcode-version }} | |
| platform: ios | |
| destination: "platform=iOS Simulator,OS=${{ matrix.ios-version }},name=${{ matrix.ios-simulator-name }}" | |
| artifact-name: ios-uitest-snapshots-${{ matrix.ios-version }}-${{ matrix.configuration.id }} | |
| update-reference: ${{ needs.prepare_uitests.outputs.update-reference == 'true' }} | |
| only-testing: ${{ needs.prepare_uitests.outputs.only-testing }} | |
| compute: ${{ matrix.configuration.compute == 1 }} | |
| - name: Require successful tests | |
| if: steps.run-tests.outputs.test-result != 'success' | |
| run: exit 1 | |
| - name: Complete PR status | |
| if: always() && github.event_name == 'issue_comment' && env.STATUS_ENABLED == 'true' && env.STATUS_SHA != '' | |
| uses: actions/github-script@v7 | |
| env: | |
| JOB_STATUS: ${{ job.status }} | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const state = process.env.JOB_STATUS === "success" ? "success" : | |
| process.env.JOB_STATUS === "cancelled" ? "error" : | |
| "failure"; | |
| const description = process.env.JOB_STATUS === "cancelled" ? | |
| "iOS UI tests were cancelled." : | |
| `iOS UI tests (${process.env.CONFIGURATION_NAME}) completed with ${state}.`; | |
| await github.rest.repos.createCommitStatus({ | |
| owner, | |
| repo, | |
| sha: process.env.STATUS_SHA, | |
| state, | |
| target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`, | |
| description, | |
| context: process.env.STATUS_CONTEXT, | |
| }); | |
| macos_uitest: | |
| name: Execute UI tests on macOS (${{ matrix.configuration.id }}) | |
| outputs: | |
| sha: ${{ steps.checkout.outputs.commit }} | |
| needs: prepare_uitests | |
| permissions: | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| statuses: write | |
| id-token: write | |
| if: >- | |
| needs.prepare_uitests.outputs.macos-requested == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [macos-15] | |
| xcode-version: ["26.6"] | |
| release: [2024] | |
| configuration: ${{ fromJSON(needs.prepare_uitests.outputs.configuration-matrix) }} | |
| runs-on: | |
| - self-hosted | |
| - ${{ matrix.os }} | |
| - xcode-26.6 | |
| env: | |
| STATUS_CONTEXT: UI Tests / macOS / ${{ matrix.configuration.id }}${{ needs.prepare_uitests.outputs.only-testing != '' && ' / Selected' || '' }} | |
| STATUS_ENABLED: ${{ needs.prepare_uitests.outputs.status-enabled }} | |
| STATUS_SHA: ${{ needs.prepare_uitests.outputs.ref }} | |
| CONFIGURATION_NAME: ${{ matrix.configuration.id }} | |
| OPENSWIFTUI_WERROR: 0 | |
| OPENSWIFTUI_OPENATTRIBUTESHIMS_ATTRIBUTEGRAPH: ${{ matrix.configuration.attributegraph }} | |
| OPENSWIFTUI_OPENATTRIBUTESHIMS_COMPUTE: ${{ matrix.configuration.compute }} | |
| OPENSWIFTUI_COMPATIBILITY_TEST: 0 | |
| OPENSWIFTUI_SWIFT_LOG: 0 | |
| OPENSWIFTUI_SWIFT_CRYPTO: 0 | |
| OPENSWIFTUI_TARGET_RELEASE: ${{ matrix.release }} | |
| OPENSWIFTUI_USE_LOCAL_DEPS: 1 | |
| OPENSWIFTUI_SWIFTUI_RENDERER: ${{ matrix.configuration.swiftui_renderer }} | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Mark PR status running | |
| if: github.event_name == 'issue_comment' && env.STATUS_ENABLED == 'true' && env.STATUS_SHA != '' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| await github.rest.repos.createCommitStatus({ | |
| owner, | |
| repo, | |
| sha: process.env.STATUS_SHA, | |
| state: "pending", | |
| target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`, | |
| description: `macOS UI tests (${process.env.CONFIGURATION_NAME}) are running.`, | |
| context: process.env.STATUS_CONTEXT, | |
| }); | |
| - uses: actions/checkout@v4 | |
| id: checkout | |
| with: | |
| repository: ${{ needs.prepare_uitests.outputs.repository || github.repository }} | |
| ref: ${{ needs.prepare_uitests.outputs.ref || github.sha }} | |
| - name: Run UI Tests | |
| id: run-tests | |
| uses: ./.github/actions/uitests | |
| with: | |
| xcode-version: ${{ matrix.xcode-version }} | |
| platform: macos | |
| destination: "platform=macOS" | |
| artifact-name: macos-uitest-snapshots-${{ matrix.configuration.id }} | |
| update-reference: ${{ needs.prepare_uitests.outputs.update-reference == 'true' }} | |
| only-testing: ${{ needs.prepare_uitests.outputs.only-testing }} | |
| compute: ${{ matrix.configuration.compute == 1 }} | |
| - name: Require successful tests | |
| if: steps.run-tests.outputs.test-result != 'success' | |
| run: exit 1 | |
| - name: Complete PR status | |
| if: always() && github.event_name == 'issue_comment' && env.STATUS_ENABLED == 'true' && env.STATUS_SHA != '' | |
| uses: actions/github-script@v7 | |
| env: | |
| JOB_STATUS: ${{ job.status }} | |
| with: | |
| script: | | |
| const { owner, repo } = context.repo; | |
| const state = process.env.JOB_STATUS === "success" ? "success" : | |
| process.env.JOB_STATUS === "cancelled" ? "error" : | |
| "failure"; | |
| const description = process.env.JOB_STATUS === "cancelled" ? | |
| "macOS UI tests were cancelled." : | |
| `macOS UI tests (${process.env.CONFIGURATION_NAME}) completed with ${state}.`; | |
| await github.rest.repos.createCommitStatus({ | |
| owner, | |
| repo, | |
| sha: process.env.STATUS_SHA, | |
| state, | |
| target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`, | |
| description, | |
| context: process.env.STATUS_CONTEXT, | |
| }); |