diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b9c2fb8838e..05c2a014711 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,6 +14,8 @@ on: permissions: contents: read pull-requests: write + statuses: write + actions: read jobs: get-branches: @@ -168,3 +170,175 @@ jobs: slack_webhook_url: ${{ secrets.EXTERNAL_SLACK_DETECTION_RULES_URL }} status: failure if: failure() + + post-pending: + if: | + github.event.pull_request.merged == true + && contains(github.event.pull_request.labels.*.name, 'backport: auto') + && ( + (github.event.action == 'labeled' && github.event.label.name == 'backport: auto') + || (github.event.action == 'closed') + ) + runs-on: ubuntu-latest + steps: + - name: Mark release-line-caught-up pending + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + STATUS_CONTEXT: release-line-caught-up + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: process.env.MERGE_SHA, + state: 'pending', + context: process.env.STATUS_CONTEXT, + description: 'Waiting for auto-backports and release-branch Unit Tests', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + + mark-caught-up-skip: + if: | + github.event.pull_request.merged == true + && contains(github.event.pull_request.labels.*.name, 'backport: skip') + && github.event.action == 'closed' + runs-on: ubuntu-latest + steps: + - name: Mark release-line-caught-up success (backport skipped) + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + STATUS_CONTEXT: release-line-caught-up + with: + script: | + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: process.env.MERGE_SHA, + state: 'success', + context: process.env.STATUS_CONTEXT, + description: 'backport: skip; no release-line wait required', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + + mark-caught-up: + needs: [get-branches, commit] + if: | + always() + && github.event.pull_request.merged == true + && contains(github.event.pull_request.labels.*.name, 'backport: auto') + && ( + (github.event.action == 'labeled' && github.event.label.name == 'backport: auto') + || (github.event.action == 'closed') + ) + && needs.commit.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 24 + steps: + - name: Wait for tip Unit Tests and mark release-line-caught-up + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + BRANCHES: ${{ toJSON(fromJSON(needs.get-branches.outputs.branches)) }} + COMMIT_RESULT: ${{ needs.commit.result }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + STATUS_CONTEXT: release-line-caught-up + WORKFLOW_FILE: pythonpackage.yml + WAIT_SECONDS: "1320" + POLL_SECONDS: "15" + with: + script: | + const { owner, repo } = context.repo; + const contextName = process.env.STATUS_CONTEXT; + const mergeSha = process.env.MERGE_SHA; + const targetUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + + async function postStatus(state, description) { + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: mergeSha, + state, + context: contextName, + description: description.slice(0, 140), + target_url: targetUrl, + }); + } + + if (process.env.COMMIT_RESULT !== 'success') { + const message = `Auto-backport commit job ${process.env.COMMIT_RESULT}`; + await postStatus('failure', message); + core.setFailed(message); + return; + } + + const branches = JSON.parse(process.env.BRANCHES); + const workflowFile = process.env.WORKFLOW_FILE; + const waitMs = Number(process.env.WAIT_SECONDS) * 1000; + const pollMs = Number(process.env.POLL_SECONDS) * 1000; + const deadline = Date.now() + waitMs; + + async function sleep() { + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + + async function unitTestRuns(sha) { + const { data } = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: workflowFile, + head_sha: sha, + per_page: 10, + }); + return data.workflow_runs || []; + } + + function classifyRuns(runs) { + if (runs.some((run) => run.status === 'completed' && run.conclusion === 'success')) { + return 'success'; + } + if (runs.some((run) => run.status === 'in_progress' || run.status === 'queued' || run.status === 'waiting' || run.status === 'pending')) { + return 'pending'; + } + if (runs.some((run) => run.status === 'completed')) { + const failed = runs.find((run) => run.status === 'completed'); + return `failed:${failed.conclusion}`; + } + return 'missing'; + } + + while (true) { + const results = []; + for (const branch of branches) { + const { data } = await github.rest.repos.getBranch({ owner, repo, branch }); + const sha = data.commit.sha; + const result = classifyRuns(await unitTestRuns(sha)); + results.push({ branch, sha, result }); + } + + const summary = results + .map((item) => `${item.branch} @ ${item.sha.slice(0, 7)}: ${item.result}`) + .join('\n'); + core.info(summary); + + if (results.every((item) => item.result === 'success')) { + await postStatus('success', 'All release targets backported; tip Unit Tests passed'); + return; + } + + const failed = results.find((item) => item.result.startsWith('failed:')); + if (failed) { + const message = `${failed.branch} Unit Tests ${failed.result}`; + await postStatus('failure', message); + core.setFailed(`${message}\n${summary}`); + return; + } + + if (Date.now() >= deadline) { + const message = `Timed out waiting for release-branch Unit Tests`; + await postStatus('failure', message); + core.setFailed(`${message}\n${summary}`); + return; + } + await sleep(); + } diff --git a/.github/workflows/branch-status-checks.yml b/.github/workflows/branch-status-checks.yml deleted file mode 100644 index e151e24e227..00000000000 --- a/.github/workflows/branch-status-checks.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Branch Version Status Checks - -on: - pull_request: - branches: [ "*" ] - -permissions: - contents: read - actions: read - -jobs: - get-branches: - uses: ./.github/workflows/get-target-branches.yml - - branch-status-checks: - needs: get-branches - runs-on: ubuntu-latest - strategy: - matrix: - target_branch: ${{ fromJSON(needs.get-branches.outputs.branches) }} - - steps: - - name: Get Backport Status - id: get_backport_status - uses: fjogeleit/http-request-action@bf78da14118941f7e940279dd58f67e863cbeff6 # v1 - with: - url: "https://api.github.com/repos/elastic/detection-rules/actions/workflows/pythonpackage.yml/runs?per_page=1&branch=${{matrix.target_branch}}" - method: 'GET' - bearerToken: ${{ github.token }} - - - name: Check Backport Status - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 - with: - script: | - const workflow_status = ${{ toJSON(fromJSON(steps.get_backport_status.outputs.response).workflow_runs[0].status) }} - const workflow_conclusion = ${{ toJSON(fromJSON(steps.get_backport_status.outputs.response).workflow_runs[0].conclusion) }} - if (workflow_status != 'completed' || - workflow_conclusion != 'success') { - core.setFailed('Recent Backport status: ' + workflow_status + ', conclusion: ' + workflow_conclusion) - } diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 916fc2171dc..1e5fe7592e9 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -10,6 +10,8 @@ on: - 'hunting/**/*.py' - 'tests/**/*.py' - 'lib/**/*.py' + merge_group: + branches: [ main ] permissions: contents: read diff --git a/.github/workflows/esql-validation.yml b/.github/workflows/esql-validation.yml index b0d7802edb6..10c2f523729 100644 --- a/.github/workflows/esql-validation.yml +++ b/.github/workflows/esql-validation.yml @@ -2,6 +2,8 @@ name: ES|QL Validation on: pull_request: branches: [ "*" ] + merge_group: + branches: [ main ] permissions: contents: read @@ -29,7 +31,8 @@ jobs: exit 0 fi - MODIFIED_FILES=$(git diff --name-only --diff-filter=AM HEAD~1 | grep '^rules/.*\.toml$' || true) + BASE_SHA="${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.pull_request.base.sha }}" + MODIFIED_FILES=$(git diff --name-only --diff-filter=AM "$BASE_SHA" HEAD | grep '^rules/.*\.toml$' || true) if [ -z "$MODIFIED_FILES" ]; then echo "No modified or new .toml files found. Skipping workflow." echo "run_esql=false" >> $GITHUB_ENV diff --git a/.github/workflows/manual-backport.yml b/.github/workflows/manual-backport.yml deleted file mode 100644 index ade86bfbc71..00000000000 --- a/.github/workflows/manual-backport.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: manual-backport -on: - workflow_dispatch: - inputs: - target_branch: - description: 'Branch to backport to (e.g. 8.0)' - required: true - commit_sha: - description: 'Sha256 hash of the merge commit to use in backporting' - required: true - exceptions: - description: 'Comma seperated list of files to skip staging e.g. detection_rules/etc/packages.yaml,detection_rules/attack.py)' - required: false - -permissions: - contents: read - -jobs: - commit: - runs-on: ubuntu-latest - - steps: - - - name: Checkout detection-rules - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - token: ${{ secrets.WRITE_TRADEBOT_DETECTION_RULES_TOKEN }} - fetch-depth: 0 - - - name: Set github config - run: | - git config --global user.email "178941316+tradebot-elastic@users.noreply.github.com" - git config --global user.name "tradebot-elastic" - - - name: Get branch histories - run: | - git fetch origin main --depth 100 - git fetch origin "${{github.event.inputs.target_branch}}" --depth 1 - git fetch origin "${{github.event.inputs.commit_sha}}" - git status - git log -1 --format='%H' - - - name: Checkout the commit into the staging area - run: | - # Checkout the merged commit - git checkout ${{github.event.inputs.commit_sha}} - - # Move it to the staging area - git reset --soft HEAD^ - - - name: Setup Python 3.12 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip cache purge - pip install .[dev] - - - name: Prune non-"${{github.event.inputs.target_branch}}" rules - env: - UNSTAGED_LIST_FILE: "../unstaged-rules.txt" - run: | - VERSION=$(cat detection_rules/etc/packages.yaml | grep -oP "(?<=name:\s')[\d\.]+[-\w\.]*(?=')") - python -m detection_rules dev unstage-incompatible-rules --target-stack-version "$VERSION" --exception-list "${{github.event.inputs.exceptions}}" - - # Track which rules were unstaged - git diff --name-only > $UNSTAGED_LIST_FILE - - # Since they've been tracked, remove any untracked files - git checkout -- . - - - name: Commit and push to "${{github.event.inputs.target_branch}}" - env: - COMMIT_MSG_FILE: "../commit-message.txt" - UNSTAGED_LIST_FILE: "../unstaged-rules.txt" - TARGET_BRANCH: "${{github.event.inputs.target_branch}}" - COMMIT_SHA: "${{github.event.inputs.commit_sha}}" - run: | - ./detection_rules/etc/commit-and-push.sh $TARGET_BRANCH $COMMIT_SHA - - - name: "Notify slack on failure" - uses: craftech-io/slack-action@fb1d4e50375d7758efb90fa0564734bae931f84f # v1 - with: - slack_webhook_url: ${{ secrets.READ_DETECTION_RULES_SLACK_WEBHOOK_TOKEN }} - status: failure - if: failure() diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index f4dbcee348f..b1845ce4978 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -5,6 +5,8 @@ on: branches: [ "main", "7.*", "8.*", "9.*" ] pull_request: branches: [ "*" ] + merge_group: + branches: [ main ] permissions: contents: read diff --git a/.github/workflows/release-line-caught-up.yml b/.github/workflows/release-line-caught-up.yml new file mode 100644 index 00000000000..86442deebe7 --- /dev/null +++ b/.github/workflows/release-line-caught-up.yml @@ -0,0 +1,150 @@ +name: Release Line Caught Up + +on: + pull_request: + branches: [main] + merge_group: + branches: [main] + +permissions: + contents: read + actions: read + statuses: read + +jobs: + get-branches: + uses: ./.github/workflows/get-target-branches.yml + + release-line-caught-up: + needs: get-branches + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Wait until origin/main is backported and tip Unit Tests passed + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + BRANCHES: ${{ toJSON(fromJSON(needs.get-branches.outputs.branches)) }} + STATUS_CONTEXT: release-line-caught-up + WORKFLOW_FILE: pythonpackage.yml + WAIT_SECONDS: "1680" + POLL_SECONDS: "15" + with: + script: | + const contextName = process.env.STATUS_CONTEXT; + const workflowFile = process.env.WORKFLOW_FILE; + const branches = JSON.parse(process.env.BRANCHES); + const waitMs = Number(process.env.WAIT_SECONDS) * 1000; + const pollMs = Number(process.env.POLL_SECONDS) * 1000; + const deadline = Date.now() + waitMs; + const { owner, repo } = context.repo; + + async function sleep() { + await new Promise((resolve) => setTimeout(resolve, pollMs)); + } + + async function mainSha() { + const { data } = await github.rest.repos.getBranch({ + owner, + repo, + branch: 'main', + }); + return data.commit.sha; + } + + async function latestStatus(sha) { + const { data } = await github.rest.repos.listCommitStatusesForRef({ + owner, + repo, + ref: sha, + per_page: 100, + }); + const match = data.find((status) => status.context === contextName); + return match ? match.state : null; + } + + async function unitTestRuns(sha) { + const { data } = await github.rest.actions.listWorkflowRuns({ + owner, + repo, + workflow_id: workflowFile, + head_sha: sha, + per_page: 10, + }); + return data.workflow_runs || []; + } + + function classifyRuns(runs) { + if (runs.some((run) => run.status === 'completed' && run.conclusion === 'success')) { + return 'success'; + } + if (runs.some((run) => run.status === 'in_progress' || run.status === 'queued' || run.status === 'waiting' || run.status === 'pending')) { + return 'pending'; + } + if (runs.some((run) => run.status === 'completed')) { + const failed = runs.find((run) => run.status === 'completed'); + return `failed:${failed.conclusion}`; + } + return 'missing'; + } + + async function classifyTips() { + const results = []; + for (const branch of branches) { + const { data } = await github.rest.repos.getBranch({ owner, repo, branch }); + const sha = data.commit.sha; + const runs = await unitTestRuns(sha); + const result = classifyRuns(runs); + results.push({ branch, sha, result }); + } + if (results.every((item) => item.result === 'success')) { + return { state: 'success', results }; + } + const failed = results.find((item) => item.result.startsWith('failed:')); + if (failed) { + return { state: 'failure', results }; + } + return { state: 'pending', results }; + } + + function formatTips(results) { + return results + .map((item) => `${item.branch} @ ${item.sha.slice(0, 7)}: ${item.result}`) + .join('\n'); + } + + while (true) { + const sha = await mainSha(); + const status = await latestStatus(sha); + + if (status === 'success') { + core.info(`${contextName} succeeded on main @ ${sha.slice(0, 7)}`); + return; + } + if (status === 'failure' || status === 'error') { + core.setFailed(`${contextName} is ${status} on main @ ${sha.slice(0, 7)}. Fix the backport or release-branch Unit Tests before merging.`); + return; + } + + if (!status) { + // Cutover: the merge that landed this workflow still ran the old + // backport.yml, so it will not post a status. Fall back to tip CI. + const tips = await classifyTips(); + if (tips.state === 'success') { + core.info(`No ${contextName} status on main @ ${sha.slice(0, 7)}; all release tips have green Unit Tests.\n${formatTips(tips.results)}`); + return; + } + if (tips.state === 'failure') { + core.setFailed(`No ${contextName} status on main @ ${sha.slice(0, 7)}, and a release tip failed Unit Tests:\n${formatTips(tips.results)}`); + return; + } + core.info(`Waiting on release tip Unit Tests (no ${contextName} status yet):\n${formatTips(tips.results)}`); + } else { + core.info(`${contextName} is ${status} on main @ ${sha.slice(0, 7)}`); + } + + if (Date.now() >= deadline) { + core.setFailed(`Timed out after ${process.env.WAIT_SECONDS}s waiting for ${contextName} on main @ ${sha.slice(0, 7)} (status=${status || 'missing'}).`); + return; + } + await sleep(); + }