diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae5534b2..59928d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -453,34 +453,55 @@ jobs: # an option: it changes the public IP of an already-registered runner # and severs its connection to GitHub (the job then hangs). So attach # the EIP to the stopped pool instance HERE, before the action starts - # it, so it boots with the whitelisted IP and registers on it. The - # filter mirrors the action's own findStoppedPoolInstance selection - # exactly (managed + repository + pool tags, stopped, matching type). - # No match => cold launch, where eip-allocation-id handles association. + # it, so it boots with the whitelisted IP and registers on it. + # + # The filter also matches `stopping` (unlike the action's own + # findStoppedPoolInstance, which matches `stopped` only): the previous + # run's stop-runner can return while its instance is still stopping, + # and a run queued right behind it would then see an "empty" pool and + # cold-launch a duplicate that nothing drains until the nightly pass + # (observed 2026-07-10: stop at 08:45:51, next run's lookup at + # 08:46:04 -> pool_empty -> second instance all day). When the chosen + # instance is still stopping, wait for it to finish stopping so the + # action's own lookup finds it. ec2-github-runner#69 fixes the same + # race action-side (mode:stop waits for `stopped`); this is the + # consumer-side belt to that suspender, and it also covers older + # action pins. No match => cold launch, where eip-allocation-id + # handles association. env: POOL_TAG: sandbox-acceptance INSTANCE_TYPE: t3.medium run: | - iid="$(aws ec2 describe-instances \ + pairs="$(aws ec2 describe-instances \ --filters "Name=tag:ec2-github-runner:managed,Values=true" \ "Name=tag:ec2-github-runner:repository,Values=${GITHUB_REPOSITORY}" \ "Name=tag:ec2-github-runner:pool,Values=${POOL_TAG}" \ - "Name=instance-state-name,Values=stopped" \ + "Name=instance-state-name,Values=stopped,stopping" \ "Name=instance-type,Values=${INSTANCE_TYPE}" \ - --query 'Reservations[].Instances[].InstanceId' --output text)" - count="$(printf '%s' "$iid" | tr '\t' '\n' | grep -c . || true)" + --query 'Reservations[].Instances[].[InstanceId,State.Name]' --output text)" + count="$(printf '%s' "$pairs" | grep -c . || true)" if [ "$count" = "0" ]; then - echo "No stopped warm-pool instance; cold launch will associate the EIP via eip-allocation-id." - else - first="$(printf '%s' "$iid" | tr '\t' '\n' | head -n1)" - if [ "$count" != "1" ]; then - echo "::warning::Found ${count} stopped warm-pool instances (${iid}); the reaper is not draining them. Pre-attaching to ${first}; the acceptance \"Verify sandbox EIP\" gate will fail cleanly if the action reuses a different one." + echo "No stopped or stopping warm-pool instance; cold launch will associate the EIP via eip-allocation-id." + exit 0 + fi + if [ "$count" != "1" ]; then + echo "::warning::Found ${count} warm-pool instances ($(printf '%s' "$pairs" | tr '\n\t' ' :')); the reaper is not draining them. Pre-attaching to the first reusable one; the acceptance \"Verify sandbox EIP\" gate will fail cleanly if the action reuses a different one." + fi + # Prefer an already-stopped instance (no wait); otherwise take the + # stopping one and wait out its transition. + target="$(printf '%s\n' "$pairs" | awk '$2 == "stopped" { print $1; exit }')" + if [ -z "$target" ]; then + target="$(printf '%s\n' "$pairs" | awk '$2 == "stopping" { print $1; exit }')" + echo "Warm-pool instance ${target} is still stopping; waiting for it to reach stopped." + if ! aws ec2 wait instance-stopped --instance-ids "$target"; then + echo "::warning::Instance ${target} did not reach stopped in time; falling back to a cold launch (eip-allocation-id associates the EIP there). The stuck instance is left for the reaper." + exit 0 fi - aws ec2 associate-address \ - --allocation-id "$SANDBOX_EIP_ALLOCATION_ID" \ - --instance-id "$first" --allow-reassociation >/dev/null - echo "Pre-attached ${SANDBOX_EIP_ALLOCATION_ID} to stopped warm-pool instance ${first}." fi + aws ec2 associate-address \ + --allocation-id "$SANDBOX_EIP_ALLOCATION_ID" \ + --instance-id "$target" --allow-reassociation >/dev/null + echo "Pre-attached ${SANDBOX_EIP_ALLOCATION_ID} to stopped warm-pool instance ${target}." - name: Start EC2 runner id: start-ec2-runner uses: namecheap/ec2-github-runner@65fbe4f0c1181c60d4df2bd3b33b8893e21fbb43 # v4.0.1 @ 2026-07-08: warm-pool reuse:stop registration fixes (#62/#64/#65/#66) diff --git a/.github/workflows/cleanup-ec2-runners.yml b/.github/workflows/cleanup-ec2-runners.yml index ffaa52a4..95f0ea0d 100644 --- a/.github/workflows/cleanup-ec2-runners.yml +++ b/.github/workflows/cleanup-ec2-runners.yml @@ -15,9 +15,14 @@ name: Cleanup EC2 runners # the EIP directly. # # Two passes share this one workflow: -# - nightly full drain (37 2 * * * UTC): tiny reaper-stopped-max-age so the -# day's pool instance -- stopped anywhere from seconds to ~24h earlier -- -# is always past the threshold and gets terminated. +# - nightly full drain (7 23 * * * UTC, i.e. right after the working day +# ends; :07 offset from the top of the hour per the same GitHub +# scheduling-congestion advice as the leak reaper below -- and observed +# schedule delays run to hours, so an early-off-peak slot keeps the drain +# landing well before the next workday starts): tiny +# reaper-stopped-max-age so the day's pool instance -- stopped anywhere +# from seconds to ~24h earlier -- is always past the threshold and gets +# terminated. # - leak reaper (7,37 * * * * UTC, i.e. every 30 min offset from :00/:30 per # GitHub's documented advice on avoiding top-of-hour/half-hour scheduling # congestion): the action's own default reaper-stopped-max-age, catching @@ -28,7 +33,7 @@ name: Cleanup EC2 runners # the schedules (see the issue's verification plan). on: schedule: - - cron: "37 2 * * *" # nightly full drain, UTC + - cron: "7 23 * * *" # nightly full drain, UTC (end of workday) - cron: "7,37 * * * *" # leak reaper, every 30 min, UTC workflow_dispatch: inputs: @@ -88,7 +93,7 @@ jobs: EVENT_SCHEDULE: ${{ github.event.schedule }} DISPATCH_MODE: ${{ inputs.mode }} run: | - if [ "${EVENT_SCHEDULE}" = "37 2 * * *" ]; then + if [ "${EVENT_SCHEDULE}" = "7 23 * * *" ]; then echo "kind=nightly-drain" >> "$GITHUB_OUTPUT" elif [ "${EVENT_SCHEDULE}" = "7,37 * * * *" ]; then echo "kind=leak-reap" >> "$GITHUB_OUTPUT" diff --git a/SECURITY_COMPLIANCE.md b/SECURITY_COMPLIANCE.md index 18897688..713d3abf 100644 --- a/SECURITY_COMPLIANCE.md +++ b/SECURITY_COMPLIANCE.md @@ -232,7 +232,7 @@ below for the reuse/cleanup/EIP mechanics: output — the exact symptom seen earlier in this PR). - **Nightly drain and leak reaper.** [`cleanup-ec2-runners.yml`](.github/workflows/cleanup-ec2-runners.yml) runs - `mode: cleanup` on two schedules: a nightly full drain at `37 2 * * *` UTC + `mode: cleanup` on two schedules: a nightly full drain at `7 23 * * *` UTC with a deliberately tiny `reaper-stopped-max-age` so the day's pool instance always ages past the threshold and is terminated, and a `7,37 * * * *` UTC leak-reaper pass at the action's own default threshold