ci: wait out the stopping-state warm-pool race; move nightly drain to 23:07 UTC

.github/workflows/ci.yml

@@ -453,34 +453,55 @@ jobs:
         # an option: it changes the public IP of an already-registered runner
         # and severs its connection to GitHub (the job then hangs). So attach
         # the EIP to the stopped pool instance HERE, before the action starts
-        # it, so it boots with the whitelisted IP and registers on it. The
-        # filter mirrors the action's own findStoppedPoolInstance selection
-        # exactly (managed + repository + pool tags, stopped, matching type).
-        # No match => cold launch, where eip-allocation-id handles association.
+        # it, so it boots with the whitelisted IP and registers on it.
+        #
+        # The filter also matches `stopping` (unlike the action's own
+        # findStoppedPoolInstance, which matches `stopped` only): the previous
+        # run's stop-runner can return while its instance is still stopping,
+        # and a run queued right behind it would then see an "empty" pool and
+        # cold-launch a duplicate that nothing drains until the nightly pass
+        # (observed 2026-07-10: stop at 08:45:51, next run's lookup at
+        # 08:46:04 -> pool_empty -> second instance all day). When the chosen
+        # instance is still stopping, wait for it to finish stopping so the
+        # action's own lookup finds it. ec2-github-runner#69 fixes the same
+        # race action-side (mode:stop waits for `stopped`); this is the
+        # consumer-side belt to that suspender, and it also covers older
+        # action pins. No match => cold launch, where eip-allocation-id
+        # handles association.
         env:
           POOL_TAG: sandbox-acceptance
           INSTANCE_TYPE: t3.medium
         run: |
-          iid="$(aws ec2 describe-instances \
+          pairs="$(aws ec2 describe-instances \
             --filters "Name=tag:ec2-github-runner:managed,Values=true" \
                       "Name=tag:ec2-github-runner:repository,Values=${GITHUB_REPOSITORY}" \
                       "Name=tag:ec2-github-runner:pool,Values=${POOL_TAG}" \
-                      "Name=instance-state-name,Values=stopped" \
+                      "Name=instance-state-name,Values=stopped,stopping" \
                       "Name=instance-type,Values=${INSTANCE_TYPE}" \
-            --query 'Reservations[].Instances[].InstanceId' --output text)"
-          count="$(printf '%s' "$iid" | tr '\t' '\n' | grep -c . || true)"
+            --query 'Reservations[].Instances[].[InstanceId,State.Name]' --output text)"
+          count="$(printf '%s' "$pairs" | grep -c . || true)"
           if [ "$count" = "0" ]; then
-            echo "No stopped warm-pool instance; cold launch will associate the EIP via eip-allocation-id."
-          else
-            first="$(printf '%s' "$iid" | tr '\t' '\n' | head -n1)"
-            if [ "$count" != "1" ]; then
-              echo "::warning::Found ${count} stopped warm-pool instances (${iid}); the reaper is not draining them. Pre-attaching to ${first}; the acceptance \"Verify sandbox EIP\" gate will fail cleanly if the action reuses a different one."
+            echo "No stopped or stopping warm-pool instance; cold launch will associate the EIP via eip-allocation-id."
+            exit 0
+          fi
+          if [ "$count" != "1" ]; then
+            echo "::warning::Found ${count} warm-pool instances ($(printf '%s' "$pairs" | tr '\n\t' ' :')); the reaper is not draining them. Pre-attaching to the first reusable one; the acceptance \"Verify sandbox EIP\" gate will fail cleanly if the action reuses a different one."
+          fi
+          # Prefer an already-stopped instance (no wait); otherwise take the
+          # stopping one and wait out its transition.
+          target="$(printf '%s\n' "$pairs" | awk '$2 == "stopped" { print $1; exit }')"
+          if [ -z "$target" ]; then
+            target="$(printf '%s\n' "$pairs" | awk '$2 == "stopping" { print $1; exit }')"
+            echo "Warm-pool instance ${target} is still stopping; waiting for it to reach stopped."
+            if ! aws ec2 wait instance-stopped --instance-ids "$target"; then
+              echo "::warning::Instance ${target} did not reach stopped in time; falling back to a cold launch (eip-allocation-id associates the EIP there). The stuck instance is left for the reaper."
+              exit 0
             fi
-            aws ec2 associate-address \
-              --allocation-id "$SANDBOX_EIP_ALLOCATION_ID" \
-              --instance-id "$first" --allow-reassociation >/dev/null
-            echo "Pre-attached ${SANDBOX_EIP_ALLOCATION_ID} to stopped warm-pool instance ${first}."
           fi
+          aws ec2 associate-address \
+            --allocation-id "$SANDBOX_EIP_ALLOCATION_ID" \
+            --instance-id "$target" --allow-reassociation >/dev/null
+          echo "Pre-attached ${SANDBOX_EIP_ALLOCATION_ID} to stopped warm-pool instance ${target}."
       - name: Start EC2 runner
         id: start-ec2-runner
         uses: namecheap/ec2-github-runner@65fbe4f0c1181c60d4df2bd3b33b8893e21fbb43 # v4.0.1 @ 2026-07-08: warm-pool reuse:stop registration fixes (#62/#64/#65/#66)

.github/workflows/cleanup-ec2-runners.yml

@@ -15,9 +15,14 @@ name: Cleanup EC2 runners
 # the EIP directly.
 #
 # Two passes share this one workflow:
-#   - nightly full drain (37 2 * * * UTC): tiny reaper-stopped-max-age so the
-#     day's pool instance -- stopped anywhere from seconds to ~24h earlier --
-#     is always past the threshold and gets terminated.
+#   - nightly full drain (7 23 * * * UTC, i.e. right after the working day
+#     ends; :07 offset from the top of the hour per the same GitHub
+#     scheduling-congestion advice as the leak reaper below -- and observed
+#     schedule delays run to hours, so an early-off-peak slot keeps the drain
+#     landing well before the next workday starts): tiny
+#     reaper-stopped-max-age so the day's pool instance -- stopped anywhere
+#     from seconds to ~24h earlier -- is always past the threshold and gets
+#     terminated.
 #   - leak reaper (7,37 * * * * UTC, i.e. every 30 min offset from :00/:30 per
 #     GitHub's documented advice on avoiding top-of-hour/half-hour scheduling
 #     congestion): the action's own default reaper-stopped-max-age, catching
@@ -28,7 +33,7 @@ name: Cleanup EC2 runners
 # the schedules (see the issue's verification plan).
 on:
   schedule:
-    - cron: "37 2 * * *" # nightly full drain, UTC
+    - cron: "7 23 * * *" # nightly full drain, UTC (end of workday)
     - cron: "7,37 * * * *" # leak reaper, every 30 min, UTC
   workflow_dispatch:
     inputs:
@@ -88,7 +93,7 @@ jobs:
           EVENT_SCHEDULE: ${{ github.event.schedule }}
           DISPATCH_MODE: ${{ inputs.mode }}
         run: |
-          if [ "${EVENT_SCHEDULE}" = "37 2 * * *" ]; then
+          if [ "${EVENT_SCHEDULE}" = "7 23 * * *" ]; then
             echo "kind=nightly-drain" >> "$GITHUB_OUTPUT"
           elif [ "${EVENT_SCHEDULE}" = "7,37 * * * *" ]; then
             echo "kind=leak-reap" >> "$GITHUB_OUTPUT"

SECURITY_COMPLIANCE.md

@@ -232,7 +232,7 @@ below for the reuse/cleanup/EIP mechanics:
     output — the exact symptom seen earlier in this PR).
 - **Nightly drain and leak reaper.**
   [`cleanup-ec2-runners.yml`](.github/workflows/cleanup-ec2-runners.yml) runs
-  `mode: cleanup` on two schedules: a nightly full drain at `37 2 * * *` UTC
+  `mode: cleanup` on two schedules: a nightly full drain at `7 23 * * *` UTC
   with a deliberately tiny `reaper-stopped-max-age` so the day's pool
   instance always ages past the threshold and is terminated, and a
   `7,37 * * * *` UTC leak-reaper pass at the action's own default threshold