diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 8ceb44b760..3d10a032ee 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -17,8 +17,9 @@ name: Add to bitácora # GraphQL pool to the ACCOUNT (not the token), so an # interactive session can drain it and the reset is up to # an hour away. Retrying would burn the job and still -# lose. bitacora-reconcile.yml heals the drop instead. -# transient -> retry with backoff (5xx, network, secondary limit). +# lose. mlorentedev/dotfiles' bitacora-reconcile.yml heals the drop instead. +# transient -> retry with backoff (5xx, network, secondary limit; +# a secondary limit waits a minute, as GitHub asks). # anything else -> red, loudly. # # Both event types now go through the same GraphQL mutation. The old issues path @@ -61,8 +62,8 @@ jobs: # Dependabot branch the actor is that human, the secrets are there, and the # job runs. A branch-name or author-string rule would have neither property. # - # Not a silent drop: bitacora-reconcile.yml adds every open issue and PR to - # the board daily, which is the healing path this workflow's header already + # Not a silent drop: bitacora-reconcile.yml (in mlorentedev/dotfiles, daily, + # every repository) adds every open issue and PR to the board, which is the healing path this workflow's header already # names for the rate-limited case. if: > (github.event_name == 'issues' || github.event.pull_request.head.repo.fork == false) @@ -83,10 +84,14 @@ jobs: # only that case. RATE_LIMITED=10 + # fd 3 is the step's stdout. gh_graphql's notices go there, so a caller + # that captures the function's output gets the response alone (mlorentedev/dotfiles#3160). + exec 3>&1 + # gh_graphql — one GraphQL call with the three-way handling # described in the header. Prints the response on success. gh_graphql() { - local attempt=1 max=3 out + local attempt=1 max=3 out wait while :; do if out=$(gh api graphql "$@" 2>&1); then printf '%s' "$out" @@ -98,14 +103,21 @@ jobs: return "$RATE_LIMITED" ;; *"secondary rate limit"*|*"abuse detection"*|*"was submitted too quickly"*|\ - *"HTTP 50"*|*"timeout"*|*"Connection reset"*|*"EOF"*) + *"HTTP 50"*|*"timeout"*|*"Connection reset"*|*"EOF"*|\ + *"connection refused"*|*"no such host"*) if [ "$attempt" -ge "$max" ]; then - printf '::error::transient failure persisted after %d attempts\n' "$max" + printf '::error::transient failure persisted after %d attempts\n' "$max" >&3 printf '%s\n' "$out" >&2 return 1 fi - printf '::warning::transient failure (attempt %d/%d), retrying\n' "$attempt" "$max" - sleep $(( attempt * 5 )) + # GitHub asks for at least a minute before retrying a secondary + # limit; a 5xx or a network blip backs off 5 s, then 10 s (mlorentedev/dotfiles#3160). + case "$out" in + *"secondary rate limit"*|*"abuse detection"*|*"was submitted too quickly"*) wait=60 ;; + *) wait=$(( attempt * 5 )) ;; + esac + printf '::warning::transient failure (attempt %d/%d), retrying in %ds\n' "$attempt" "$max" "$wait" >&3 + sleep "$wait" attempt=$(( attempt + 1 )) ;; *) @@ -126,6 +138,8 @@ jobs: query { user(login: "mlorentedev") { projectV2(number: 1) { id } } } ' --jq '.data.user.projectV2.id') || rc=$? + # --jq prints `null` for a missing project: that is no id either. + [ "${PROJECT_ID}" = null ] && PROJECT_ID= if [ "$rc" -eq 0 ] && [ -n "${PROJECT_ID}" ]; then # $project and $contentId are GraphQL variables, not shell ones. # shellcheck disable=SC2016 @@ -149,14 +163,15 @@ jobs: # Green on purpose. The item IS off the board, and that is recorded # for the reconciler rather than pretended away — failing the job # here would only add a red run nobody reads to a drop nobody sees. - printf '%s%s\n' \ + printf '%s%s%s\n' \ '::warning::primary GraphQL pool exhausted (billed per account, not per token) — ' \ - 'this item is NOT on the board; bitacora-reconcile.yml will add it within a day' + 'this item is NOT on the board; if it is still open, ' \ + 'mlorentedev/dotfiles bitacora-reconcile.yml adds it within a day' ;; *) printf '%s%s\n' \ - '::error::could not add this item to the bitácora board — ' \ - 'this is not a rate limit, so it will not self-heal' + '::error::could not add this item to the bitácora board (not a rate limit). ' \ + 'An outage heals at the daily reconcile if the item is still open; a token or board error does not' exit 1 ;; esac diff --git a/.github/workflows/bitacora-status.yml b/.github/workflows/bitacora-status.yml index 236079a10b..c6c7cc3bba 100644 --- a/.github/workflows/bitacora-status.yml +++ b/.github/workflows/bitacora-status.yml @@ -20,10 +20,17 @@ name: Bitácora status — In Progress on assign # Uses actions/github-script for the Projects v2 GraphQL (the `gh project` CLI # fails with "unknown owner type" under a fine-grained PAT). # +# The guard and the write both go by id, so renaming the field or an option +# cannot make the guard read a parked issue as unset (dotfiles#3160 item 4). +# A transient failure retries the whole read-then-write, so each attempt +# re-checks the live state and the Status; one blip used to leave the issue +# where it was (item 3). +# # Canonical IDs (re-list: gh project field-list 1 --owner mlorentedev --format json): # Project PVT_kwHOAM7xJs4BZ6GY -# Status PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI (option "In Progress" = 6c133cc8) -# Full reference: vault 00_meta/runbooks/bitacora-project-setup.md §2 / §5 / §7. +# Status PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI (options "Backlog" = f65b1174, "In Progress" = 6c133cc8) +# This file is copied to every repository by mlorentedev/dotfiles scripts/bitacora-rollout.sh; +# change it there. on: issues: @@ -45,47 +52,105 @@ jobs: script: | const projectId = 'PVT_kwHOAM7xJs4BZ6GY'; const statusFieldId = 'PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI'; + const backlogId = 'f65b1174'; const inProgressId = '6c133cc8'; - const contentId = context.payload.issue.node_id; + const { number, node_id: contentId } = context.payload.issue; - // addProjectV2ItemById is idempotent: returns the existing item id - // if the issue is already on the board. - const added = await github.graphql( - `mutation($projectId: ID!, $contentId: ID!) { - addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { - item { id } + // A transient failure (5xx, network, rate limit) is retried; anything + // else, or the third failure, fails the run. octokit reports a network + // failure as status 500, so an error with no status is the script's own + // and is not retried. A 403 is a rate limit only when GitHub says so: + // otherwise it is a missing permission. A GraphQL error arrives with + // HTTP 200, so its type decides. + async function withRetry(call) { + for (let attempt = 1; ; attempt++) { + try { + return await call(); + } catch (err) { + const status = err.status ?? err.response?.status; + // REST errors carry the headers on err.response; @octokit/graphql + // keeps the parsed body there and the headers on err.headers. + const headers = err.response?.headers ?? err.headers ?? {}; + const after = Number(headers['retry-after']) || 0; + const limited = status === 429 + || (status === 403 && (after > 0 || /rate limit/i.test(err.message))) + || (err.errors ?? []).some((e) => /^RATE_LIMIT(ED)?$/.test(e.type)); + const transient = limited || status >= 500; + if (attempt === 3 || !transient) throw err; + // GitHub asks for retry-after when given. Without it, a spent + // primary pool (x-ratelimit-remaining 0) waits for its reset; any + // other limit waits a minute. A secondary limit arrives while the + // primary reset is still up to an hour away, so that reset is not + // its wait. A wait past five minutes would outlast the job: fail + // now, and the reconciler heals it. + const spent = String(headers['x-ratelimit-remaining']) === '0'; + const reset = spent ? Number(headers['x-ratelimit-reset']) || 0 : 0; + const untilReset = reset ? Math.ceil(reset - Date.now() / 1000) : 0; + const wait = limited ? after || (untilReset > 0 ? untilReset : 60) : attempt * 5; + if (wait > 300) throw err; + core.warning(`transient failure (attempt ${attempt}/3, ${status ?? err.message}); retrying in ${wait}s`); + await new Promise((r) => setTimeout(r, wait * 1000)); } - }`, - { projectId, contentId }, - ); - const itemId = added.addProjectV2ItemById.item.id; + } + } + + // One attempt reads everything it decides on: a retry can wait minutes, + // and the issue may close or be parked during the wait. Each attempt + // re-reads the live state (the event carries the state at assignment + // time, dotfiles#3160 item 8) and the Status before it writes. + async function attempt() { + const { data: live } = await github.rest.issues.get({ ...context.repo, issue_number: number }); + if (live.state !== 'open') return `Issue #${number} is ${live.state}; left off the board`; + + // addProjectV2ItemById is idempotent: returns the existing item id + // if the issue is already on the board. + const added = await github.graphql( + `mutation($projectId: ID!, $contentId: ID!) { + addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { + item { id } + } + }`, + { projectId, contentId }, + ); + const itemId = added.addProjectV2ItemById.item.id; - const current = await github.graphql( - `query($itemId: ID!) { - node(id: $itemId) { - ... on ProjectV2Item { - fieldValueByName(name: "Status") { - ... on ProjectV2ItemFieldSingleSelectValue { name } + const current = await github.graphql( + `query($itemId: ID!) { + node(id: $itemId) { + ... on ProjectV2Item { + fieldValues(first: 100) { + pageInfo { hasNextPage } + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + optionId name + field { ... on ProjectV2SingleSelectField { id } } + } + } + } } } - } - }`, - { itemId }, - ); - const status = current.node?.fieldValueByName?.name; - if (status && status !== 'Backlog') { - core.info(`Issue #${context.payload.issue.number} is "${status}"; left as is`); - return; - } + }`, + { itemId }, + ); + const values = current.node?.fieldValues; + const status = (values?.nodes ?? []).find((v) => v?.field?.id === statusFieldId); + // A project holds at most 50 fields, so one page of 100 is all of them. + // Should that change, an unseen Status must not read as unset. + if (!status && values?.pageInfo?.hasNextPage) { + throw new Error(`Issue #${number}: Status not in the first 100 field values; not moving it`); + } + if (status && status.optionId !== backlogId) return `Issue #${number} is "${status.name}"; left as is`; - await github.graphql( - `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId, itemId: $itemId, fieldId: $fieldId, - value: { singleSelectOptionId: $optionId }, - }) { projectV2Item { id } } - }`, - { projectId, itemId, fieldId: statusFieldId, optionId: inProgressId }, - ); + await github.graphql( + `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { + updateProjectV2ItemFieldValue(input: { + projectId: $projectId, itemId: $itemId, fieldId: $fieldId, + value: { singleSelectOptionId: $optionId }, + }) { projectV2Item { id } } + }`, + { projectId, itemId, fieldId: statusFieldId, optionId: inProgressId }, + ); + return `Moved issue #${number} → In Progress (item ${itemId})`; + } - core.info(`Moved issue #${context.payload.issue.number} → In Progress (item ${itemId})`); + core.info(await withRetry(attempt));