From 97fc923941542297f54c40885187f552ad5f14ad Mon Sep 17 00:00:00 2001 From: Manu Lorente Date: Wed, 4 Nov 2026 23:36:35 -0700 Subject: [PATCH 1/4] =?UTF-8?q?ci:=20deploy=20bit=C3=A1cora=20workflow=20a?= =?UTF-8?q?dd-to-project.yml=20(OPS-002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/add-to-project.yml | 34 ++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 8ceb44b760..3049d81028 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -17,8 +17,9 @@ name: Add to bitácora # GraphQL pool to the ACCOUNT (not the token), so an # interactive session can drain it and the reset is up to # an hour away. Retrying would burn the job and still -# lose. bitacora-reconcile.yml heals the drop instead. -# transient -> retry with backoff (5xx, network, secondary limit). +# lose. mlorentedev/dotfiles' bitacora-reconcile.yml heals the drop instead. +# transient -> retry with backoff (5xx, network, secondary limit; +# a secondary limit waits a minute, as GitHub asks). # anything else -> red, loudly. # # Both event types now go through the same GraphQL mutation. The old issues path @@ -61,8 +62,8 @@ jobs: # Dependabot branch the actor is that human, the secrets are there, and the # job runs. A branch-name or author-string rule would have neither property. # - # Not a silent drop: bitacora-reconcile.yml adds every open issue and PR to - # the board daily, which is the healing path this workflow's header already + # Not a silent drop: bitacora-reconcile.yml (in mlorentedev/dotfiles, daily, + # every repository) adds every open issue and PR to the board, which is the healing path this workflow's header already # names for the rate-limited case. if: > (github.event_name == 'issues' || github.event.pull_request.head.repo.fork == false) @@ -83,10 +84,14 @@ jobs: # only that case. RATE_LIMITED=10 + # fd 3 is the step's stdout. gh_graphql's notices go there, so a caller + # that captures the function's output gets the response alone (#3160). + exec 3>&1 + # gh_graphql — one GraphQL call with the three-way handling # described in the header. Prints the response on success. gh_graphql() { - local attempt=1 max=3 out + local attempt=1 max=3 out wait while :; do if out=$(gh api graphql "$@" 2>&1); then printf '%s' "$out" @@ -98,14 +103,21 @@ jobs: return "$RATE_LIMITED" ;; *"secondary rate limit"*|*"abuse detection"*|*"was submitted too quickly"*|\ - *"HTTP 50"*|*"timeout"*|*"Connection reset"*|*"EOF"*) + *"HTTP 50"*|*"timeout"*|*"Connection reset"*|*"EOF"*|\ + *"connection refused"*|*"no such host"*) if [ "$attempt" -ge "$max" ]; then - printf '::error::transient failure persisted after %d attempts\n' "$max" + printf '::error::transient failure persisted after %d attempts\n' "$max" >&3 printf '%s\n' "$out" >&2 return 1 fi - printf '::warning::transient failure (attempt %d/%d), retrying\n' "$attempt" "$max" - sleep $(( attempt * 5 )) + # GitHub asks for at least a minute before retrying a secondary + # limit; a 5xx or a network blip backs off 5 s, then 10 s (#3160). + case "$out" in + *"secondary rate limit"*|*"abuse detection"*|*"was submitted too quickly"*) wait=60 ;; + *) wait=$(( attempt * 5 )) ;; + esac + printf '::warning::transient failure (attempt %d/%d), retrying in %ds\n' "$attempt" "$max" "$wait" >&3 + sleep "$wait" attempt=$(( attempt + 1 )) ;; *) @@ -126,6 +138,8 @@ jobs: query { user(login: "mlorentedev") { projectV2(number: 1) { id } } } ' --jq '.data.user.projectV2.id') || rc=$? + # --jq prints `null` for a missing project: that is no id either. + [ "${PROJECT_ID}" = null ] && PROJECT_ID= if [ "$rc" -eq 0 ] && [ -n "${PROJECT_ID}" ]; then # $project and $contentId are GraphQL variables, not shell ones. # shellcheck disable=SC2016 @@ -151,7 +165,7 @@ jobs: # here would only add a red run nobody reads to a drop nobody sees. printf '%s%s\n' \ '::warning::primary GraphQL pool exhausted (billed per account, not per token) — ' \ - 'this item is NOT on the board; bitacora-reconcile.yml will add it within a day' + 'this item is NOT on the board; mlorentedev/dotfiles bitacora-reconcile.yml will add it within a day' ;; *) printf '%s%s\n' \ From 81f9de3ab317bb3faf740a8854fc57a44d69d288 Mon Sep 17 00:00:00 2001 From: Manu Lorente Date: Wed, 4 Nov 2026 23:36:37 -0700 Subject: [PATCH 2/4] =?UTF-8?q?ci:=20deploy=20bit=C3=A1cora=20workflow=20b?= =?UTF-8?q?itacora-status.yml=20(OPS-002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/bitacora-status.yml | 82 +++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/.github/workflows/bitacora-status.yml b/.github/workflows/bitacora-status.yml index 236079a10b..adf7c0f538 100644 --- a/.github/workflows/bitacora-status.yml +++ b/.github/workflows/bitacora-status.yml @@ -20,10 +20,16 @@ name: Bitácora status — In Progress on assign # Uses actions/github-script for the Projects v2 GraphQL (the `gh project` CLI # fails with "unknown owner type" under a fine-grained PAT). # +# The guard and the write both go by id, so renaming the field or an option +# cannot make the guard read a parked issue as unset (dotfiles#3160 item 4). +# Every GraphQL call retries a transient failure; one blip used to leave the +# issue where it was (item 3). +# # Canonical IDs (re-list: gh project field-list 1 --owner mlorentedev --format json): # Project PVT_kwHOAM7xJs4BZ6GY -# Status PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI (option "In Progress" = 6c133cc8) -# Full reference: vault 00_meta/runbooks/bitacora-project-setup.md §2 / §5 / §7. +# Status PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI (options "Backlog" = f65b1174, "In Progress" = 6c133cc8) +# This file is copied to every repository by mlorentedev/dotfiles scripts/bitacora-rollout.sh; +# change it there. on: issues: @@ -45,12 +51,52 @@ jobs: script: | const projectId = 'PVT_kwHOAM7xJs4BZ6GY'; const statusFieldId = 'PVTSSF_lAHOAM7xJs4BZ6GYzhU1dtI'; + const backlogId = 'f65b1174'; const inProgressId = '6c133cc8'; - const contentId = context.payload.issue.node_id; + const { number, node_id: contentId } = context.payload.issue; + + // A transient failure (5xx, network, rate limit) is retried; anything + // else, or the third failure, fails the run. A 403 is a rate limit only + // when GitHub says so: otherwise it is a missing permission. A GraphQL + // error arrives with HTTP 200, so its type decides. + async function withRetry(call) { + for (let attempt = 1; ; attempt++) { + try { + return await call(); + } catch (err) { + const status = err.status ?? err.response?.status; + const headers = err.response?.headers ?? {}; + const after = Number(headers['retry-after']) || 0; + const limited = status === 429 + || (status === 403 && (after > 0 || /rate limit/i.test(err.message))) + || (err.errors ?? []).some((e) => e.type === 'RATE_LIMITED'); + const transient = limited || status >= 500 || (status === undefined && !err.errors); + if (attempt === 3 || !transient) throw err; + // GitHub asks for retry-after when given, then the primary limit's + // reset time, else at least a minute. A wait past five minutes + // would outlast the job: fail now, and the reconciler heals it. + const reset = Number(headers['x-ratelimit-reset']) || 0; + const untilReset = reset ? Math.ceil(reset - Date.now() / 1000) : 0; + const wait = limited ? Math.max(after, untilReset, 60) : attempt * 5; + if (wait > 300) throw err; + core.warning(`transient failure (attempt ${attempt}/3, ${status ?? err.message}); retrying in ${wait}s`); + await new Promise((r) => setTimeout(r, wait * 1000)); + } + } + } + const graphql = (query, vars) => withRetry(() => github.graphql(query, vars)); + + // The event carries the state at assignment time; the issue may have + // closed while this run sat in the queue (dotfiles#3160 item 8). + const { data: live } = await withRetry(() => github.rest.issues.get({ ...context.repo, issue_number: number })); + if (live.state !== 'open') { + core.info(`Issue #${number} is ${live.state}; left off the board`); + return; + } // addProjectV2ItemById is idempotent: returns the existing item id // if the issue is already on the board. - const added = await github.graphql( + const added = await graphql( `mutation($projectId: ID!, $contentId: ID!) { addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { item { id } @@ -60,25 +106,37 @@ jobs: ); const itemId = added.addProjectV2ItemById.item.id; - const current = await github.graphql( + const current = await graphql( `query($itemId: ID!) { node(id: $itemId) { ... on ProjectV2Item { - fieldValueByName(name: "Status") { - ... on ProjectV2ItemFieldSingleSelectValue { name } + fieldValues(first: 100) { + pageInfo { hasNextPage } + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + optionId name + field { ... on ProjectV2SingleSelectField { id } } + } + } } } } }`, { itemId }, ); - const status = current.node?.fieldValueByName?.name; - if (status && status !== 'Backlog') { - core.info(`Issue #${context.payload.issue.number} is "${status}"; left as is`); + const values = current.node?.fieldValues; + const status = (values?.nodes ?? []).find((v) => v?.field?.id === statusFieldId); + // A project holds at most 50 fields, so one page of 100 is all of them. + // Should that change, an unseen Status must not read as unset. + if (!status && values?.pageInfo?.hasNextPage) { + throw new Error(`Issue #${number}: Status not in the first 100 field values; not moving it`); + } + if (status && status.optionId !== backlogId) { + core.info(`Issue #${number} is "${status.name}"; left as is`); return; } - await github.graphql( + await graphql( `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { updateProjectV2ItemFieldValue(input: { projectId: $projectId, itemId: $itemId, fieldId: $fieldId, @@ -88,4 +146,4 @@ jobs: { projectId, itemId, fieldId: statusFieldId, optionId: inProgressId }, ); - core.info(`Moved issue #${context.payload.issue.number} → In Progress (item ${itemId})`); + core.info(`Moved issue #${number} → In Progress (item ${itemId})`); From 725fddc0dc333fc4f044c34a4da9df9daa806b2d Mon Sep 17 00:00:00 2001 From: Manu Lorente Date: Thu, 5 Nov 2026 01:48:30 -0700 Subject: [PATCH 3/4] =?UTF-8?q?ci:=20deploy=20bit=C3=A1cora=20workflow=20a?= =?UTF-8?q?dd-to-project.yml=20(OPS-002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/add-to-project.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 3049d81028..3d10a032ee 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -85,7 +85,7 @@ jobs: RATE_LIMITED=10 # fd 3 is the step's stdout. gh_graphql's notices go there, so a caller - # that captures the function's output gets the response alone (#3160). + # that captures the function's output gets the response alone (mlorentedev/dotfiles#3160). exec 3>&1 # gh_graphql — one GraphQL call with the three-way handling @@ -111,7 +111,7 @@ jobs: return 1 fi # GitHub asks for at least a minute before retrying a secondary - # limit; a 5xx or a network blip backs off 5 s, then 10 s (#3160). + # limit; a 5xx or a network blip backs off 5 s, then 10 s (mlorentedev/dotfiles#3160). case "$out" in *"secondary rate limit"*|*"abuse detection"*|*"was submitted too quickly"*) wait=60 ;; *) wait=$(( attempt * 5 )) ;; @@ -163,14 +163,15 @@ jobs: # Green on purpose. The item IS off the board, and that is recorded # for the reconciler rather than pretended away — failing the job # here would only add a red run nobody reads to a drop nobody sees. - printf '%s%s\n' \ + printf '%s%s%s\n' \ '::warning::primary GraphQL pool exhausted (billed per account, not per token) — ' \ - 'this item is NOT on the board; mlorentedev/dotfiles bitacora-reconcile.yml will add it within a day' + 'this item is NOT on the board; if it is still open, ' \ + 'mlorentedev/dotfiles bitacora-reconcile.yml adds it within a day' ;; *) printf '%s%s\n' \ - '::error::could not add this item to the bitácora board — ' \ - 'this is not a rate limit, so it will not self-heal' + '::error::could not add this item to the bitácora board (not a rate limit). ' \ + 'An outage heals at the daily reconcile if the item is still open; a token or board error does not' exit 1 ;; esac From 2043fa5cdd62165a390c568f81e093b1745574bb Mon Sep 17 00:00:00 2001 From: Manu Lorente Date: Thu, 5 Nov 2026 01:48:32 -0700 Subject: [PATCH 4/4] =?UTF-8?q?ci:=20deploy=20bit=C3=A1cora=20workflow=20b?= =?UTF-8?q?itacora-status.yml=20(OPS-002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/bitacora-status.yml | 141 ++++++++++++++------------ 1 file changed, 74 insertions(+), 67 deletions(-) diff --git a/.github/workflows/bitacora-status.yml b/.github/workflows/bitacora-status.yml index adf7c0f538..c6c7cc3bba 100644 --- a/.github/workflows/bitacora-status.yml +++ b/.github/workflows/bitacora-status.yml @@ -22,8 +22,9 @@ name: Bitácora status — In Progress on assign # # The guard and the write both go by id, so renaming the field or an option # cannot make the guard read a parked issue as unset (dotfiles#3160 item 4). -# Every GraphQL call retries a transient failure; one blip used to leave the -# issue where it was (item 3). +# A transient failure retries the whole read-then-write, so each attempt +# re-checks the live state and the Status; one blip used to leave the issue +# where it was (item 3). # # Canonical IDs (re-list: gh project field-list 1 --owner mlorentedev --format json): # Project PVT_kwHOAM7xJs4BZ6GY @@ -56,94 +57,100 @@ jobs: const { number, node_id: contentId } = context.payload.issue; // A transient failure (5xx, network, rate limit) is retried; anything - // else, or the third failure, fails the run. A 403 is a rate limit only - // when GitHub says so: otherwise it is a missing permission. A GraphQL - // error arrives with HTTP 200, so its type decides. + // else, or the third failure, fails the run. octokit reports a network + // failure as status 500, so an error with no status is the script's own + // and is not retried. A 403 is a rate limit only when GitHub says so: + // otherwise it is a missing permission. A GraphQL error arrives with + // HTTP 200, so its type decides. async function withRetry(call) { for (let attempt = 1; ; attempt++) { try { return await call(); } catch (err) { const status = err.status ?? err.response?.status; - const headers = err.response?.headers ?? {}; + // REST errors carry the headers on err.response; @octokit/graphql + // keeps the parsed body there and the headers on err.headers. + const headers = err.response?.headers ?? err.headers ?? {}; const after = Number(headers['retry-after']) || 0; const limited = status === 429 || (status === 403 && (after > 0 || /rate limit/i.test(err.message))) - || (err.errors ?? []).some((e) => e.type === 'RATE_LIMITED'); - const transient = limited || status >= 500 || (status === undefined && !err.errors); + || (err.errors ?? []).some((e) => /^RATE_LIMIT(ED)?$/.test(e.type)); + const transient = limited || status >= 500; if (attempt === 3 || !transient) throw err; - // GitHub asks for retry-after when given, then the primary limit's - // reset time, else at least a minute. A wait past five minutes - // would outlast the job: fail now, and the reconciler heals it. - const reset = Number(headers['x-ratelimit-reset']) || 0; + // GitHub asks for retry-after when given. Without it, a spent + // primary pool (x-ratelimit-remaining 0) waits for its reset; any + // other limit waits a minute. A secondary limit arrives while the + // primary reset is still up to an hour away, so that reset is not + // its wait. A wait past five minutes would outlast the job: fail + // now, and the reconciler heals it. + const spent = String(headers['x-ratelimit-remaining']) === '0'; + const reset = spent ? Number(headers['x-ratelimit-reset']) || 0 : 0; const untilReset = reset ? Math.ceil(reset - Date.now() / 1000) : 0; - const wait = limited ? Math.max(after, untilReset, 60) : attempt * 5; + const wait = limited ? after || (untilReset > 0 ? untilReset : 60) : attempt * 5; if (wait > 300) throw err; core.warning(`transient failure (attempt ${attempt}/3, ${status ?? err.message}); retrying in ${wait}s`); await new Promise((r) => setTimeout(r, wait * 1000)); } } } - const graphql = (query, vars) => withRetry(() => github.graphql(query, vars)); - // The event carries the state at assignment time; the issue may have - // closed while this run sat in the queue (dotfiles#3160 item 8). - const { data: live } = await withRetry(() => github.rest.issues.get({ ...context.repo, issue_number: number })); - if (live.state !== 'open') { - core.info(`Issue #${number} is ${live.state}; left off the board`); - return; - } + // One attempt reads everything it decides on: a retry can wait minutes, + // and the issue may close or be parked during the wait. Each attempt + // re-reads the live state (the event carries the state at assignment + // time, dotfiles#3160 item 8) and the Status before it writes. + async function attempt() { + const { data: live } = await github.rest.issues.get({ ...context.repo, issue_number: number }); + if (live.state !== 'open') return `Issue #${number} is ${live.state}; left off the board`; - // addProjectV2ItemById is idempotent: returns the existing item id - // if the issue is already on the board. - const added = await graphql( - `mutation($projectId: ID!, $contentId: ID!) { - addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { - item { id } - } - }`, - { projectId, contentId }, - ); - const itemId = added.addProjectV2ItemById.item.id; + // addProjectV2ItemById is idempotent: returns the existing item id + // if the issue is already on the board. + const added = await github.graphql( + `mutation($projectId: ID!, $contentId: ID!) { + addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { + item { id } + } + }`, + { projectId, contentId }, + ); + const itemId = added.addProjectV2ItemById.item.id; - const current = await graphql( - `query($itemId: ID!) { - node(id: $itemId) { - ... on ProjectV2Item { - fieldValues(first: 100) { - pageInfo { hasNextPage } - nodes { - ... on ProjectV2ItemFieldSingleSelectValue { - optionId name - field { ... on ProjectV2SingleSelectField { id } } + const current = await github.graphql( + `query($itemId: ID!) { + node(id: $itemId) { + ... on ProjectV2Item { + fieldValues(first: 100) { + pageInfo { hasNextPage } + nodes { + ... on ProjectV2ItemFieldSingleSelectValue { + optionId name + field { ... on ProjectV2SingleSelectField { id } } + } } } } } - } - }`, - { itemId }, - ); - const values = current.node?.fieldValues; - const status = (values?.nodes ?? []).find((v) => v?.field?.id === statusFieldId); - // A project holds at most 50 fields, so one page of 100 is all of them. - // Should that change, an unseen Status must not read as unset. - if (!status && values?.pageInfo?.hasNextPage) { - throw new Error(`Issue #${number}: Status not in the first 100 field values; not moving it`); - } - if (status && status.optionId !== backlogId) { - core.info(`Issue #${number} is "${status.name}"; left as is`); - return; - } + }`, + { itemId }, + ); + const values = current.node?.fieldValues; + const status = (values?.nodes ?? []).find((v) => v?.field?.id === statusFieldId); + // A project holds at most 50 fields, so one page of 100 is all of them. + // Should that change, an unseen Status must not read as unset. + if (!status && values?.pageInfo?.hasNextPage) { + throw new Error(`Issue #${number}: Status not in the first 100 field values; not moving it`); + } + if (status && status.optionId !== backlogId) return `Issue #${number} is "${status.name}"; left as is`; - await graphql( - `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { - updateProjectV2ItemFieldValue(input: { - projectId: $projectId, itemId: $itemId, fieldId: $fieldId, - value: { singleSelectOptionId: $optionId }, - }) { projectV2Item { id } } - }`, - { projectId, itemId, fieldId: statusFieldId, optionId: inProgressId }, - ); + await github.graphql( + `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { + updateProjectV2ItemFieldValue(input: { + projectId: $projectId, itemId: $itemId, fieldId: $fieldId, + value: { singleSelectOptionId: $optionId }, + }) { projectV2Item { id } } + }`, + { projectId, itemId, fieldId: statusFieldId, optionId: inProgressId }, + ); + return `Moved issue #${number} → In Progress (item ${itemId})`; + } - core.info(`Moved issue #${number} → In Progress (item ${itemId})`); + core.info(await withRetry(attempt));