diff --git a/CHANGELOG.md b/CHANGELOG.md index 67458724..ff5d633d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ It is not an exhaustive repository log and does not represent a root package ver ## Unreleased +- Fix the per-task PR cost report to include every Drill pipeline invocation, attribute usage by provider-qualified model, render money to cents labeled provider-recorded versus estimate, and never present OpenCode Go flat-rate subscription usage as spend. - Add a `--grimoire-output` mode to the skills registry generator that emits a schema-compliant registry for the Grimoire skills MCP from public `skills/` only, plus an optional `--grimoire-payload-dir` payload and a manual GitHub Pages workflow, leaving the legacy `skills-registry.json` unchanged. - Fix the skills registry publishing an empty description for most entries whose `SKILL.md` declares it as a YAML block scalar; folded and literal block scalars are now read in full while a genuinely absent description still publishes empty. - Fix the always-on sentry re-escalating a declared `paused:` (or commander-held) operator as a possible wedge whenever its pane stayed busy; the declared pause now keeps the bounded long-pause recheck cadence. diff --git a/bin/sq-cost-lib.sh b/bin/sq-cost-lib.sh index 1e3c9734..17120461 100644 --- a/bin/sq-cost-lib.sh +++ b/bin/sq-cost-lib.sh @@ -5,8 +5,9 @@ # (MIT, Chaitanya Giri). Reads real operator transcripts (JSONL) and prices # them per model to get per-operation cost. # -# Supports Claude Code JSONL transcripts and Pi session JSONL files. Other harnesses -# (opencode, codex, grok, kimi) are estimated from token counts when available. +# Supports Claude Code JSONL transcripts, Pi session JSONL files, and Drill +# agent-invocation records. Other harnesses (opencode, codex, grok, kimi) are +# estimated from token counts when available. # Pi attribution is exact: a session is eligible when its session header cwd # equals any recorded task execution workspace and its record carries an exact # task-attribution entry. The recorded execution-attempt window remains the @@ -14,7 +15,8 @@ # recorded harness must be pi or pi-signed. # The session directory is scoped by SQUAD_PI_SESSION_DIR (or ~/.pi/agent/sessions), # so another base and the primary session cannot be counted accidentally. No -# prompt or response content is read. +# prompt or response content is read. Drill invocations are selected by the exact +# `sq/` branch from `~/.drill/state.sqlite` or SQUAD_DRILL_STATE. # # Usage: # . bin/sq-cost-lib.sh @@ -224,6 +226,42 @@ sq_cost_from_transcript() { echo "${in}|${out}|${cr}|${cw}|${model}|${cost}" } +# ── Drill invocation reporting ──────────────────────────────────────────── + +# sq_cost_drill_task_json — return invocations for the task's exact Drill branch. +# Args: $1=task id, $2=Drill SQLite database path. +# Each row includes invocation identity, provider/model, step, timestamp, and tokens. +sq_cost_drill_task_json() { + local task_id="${1:?task-id required}" database="${2:?Drill database required}" + local branch="sq/$task_id" escaped_branch + escaped_branch=${branch//\'/\'\'} + [ -f "$database" ] || { printf '[]\n'; return 0; } + local rows status err detail + err=$(mktemp "${TMPDIR:-/tmp}/sq-cost-drill.XXXXXX") + if rows=$(sqlite3 -json "$database" \ + "SELECT ai.agent, COALESCE(NULLIF(ai.model, ''), 'unknown') AS model, + COALESCE(NULLIF(ai.model_provider, ''), 'unknown') AS model_provider, + ai.step_name, ai.started_at, + COALESCE(ai.input_tokens, 0) AS input, + COALESCE(ai.output_tokens, 0) AS output, + COALESCE(ai.cache_read_tokens, 0) AS cache_read, + COALESCE(ai.cache_creation_tokens, 0) AS cache_write + FROM agent_invocations AS ai + JOIN runs AS r ON r.id = ai.run_id + WHERE r.branch = '$escaped_branch' + ORDER BY ai.started_at, ai.id;" 2>"$err"); then + : + else + status=$? + detail=$(tr '\n' ' ' < "$err" | sed 's/[[:space:]]*$//') + printf 'sq-cost: warning: could not read Drill invocations from %s: %s\n' \ + "$database" "${detail:-sqlite3 exited with status $status}" >&2 + rows="" + fi + rm -f "$err" + printf '%s\n' "${rows:-[]}" +} + # ── Pi task reporting ───────────────────────────────────────────────────── # sq_cost_pi_task_json — return privacy-safe aggregated usage for a Pi task. @@ -288,23 +326,36 @@ sq_cost_pi_task_json() { any(.[]; .type == "custom" and .customType == "squad-task-attribution" and .data.taskId? == $task); def has_any_task_identity: any(.[]; .type == "custom" and .customType == "squad-task-attribution"); + def session_rows($session): + reduce $session[] as $event + ({current_model:$configured_model,current_provider:"",rows:[]}; + if $event.type == "model_change" then + .current_model = ($event.modelId // .current_model) | + .current_provider = ($event.provider // "") + elif $event.type == "message" and $event.message.role == "assistant" and $event.message.usage != null then + .rows += [{model:($event.message.model // .current_model),provider:($event.message.provider // .current_provider), + session:($session[0].id // "unknown"),started:($session[0].timestamp // ""), + input:($event.message.usage.input // 0),output:($event.message.usage.output // 0), + cache_read:($event.message.usage.cacheRead // 0),cache_write:($event.message.usage.cacheWrite // 0), + total:($event.message.usage.totalTokens // (($event.message.usage.input // 0)+($event.message.usage.output // 0)+($event.message.usage.cacheRead // 0)+($event.message.usage.cacheWrite // 0))), + reported_cost:($event.message.usage.cost.total // null)}] + else . end) | .rows; def report($matched): - [ $matched[] as $s | $s[] | select(.type == "message" and .message.role == "assistant" and .message.usage != null) | - {model:(.message.model // (($s | map(select(.type == "model_change") | .modelId) | last) // $configured_model)), - provider:(($s | map(select(.type == "model_change") | .provider) | last) // ""), session:($s[0].id // "unknown"), - started:($s[0].timestamp // ""), input:(.message.usage.input // 0), output:(.message.usage.output // 0), - cache_read:(.message.usage.cacheRead // 0), cache_write:(.message.usage.cacheWrite // 0), - total:(.message.usage.totalTokens // ((.message.usage.input // 0)+(.message.usage.output // 0)+(.message.usage.cacheRead // 0)+(.message.usage.cacheWrite // 0))), - reported_cost:(.message.usage.cost.total // null)} ] as $rows | + ([$matched[] | session_rows(.)] | add // []) as $rows | {found:($matched|length > 0), task:$task, agent:$agent, worktree:($workspaces | last), sessions:($matched|length), started:($matched|map(.[0].timestamp // "")|min // ""), - models:([ $rows[].model ] | unique | map(. as $m | {model:$m, - sessions:([$rows[] | select(.model == $m) | .session] | unique | length), - input:([$rows[] | select(.model == $m) | .input] | add // 0), output:([$rows[] | select(.model == $m) | .output] | add // 0), - cache_read:([$rows[] | select(.model == $m) | .cache_read] | add // 0), cache_write:([$rows[] | select(.model == $m) | .cache_write] | add // 0), - total:([$rows[] | select(.model == $m) | .total] | add // 0), - reported_cost:([$rows[] | select(.model == $m) | .reported_cost] | map(select(. != null)) | add // null), - provider:([$rows[] | select(.model == $m) | .provider] | map(select(. != "")) | first // "")}))}; + models:([ $rows[] | [.model,.provider] ] | unique | map(. as $key | {model:$key[0],provider:$key[1], + sessions:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .session] | unique | length), + input:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .input] | add // 0), + output:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .output] | add // 0), + cache_read:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .cache_read] | add // 0), + cache_write:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .cache_write] | add // 0), + total:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .total] | add // 0), + estimate_input:([$rows[] | select(.model == $key[0] and .provider == $key[1] and .reported_cost == null) | .input] | add // 0), + estimate_output:([$rows[] | select(.model == $key[0] and .provider == $key[1] and .reported_cost == null) | .output] | add // 0), + estimate_cache_read:([$rows[] | select(.model == $key[0] and .provider == $key[1] and .reported_cost == null) | .cache_read] | add // 0), + estimate_cache_write:([$rows[] | select(.model == $key[0] and .provider == $key[1] and .reported_cost == null) | .cache_write] | add // 0), + reported_cost:([$rows[] | select(.model == $key[0] and .provider == $key[1]) | .reported_cost] | map(select(. != null)) | add // null)}))}; ($sessions | map(select(.[0].type == "session" and (.[0].cwd as $session_cwd | $workspaces | index($session_cwd) != null)))) as $workspace_sessions | ($workspace_sessions | map(select(has_task_identity))) as $identity_matched | diff --git a/bin/sq-cost.sh b/bin/sq-cost.sh index 1b845ada..eb0e0940 100755 --- a/bin/sq-cost.sh +++ b/bin/sq-cost.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # sq-cost.sh — Transcript-derived cost estimation for Squad tasks. # -# Reads real operator transcripts and prices them per model to get -# per-operation cost. Degrades gracefully when no transcript exists. +# Reads attributable Pi sessions and Drill agent invocations, then prices usage +# per provider/model. Degrades gracefully when neither source has task usage. # # Ported from munder-difflin's src/main/transcript.ts + src/main/pricing.ts # (MIT, Chaitanya Giri). Adapted for Squad. @@ -21,7 +21,9 @@ # # `task ` preserves that legacy line unless `--json` is supplied. # `report ` (or `task --json`) emits the complete privacy-safe -# report and never includes prompt or response content. Pi sessions are counted +# report and never includes prompt or response content. Drill invocations are +# selected from ~/.drill/state.sqlite by exact branch `sq/`; override +# that path with SQUAD_DRILL_STATE. Pi sessions are counted # only when their session header cwd exactly matches a recorded execution # workspace and the session carries an exact task-attribution metadata entry; # legacy sessions without any task identity use the recorded execution window. @@ -105,35 +107,100 @@ cmd_report() { local task_id="${1:?task-id required}" state_dir="${SQUAD_STATE_OVERRIDE:-${SQUAD_BASE:-${SQUAD_HOME:-.}}/state}" local session_root="${SQUAD_PI_SESSION_DIR:-$HOME/.pi/agent/sessions}" raw raw=$(sq_cost_pi_task_json "$task_id" "$state_dir" "$session_root") - if [ "${2:-}" = "--json" ]; then - printf '%s\n' "$raw" - return 0 - fi + local pipeline + pipeline=$(sq_cost_drill_task_json "$task_id" "${SQUAD_DRILL_STATE:-$HOME/.drill/state.sqlite}") + raw=$(jq --argjson pipeline "$pipeline" --arg task "$task_id" ' + ($pipeline | map(select(.model != null and .model != "")) | + group_by([.model_provider // "", .model]) | map(.[0] as $first | { + model:$first.model, provider:($first.model_provider // ""), sessions:0, + invocations:length, input:(map(.input)|add // 0), output:(map(.output)|add // 0), + cache_read:(map(.cache_read)|add // 0), cache_write:(map(.cache_write)|add // 0), + estimate_input:(map(.input)|add // 0), estimate_output:(map(.output)|add // 0), + estimate_cache_read:(map(.cache_read)|add // 0), estimate_cache_write:(map(.cache_write)|add // 0), + total:(map(.input + .output + .cache_read + .cache_write)|add // 0), reported_cost:null + })) as $drill_models | + (if .found == true then . else + {found:($drill_models|length > 0),task:$task,agent:"drill",sessions:0, + started:($pipeline|map(.started_at|tostring)|min // ""),models:[],reason:.reason} + end) | + .models = (.models + $drill_models) | + .models = ([.models | group_by([.provider // "", .model])[] | + .[0] as $first | {model:$first.model,provider:($first.provider // ""), + sessions:(map(.sessions // 0)|add // 0),invocations:(map(.invocations // 0)|add // 0), + input:(map(.input)|add // 0),output:(map(.output)|add // 0), + cache_read:(map(.cache_read)|add // 0),cache_write:(map(.cache_write)|add // 0), + estimate_input:(map(.estimate_input // 0)|add // 0),estimate_output:(map(.estimate_output // 0)|add // 0), + estimate_cache_read:(map(.estimate_cache_read // 0)|add // 0),estimate_cache_write:(map(.estimate_cache_write // 0)|add // 0), + total:(map(.total)|add // 0), + reported_cost:(map(.reported_cost)|map(select(. != null))|add // null)}]) + ' <<<"$raw") if ! jq -e '.found == true' >/dev/null 2>&1 <<<"$raw"; then + if [ "${2:-}" = "--json" ]; then + printf '%s\n' "$raw" + return 0 + fi local reason reason=$(jq -r '.reason // empty' <<<"$raw") printf '## Coding agent usage on this pull request\n\nUsage unavailable: %s.\n' "${reason:-no attributable sessions}" return 0 fi - local enriched model cost mode provider in_tokens out_tokens cache_read cache_write + local enriched entry model provider reported cost estimate_cost mode in_tokens out_tokens cache_read cache_write enriched=$(mktemp "${TMPDIR:-/tmp}/sq-cost-report.XXXXXX") - trap 'rm -f "$enriched"' RETURN - while IFS=$'\t' read -r model provider reported; do - if [ "$provider" = "opencode-go" ] || [[ "$model" == *opencode-go* ]]; then - cost="null"; mode="flat-rate subscription" - elif [ "$reported" != "null" ]; then - cost="$reported"; mode="provider-recorded" + trap 'rm -f "${enriched:-}"; trap - RETURN' RETURN + while IFS= read -r entry; do + model=$(jq -r '.model // ""' <<<"$entry") + provider=$(jq -r '.provider // ""' <<<"$entry") + reported=$(jq -r '.reported_cost | tojson' <<<"$entry") + if [ "$provider" = "opencode-go" ] || [[ "$model" == opencode-go* ]]; then + cost="null"; reported="null"; estimate_cost="null"; mode="flat-rate subscription" else - in_tokens=$(jq -r --arg m "$model" '.models[]|select(.model==$m)|.input' <<<"$raw") - out_tokens=$(jq -r --arg m "$model" '.models[]|select(.model==$m)|.output' <<<"$raw") - cache_read=$(jq -r --arg m "$model" '.models[]|select(.model==$m)|.cache_read' <<<"$raw") - cache_write=$(jq -r --arg m "$model" '.models[]|select(.model==$m)|.cache_write' <<<"$raw") - cost=$(sq_cost_estimate "$model" "$in_tokens" "$out_tokens" "$cache_read" "$cache_write") - mode="estimate" + in_tokens=$(jq -r --arg m "$model" --arg p "$provider" '.models[]|select(.model==$m and .provider==$p)|.estimate_input' <<<"$raw") + out_tokens=$(jq -r --arg m "$model" --arg p "$provider" '.models[]|select(.model==$m and .provider==$p)|.estimate_output' <<<"$raw") + cache_read=$(jq -r --arg m "$model" --arg p "$provider" '.models[]|select(.model==$m and .provider==$p)|.estimate_cache_read' <<<"$raw") + cache_write=$(jq -r --arg m "$model" --arg p "$provider" '.models[]|select(.model==$m and .provider==$p)|.estimate_cache_write' <<<"$raw") + estimate_cost="null" + if [ "$((in_tokens + out_tokens + cache_read + cache_write))" -gt 0 ]; then + estimate_cost=$(sq_cost_estimate "$model" "$in_tokens" "$out_tokens" "$cache_read" "$cache_write") + fi + if [ "$estimate_cost" = "null" ]; then + cost="$reported" + elif [ "$reported" = "null" ]; then + cost="$estimate_cost" + else + cost=$(awk -v recorded="$reported" -v estimated="$estimate_cost" 'BEGIN { printf "%.6f", recorded + estimated }') + fi + if [ "$reported" != "null" ] && [ "$estimate_cost" != "null" ]; then + mode="provider-recorded + estimate" + elif [ "$reported" != "null" ]; then + mode="provider-recorded" + else + mode="estimate" + fi fi - jq --arg m "$model" --arg mode "$mode" --argjson cost "$cost" '.models |= map(if .model == $m then . + {cost:$cost,cost_basis:$mode} else . end)' <<<"$raw" > "$enriched" - raw=$(cat "$enriched") - done < <(jq -r '.models[] | [.model,.provider,(.reported_cost|tojson)] | @tsv' <<<"$raw") + jq --arg m "$model" --arg p "$provider" --arg mode "$mode" --argjson cost "$cost" \ + --argjson recorded "$reported" --argjson estimated "$estimate_cost" \ + '.models |= map(if .model == $m and .provider == $p then . + {cost:$cost,cost_basis:$mode,reported_cost:$recorded,estimate_cost:$estimated} else . end)' \ + <<<"$raw" > "$enriched" + raw=$(<"$enriched") + done < <(jq -c '.models[]' <<<"$raw") + if [ "${2:-}" = "--json" ]; then + jq ' + def money: + if . == null then null + else ((. * 100 | round) as $c | "$\(($c / 100 | floor)).\(if ($c % 100) < 10 then "0" else "" end)\(($c % 100))") end; + ([.models[] | .reported_cost // 0] | add // 0) as $recorded | + ([.models[] | .estimate_cost // 0] | add // 0) as $estimated | + ([.models[] | select(.cost_basis == "flat-rate subscription")] | length > 0) as $subscription | + .total_cost = { + combined:((if ($recorded + $estimated) == 0 then null else $recorded + $estimated end) | money), + provider_recorded:(if [.models[] | select(.reported_cost != null)] | length > 0 then ($recorded | money) else null end), + estimate:(if [.models[] | select(.estimate_cost != null)] | length > 0 then ($estimated | money) else null end), + flat_rate_subscription:(if $subscription then "not spend" else null end) + } | + .models |= map(.cost = (.cost | money) | .reported_cost = (.reported_cost | money) | .estimate_cost = (.estimate_cost | money)) + ' <<<"$raw" + return 0 + fi local jq_program jq_program=$(cat <<'JQ' def humanize: @@ -146,14 +213,31 @@ cmd_report() { {pi:"Pi", "pi-signed":"Pi", claude:"Claude Code", codex:"Codex", opencode:"OpenCode", grok:"Grok", kimi:"Kimi", muse:"Muse"} as $labels | if $labels[.] then $labels[.] else (split("[-_]") | map((.[0:1] | ascii_upcase) + .[1:]) | join(" ")) end; - def money: if . == null then "not applicable" else ("$" + (.|tostring)) end; + def money: + if . == null then "not applicable" + else ((. * 100 | round) as $c | "$\(($c / 100 | floor)).\(if ($c % 100) < 10 then "0" else "" end)\(($c % 100))") end; + def qualified_model: if (.provider // "") == "" or .provider == "unknown" then .model else (.provider + "/" + .model) end; + def cost_label: + if .cost_basis == "flat-rate subscription" then "flat-rate subscription: not spend" + else ([if .reported_cost != null then "provider-recorded: \(.reported_cost | money)" else empty end, + if .estimate_cost != null then "estimate: \(.estimate_cost | money)" else empty end] | join("; ")) end; (.models | map(.total) | add // 0) as $total | + ([.models[] | .reported_cost // 0] | add // 0) as $recorded | + ([.models[] | .estimate_cost // 0] | add // 0) as $estimated | + ([.models[] | select(.cost_basis == "flat-rate subscription")] | length > 0) as $subscription | + any(.models[]; .reported_cost != null) as $has_recorded | + any(.models[]; .estimate_cost != null) as $has_estimated | + ([if $has_recorded and $has_estimated then "provider-recorded + estimate total: \(($recorded + $estimated) | money)" + elif $has_recorded or $has_estimated then "total: \(($recorded + $estimated) | money)" else empty end, + if [.models[] | select(.reported_cost != null)] | length > 0 then "provider-recorded: \($recorded | money)" else empty end, + if [.models[] | select(.estimate_cost != null)] | length > 0 then "estimate: \($estimated | money)" else empty end, + if $subscription then "flat-rate subscription: not spend" else empty end] | join("; ")) as $cost_summary | "## Coding agent usage on this pull request\n\n" + - "| Contributor | Agent | Sessions | Total tokens | Estimated cost |\n|---|---|---:|---:|---:|\n" + - ("| Squad task \(.task) | \(.agent | agent_label) | \(.sessions) | \($total | humanize) | " + (([.models[].cost] | map(select(. != null)) | add) | money) + " |\n\n") + - "### Token and model breakdown\n\n| Model | Input | Output | Cache read | Cache write | Total tokens | Estimated cost |\n|---|---:|---:|---:|---:|---:|---:|\n" + - ([.models[] | "| \(.model) | \(.input | humanize) | \(.output | humanize) | \(.cache_read | humanize) | \(.cache_write | humanize) | \(.total | humanize) | \(.cost_basis): \(.cost // $na) |\n"] | join("")) + - "\n_Source: Pi session JSONL usage records, covering the task lifetime from \(.started) through report generation. Costs are provider-recorded where available, otherwise list-price estimates; subscription usage is not represented as spend._" + "| Contributor | Agent | Models | Sessions | Total tokens | Cost |\n|---|---|---|---:|---:|---|\n" + + ("| Squad task \(.task) | \(.agent | agent_label) | " + ([.models[] | qualified_model] | unique | join(", ")) + " | \(.sessions) | \($total | humanize) | \($cost_summary) |\n\n") + + "### Token and model breakdown\n\n| Model | Input | Output | Cache read | Cache write | Total tokens | Cost |\n|---|---:|---:|---:|---:|---:|---|\n" + + ([.models[] | "| \(qualified_model) | \(.input | humanize) | \(.output | humanize) | \(.cache_read | humanize) | \(.cache_write | humanize) | \(.total | humanize) | \(cost_label) |\n"] | join("")) + + "\n_Source: Pi session JSONL usage records and Drill agent-invocation records, covering the task lifetime through report generation. Provider-recorded costs and list-price estimates are labeled separately; subscription usage is not represented as spend._" JQ ) jq -r --arg na "not applicable" "$jq_program" <<<"$raw" diff --git a/docs/cost-observability.md b/docs/cost-observability.md index 8c074a1f..46a067a7 100644 --- a/docs/cost-observability.md +++ b/docs/cost-observability.md @@ -1,11 +1,13 @@ # Task cost observability -`bin/sq-cost.sh report ` renders a pull-request-ready usage report. +`bin/sq-cost.sh report ` renders a pull-request-ready usage report combining attributable operator sessions with every Drill agent invocation on branch `sq/`. Pass `--json` to `report`, or to `task --json`, for machine consumption. -The JSON contains agent, session, model, token, timestamp, provider-cost, estimate-basis, and subscription fields only. -Markdown output renders token counts as humanized strings (e.g. `2.6 billion`). +The JSON contains agent, session, invocation, model, token, timestamp, provider-cost, estimate-basis, and subscription fields only. +Markdown output renders token counts as humanized strings (e.g. `2.6 billion`) and money to cents with each amount labeled provider-recorded or estimate. +JSON money fields and the combined total are cents-formatted strings, with `cost_basis` set to `provider-recorded`, `estimate`, `provider-recorded + estimate`, or `flat-rate subscription`. Raw integer values are preserved only in the JSON output. +Drill records are read from `~/.drill/state.sqlite`; `SQUAD_DRILL_STATE` overrides this path for tests. Recorded harness identifiers (`pi`, `pi-signed`, `claude`, `codex`, `opencode`, `grok`, `kimi`, `muse`) are mapped to product labels in the Markdown report. Unrecognized identifiers fall back to a title-cased form. @@ -22,9 +24,11 @@ When the execution window is missing or invalid (non-numeric timestamps, or end This rule excludes the primary session, other tasks, and sessions from another base without widening a pooled workspace's time range. When no attributable session exists, the report says why instead of fabricating zero usage. -Provider-recorded costs are preferred. +Each Pi assistant usage row uses its message-level provider when present, falling back to the preceding model-change record. +Provider-recorded costs are preferred when a record supplies them. List-price calculations are labeled estimates. OpenCode Go usage is labeled flat-rate subscription usage and is not presented as token spend. +The summary lists provider-qualified models and includes both operator and pipeline token totals and costs. `sq-cost.sh publish ` publishes one marked comment through `sq-gh`. The command updates that comment on later runs rather than creating duplicates. diff --git a/docs/pt-BR/scripts.md b/docs/pt-BR/scripts.md index 70320d6b..d83e9381 100644 --- a/docs/pt-BR/scripts.md +++ b/docs/pt-BR/scripts.md @@ -95,8 +95,8 @@ A recusa compartilhada do gate drill para entrypoints do ciclo de vida da unidad | `sq-crew-state.sh` | Imprimir uma linha determinística de estado atual para um operador | | `sq-breaker.sh` | Avaliar os sinais de circuit-breaker de uma tarefa num veredito healthy/steering/constrained/stopped com ação e motivos | | `sq-breaker-lib.sh` | Política compartilhada de escada de circuit-breaker sem efeitos colaterais usada por `sq-breaker.sh` | -| `sq-cost.sh` | Precificar transcripts reais de operadores por modelo, renderizar relatórios de uso por tarefa e publicar um comentário idempotente no PR | -| `sq-cost-lib.sh` | Parsing compartilhado de transcrições Claude e sessões Pi, atribuição, normalização de modelo e preços por modelo usada por `sq-cost.sh` | +| `sq-cost.sh` | Precificar sessões de operadores e invocações do pipeline Drill por provedor/modelo, renderizar relatórios de uso por tarefa e publicar um comentário idempotente no PR | +| `sq-cost-lib.sh` | Parsing compartilhado de transcrições Claude, sessões Pi e invocações Drill, atribuição, normalização de modelo e preços por modelo usada por `sq-cost.sh` | | `sq-drill-run-lib.sh` | Atribuição compartilhada de identidade de branch-e-código para execuções drill | | `sq-tangle-lib.sh` | Resolução compartilhada de branch default e classificação de tangle de checkout primário | | `sq-timeout-lib.sh` | Único dono da execução de comandos com limite rígido e seu watchdog fallback | diff --git a/docs/scripts.md b/docs/scripts.md index b0d8c6ab..77cbc70a 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -86,8 +86,8 @@ The shared drill gate refusal for unit lifecycle entrypoints is summarized in [a | `sq-crew-state.sh` | Print one deterministic current-state line for an operator | | `sq-breaker.sh` | Evaluate a task's circuit-breaker signals to a healthy/steering/constrained/stopped verdict with action and reasons | | `sq-breaker-lib.sh` | Side-effect-free circuit-breaker ladder policy shared by `sq-breaker.sh` | -| `sq-cost.sh` | Price real operator transcripts per model, render task usage reports, and publish an idempotent PR comment | -| `sq-cost-lib.sh` | Shared Claude transcript and Pi session parsing, attribution, model normalization, and per-model pricing used by `sq-cost.sh` | +| `sq-cost.sh` | Price operator sessions and Drill pipeline invocations per provider/model, render task usage reports, and publish an idempotent PR comment | +| `sq-cost-lib.sh` | Shared Claude transcript, Pi session, and Drill invocation parsing and attribution, model normalization, and per-model pricing used by `sq-cost.sh` | | `sq-evidence-receipt.sh` | Create, verify, and render private deterministic evidence receipts for local source citations | | `sq-observation-pack.sh` | Store, compactly display, and page private task output artifacts; see `sq-observation-pack.py` for the implementation | | `sq-observation-pack.py` | Python implementation of observation packs: create --task ID --source PATH, card PACK, and read PACK --offset N --limit M [--receipt] | diff --git a/tests/sq-cost.test.sh b/tests/sq-cost.test.sh index 1fb2dda3..d3645de7 100755 --- a/tests/sq-cost.test.sh +++ b/tests/sq-cost.test.sh @@ -19,6 +19,7 @@ set -u COST_LIB="$ROOT/bin/sq-cost-lib.sh" COST_CLI="$ROOT/bin/sq-cost.sh" TMP_ROOT=$(fm_test_tmproot sq-cost) +export SQUAD_DRILL_STATE="$TMP_ROOT/drill-state.sqlite" # Source the library for direct function tests # shellcheck disable=SC1090 # test source @@ -375,7 +376,7 @@ EOF output=$(SQUAD_STATE_OVERRIDE="$state" SQUAD_PI_SESSION_DIR="$pi_root" "$COST_CLI" report pi-task --json) assert_contains "$output" '"input": 2600000400' "Pi report finds matching sessions across attempts" assert_contains "$output" '"sessions": 2' "Pi report counts both attempts for the mission" - assert_contains "$output" '"reported_cost": 0.05' "Pi report preserves provider cost across attempts" + assert_contains "$output" "\"reported_cost\": \"\$0.05\"" "Pi report preserves provider cost across attempts in cents" if printf '%s' "$output" | grep -q '9999'; then fail "Pi report counted another worktree"; fi if printf '%s' "$output" | grep -q 'gpt-6-astra\|8888\|12.59'; then fail "Pi report counted a prior mission from the reused worktree slot" @@ -388,18 +389,178 @@ EOF cat > "$pi_dir/subscription.jsonl" < --json should exit 0 for a found report, got: $task_rc" + [ "$(jq -r '.found' <<<"$task_json")" = "true" ] || fail "task --json should emit the found report" pass "Pi task report attributes sessions exactly and avoids zero-result regression" } test_pi_task_report +# ── (h2c) Pi sessions without a provider record ──────────────────────────── + +test_pi_empty_provider() { + local state="$TMP_ROOT/empty-provider-state" pi_root="$TMP_ROOT/empty-provider-pi" wt="$TMP_ROOT/empty-provider-worktree" + mkdir -p "$state" "$pi_root/fixture" "$wt" + printf 'window=sq:empty-provider\nharness=pi\nworktree=%s\nmodel=default\n' "$wt" > "$state/empty-provider.meta" + cat > "$pi_root/fixture/session.jsonl" <&1) && rc=$? || rc=$? + [ "$rc" -eq 0 ] || fail "provider-less Pi session report should exit 0, got: $rc" + assert_contains "$output" '"found": true' "provider-less Pi session is still reported" + assert_contains "$output" '"model": "claude-sonnet-4"' "provider-less Pi model is preserved" + assert_contains "$output" "\"combined\": \"\$0.01\"" "provider-less Pi session is costed in cents" + pass "Pi sessions without a provider record do not break the report" +} + +test_pi_empty_provider + +# ── (h2d) message-level provider wins over the preceding model_change ────── + +test_pi_message_provider_wins() { + local state="$TMP_ROOT/provider-mismatch-state" pi_root="$TMP_ROOT/provider-mismatch-pi" wt="$TMP_ROOT/provider-mismatch-worktree" + mkdir -p "$state" "$pi_root/fixture" "$wt" + printf 'window=sq:provider-mismatch\nharness=pi\nworktree=%s\nmodel=default\n' "$wt" > "$state/provider-mismatch.meta" + cat > "$pi_root/fixture/session.jsonl" < "$state/pipeline-task.meta" + cat > "$pi_root/fixture/operator.jsonl" < "$state/drill-fail.meta" + cat > "$pi_root/fixture/operator.jsonl" < "$bad_db" + stdout=$(sq_cost_drill_task_json drill-fail "$bad_db" 2>"$TMP_ROOT/drill-fail-read.stderr") + stderr=$(cat "$TMP_ROOT/drill-fail-read.stderr") + [ "$(jq 'length' <<<"$stdout")" = "0" ] || fail "unreadable Drill database should yield an empty invocation list" + assert_contains "$stderr" "could not read Drill invocations" "unreadable Drill database emits a diagnostic on stderr" + local output report_stderr + output=$(SQUAD_STATE_OVERRIDE="$state" SQUAD_PI_SESSION_DIR="$pi_root" SQUAD_DRILL_STATE="$bad_db" \ + "$COST_CLI" report drill-fail --json 2>"$TMP_ROOT/drill-fail-report.stderr") + report_stderr=$(cat "$TMP_ROOT/drill-fail-report.stderr") + assert_contains "$report_stderr" "could not read Drill invocations" "report surfaces the Drill read failure instead of dropping pipeline cost silently" + assert_contains "$output" '"found": true' "Pi usage is still reported when the Drill database is unreadable" + pass "Drill read failures are observable instead of silently dropping pipeline cost" +} + +test_drill_read_failure_is_observable + # ── (h2b) Pi execution-window requirement ───────────────────────────────── test_pi_requires_execution_window() { @@ -414,7 +575,7 @@ EOF local output output=$(SQUAD_STATE_OVERRIDE="$state" SQUAD_PI_SESSION_DIR="$pi_root" \ "$COST_CLI" report no-exec --json) - assert_contains "$output" '"found":false' "Pi report refuses path-only attribution without an execution window" + assert_contains "$output" '"found": false' "Pi report refuses path-only attribution without an execution window" assert_contains "$output" 'execution window is unavailable' "missing execution window explains unavailable attribution" cat >> "$pi_root/fixture/session.jsonl" <<'EOF' {"type":"custom","customType":"squad-task-attribution","data":{"taskId":"no-exec"}} diff --git a/tests/sq-trajectory.test.sh b/tests/sq-trajectory.test.sh index 5bedf17c..91aaa8cb 100755 --- a/tests/sq-trajectory.test.sh +++ b/tests/sq-trajectory.test.sh @@ -7,6 +7,7 @@ trap 'rm -rf "$TMP"' EXIT BASE="$TMP/base" mkdir -p "$BASE/state" "$BASE/data" export SQUAD_BASE="$BASE" SQUAD_STATE_OVERRIDE="$BASE/state" SQUAD_DATA_OVERRIDE="$BASE/data" +export SQUAD_DRILL_STATE="$TMP/drill-state.sqlite" SQUAD_PI_SESSION_DIR="$TMP/pi-sessions" fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } pass() { printf 'ok - %s\n' "$*"; } fixture() {