diff --git a/.github/kaibench-flaky-questions.txt b/.github/kaibench-flaky-questions.txt new file mode 100644 index 000000000..6b6a53e91 --- /dev/null +++ b/.github/kaibench-flaky-questions.txt @@ -0,0 +1,19 @@ +# Questions excluded from the KaiBench regression gate. +# +# A question belongs here only when it has been MEASURED as unstable against an unmodified +# server — not merely observed failing once. The gate compares a single trial per question, +# so a question with a base pass rate meaningfully below 100% produces phantom regressions +# at that rate and trains reviewers to ignore the check. +# +# Exclusions are reported as warnings on every gated run, so coverage lost here stays +# visible rather than silently narrowing. Remove an entry once the underlying question is +# made deterministic. +# +# Format: one question ID per line. Blank lines and #-comments ignored. + +# ~50% pass rate on unmodified main (6/10 vs 4/10 on a branch, Fisher exact p=0.66 — the +# instability is the question's own, not any PR's). Six-value all-or-nothing set_comparison +# over an underspecified multi-source join: date-overlap semantics, weekday/weekend boundary, +# and aggregation grain are all undefined. Tracked in keboola/KaiBench#80 as the reference +# case for the semantic layer; remove this entry once those definitions are modelled. +12 diff --git a/.github/scripts/kaibench-parse-results.py b/.github/scripts/kaibench-parse-results.py index 6e2893e11..5c7547878 100644 --- a/.github/scripts/kaibench-parse-results.py +++ b/.github/scripts/kaibench-parse-results.py @@ -31,11 +31,31 @@ status = 'passed' if m['failed'] == 0 and m.get('errors', 0) == 0 and partial_count == 0 else 'failed' print(f"status={status}") +# Questions measured as unstable against an unmodified server. A single-trial comparison on a +# question whose base pass rate is well under 100% produces phantom regressions at that rate, so +# these are counted separately instead of failing the gate. Reported, never silently dropped. +flaky_path = Path(__file__).parent.parent / 'kaibench-flaky-questions.txt' +flaky_qids = set() +if flaky_path.exists(): + for raw in flaky_path.read_text().splitlines(): + entry = raw.split('#', 1)[0].strip() + if entry: + flaky_qids.add(entry) + # Count regressions vs previous run (downloaded into prev-results/) # `baseline_run` stays empty when no comparison happened, so callers can tell "0 regressions" # apart from "never compared" — the two look identical otherwise. regressions = 0 +regressed_qids = [] +flaky_regressions = 0 +flaky_regressed_qids = [] baseline_run = '' +# Share of this run's questions the baseline actually covers. A targeted run (say a single +# question dispatched with --questions) produces a perfectly valid artifact that nonetheless +# makes a near-empty baseline, which would otherwise yield "0 regressions" and a green check +# while verifying almost nothing. +baseline_overlap = 0 +baseline_shared = 0 prev_runs = sorted(Path('prev-results').glob('run_*'), key=lambda p: p.stat().st_mtime) if Path('prev-results').exists() else [] if prev_runs: prev_file = prev_runs[-1] / 'results.jsonl' @@ -49,10 +69,24 @@ except json.JSONDecodeError: continue prev_by_qid[str(pr.get('question_id', ''))] = pr + candidate_qids = {str(r.get('question_id', '')) for r in evaluated} + baseline_shared = len(candidate_qids & set(prev_by_qid)) + if candidate_qids: + baseline_overlap = round(100 * baseline_shared / len(candidate_qids)) for r in evaluated: qid = str(r.get('question_id', '')) if qid in prev_by_qid: if prev_by_qid[qid].get('status') == 'passed' and r.get('status') not in ('passed', 'skipped'): - regressions += 1 + if qid in flaky_qids: + flaky_regressions += 1 + flaky_regressed_qids.append(qid) + else: + regressions += 1 + regressed_qids.append(qid) print(f"regressions={regressions}") +print(f"regressed_qids={','.join(sorted(regressed_qids))}") +print(f"flaky_regressions={flaky_regressions}") +print(f"flaky_regressed_qids={','.join(sorted(flaky_regressed_qids))}") +print(f"baseline_overlap={baseline_overlap}") +print(f"baseline_shared={baseline_shared}") print(f"baseline_run={baseline_run}") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78aa2e823..c87f1c543 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,12 @@ jobs: env: EVAL_RESULT: ${{ needs.kaibench.result }} REGRESSIONS: ${{ needs.kaibench.outputs.regressions }} + REGRESSED_QIDS: ${{ needs.kaibench.outputs.regressed_qids }} + FLAKY_REGRESSIONS: ${{ needs.kaibench.outputs.flaky_regressions }} + FLAKY_REGRESSED_QIDS: ${{ needs.kaibench.outputs.flaky_regressed_qids }} BASELINE_RUN: ${{ needs.kaibench.outputs.baseline_run }} + BASELINE_OVERLAP: ${{ needs.kaibench.outputs.baseline_overlap }} + BASELINE_SHARED: ${{ needs.kaibench.outputs.baseline_shared }} PASSED: ${{ needs.kaibench.outputs.passed }} TOTAL: ${{ needs.kaibench.outputs.total }} PASS_RATE: ${{ needs.kaibench.outputs.pass_rate }} @@ -206,9 +211,20 @@ jobs: echo "::warning::No baseline artifact available, so no regression comparison was made — this check passing does NOT mean the PR is regression-free" exit 0 fi - echo "Compared against baseline run: $BASELINE_RUN" + echo "Compared against baseline run: $BASELINE_RUN (covers ${BASELINE_SHARED:-?} of $TOTAL questions, ${BASELINE_OVERLAP:-?}%)" + # A targeted run (dispatched with `questions`) yields a valid artifact that is nonetheless a + # near-empty baseline. Real regressions inside a thin overlap are still worth failing on, but a + # pass must not read as full coverage. + if [ "${BASELINE_OVERLAP:-0}" -lt 50 ]; then + echo "::warning::Baseline covers only ${BASELINE_OVERLAP:-0}% of this run's questions (${BASELINE_SHARED:-0}/$TOTAL) — most questions were NOT compared against anything. Treat a pass here as unverified; run the full suite on main to establish a comparable baseline." + fi + # Surface excluded questions every time, so coverage given up here stays visible + # instead of quietly shrinking what this check actually verifies. + if [ "${FLAKY_REGRESSIONS:-0}" -gt 0 ]; then + echo "::warning::Ignored $FLAKY_REGRESSIONS regression(s) on known-unstable question(s): ${FLAKY_REGRESSED_QIDS} — these are excluded via .github/kaibench-flaky-questions.txt and are NOT verified by this check" + fi if [ "${REGRESSIONS:-0}" -gt 0 ]; then - echo "::error::$REGRESSIONS question(s) regressed vs the previous run — see the step summary for details" + echo "::error::$REGRESSIONS question(s) regressed vs the previous run: ${REGRESSED_QIDS} — see the step summary for details" exit 1 fi echo "No regressions." diff --git a/.github/workflows/kaibench.yml b/.github/workflows/kaibench.yml index a9b1c10af..ae117e9f2 100644 --- a/.github/workflows/kaibench.yml +++ b/.github/workflows/kaibench.yml @@ -25,13 +25,23 @@ on: required: false type: string default: '' + questions: + description: 'Specific question IDs (comma-separated). Intersects with the other filters.' + required: false + type: string + default: '' + repeat: + description: 'Trials per question, for consistency/noise measurement (default 1)' + required: false + type: string + default: '' workflow_call: jobs: evaluate: name: Run KaiBench evaluation runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 120 continue-on-error: ${{ github.event_name == 'workflow_call' }} outputs: status: ${{ steps.parse.outputs.status }} @@ -41,7 +51,12 @@ jobs: pass_rate: ${{ steps.parse.outputs.pass_rate }} duration: ${{ steps.parse.outputs.duration }} regressions: ${{ steps.parse.outputs.regressions }} + regressed_qids: ${{ steps.parse.outputs.regressed_qids }} + flaky_regressions: ${{ steps.parse.outputs.flaky_regressions }} + flaky_regressed_qids: ${{ steps.parse.outputs.flaky_regressed_qids }} baseline_run: ${{ steps.parse.outputs.baseline_run }} + baseline_overlap: ${{ steps.parse.outputs.baseline_overlap }} + baseline_shared: ${{ steps.parse.outputs.baseline_shared }} services: postgres: @@ -209,6 +224,8 @@ jobs: KAIBENCH_EVAL_PARALLEL_WORKERS: '4' KAIBENCH_EVAL_KAI_BACKEND_URL: http://localhost:3000 QUESTION_TYPES: ${{ inputs.question_types }} + QUESTION_IDS: ${{ inputs.questions }} + REPEAT: ${{ inputs.repeat }} run: | CMD_ARGS=() if [ -n "$QUESTION_TYPES" ]; then @@ -220,11 +237,22 @@ jobs: else CMD_ARGS=(-t "Data Analysis Query" -t "Configuration Reasoning" -t "Storage Object Reasoning" -t "MCP Tool Validation") fi + if [ -n "$QUESTION_IDS" ]; then + IFS=',' read -ra QIDS <<< "$QUESTION_IDS" + for q in "${QIDS[@]}"; do + trimmed=$(echo "$q" | xargs) + CMD_ARGS+=(--question "$trimmed") + done + fi + if [ -n "$REPEAT" ]; then + CMD_ARGS+=(--repeat "$REPEAT") + fi if [ "${{ inputs.regression_only }}" = "true" ]; then # the CLI option is `--regression` (see kaibench/cli.py); `--regression-only` is rejected by # typer as an unknown option, and because this step is continue-on-error it failed silently CMD_ARGS+=(--regression) fi + echo "Running: kaibench run ${CMD_ARGS[*]}" uv run kaibench run "${CMD_ARGS[@]}" continue-on-error: true