codereview.py

import logging
import subprocess
from pathlib import Path
from typing import override

from bcbench_core.git import apply_patch, fetch_commit_if_missing

from bcbench.dataset.codereview import CodeReviewEntry, ReviewComment
from bcbench.evaluate.base import EvaluationPipeline
from bcbench.evaluate.codereview_judge import judge_expected_and_ignored
from bcbench.evaluate.review_parsing import parse_review_output
from bcbench.operations import setup_repo_prebuild
from bcbench.results.codereview import CodeReviewResult, candidate_comment_pairs
from bcbench.types import EvaluationContext

logger = logging.getLogger(__name__)

REVIEW_OUTPUT_FILE = "review.json"

__all__ = ["CodeReviewPipeline"]


def _patched_paths(patch: str) -> list[str]:
    return [line[6:].strip() for line in patch.splitlines() if line.startswith("+++ b/")]


class CodeReviewPipeline(EvaluationPipeline[CodeReviewEntry]):
    """Pipeline for code-review evaluation category.

    Code review does not require a BC container. We materialize the dataset patch
    as local git changes so the agent can review the branch diff directly.
    """

    @override
    def setup_workspace(self, entry: CodeReviewEntry, repo_path: Path) -> None:
        """Setup workspace for code review by applying the entry patch as local changes."""
        # Code-review base commits are pre-squash PR commits, so they might be missing from local dev setups.
        fetch_commit_if_missing(repo_path, entry.base_commit)
        setup_repo_prebuild(entry, repo_path)
        apply_patch(repo_path, entry.patch, f"{entry.instance_id} review patch")
        # Mark newly added files as intent-to-add so they appear in `git diff HEAD`;
        # all dataset patches are new-file diffs, which are otherwise untracked and invisible to the agent.
        if paths := _patched_paths(entry.patch):
            subprocess.run(
                ["git", "add", "-N", "--", *paths],
                cwd=repo_path,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
                check=True,
            )

    @override
    def setup(self, context: EvaluationContext[CodeReviewEntry]) -> None:
        self.setup_workspace(context.entry, context.repo_path)

    @override
    def evaluate(self, context: EvaluationContext[CodeReviewEntry]) -> None:
        review_output_file: Path = context.repo_path / REVIEW_OUTPUT_FILE

        if not review_output_file.exists():
            logger.error(f"No review generated for {context.entry.instance_id}")
            raise RuntimeError(f"No review generated for {context.entry.instance_id}")
        output: str = review_output_file.read_text(encoding="utf-8")

        generated_comments: list[ReviewComment] | None = parse_review_output(output)

        if generated_comments is None:
            logger.warning(f"Invalid review output for {context.entry.instance_id}")
            result = CodeReviewResult.create_invalid(context, output, context.entry.expected_comments)
        else:
            expected_candidates = candidate_comment_pairs(context.entry.expected_comments, generated_comments)
            ignored_candidates = candidate_comment_pairs(context.entry.ignored_comments, generated_comments)
            validated_matches, ignored_matches = judge_expected_and_ignored(
                expected_candidates,
                ignored_candidates,
                work_dir=context.repo_path,
            )
            result = CodeReviewResult.create(
                context,
                output=output,
                expected_comments=context.entry.expected_comments,
                generated_comments=generated_comments,
                matched_pairs=validated_matches,
                ignored_comments=context.entry.ignored_comments,
                ignored_matched_pairs=ignored_matches,
            )
        logger.info(f"Parsed {len(result.generated_comments)} comments from {REVIEW_OUTPUT_FILE}")
        logger.info(
            f"Code review metrics: matched={result.matched_comment_count}, "
            f"incorrect={result.incorrect_comment_count}, missed={result.missed_comment_count}, "
            f"ignored={result.ignored_matched_comment_count}, "
            f"precision={result.precision:.3f}, recall={result.recall:.3f}, f1={result.f1:.3f}"
        )

        self.save_result(context, result)