diff --git a/rdagent/components/coder/data_science/share/eval.py b/rdagent/components/coder/data_science/share/eval.py index d3b95f5f42..af6ec425b1 100644 --- a/rdagent/components/coder/data_science/share/eval.py +++ b/rdagent/components/coder/data_science/share/eval.py @@ -166,11 +166,16 @@ def evaluate( # Read the content of files submission.csv and scores.csv after execution # Check if the content has changed # excactly same checking. But it will take more user's time + change_messages = [] if scores_content_before != scores_content_after: - return_msg = "\n[Error] The content of scores.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it." - return_msg += f"\nBefore:\n{scores_content_before}\nAfter:\n{scores_content_after}" - if submission_content_before != submission_content_after: - # If the scores file changes, display the two contents and append it into the return_checking - return_msg = "[Error] The content of submission.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it." - csfb.return_checking = (csfb.return_checking or "") + return_msg + change_messages.append( + "\n[Error] The content of scores.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it." + f"\nBefore:\n{scores_content_before}\nAfter:\n{scores_content_after}", + ) + if submission_content_before != submission_content_after: + change_messages.append( + "\n[Error] The content of submission.csv has changed. Please check the code to ensure that the model is dumped correctly, and rerun the code to use the model directly without retraining it.", + ) + if change_messages: + csfb.return_checking = (csfb.return_checking or "") + "".join(change_messages) return csfb diff --git a/test/scenarios/data_science/test_model_dump_high_diagnostics.py b/test/scenarios/data_science/test_model_dump_high_diagnostics.py new file mode 100644 index 0000000000..e8d0a5d2ac --- /dev/null +++ b/test/scenarios/data_science/test_model_dump_high_diagnostics.py @@ -0,0 +1,95 @@ +from types import SimpleNamespace + +import pytest + +import rdagent.components.coder.data_science.share.eval as share_eval +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback +from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator + + +class FakeTemplate: + def r(self, *args, **kwargs): + return "/input" + + +@pytest.mark.parametrize( + ("change_scores", "change_submission", "expected_messages"), + [ + (True, False, ("scores.csv",)), + (False, True, ("submission.csv",)), + (True, True, ("scores.csv", "submission.csv")), + (False, False, ()), + ], +) +def test_model_dump_high_check_reports_each_changed_artifact( + tmp_path, + monkeypatch, + change_scores, + change_submission, + expected_messages, +): + monkeypatch.setattr(share_eval, "T", lambda *args, **kwargs: FakeTemplate()) + monkeypatch.setattr(share_eval, "get_ds_env", lambda *args, **kwargs: object()) + monkeypatch.setattr(share_eval, "get_clear_ws_cmd", lambda *args, **kwargs: "clear") + monkeypatch.setattr(share_eval, "remove_eda_part", lambda stdout: stdout) + monkeypatch.setattr(share_eval.DS_RD_SETTING, "model_dump_check_level", "high") + monkeypatch.setattr( + share_eval, + "build_cls_from_json_with_retry", + lambda *args, **kwargs: CoSTEERSingleFeedback( + execution="ok", + return_checking="existing diagnostics", + code="ok", + final_decision=True, + ), + ) + + (tmp_path / "models").mkdir() + (tmp_path / "models" / "model.bin").write_text("model", encoding="utf-8") + + scores_before = "model,score\nensemble,0.1\n" + scores_after = "model,score\nensemble,0.2\n" + submission_before = "id,pred\n1,0.1\n" + submission_after = "id,pred\n1,0.2\n" + (tmp_path / "scores.csv").write_text(scores_before, encoding="utf-8") + (tmp_path / "submission.csv").write_text(submission_before, encoding="utf-8") + + class FakeImplementation: + workspace_path = tmp_path + all_codes = {"main.py": "pass"} + + def execute(self, env=None, entry=None): + if entry == "clear": + (tmp_path / "scores.csv").unlink(missing_ok=True) + (tmp_path / "submission.csv").unlink(missing_ok=True) + return "cleared" + + assert "--inference" in entry + (tmp_path / "scores.csv").write_text( + scores_after if change_scores else scores_before, + encoding="utf-8", + ) + (tmp_path / "submission.csv").write_text( + submission_after if change_submission else submission_before, + encoding="utf-8", + ) + return "inference finished" + + scen = SimpleNamespace( + competition="demo-competition", + debug_path="/tmp/demo-input", + real_debug_timeout=lambda: 1, + real_full_timeout=lambda: 1, + ) + feedback = ModelDumpEvaluator(scen, data_type="sample").evaluate( + None, + FakeImplementation(), + None, + ) + + diagnostics = feedback.return_checking or "" + assert "existing diagnostics" in diagnostics + for filename in expected_messages: + assert f"content of {filename} has changed" in diagnostics + for filename in {"scores.csv", "submission.csv"} - set(expected_messages): + assert f"content of {filename} has changed" not in diagnostics