From fa576fcb3fc9757e4798f5fcc03886f1d05e931d Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Fri, 4 Sep 2026 18:29:39 -0700 Subject: [PATCH] fix(eval): capture stdout when a scenario subprocess fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every scenario in a failing eval run reported ERRORED with an empty error field, so a CI failure said nothing about what actually went wrong. The runner read only stderr, but `--output-format json` puts the CLI's own error on stdout — stderr is routinely empty on exactly the failures worth reading. It now reports both streams, and falls back to naming the exit code when a process dies silently rather than storing an empty string. --- src/gaia/eval/runner.py | 14 ++++++++++++-- tests/test_eval.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/gaia/eval/runner.py b/src/gaia/eval/runner.py index 1a01fdd4c..5a7affad3 100644 --- a/src/gaia/eval/runner.py +++ b/src/gaia/eval/runner.py @@ -984,16 +984,26 @@ def run_scenario_subprocess( elapsed = time.time() - start if proc.returncode != 0: + # `--output-format json` puts the CLI's own error on stdout, so stderr + # is routinely empty here — capture both or the failure is unreadable. + detail = ( + "\n".join( + f"{name}: {text.strip()[:500]}" + for name, text in (("stderr", proc.stderr), ("stdout", proc.stdout)) + if text and text.strip() + ) + or f"no output on either stream (exit {proc.returncode})" + ) print( f"[ERROR] {scenario_id} — exit code {proc.returncode}", file=sys.stderr ) - print(proc.stderr[:500], file=sys.stderr) + print(detail, file=sys.stderr) result = { "scenario_id": scenario_id, "status": "ERRORED", "overall_score": None, "turns": [], - "error": proc.stderr[:500], + "error": detail, "elapsed_s": elapsed, "cost_estimate": {"turns": 0, "estimated_usd": 0.0}, } diff --git a/tests/test_eval.py b/tests/test_eval.py index 890d71131..376f1fc92 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -1020,14 +1020,14 @@ def _minimal_scenario(self): "turns": [{"turn": 1, "objective": "x", "success_criteria": "ok"}], } - def _run(self, mocker, stdout, returncode=0): + def _run(self, mocker, stdout, returncode=0, stderr=""): import tempfile from gaia.eval.runner import run_scenario_subprocess mock_proc = mocker.MagicMock() mock_proc.stdout = stdout - mock_proc.stderr = "" + mock_proc.stderr = stderr mock_proc.returncode = returncode mocker.patch("subprocess.run", return_value=mock_proc) @@ -1074,6 +1074,31 @@ def test_nonzero_exit_returns_errored(self, mocker): assert result["status"] == "ERRORED" assert result["overall_score"] is None + def test_nonzero_exit_captures_stdout(self, mocker): + """`--output-format json` puts the CLI's error on stdout, not stderr. + + Regression guard for the CI signature where every scenario ERRORED with + an empty `error` field, making the failure impossible to triage. + """ + result = self._run( + mocker, + '{"type":"result","subtype":"error_during_execution"}', + returncode=1, + ) + assert result["status"] == "ERRORED" + assert "error_during_execution" in result["error"] + + def test_nonzero_exit_captures_both_streams(self, mocker): + result = self._run(mocker, "on-stdout", returncode=1, stderr="on-stderr") + assert "on-stderr" in result["error"] + assert "on-stdout" in result["error"] + + def test_nonzero_exit_error_never_empty(self, mocker): + """A silent exit must still say something — an empty string explains nothing.""" + result = self._run(mocker, "", returncode=3, stderr="") + assert result["error"].strip() + assert "exit 3" in result["error"] + def test_missing_status_field_defaulted(self, mocker): """Eval agent returning JSON without 'status' should be defaulted to ERRORED.""" payload = {

AltStyle によって変換されたページ (->オリジナル) /