diff --git a/docs/sdk/core/agent-system.mdx b/docs/sdk/core/agent-system.mdx
index 612c9cc2f..2679db4d6 100644
--- a/docs/sdk/core/agent-system.mdx
+++ b/docs/sdk/core/agent-system.mdx
@@ -869,6 +869,52 @@ walkthrough of the registry entry.
---
+## Verification Scope
+
+Every answer the agent loop emits ends with one line saying whether the result
+was actually checked. Without it, "done" reads the same whether the agent ran
+the test suite or ran nothing at all, and the only rational response is to
+distrust every completion message.
+
+```text
+Fixed the off-by-one in the pagination helper.
+
+Verification: verified — pytest ran and passed.
+```
+
+Three states, derived from the tool calls that actually executed during the
+turn — no extra model call:
+
+| State | When |
+|-------|------|
+| `verified` | At least one test / lint / build ran, and every one passed. |
+| `partially verified` | Checks ran, but not all of them passed. |
+| `unverified` | No check ran. Names how many tools did run, if any. |
+
+A call counts as a check when the tool is one of `run_tests`, `run_lint`,
+`typecheck`, or `build`, or when its `command` argument names a known runner
+(`pytest`, `ruff`, `npm test`, `cargo test`, `go test`, `tsc`, ...). The
+classifier is deliberately conservative: a missed runner reads `unverified`,
+which is cautious rather than wrong.
+
+The statement rides in the emitted answer, so it reaches every surface — the
+CLI console, the SSE `answer` event, and `process_query`'s return value — and
+it is capped at `VERIFICATION_SCOPE_MAX_CHARS` (200) so it cannot grow the
+prompt as conversation history accumulates.
+
+```python
+from gaia.agents.base.verification import strip_verification_scope
+
+answer_only = strip_verification_scope(result["result"])
+```
+
+
+A turn cancelled from the UI returns an empty result on purpose, and carries no
+statement — filling it in would make a cancelled turn look like a completed one.
+
+
+---
+
## Proactive Lifecycle Hooks
Beyond responding to a user prompt, an agent can act **proactively** — proposing
diff --git a/hub/agents/email/python/gaia_agent_email/agent.py b/hub/agents/email/python/gaia_agent_email/agent.py
index 016e8a11c..b0741407f 100644
--- a/hub/agents/email/python/gaia_agent_email/agent.py
+++ b/hub/agents/email/python/gaia_agent_email/agent.py
@@ -94,6 +94,7 @@ class never passes ``use_claude=True`` / ``use_chatgpt=True`` to
from gaia.agents.base.console import AgentConsole
from gaia.agents.base.memory import MemoryMixin
from gaia.agents.base.tools import _TOOL_REGISTRY
+from gaia.agents.base.verification import strip_verification_scope
from gaia.agents.registry import get_embedding_model_for_device
from gaia.connectors.errors import ConnectorsError
from gaia.connectors.formatting import format_connector_error
@@ -1229,7 +1230,12 @@ def process_query(self, user_input: str, *args, **kwargs):
# consumers never see raw TeX in the final answer (#2115).
if isinstance(result, dict) and isinstance(result.get("result"), str):
result["result"] = _normalize_plain_text_answer(result["result"])
- if isinstance(result, dict) and result.get("result") != self._grounded_answer:
+ if (
+ isinstance(result, dict)
+ # The loop appends a verification-scope line after finalize_answer
+ # (#3376); compare the answer text itself.
+ and strip_verification_scope(result.get("result")) != self._grounded_answer
+ ):
# Normally finalize_answer already grounded this text before the
# loop emitted it. This covers the branches that never reach that
# call — the loop setting an actionable answer on an internal error
diff --git a/src/gaia/agents/base/agent.py b/src/gaia/agents/base/agent.py
index 70d717087..f1d57d967 100644
--- a/src/gaia/agents/base/agent.py
+++ b/src/gaia/agents/base/agent.py
@@ -39,6 +39,10 @@
from gaia.agents.base.console import AgentConsole, SilentConsole
from gaia.agents.base.errors import format_execution_trace
from gaia.agents.base.tools import _TOOL_REGISTRY
+from gaia.agents.base.verification import (
+ build_verification_scope,
+ verification_check_label,
+)
# First-party imports
from gaia.chat.sdk import AgentConfig, AgentSDK
@@ -745,6 +749,8 @@ def __init__(
# if called outside the normal process_query loop (e.g. directly in a
# test); _process_query_impl resets this per-turn (#2899).
self._tool_reported_usage: List[Dict[str, Any]] = []
+ # Same rationale for the verification-scope log (#3376).
+ self._turn_tool_executions: List[Dict[str, Any]] = []
self.conversation_history = (
[]
) # Store conversation history for session persistence
@@ -3328,7 +3334,9 @@ def _execute_tool_timed(self, tool_name: str, tool_args: Dict[str, Any]) -> Any:
# (CodeAgent's orchestration does); timing both would count the inner
# one's seconds twice and push tool_s past the turn's own total.
if recorder is None or getattr(self, "_tool_timing_depth", 0):
- return self._execute_tool(tool_name, tool_args)
+ result = self._execute_tool(tool_name, tool_args)
+ self._note_verification_signal(tool_name, tool_args, result)
+ return result
started = time.perf_counter()
# Default False, set only on a clean return: a tool that RAISES must not
@@ -3339,6 +3347,7 @@ def _execute_tool_timed(self, tool_name: str, tool_args: Dict[str, Any]) -> Any:
try:
result = self._execute_tool(tool_name, tool_args)
ok = not self._is_error_result(result)
+ self._note_verification_signal(tool_name, tool_args, result)
return result
finally:
self._tool_timing_depth = 0
@@ -4439,6 +4448,41 @@ def finalize_answer(self, answer: str, _conversation: Any) -> str:
"""
return answer
+ def _note_verification_signal(
+ self, tool_name: str, tool_args: Dict[str, Any], result: Any
+ ) -> None:
+ """Record one executed tool call for this turn's verification scope.
+
+ Called from the single execution seam so every loop path — legacy,
+ native tool-calling, and the forced-call branch — is covered.
+ """
+ log = getattr(self, "_turn_tool_executions", None)
+ if log is None:
+ return
+ log.append(
+ {
+ "tool": tool_name,
+ "check_label": verification_check_label(tool_name, tool_args),
+ "failed": self._is_error_result(result),
+ }
+ )
+
+ def verification_scope_statement(self) -> str:
+ """This turn's bounded verified / partially verified / unverified line."""
+ return build_verification_scope(
+ getattr(self, "_turn_tool_executions", None) or []
+ )
+
+ def _with_verification_scope(self, answer: Optional[str]) -> Optional[str]:
+ """Append the scope statement to a non-empty answer (#3376).
+
+ Empty stays empty — a blank answer is a signal downstream (cancelled
+ turns skip persistence), and a scope line would make it non-blank.
+ """
+ if not answer or not answer.strip():
+ return answer
+ return f"{answer.rstrip()}\n\n{self.verification_scope_statement()}"
+
def process_query(
self,
user_input: str,
@@ -4556,6 +4600,12 @@ def _process_query_impl(
# reset per-turn since an Agent instance persists across queries in
# an interactive session.
self._tool_reported_usage: List[Dict[str, Any]] = []
+ # Executed tool calls this turn, classified for the verification-scope
+ # statement (#3376). Per-turn: an instance persists across queries.
+ self._turn_tool_executions: List[Dict[str, Any]] = []
+ # True once the emitted answer carries its scope line, so the post-loop
+ # catch-all below never appends a second one.
+ verification_scope_applied = False
# Add user query to the conversation history
conversation.append({"role": "user", "content": user_input})
@@ -6428,7 +6478,12 @@ def _process_query_impl(
"start GAIA with the `--sd` flag to enable it."
)
- final_answer = self.finalize_answer(answer_candidate, conversation)
+ # Scope line goes on AFTER the subclass hook: a subclass that
+ # rewrites the answer must not be able to drop it (#3376).
+ final_answer = self._with_verification_scope(
+ self.finalize_answer(answer_candidate, conversation)
+ )
+ verification_scope_applied = True
self.execution_state = self.STATE_COMPLETION
# Compute the real token total BEFORE printing the answer so it
# can ride the same event, instead of the post-loop aggregation
@@ -6524,6 +6579,16 @@ def _process_query_impl(
conversation, self._tool_reported_usage
)
+ # Every exit other than the parsed-answer seam sets ``final_answer``
+ # directly — cancel-event timeout, LLM connection error, context
+ # overflow, typed Lemonade error, parse give-up, loop-break summary —
+ # or leaves it None for the max-steps message below. Those are
+ # disproportionately the runs that went wrong, so they need the scope
+ # line most (#3376). The console-cancellation path returns above with a
+ # deliberately empty result and is excluded (#3386).
+ if not verification_scope_applied:
+ final_answer = self._with_verification_scope(final_answer)
+
# Return the result
has_errors = len(self.error_history)> 0
has_valid_answer = (
@@ -6538,8 +6603,10 @@ def _process_query_impl(
"result": (
final_answer
if final_answer
- else self._generate_max_steps_message(
- conversation, steps_taken, steps_limit
+ else self._with_verification_scope(
+ self._generate_max_steps_message(
+ conversation, steps_taken, steps_limit
+ )
)
),
"system_prompt": self.system_prompt, # Include system prompt in the result
diff --git a/src/gaia/agents/base/verification.py b/src/gaia/agents/base/verification.py
new file mode 100644
index 000000000..a1cd1ba5f
--- /dev/null
+++ b/src/gaia/agents/base/verification.py
@@ -0,0 +1,148 @@
+# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: MIT
+"""Verification-scope statement appended to every emitted answer (#3376).
+
+The agent loop used to report "done" in the same confident language whether it
+ran the test suite or ran nothing at all. Every emitted answer now carries one
+line saying which — derived from the turn's own tool-execution log, so it costs
+no extra model call.
+
+The line rides in the answer, which the surfaces persist and re-send as
+conversation history, so it is HARD-CAPPED at ``VERIFICATION_SCOPE_MAX_CHARS``.
+:func:`strip_verification_scope` removes it again for consumers that need the
+answer text alone.
+
+Pure and dependency-free on purpose: the agent loop, the Agent-UI SSE handler,
+and hub agents all consume it.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, FrozenSet, List, Optional, Tuple
+
+VERIFICATION_SCOPE_PREFIX = "Verification: "
+VERIFICATION_SCOPE_MAX_CHARS = 200
+
+# Tools that ARE a check by name, whatever their arguments.
+_CHECK_TOOLS: FrozenSet[str] = frozenset(
+ {
+ "build",
+ "lint",
+ "run_lint",
+ "run_test_suite",
+ "run_tests",
+ "typecheck",
+ }
+)
+
+# A shell-style call is a check when its command names a test / lint / build
+# runner. Deliberately conservative: a miss reads "unverified" (honest and
+# cautious), a false positive would claim a check that never ran.
+_CHECK_COMMAND_RE = re.compile(
+ r"\b("
+ r"pytest|py\.test|tox|nox"
+ r"|python\s+-m\s+(?:pytest|unittest)"
+ r"|npm\s+(?:run\s+)?(?:test|lint|build|typecheck)"
+ r"|yarn\s+(?:test|lint|build)"
+ r"|pnpm\s+(?:run\s+)?(?:test|lint|build)"
+ r"|go\s+(?:test|vet|build)"
+ r"|cargo\s+(?:test|clippy|check|build)"
+ r"|dotnet\s+(?:test|build)"
+ r"|mvn\s+(?:test|verify)"
+ r"|make\s+(?:test|check|lint|build)"
+ r"|ctest|jest|vitest|mocha"
+ r"|ruff|flake8|pylint|mypy|pyright|eslint|tsc|shellcheck"
+ r"|util[/\\]lint\.py"
+ r")\b",
+ re.IGNORECASE,
+)
+
+# Argument keys that carry a shell command, in priority order.
+_COMMAND_KEYS: Tuple[str, ...] = ("command", "cmd", "script")
+
+_SCOPE_LINE_RE = re.compile(
+ r"\n{1,2}" + re.escape(VERIFICATION_SCOPE_PREFIX) + r"[^\n]*\s*\Z"
+)
+
+
+def verification_check_label(tool_name: str, tool_args: Any) -> Optional[str]:
+ """Short label when this call is a verification check, else ``None``.
+
+ ``pytest tests/unit -q`` → ``"pytest"``; ``read_file`` → ``None``.
+ """
+ name = (tool_name or "").strip()
+ if name in _CHECK_TOOLS:
+ return name
+ if not isinstance(tool_args, dict):
+ return None
+ for key in _COMMAND_KEYS:
+ command = tool_args.get(key)
+ if isinstance(command, str) and command.strip():
+ match = _CHECK_COMMAND_RE.search(command)
+ return " ".join(match.group(0).split()).lower() if match else None
+ return None
+
+
+def _names(executions: List[Dict[str, Any]], limit: int = 3) -> str:
+ """Deduped, order-preserving, count-capped label list."""
+ labels: List[str] = []
+ for execution in executions:
+ label = execution.get("check_label")
+ if label and label not in labels:
+ labels.append(label)
+ if not labels:
+ return "a check"
+ shown = ", ".join(labels[:limit])
+ extra = len(labels) - limit
+ return f"{shown} +{extra} more" if extra> 0 else shown
+
+
+def build_verification_scope(executions: List[Dict[str, Any]]) -> str:
+ """One bounded line naming what ran, what passed, and what went unchecked.
+
+ Three distinguishable states: ``verified`` (checks ran and every one
+ passed), ``partially verified`` (checks ran, not all passed), and
+ ``unverified`` (no check ran at all).
+
+ Each execution is ``{"tool": str, "check_label": str | None,
+ "failed": bool}`` — see ``Agent._note_verification_signal``.
+ """
+ executions = list(executions or [])
+ checks = [e for e in executions if e.get("check_label")]
+ if not checks:
+ total = len(executions)
+ if total == 0:
+ body = "unverified — no tools ran, so nothing was checked."
+ else:
+ plural = "" if total == 1 else "s"
+ body = (
+ f"unverified — {total} tool call{plural} ran, none of them a "
+ "test, lint, or build."
+ )
+ else:
+ passed = [e for e in checks if not e.get("failed")]
+ failed = [e for e in checks if e.get("failed")]
+ if not failed:
+ body = f"verified — {_names(passed)} ran and passed."
+ elif not passed:
+ body = (
+ f"partially verified — {_names(failed)} ran and did not pass; "
+ "nothing else was checked."
+ )
+ else:
+ body = (
+ f"partially verified — {_names(passed)} passed, "
+ f"{_names(failed)} did not."
+ )
+ statement = VERIFICATION_SCOPE_PREFIX + body
+ if len(statement)> VERIFICATION_SCOPE_MAX_CHARS:
+ statement = statement[: VERIFICATION_SCOPE_MAX_CHARS - 1].rstrip() + "..."
+ return statement
+
+
+def strip_verification_scope(text: str) -> str:
+ """Remove a trailing verification-scope line added by the agent loop."""
+ if not isinstance(text, str) or VERIFICATION_SCOPE_PREFIX not in text:
+ return text
+ return _SCOPE_LINE_RE.sub("", text)
diff --git a/src/gaia/ui/sse_handler.py b/src/gaia/ui/sse_handler.py
index aa25b629a..71c0046e5 100644
--- a/src/gaia/ui/sse_handler.py
+++ b/src/gaia/ui/sse_handler.py
@@ -25,6 +25,7 @@
from gaia.agents.base.tool_grants import grant_scope
from gaia.agents.base.tools import get_tool_display_label, get_tool_metadata
from gaia.agents.base.turn_metrics import turn_log_path
+from gaia.agents.base.verification import strip_verification_scope
from gaia.ui.event_narration import DEBUG_CHANNEL, format_count
logger = logging.getLogger(__name__)
@@ -608,6 +609,14 @@ def print_final_answer(
ttft_seconds: Optional[float] = None,
):
if answer:
+ scope_line = ""
+ # Set aside the verification-scope line before the cleaners run: an
+ # answer they strip to nothing (a card-echo) must stay empty, not
+ # arrive as a message consisting only of the scope line (#3376).
+ cleaned_of_scope = strip_verification_scope(answer)
+ if cleaned_of_scope != answer:
+ scope_line = answer[len(cleaned_of_scope) :].strip()
+ answer = cleaned_of_scope
answer = _THINK_TAG_SUB_RE.sub("", answer)
# Extract answer text from {"thought":..., "answer":...} JSON before
# the regex cleaners run. _THOUGHT_JSON_SUB_RE would otherwise strip
@@ -620,6 +629,8 @@ def print_final_answer(
answer = _TOOL_CALL_JSON_SUB_RE.sub("", answer)
answer = _THOUGHT_JSON_SUB_RE.sub("", answer)
answer = answer.strip()
+ if answer and scope_line:
+ answer = f"{answer}\n\n{scope_line}"
event: Dict[str, Any] = {
"type": "answer",
"content": _fix_double_escaped(answer) if answer else answer,
diff --git a/tests/unit/agents/test_parse_error_recovery.py b/tests/unit/agents/test_parse_error_recovery.py
index b8f6a0db9..d7b7c0338 100644
--- a/tests/unit/agents/test_parse_error_recovery.py
+++ b/tests/unit/agents/test_parse_error_recovery.py
@@ -20,6 +20,7 @@
import pytest
from gaia.agents.base.agent import _CONTEXT_STILL_OVERFLOWING_MESSAGE, Agent
+from gaia.agents.base.verification import strip_verification_scope
class _DummyAgent(Agent):
@@ -221,7 +222,8 @@ def _send(*_, **__):
# ``process_query`` returns ``{"status": ..., "result": , ...}``
text = result["result"] if isinstance(result, dict) else str(result)
assert text, "expected the friendly trim-exhausted fallback text"
- assert text == _CONTEXT_STILL_OVERFLOWING_MESSAGE
+ # The loop appends a verification-scope line to every answer (#3376).
+ assert strip_verification_scope(text) == _CONTEXT_STILL_OVERFLOWING_MESSAGE
assert "Max length reached" not in text
assert "Sorry, I ran into" not in text
@@ -353,7 +355,8 @@ def _send_stream(*_, **__):
assert call_count["n"] == 2 # initial attempt + one trim-and-retry
text = result["result"] if isinstance(result, dict) else str(result)
- assert text == _CONTEXT_STILL_OVERFLOWING_MESSAGE
+ # The loop appends a verification-scope line to every answer (#3376).
+ assert strip_verification_scope(text) == _CONTEXT_STILL_OVERFLOWING_MESSAGE
assert "exceeds the available context size" not in text
assert "Sorry, I ran into" not in text
diff --git a/tests/unit/agents/test_verification_scope.py b/tests/unit/agents/test_verification_scope.py
new file mode 100644
index 000000000..917e1474d
--- /dev/null
+++ b/tests/unit/agents/test_verification_scope.py
@@ -0,0 +1,487 @@
+# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: MIT
+"""Tests for the verification-scope statement on emitted answers (#3376).
+
+The loop used to say "done" identically whether it ran the test suite or ran
+nothing at all. Every emitted answer now carries one bounded line naming which
+of three states applies — verified / partially verified / unverified — derived
+from the turn's own tool-execution log.
+
+Two layers of coverage:
+
+* the pure classifier/builder, exercised directly;
+* every exit path in ``_process_query_impl`` that produces a final answer,
+ driven through the real loop with a stubbed chat client. The direct-set
+ paths (LLM error, context overflow, cancel-event timeout, parse give-up,
+ loop break, max steps) are the point of the issue — they bypass
+ ``finalize_answer``, and they are disproportionately the runs that went
+ wrong.
+
+The one deliberate exclusion is the console-cancellation path, which returns
+an empty result on purpose (#3386).
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from gaia.agents.base.agent import Agent
+from gaia.agents.base.tools import tool
+from gaia.agents.base.verification import (
+ VERIFICATION_SCOPE_MAX_CHARS,
+ VERIFICATION_SCOPE_PREFIX,
+ build_verification_scope,
+ strip_verification_scope,
+ verification_check_label,
+)
+
+_SANDBOX_SHELL = "sandbox_shell_for_verification_scope_test"
+
+
+class _DummyAgent(Agent):
+ """Minimal concrete Agent — same pattern as test_loop_break_truthful."""
+
+ #: Set per-test; the stub tool returns it verbatim.
+ shell_result = {"status": "success", "return_code": 0, "stdout": "ok"}
+
+ def _get_system_prompt(self) -> str:
+ return "test"
+
+ def _register_tools(self) -> None:
+ agent = self
+
+ # Deliberately NOT named ``run_shell_command``: that name is
+ # confirmation-gated, and the classifier keys on the ``command``
+ # argument, not the tool name. The real name is covered by the
+ # classifier tests below.
+ @tool
+ def sandbox_shell_for_verification_scope_test(command: str) -> dict:
+ """Run a command in a sandbox."""
+ del command
+ return agent.shell_result
+
+ def _create_console(self):
+ from gaia.agents.base.console import AgentConsole
+
+ return AgentConsole()
+
+
+@pytest.fixture
+def agent():
+ with patch("gaia.agents.base.agent.AgentSDK"):
+ a = _DummyAgent(silent_mode=True, skip_lemonade=True)
+ a.streaming = False
+ return a
+
+
+def _stub_chat(agent_, *responses):
+ """Replace ``agent.chat`` with a stub yielding *responses* in order."""
+ queue = list(responses)
+ chat = MagicMock()
+
+ def _send(*_, **__):
+ payload = queue.pop(0) if queue else queue_exhausted()
+ if isinstance(payload, Exception):
+ raise payload
+ resp = MagicMock()
+ resp.text = payload
+ resp.stats = {}
+ return resp
+
+ def queue_exhausted():
+ raise AssertionError("chat stub ran out of scripted responses")
+
+ chat.send_messages = MagicMock(side_effect=_send)
+ agent_.chat = chat
+ return chat
+
+
+def _answer(text: str) -> str:
+ return json.dumps({"thought": "done", "answer": text})
+
+
+def _tool_call(command: str) -> str:
+ return json.dumps(
+ {
+ "thought": "checking",
+ "tool": _SANDBOX_SHELL,
+ "tool_args": {"command": command},
+ }
+ )
+
+
+def _scope_line(text: str) -> str:
+ """The verification line from an emitted answer (asserts there is one)."""
+ lines = [
+ line for line in text.splitlines() if line.startswith(VERIFICATION_SCOPE_PREFIX)
+ ]
+ assert len(lines) == 1, f"expected exactly one scope line in:\n{text}"
+ return lines[0]
+
+
+# ---------------------------------------------------------------------------
+# Classifier
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+ "command,expected",
+ [
+ ("pytest tests/unit -q", "pytest"),
+ ("python -m pytest tests/", "python -m pytest"),
+ ("python util/lint.py --all", "util/lint.py"),
+ ("npm run test", "npm run test"),
+ ("cargo clippy -- -D warnings", "cargo clippy"),
+ ("go test ./...", "go test"),
+ ("ruff check src/", "ruff"),
+ ("tsc --noEmit", "tsc"),
+ ("git commit -m 'wip'", None),
+ ("ls -la", None),
+ ("", None),
+ ],
+)
+def test_command_classification(command, expected):
+ assert verification_check_label("run_shell_command", {"command": command}) == (
+ expected
+ )
+
+
+def test_tool_name_alone_can_be_a_check():
+ assert verification_check_label("run_tests", {}) == "run_tests"
+
+
+def test_non_check_tool_is_not_a_check():
+ assert verification_check_label("read_file", {"file_path": "pytest.ini"}) is None
+
+
+def test_non_dict_args_do_not_raise():
+ assert verification_check_label("read_file", "pytest") is None
+
+
+# ---------------------------------------------------------------------------
+# Three states
+# ---------------------------------------------------------------------------
+
+
+def _execution(label=None, failed=False, name="some_tool"):
+ return {"tool": name, "check_label": label, "failed": failed}
+
+
+def test_no_tools_at_all_is_unverified():
+ statement = build_verification_scope([])
+ assert statement.startswith(VERIFICATION_SCOPE_PREFIX)
+ assert "unverified" in statement
+ assert "no tools ran" in statement
+
+
+def test_tools_but_no_checks_is_unverified_and_says_how_many():
+ statement = build_verification_scope([_execution(), _execution()])
+ assert "unverified" in statement
+ assert "2 tool calls ran" in statement
+
+
+def test_singular_tool_call_wording():
+ assert "1 tool call ran" in build_verification_scope([_execution()])
+
+
+def test_all_checks_passed_is_verified():
+ statement = build_verification_scope(
+ [_execution("pytest"), _execution("ruff"), _execution()]
+ )
+ assert "verified —" in statement
+ assert "partially" not in statement
+ assert "pytest, ruff" in statement
+
+
+def test_mixed_check_outcomes_is_partially_verified():
+ statement = build_verification_scope(
+ [_execution("ruff"), _execution("pytest", failed=True)]
+ )
+ assert "partially verified" in statement
+ assert "ruff passed" in statement
+ assert "pytest did not" in statement
+
+
+def test_all_checks_failed_is_partially_verified_not_verified():
+ statement = build_verification_scope([_execution("pytest", failed=True)])
+ assert "partially verified" in statement
+ assert "did not pass" in statement
+
+
+def test_the_three_states_are_distinguishable():
+ unverified = build_verification_scope([])
+ verified = build_verification_scope([_execution("pytest")])
+ partial = build_verification_scope(
+ [_execution("pytest"), _execution("ruff", failed=True)]
+ )
+ assert len({unverified, verified, partial}) == 3
+ # "verified" is a substring of "unverified", so the states must be told
+ # apart on their own terms, not by substring containment.
+ assert unverified.startswith(f"{VERIFICATION_SCOPE_PREFIX}unverified")
+ assert verified.startswith(f"{VERIFICATION_SCOPE_PREFIX}verified")
+ assert partial.startswith(f"{VERIFICATION_SCOPE_PREFIX}partially verified")
+
+
+# ---------------------------------------------------------------------------
+# Bounded: the statement rides in conversation history
+# ---------------------------------------------------------------------------
+
+
+def test_statement_is_bounded_even_with_many_distinct_checks():
+ executions = [_execution(f"checker-{i:03d}-with-a-long-name") for i in range(200)]
+ assert len(build_verification_scope(executions)) <= VERIFICATION_SCOPE_MAX_CHARS + + +def test_label_list_is_capped_with_a_count(): + statement = build_verification_scope( + [ + _execution("pytest"), + _execution("ruff"), + _execution("mypy"), + _execution("tsc"), + ] + ) + assert "+1 more" in statement + + +def test_duplicate_labels_collapse(): + statement = build_verification_scope([_execution("pytest")] * 5) + assert statement.count("pytest") == 1 + + +def test_every_state_fits_the_bound(): + for executions in ( + [], + [_execution()], + [_execution("pytest")], + [_execution("pytest", failed=True)], + [_execution("pytest"), _execution("ruff", failed=True)], + ): + assert len(build_verification_scope(executions)) <= VERIFICATION_SCOPE_MAX_CHARS + + +def test_strip_removes_the_statement_and_leaves_the_answer(): + answer = "All set." + emitted = f"{answer}\n\n{build_verification_scope([])}" + assert strip_verification_scope(emitted) == answer + + +def test_strip_is_a_no_op_without_a_statement(): + assert strip_verification_scope("plain answer") == "plain answer" + + +# --------------------------------------------------------------------------- +# Exit path: parsed answer (the finalize_answer seam) +# --------------------------------------------------------------------------- + + +def test_parsed_answer_with_no_tools_is_unverified(agent): + _stub_chat(agent, _answer("Paris is the capital of France.")) + result = agent.process_query("capital of france?", max_steps=3) + assert "Paris" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_parsed_answer_after_a_passing_check_is_verified(agent): + agent.shell_result = {"status": "success", "return_code": 0, "stdout": "3 passed"} + _stub_chat(agent, _tool_call("pytest tests/unit -q"), _answer("Tests pass.")) + result = agent.process_query("run the tests", max_steps=5) + line = _scope_line(result["result"]) + assert line.startswith(f"{VERIFICATION_SCOPE_PREFIX}verified") + assert "pytest" in line + + +def test_parsed_answer_after_a_failing_check_is_partially_verified(agent): + agent.shell_result = {"status": "error", "error": "1 failed", "return_code": 1} + _stub_chat(agent, _tool_call("pytest tests/unit -q"), _answer("One test failed.")) + result = agent.process_query("run the tests", max_steps=5) + assert "partially verified" in _scope_line(result["result"]) + + +def test_non_check_tool_still_reads_unverified(agent): + _stub_chat(agent, _tool_call("ls -la"), _answer("Here are the files.")) + result = agent.process_query("list files", max_steps=5) + line = _scope_line(result["result"]) + assert "unverified" in line + assert "1 tool call ran" in line + + +def test_statement_survives_a_subclass_rewriting_the_answer(agent): + """``finalize_answer`` runs first; a subclass must not be able to drop it.""" + agent.finalize_answer = lambda answer, _conversation: "REWRITTEN" + _stub_chat(agent, _answer("original")) + result = agent.process_query("hello", max_steps=3) + assert result["result"].startswith("REWRITTEN") + assert "unverified" in _scope_line(result["result"]) + + +def test_statement_is_emitted_to_the_console_not_only_returned(agent): + """The scope line must reach the SSE/console surface, not just the dict.""" + _stub_chat(agent, _answer("Done.")) + agent.console.print_final_answer = MagicMock() + agent.process_query("hello", max_steps=3) + printed = agent.console.print_final_answer.call_args[0][0] + assert VERIFICATION_SCOPE_PREFIX in printed + + +def test_scope_is_reset_between_turns(agent): + agent.shell_result = {"status": "success", "return_code": 0} + _stub_chat( + agent, + _tool_call("pytest -q"), + _answer("first"), + _answer("second"), + ) + first = agent.process_query("run tests", max_steps=5) + second = agent.process_query("say hi", max_steps=3) + assert _scope_line(first["result"]).startswith( + f"{VERIFICATION_SCOPE_PREFIX}verified" + ) + assert "unverified" in _scope_line(second["result"]) + + +# --------------------------------------------------------------------------- +# Exit paths that set ``final_answer`` directly — the substance of the issue +# --------------------------------------------------------------------------- + + +def test_cancel_event_timeout_path_carries_the_statement(agent): + event = threading.Event() + event.set() + agent._cancel_event = event + _stub_chat(agent, _answer("never reached")) + result = agent.process_query("do something", max_steps=5) + assert "exceeded the allowed" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_llm_connection_error_path_carries_the_statement(agent): + _stub_chat(agent, ConnectionError("connection refused")) + result = agent.process_query("hello", max_steps=3) + assert "trouble reaching the language model" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_context_overflow_path_carries_the_statement(agent): + overflow = RuntimeError( + "request (99999 tokens) exceeds the available context size (65536 tokens)" + ) + _stub_chat(agent, overflow, overflow) + with patch.object(_DummyAgent, "_is_loaded_ctx_too_small", return_value=False): + result = agent.process_query("summarize everything", max_steps=3) + assert "context window" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_generic_llm_error_path_carries_the_statement(agent): + _stub_chat(agent, RuntimeError("kaboom")) + with patch.object(_DummyAgent, "_is_loaded_ctx_too_small", return_value=False): + result = agent.process_query("hello", max_steps=3) + assert "unexpected problem" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_parse_give_up_path_carries_the_statement(agent): + bad = '{"__tool_calls__": [{"function": {"name": "x", "arguments": "{' + _stub_chat(agent, bad, bad, bad, bad, bad) + result = agent.process_query("do it", max_steps=10) + assert "trouble formatting my tool call" in result["result"] + assert "unverified" in _scope_line(result["result"]) + + +def test_loop_break_summary_path_carries_the_statement(agent): + """A repeated failing check breaks the loop — and still states its scope. + + This exit is the clearest case for the feature: the loop-break summary + reads "Task completed with " even though every call errored, and
+ the scope line is what tells the user the check did not pass. Correcting
+ that summary itself belongs to the answer-guard work in #3381.
+ """
+ agent.max_consecutive_repeats = 2
+ agent.shell_result = {"status": "error", "error": "boom", "return_code": 1}
+ call = _tool_call("pytest -q")
+ _stub_chat(agent, call, call, call, call)
+ result = agent.process_query("run the tests", max_steps=6)
+ assert "Task completed with" in result["result"]
+ # Checks ran and did not pass — not "unverified", not "verified".
+ assert "partially verified" in _scope_line(result["result"])
+
+
+def test_max_steps_path_carries_the_statement(agent):
+ """No answer was ever produced; the max-steps message still states scope."""
+ _stub_chat(agent, _tool_call("ls -la"), _tool_call("ls -la"))
+ result = agent.process_query("browse", max_steps=1)
+ assert "Reached maximum steps limit" in result["result"]
+ assert "unverified" in _scope_line(result["result"])
+
+
+# ---------------------------------------------------------------------------
+# The one deliberate exclusion (#3386)
+# ---------------------------------------------------------------------------
+
+
+def test_console_cancellation_stays_empty(agent):
+ """A cancelled turn returns an empty result on purpose — do not fill it."""
+ agent.streaming = True
+ agent.console.cancelled = threading.Event()
+ agent.console.cancelled.set()
+
+ chat = MagicMock()
+
+ def _stream(*_, **__):
+ chunk = MagicMock()
+ chunk.is_complete = False
+ chunk.text = "partial "
+ yield chunk
+
+ chat.send_messages_stream = MagicMock(side_effect=_stream)
+ agent.chat = chat
+
+ result = agent.process_query("hello", max_steps=3)
+ assert result["status"] == "cancelled"
+ assert result["result"] == ""
+ assert VERIFICATION_SCOPE_PREFIX not in result["result"]
+
+
+def test_empty_answer_is_left_empty(agent):
+ """``_with_verification_scope`` never turns a blank answer non-blank."""
+ assert agent._with_verification_scope("") == ""
+ assert agent._with_verification_scope(" ") == " "
+ assert agent._with_verification_scope(None) is None
+
+
+# ---------------------------------------------------------------------------
+# Agent-UI surface: the SSE cleaners must not leave a scope-line-only message
+# ---------------------------------------------------------------------------
+
+
+def _sse_answer(raw: str) -> str:
+ """The ``answer`` event content the Agent UI would emit for *raw*."""
+ from gaia.ui.sse_handler import SSEOutputHandler
+
+ captured: list = []
+ handler = SSEOutputHandler.__new__(SSEOutputHandler)
+ handler._emit = captured.append
+ handler._elapsed = lambda: 0.0
+ handler._step_count = 0
+ handler._tool_count = 0
+ handler._turn_metrics = None
+ handler.print_final_answer(raw)
+ return captured[0]["content"]
+
+
+def test_sse_keeps_the_scope_line_on_a_real_answer():
+ raw = f"Here is the answer.\n\n{build_verification_scope([])}"
+ content = _sse_answer(raw)
+ assert content.startswith("Here is the answer.")
+ assert VERIFICATION_SCOPE_PREFIX in content
+
+
+def test_sse_card_echo_stays_empty_rather_than_scope_line_only():
+ """An answer the cleaners strip to nothing must not become a scope line."""
+ raw = f'{{"thought": "done", "answer": ""}}\n\n{build_verification_scope([])}'
+ assert _sse_answer(raw) == ""