feat(agent): state whether an answer was verified, on every exit path #3397

Original file line number	Diff line number	Diff line change
Expand Up		@@ -869,6 +869,52 @@ walkthrough of the registry entry.

		---

	## Verification Scope

	Every answer the agent loop emits ends with one line saying whether the result
	was actually checked. Without it, "done" reads the same whether the agent ran
	the test suite or ran nothing at all, and the only rational response is to
	distrust every completion message.

	```text
	Fixed the off-by-one in the pagination helper.

	Verification: verified — pytest ran and passed.
	```

	Three states, derived from the tool calls that actually executed during the
	turn — no extra model call:

	\| State \| When \|
	\|-------\|------\|
	\| `verified` \| At least one test / lint / build ran, and every one passed. \|
	\| `partially verified` \| Checks ran, but not all of them passed. \|
	\| `unverified` \| No check ran. Names how many tools did run, if any. \|

	A call counts as a check when the tool is one of `run_tests`, `run_lint`,
	`typecheck`, or `build`, or when its `command` argument names a known runner
	(`pytest`, `ruff`, `npm test`, `cargo test`, `go test`, `tsc`, ...). The
	classifier is deliberately conservative: a missed runner reads `unverified`,
	which is cautious rather than wrong.

	The statement rides in the emitted answer, so it reaches every surface — the
	CLI console, the SSE `answer` event, and `process_query`'s return value — and
	it is capped at `VERIFICATION_SCOPE_MAX_CHARS` (200) so it cannot grow the
	prompt as conversation history accumulates.

	```python
	from gaia.agents.base.verification import strip_verification_scope

	answer_only = strip_verification_scope(result["result"])
	```

	<Note>
	A turn cancelled from the UI returns an empty result on purpose, and carries no
	statement — filling it in would make a cancelled turn look like a completed one.
	</Note>

	---

		## Proactive Lifecycle Hooks

		Beyond responding to a user prompt, an agent can act proactively — proposing
Expand Down

8 changes: 7 additions & 1 deletion hub/agents/email/python/gaia_agent_email/agent.py

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
Expand Up		@@ -94,6 +94,7 @@ class never passes ``use_claude=True`` / ``use_chatgpt=True`` to
		from gaia.agents.base.console import AgentConsole
		from gaia.agents.base.memory import MemoryMixin
		from gaia.agents.base.tools import _TOOL_REGISTRY
	from gaia.agents.base.verification import strip_verification_scope
		from gaia.agents.registry import get_embedding_model_for_device
		from gaia.connectors.errors import ConnectorsError
		from gaia.connectors.formatting import format_connector_error
Expand Down Expand Up		@@ -1229,7 +1230,12 @@ def process_query(self, user_input: str, args, *kwargs):
		# consumers never see raw TeX in the final answer (#2115).
		if isinstance(result, dict) and isinstance(result.get("result"), str):
		result["result"] = _normalize_plain_text_answer(result["result"])
	if isinstance(result, dict) and result.get("result") != self._grounded_answer:
	if (
	isinstance(result, dict)
	# The loop appends a verification-scope line after finalize_answer
	# (#3376); compare the answer text itself.
	and strip_verification_scope(result.get("result")) != self._grounded_answer
	):
		# Normally finalize_answer already grounded this text before the
		# loop emitted it. This covers the branches that never reach that
		# call — the loop setting an actionable answer on an internal error
Expand Down

75 changes: 71 additions & 4 deletions src/gaia/agents/base/agent.py

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
Expand Up		@@ -39,6 +39,10 @@
		from gaia.agents.base.console import AgentConsole, SilentConsole
		from gaia.agents.base.errors import format_execution_trace
		from gaia.agents.base.tools import _TOOL_REGISTRY
	from gaia.agents.base.verification import (
	build_verification_scope,
	verification_check_label,
	)

		# First-party imports
		from gaia.chat.sdk import AgentConfig, AgentSDK
Expand Down Expand Up		@@ -745,6 +749,8 @@ def __init__(
		# if called outside the normal process_query loop (e.g. directly in a
		# test); _process_query_impl resets this per-turn (#2899).
		self._tool_reported_usage: List[Dict[str, Any]] = []
	# Same rationale for the verification-scope log (#3376).
	self._turn_tool_executions: List[Dict[str, Any]] = []
		self.conversation_history = (
		[]
		) # Store conversation history for session persistence
Expand Down Expand Up		@@ -3328,7 +3334,9 @@ def _execute_tool_timed(self, tool_name: str, tool_args: Dict[str, Any]) -> Any:
		# (CodeAgent's orchestration does); timing both would count the inner
		# one's seconds twice and push tool_s past the turn's own total.
		if recorder is None or getattr(self, "_tool_timing_depth", 0):
	return self._execute_tool(tool_name, tool_args)
	result = self._execute_tool(tool_name, tool_args)
	self._note_verification_signal(tool_name, tool_args, result)
	return result

		started = time.perf_counter()
		# Default False, set only on a clean return: a tool that RAISES must not
Expand All		@@ -3339,6 +3347,7 @@ def _execute_tool_timed(self, tool_name: str, tool_args: Dict[str, Any]) -> Any:
		try:
		result = self._execute_tool(tool_name, tool_args)
		ok = not self._is_error_result(result)
	self._note_verification_signal(tool_name, tool_args, result)
		return result
		finally:
		self._tool_timing_depth = 0
Expand Down Expand Up		@@ -4439,6 +4448,41 @@ def finalize_answer(self, answer: str, _conversation: Any) -> str:
		"""
		return answer

	def _note_verification_signal(
	self, tool_name: str, tool_args: Dict[str, Any], result: Any
	) -> None:
	"""Record one executed tool call for this turn's verification scope.

	Called from the single execution seam so every loop path — legacy,
	native tool-calling, and the forced-call branch — is covered.
	"""
	log = getattr(self, "_turn_tool_executions", None)
	if log is None:
	return
	log.append(
	{
	"tool": tool_name,
	"check_label": verification_check_label(tool_name, tool_args),
	"failed": self._is_error_result(result),
	}
	)

	def verification_scope_statement(self) -> str:
	"""This turn's bounded verified / partially verified / unverified line."""
	return build_verification_scope(
	getattr(self, "_turn_tool_executions", None) or []
	)

	def _with_verification_scope(self, answer: Optional[str]) -> Optional[str]:
	"""Append the scope statement to a non-empty answer (#3376).

	Empty stays empty — a blank answer is a signal downstream (cancelled
	turns skip persistence), and a scope line would make it non-blank.
	"""
	if not answer or not answer.strip():
	return answer
	return f"{answer.rstrip()}\n\n{self.verification_scope_statement()}"

		def process_query(
		self,
		user_input: str,
Expand Down Expand Up		@@ -4556,6 +4600,12 @@ def _process_query_impl(
		# reset per-turn since an Agent instance persists across queries in
		# an interactive session.
		self._tool_reported_usage: List[Dict[str, Any]] = []
	# Executed tool calls this turn, classified for the verification-scope
	# statement (#3376). Per-turn: an instance persists across queries.
	self._turn_tool_executions: List[Dict[str, Any]] = []
	# True once the emitted answer carries its scope line, so the post-loop
	# catch-all below never appends a second one.
	verification_scope_applied = False

		# Add user query to the conversation history
		conversation.append({"role": "user", "content": user_input})
Expand Down Expand Up		@@ -6428,7 +6478,12 @@ def _process_query_impl(
		"start GAIA with the `--sd` flag to enable it."
		)

	final_answer = self.finalize_answer(answer_candidate, conversation)
	# Scope line goes on AFTER the subclass hook: a subclass that
	# rewrites the answer must not be able to drop it (#3376).
	final_answer = self._with_verification_scope(
	self.finalize_answer(answer_candidate, conversation)
	)
	verification_scope_applied = True
		self.execution_state = self.STATE_COMPLETION
		# Compute the real token total BEFORE printing the answer so it
		# can ride the same event, instead of the post-loop aggregation
Expand Down Expand Up		@@ -6524,6 +6579,16 @@ def _process_query_impl(
		conversation, self._tool_reported_usage
		)

	# Every exit other than the parsed-answer seam sets ``final_answer``
	# directly — cancel-event timeout, LLM connection error, context
	# overflow, typed Lemonade error, parse give-up, loop-break summary —
	# or leaves it None for the max-steps message below. Those are
	# disproportionately the runs that went wrong, so they need the scope
	# line most (#3376). The console-cancellation path returns above with a
	# deliberately empty result and is excluded (#3386).
	if not verification_scope_applied:
	final_answer = self._with_verification_scope(final_answer)

		# Return the result
		has_errors = len(self.error_history) > 0
		has_valid_answer = (
Expand All		@@ -6538,8 +6603,10 @@ def _process_query_impl(
		"result": (
		final_answer
		if final_answer
	else self._generate_max_steps_message(
	conversation, steps_taken, steps_limit
	else self._with_verification_scope(
	self._generate_max_steps_message(
	conversation, steps_taken, steps_limit
	)
		)
		),
		"system_prompt": self.system_prompt, # Include system prompt in the result
Expand Down

148 changes: 148 additions & 0 deletions src/gaia/agents/base/verification.py

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,148 @@
	# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
	# SPDX-License-Identifier: MIT
	"""Verification-scope statement appended to every emitted answer (#3376).

	The agent loop used to report "done" in the same confident language whether it
	ran the test suite or ran nothing at all. Every emitted answer now carries one
	line saying which — derived from the turn's own tool-execution log, so it costs
	no extra model call.

	The line rides in the answer, which the surfaces persist and re-send as
	conversation history, so it is HARD-CAPPED at ``VERIFICATION_SCOPE_MAX_CHARS``.
	:func:`strip_verification_scope` removes it again for consumers that need the
	answer text alone.

	Pure and dependency-free on purpose: the agent loop, the Agent-UI SSE handler,
	and hub agents all consume it.
	"""

	from __future__ import annotations

	import re
	from typing import Any, Dict, FrozenSet, List, Optional, Tuple

	VERIFICATION_SCOPE_PREFIX = "Verification: "
	VERIFICATION_SCOPE_MAX_CHARS = 200

	# Tools that ARE a check by name, whatever their arguments.
	_CHECK_TOOLS: FrozenSet[str] = frozenset(
	{
	"build",
	"lint",
	"run_lint",
	"run_test_suite",
	"run_tests",
	"typecheck",
	}
	)

	# A shell-style call is a check when its command names a test / lint / build
	# runner. Deliberately conservative: a miss reads "unverified" (honest and
	# cautious), a false positive would claim a check that never ran.
	_CHECK_COMMAND_RE = re.compile(
	r"\b("
	r"pytest\|py\.test\|tox\|nox"
	r"\|python\s+-m\s+(?:pytest\|unittest)"
	r"\|npm\s+(?:run\s+)?(?:test\|lint\|build\|typecheck)"
	r"\|yarn\s+(?:test\|lint\|build)"
	r"\|pnpm\s+(?:run\s+)?(?:test\|lint\|build)"
	r"\|go\s+(?:test\|vet\|build)"
	r"\|cargo\s+(?:test\|clippy\|check\|build)"
	r"\|dotnet\s+(?:test\|build)"
	r"\|mvn\s+(?:test\|verify)"
	r"\|make\s+(?:test\|check\|lint\|build)"
	r"\|ctest\|jest\|vitest\|mocha"
	r"\|ruff\|flake8\|pylint\|mypy\|pyright\|eslint\|tsc\|shellcheck"
	r"\|util[/\\]lint\.py"
	r")\b",
	re.IGNORECASE,
	)

	# Argument keys that carry a shell command, in priority order.
	_COMMAND_KEYS: Tuple[str, ...] = ("command", "cmd", "script")

	_SCOPE_LINE_RE = re.compile(
	r"\n{1,2}" + re.escape(VERIFICATION_SCOPE_PREFIX) + r"[^\n]\s\Z"
	)


	def verification_check_label(tool_name: str, tool_args: Any) -> Optional[str]:
	"""Short label when this call is a verification check, else ``None``.

	``pytest tests/unit -q`` → ``"pytest"``; ``read_file`` → ``None``.
	"""
	name = (tool_name or "").strip()
	if name in _CHECK_TOOLS:
	return name
	if not isinstance(tool_args, dict):
	return None
	for key in _COMMAND_KEYS:
	command = tool_args.get(key)
	if isinstance(command, str) and command.strip():
	match = _CHECK_COMMAND_RE.search(command)
	return " ".join(match.group(0).split()).lower() if match else None
	return None


	def _names(executions: List[Dict[str, Any]], limit: int = 3) -> str:
	"""Deduped, order-preserving, count-capped label list."""
	labels: List[str] = []
	for execution in executions:
	label = execution.get("check_label")
	if label and label not in labels:
	labels.append(label)
	if not labels:
	return "a check"
	shown = ", ".join(labels[:limit])
	extra = len(labels) - limit
	return f"{shown} +{extra} more" if extra > 0 else shown


	def build_verification_scope(executions: List[Dict[str, Any]]) -> str:
	"""One bounded line naming what ran, what passed, and what went unchecked.

	Three distinguishable states: ``verified`` (checks ran and every one
	passed), ``partially verified`` (checks ran, not all passed), and
	``unverified`` (no check ran at all).

	Each execution is ``{"tool": str, "check_label": str \| None,
	"failed": bool}`` — see ``Agent._note_verification_signal``.
	"""
	executions = list(executions or [])
	checks = [e for e in executions if e.get("check_label")]
	if not checks:
	total = len(executions)
	if total == 0:
	body = "unverified — no tools ran, so nothing was checked."
	else:
	plural = "" if total == 1 else "s"
	body = (
	f"unverified — {total} tool call{plural} ran, none of them a "
	"test, lint, or build."
	)
	else:
	passed = [e for e in checks if not e.get("failed")]
	failed = [e for e in checks if e.get("failed")]
	if not failed:
	body = f"verified — {_names(passed)} ran and passed."
	elif not passed:
	body = (
	f"partially verified — {_names(failed)} ran and did not pass; "
	"nothing else was checked."
	)
	else:
	body = (
	f"partially verified — {_names(passed)} passed, "
	f"{_names(failed)} did not."
	)
	statement = VERIFICATION_SCOPE_PREFIX + body
	if len(statement) > VERIFICATION_SCOPE_MAX_CHARS:
	statement = statement[: VERIFICATION_SCOPE_MAX_CHARS - 1].rstrip() + "..."
	return statement


	def strip_verification_scope(text: str) -> str:
	"""Remove a trailing verification-scope line added by the agent loop."""
	if not isinstance(text, str) or VERIFICATION_SCOPE_PREFIX not in text:
	return text
	return _SCOPE_LINE_RE.sub("", text)

11 changes: 11 additions & 0 deletions src/gaia/ui/sse_handler.py

Show comments View file Open in desktop

Original file line number	Diff line number	Diff line change
Expand Up		@@ -25,6 +25,7 @@
		from gaia.agents.base.tool_grants import grant_scope
		from gaia.agents.base.tools import get_tool_display_label, get_tool_metadata
		from gaia.agents.base.turn_metrics import turn_log_path
	from gaia.agents.base.verification import strip_verification_scope
		from gaia.ui.event_narration import DEBUG_CHANNEL, format_count

		logger = logging.getLogger(__name__)
Expand Down Expand Up		@@ -608,6 +609,14 @@ def print_final_answer(
		ttft_seconds: Optional[float] = None,
		):
		if answer:
	scope_line = ""
	# Set aside the verification-scope line before the cleaners run: an
	# answer they strip to nothing (a card-echo) must stay empty, not
	# arrive as a message consisting only of the scope line (#3376).
	cleaned_of_scope = strip_verification_scope(answer)
	if cleaned_of_scope != answer:
	scope_line = answer[len(cleaned_of_scope) :].strip()
	answer = cleaned_of_scope
		answer = _THINK_TAG_SUB_RE.sub("", answer)
		# Extract answer text from {"thought":..., "answer":...} JSON before
		# the regex cleaners run. _THOUGHT_JSON_SUB_RE would otherwise strip
Expand All		@@ -620,6 +629,8 @@ def print_final_answer(
		answer = _TOOL_CALL_JSON_SUB_RE.sub("", answer)
		answer = _THOUGHT_JSON_SUB_RE.sub("", answer)
		answer = answer.strip()
	if answer and scope_line:
	answer = f"{answer}\n\n{scope_line}"
		event: Dict[str, Any] = {
		"type": "answer",
		"content": _fix_double_escaped(answer) if answer else answer,
Expand Down

Navigation Menu

Uh oh!

feat(agent): state whether an answer was verified, on every exit path #3397

Are you sure you want to change the base?

Uh oh!

feat(agent): state whether an answer was verified, on every exit path #3397

Filter by extension

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!