From 23573da87b85b19ca723d4f2c5d76924bfdf45c8 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Sat, 5 Sep 2026 06:45:47 -0700 Subject: [PATCH 1/2] feat(agent): materialize a project map at task start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flagship opened every task blind — it guessed directory names, guessed which commands existed, and guessed which shell it was talking to, and each wrong guess cost a full round trip to learn something one orientation pass establishes once. It now gets a 600-token block in the system prompt naming the project root, the directory shape two levels deep, the likely entry points, which commands are installed (and which are not), the three platform differences that change command syntax, and whether the semantic code index is built. The binary half extends the existing day-0 system-context probe rather than adding a second one: `probe_binaries` is now the single PATH probe, shared by `collect_system_info` and the map, and `DEV_TOOL_PROBES` widens the seven desktop-app markers to the developer toolchain. `index_codebase` gains its first automatic trigger. When the root satisfies `is_code_repository` — a VCS directory or a recognised manifest, non-recursive — and no index exists, the map starts one in a background thread and says so in the prompt. `GAIA_PROJECT_MAP_AUTO_INDEX=0` turns it off. Budget is 600 tokens, 1.8% of the 32K NPU window, enforced by `render_project_map` on every render and pinned by a test against a 200-directory repository. The map is cached per root and invalidated by a fingerprint over the top-level listing, manifest contents, VCS head and PATH. --- hub/agents/gaia/python/gaia_agent/agent.py | 34 +- src/gaia/agents/base/agent.py | 15 + src/gaia/agents/base/project_map.py | 696 +++++++++++++++++++++ src/gaia/agents/base/system_context.py | 117 +++- src/gaia/agents/base/turn_metrics.py | 5 + tests/unit/test_project_map.py | 393 ++++++++++++ 6 files changed, 1234 insertions(+), 26 deletions(-) create mode 100644 src/gaia/agents/base/project_map.py create mode 100644 tests/unit/test_project_map.py diff --git a/hub/agents/gaia/python/gaia_agent/agent.py b/hub/agents/gaia/python/gaia_agent/agent.py index 111d8bdd4..694620c5c 100644 --- a/hub/agents/gaia/python/gaia_agent/agent.py +++ b/hub/agents/gaia/python/gaia_agent/agent.py @@ -49,6 +49,7 @@ from gaia_agent_chat.agent import ChatAgent, ChatAgentConfig +from gaia.agents.base.project_map import ProjectMapMixin, resolve_project_root from gaia.agents.base.skill_discovery import ( DISCOVERY_THRESHOLD_ENV, SkillDiscovery, @@ -204,11 +205,24 @@ class GaiaAgentConfig(ChatAgentConfig): default_factory=lambda: [str(Path.home())] ) + # The project this task is about (#3379). ``None`` resolves through + # ``GAIA_PROJECT_ROOT``, then the working directory or the nearest + # repository above it — and stays ``None`` when neither is a repository, + # which is the common case for a sidecar launched from its package dir. + project_root: Optional[str] = None -# Base agent first, tool mixins after — the repo's MRO convention for every -# hub agent. Neither mixin overrides anything today; this order keeps a future -# mixin method from silently winning over ChatAgent's. -class GaiaAgent(ChatAgent, SkillLibraryToolsMixin, CodeIndexToolsMixin): + # Build the semantic code index at task start when the project is a + # repository and has none. Off-switch: ``GAIA_PROJECT_MAP_AUTO_INDEX=0``, + # for a monorepo where a full embed pass is not worth it. + auto_index: bool = True + + +# ``ProjectMapMixin`` is the one exception to "base agent first": it overrides +# ``_on_task_start`` and calls ``super()``, and ``Agent``'s no-op default sits +# ahead of every trailing mixin in the MRO — listed after ChatAgent it would +# never run. Tool mixins keep their usual place at the back, where neither +# overrides anything and a future method cannot silently win over ChatAgent's. +class GaiaAgent(ProjectMapMixin, ChatAgent, SkillLibraryToolsMixin, CodeIndexToolsMixin): """The flagship GAIA agent — conversation, documents, data, web, and skills.""" SKILL_DIRS: ClassVar[List[str]] = _bundled_skill_roots() @@ -264,12 +278,14 @@ def _register_tools(self) -> None: self.skill_loader = self._maybe_build_skill_loader() self._skill_discovery = self._maybe_build_skill_discovery() self.register_skill_library_tools() - # Same scope as allowed_paths, for the same reason that field rejects - # cwd: the daemon launches this sidecar with cwd = the package - # directory, so cwd would sandbox code search to the agent's own - # source tree — and index it by default. + # The project map's root when there is one, so "is the index built?" + # and "index it" both mean the repository the task is about. Falling + # back to allowed_paths for the same reason that field rejects cwd: the + # daemon launches this sidecar with cwd = the package directory, so cwd + # would sandbox code search to the agent's own source tree. allowed = getattr(self.config, "allowed_paths", None) or [str(Path.home())] - self._init_code_index_state(repo_path=allowed[0]) + index_root = resolve_project_root(self.config.project_root) or allowed[0] + self._init_code_index_state(repo_path=index_root) self.register_code_index_tools() super()._register_tools() diff --git a/src/gaia/agents/base/agent.py b/src/gaia/agents/base/agent.py index 70d717087..628268613 100644 --- a/src/gaia/agents/base/agent.py +++ b/src/gaia/agents/base/agent.py @@ -1248,6 +1248,17 @@ def _select_tools_for_turn( # pylint: disable=unused-argument """ return None + def _on_task_start( # pylint: disable=unused-argument + self, user_input: str + ) -> None: + """Hook run at the top of every turn, before the prompt is composed. + + Default: no-op. Mixins that orient the agent in its environment — the + project map (#3379) is the first — override this and must call + ``super()._on_task_start(user_input)``. Anything expensive belongs + behind a once-per-session guard inside the override, not here. + """ + def _on_tool_invoked(self, tool_name: str) -> None: """Hook called when a tool is about to execute (after registry lookup). @@ -4486,6 +4497,10 @@ def _process_query_impl( self._current_query = user_input self._single_tool_done = False + # Orientation. Runs before the prompt is composed so anything it + # establishes is in the prompt on the turn that established it. + self._on_task_start(user_input) + # Proactive skill discovery: a skill the user never named can become # loaded here, registering its tools — so it must run BEFORE the tool # filter, or those tools are invisible on the very turn that loaded the diff --git a/src/gaia/agents/base/project_map.py b/src/gaia/agents/base/project_map.py new file mode 100644 index 000000000..b9bdba813 --- /dev/null +++ b/src/gaia/agents/base/project_map.py @@ -0,0 +1,696 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Task-start orientation: a project map injected into the system prompt. + +The agent otherwise opens every task blind — it guesses directory names, guesses +which commands exist, and guesses which shell it is talking to. Each wrong guess +costs a full round trip to learn something one orientation pass establishes once. + +What the map carries, and why each part is there: + +* **Directory shape and likely entry points** — the genuinely new half. Nothing + else in GAIA states them without the model choosing to call ``tree``. +* **Which binaries are present** — from :func:`gaia.agents.base.system_context.probe_binaries`, + the same probe that backs day-0 memory, widened to the developer toolchain and + to the shell tool's own allowlist. Absent commands are named too: "do not run + ``cargo``" saves the round trip that "command not found" would have cost. +* **Three platform quirks** — path separator, quoting for spaces, shell dialect. + Exactly three, enumerated in :class:`PlatformQuirks`. + +Budget: :data:`PROJECT_MAP_TOKEN_BUDGET` tokens, enforced by +:func:`render_project_map` on every render. The figure is sized against the +smaller of the two device profiles (``NPU_CTX_SIZE`` = 32,768), not the 64K one. + +Caching: :func:`build_project_map` is memoised per root and invalidated by a +filesystem fingerprint (see :func:`_fingerprint`), so it is rebuilt when the +project changes rather than on every query. +""" + +from __future__ import annotations + +import json +import os +import platform +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple + +from gaia.agents.base.system_context import DEV_TOOL_PROBES, probe_binaries +from gaia.agents.base.turn_metrics import count_tokens +from gaia.logger import get_logger + +logger = get_logger(__name__) + +# ── budget ──────────────────────────────────────────────────────────────── + +#: Hard ceiling on the rendered map, in estimated tokens. +#: +#: 600 tokens is 1.8% of the NPU profile's 32,768-token window (``NPU_CTX_SIZE``) +#: and 0.9% of the GPU profile's 65,536. Sized against the NPU deliberately: the +#: smaller window is the one that has to survive the addition, and a budget that +#: only holds on 64K is not a budget. +PROJECT_MAP_TOKEN_BUDGET = 600 + +#: Per-section sub-caps, as a fraction of the total budget. Without these the +#: directory listing — the one unbounded section — eats the whole allowance and +#: the platform quirks fall off the end. +_DIR_SHAPE_SHARE = 0.40 +_COMMANDS_SHARE = 0.30 + +# ── the "code repository" predicate ─────────────────────────────────────── + +#: A version-control directory at the root. First half of the predicate. +VCS_DIRS: Tuple[str, ...] = (".git", ".hg", ".svn") + +#: A recognised build/package manifest at the root. Second half of the predicate. +PROJECT_MANIFESTS: Tuple[str, ...] = ( + "pyproject.toml", + "setup.py", + "setup.cfg", + "requirements.txt", + "package.json", + "Cargo.toml", + "go.mod", + "pom.xml", + "build.gradle", + "build.gradle.kts", + "CMakeLists.txt", + "Makefile", + "Gemfile", + "composer.json", +) + + +def is_code_repository(root: os.PathLike | str) -> bool: + """Is *root* a code repository? + + The predicate, stated so it can be checked rather than judged: **a directory + from** :data:`VCS_DIRS` **is present at the root, or a file from** + :data:`PROJECT_MANIFESTS` **is present at the root.** Non-recursive, so a + home directory that happens to contain repositories is not itself one. + """ + path = Path(root) + try: + if not path.is_dir(): + return False + return any((path / d).is_dir() for d in VCS_DIRS) or any( + (path / f).is_file() for f in PROJECT_MANIFESTS + ) + except OSError as e: + logger.debug("is_code_repository(%s) could not stat the path: %s", root, e) + return False + + +# ── platform quirks: exactly three, enumerated ──────────────────────────── + + +@dataclass(frozen=True) +class PlatformQuirks: + """The closed set of platform differences that change command syntax. + + Three, and only three. Each is something the model gets wrong by default on + the other platform, and each changes the text of a command it emits. + """ + + #: 1. Separator between path components — ``\\`` or ``/``. + path_separator: str + #: 2. How to quote a path containing spaces. + path_quoting: str + #: 3. Which shell interprets the command line. + shell_dialect: str + + +def detect_platform_quirks() -> PlatformQuirks: + """Resolve the three quirks for the running host.""" + if platform.system() == "Windows": + comspec = os.environ.get("COMSPEC", "") + dialect = "PowerShell" if "powershell" in comspec.lower() else "cmd.exe" + return PlatformQuirks( + path_separator="\\", + path_quoting='wrap in double quotes: "C:\\Program Files\\app"', + shell_dialect=dialect, + ) + shell = os.environ.get("SHELL", "") + return PlatformQuirks( + path_separator="/", + path_quoting="wrap in single quotes: '/home/me/My Docs'", + shell_dialect=Path(shell).name if shell else "sh", + ) + + +# ── directory shape and entry points ────────────────────────────────────── + +#: Directories never worth a slot in the map — build output, caches, vendored +#: dependencies. Closed list. +IGNORED_DIRS: frozenset = frozenset( + { + ".git", + ".hg", + ".svn", + ".idea", + ".vscode", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".tox", + ".venv", + ".next", + ".gaia", + "__pycache__", + "node_modules", + "venv", + "env", + "dist", + "build", + "target", + "coverage", + "htmlcov", + "site-packages", + "vendor", + } +) + +#: Files that are, when present, a likely way to start or drive the project. +#: Closed list — checked as literal relative paths, no globbing. +ENTRY_POINT_CANDIDATES: Tuple[str, ...] = ( + "main.py", + "__main__.py", + "app.py", + "cli.py", + "run.py", + "manage.py", + "src/main.py", + "src/cli.py", + "src/app.py", + "index.js", + "index.ts", + "server.js", + "src/index.js", + "src/index.ts", + "main.go", + "cmd/main.go", + "src/main.rs", + "Makefile", + "Dockerfile", + "docker-compose.yml", +) + +_MAX_TOP_LEVEL_DIRS = 24 +_MAX_SUBDIRS_PER_DIR = 8 +_MAX_EXPANDED_DIRS = 6 + +#: Only a few directories can afford a second level within the budget. These +#: names get it first — alphabetical order would spend the whole allowance on +#: ``cpp/``, ``data/``, ``docs/`` and never reach ``src/``. +_EXPAND_FIRST: Tuple[str, ...] = ( + "src", + "lib", + "app", + "apps", + "packages", + "pkg", + "cmd", + "internal", + "hub", + "tests", + "test", +) + + +@dataclass +class ProjectMap: + """One project's shape, as of the fingerprint it was built at.""" + + root: str + is_repository: bool + vcs: Optional[str] + manifests: List[str] = field(default_factory=list) + top_level_dirs: List[str] = field(default_factory=list) + subdirs: Dict[str, List[str]] = field(default_factory=dict) + entry_points: List[str] = field(default_factory=list) + commands_present: List[str] = field(default_factory=list) + commands_absent: List[str] = field(default_factory=list) + quirks: PlatformQuirks = field(default_factory=detect_platform_quirks) + fingerprint: str = "" + + +def _shell_allowlisted_commands() -> Tuple[str, ...]: + """Commands ``run_shell_command`` will accept, as a sorted tuple. + + Imported lazily: the shell tool module is heavy and an agent without it + should still get a project map. + """ + from gaia.agents.tools.shell_tools import ALLOWED_COMMANDS + + return tuple(sorted(ALLOWED_COMMANDS)) + + +def _fingerprint(root: Path) -> str: + """Cheap change detector for *root*. + + Covers the three things that make a map stale: the top-level listing (a new + directory appears), any manifest's content (dependencies or scripts change), + and the VCS head (a branch switch rewrites the tree). Reading the whole tree + to detect a change would cost as much as rebuilding the map. + """ + parts: List[str] = [] + try: + entries = sorted(e.name for e in os.scandir(root)) + except OSError as e: + logger.debug("fingerprint scandir(%s) failed: %s", root, e) + entries = [] + parts.append(",".join(entries)) + + for name in PROJECT_MANIFESTS: + p = root / name + try: + st = p.stat() + parts.append(f"{name}:{st.st_mtime_ns}:{st.st_size}") + except OSError: + continue + + for vcs in VCS_DIRS: + head = root / vcs / "HEAD" + try: + parts.append(f"{vcs}:{head.stat().st_mtime_ns}") + except OSError: + continue + + parts.append(os.environ.get("PATH", "")) + return "|".join(parts) + + +#: ``absolute root -> (fingerprint, map)``. Module-level so a process that +#: builds several agents over the same project pays for the walk once. +_MAP_CACHE: Dict[str, Tuple[str, ProjectMap]] = {} + + +def clear_project_map_cache() -> None: + """Drop every cached map. For tests and for an explicit re-orientation.""" + _MAP_CACHE.clear() + + +def build_project_map(root: os.PathLike | str) -> ProjectMap: + """Build (or return the cached) map for *root*. + + Cached per absolute root and invalidated when :func:`_fingerprint` changes, + so repeated queries against an unchanged project reuse one walk. + """ + path = Path(root).expanduser().resolve() + key = str(path) + fp = _fingerprint(path) + + cached = _MAP_CACHE.get(key) + if cached is not None and cached[0] == fp: + return cached[1] + + pm = _collect(path, fp) + _MAP_CACHE[key] = (fp, pm) + logger.debug( + "[project-map] built for %s (%d top-level dirs, %d entry points)", + key, + len(pm.top_level_dirs), + len(pm.entry_points), + ) + return pm + + +def _collect(path: Path, fingerprint: str) -> ProjectMap: + """Walk *path* two levels deep and probe the toolchain.""" + vcs = next((d for d in VCS_DIRS if (path / d).is_dir()), None) + manifests = [f for f in PROJECT_MANIFESTS if (path / f).is_file()] + + top_dirs: List[str] = [] + try: + for entry in sorted(os.scandir(path), key=lambda e: e.name): + if len(top_dirs)>= _MAX_TOP_LEVEL_DIRS: + break + if not entry.is_dir(follow_symlinks=False): + continue + if entry.name in IGNORED_DIRS or entry.name.startswith("."): + continue + top_dirs.append(entry.name) + except OSError as e: + logger.debug("[project-map] cannot list %s: %s", path, e) + + expand_order = sorted( + top_dirs, + key=lambda n: ( + _EXPAND_FIRST.index(n) if n in _EXPAND_FIRST else len(_EXPAND_FIRST), + n, + ), + ) + subdirs: Dict[str, List[str]] = {} + for name in expand_order[:_MAX_EXPANDED_DIRS]: + children: List[str] = [] + try: + for entry in sorted(os.scandir(path / name), key=lambda e: e.name): + if len(children)>= _MAX_SUBDIRS_PER_DIR: + break + if not entry.is_dir(follow_symlinks=False): + continue + if entry.name in IGNORED_DIRS or entry.name.startswith("."): + continue + children.append(entry.name) + except OSError: + continue + if children: + subdirs[name] = children + + entry_points = [c for c in ENTRY_POINT_CANDIDATES if (path / c).is_file()] + entry_points.extend(_declared_npm_scripts(path)) + + probed = probe_binaries(set(DEV_TOOL_PROBES) | set(_shell_allowlisted_commands())) + present = sorted(n for n, ok in probed.items() if ok) + # Only toolchain absences are reported: that list is closed and bounded, + # whereas "every allowlisted command not installed" is mostly platform noise + # (a Windows box is never going to have ``lspci``). + absent = sorted(n for n in DEV_TOOL_PROBES if not probed.get(n)) + + return ProjectMap( + root=str(path), + is_repository=bool(vcs) or bool(manifests), + vcs=vcs, + manifests=manifests, + top_level_dirs=top_dirs, + subdirs=subdirs, + entry_points=entry_points, + commands_present=present, + commands_absent=absent, + quirks=detect_platform_quirks(), + fingerprint=fingerprint, + ) + + +def _declared_npm_scripts(path: Path) -> List[str]: + """``npm run `` targets declared in ``package.json``, capped at 8.""" + manifest = path / "package.json" + if not manifest.is_file(): + return [] + try: + data = json.loads(manifest.read_text(encoding="utf-8", errors="replace")) + except (OSError, ValueError) as e: + logger.warning("[project-map] %s is not readable JSON: %s", manifest, e) + return [] + scripts = data.get("scripts") + if not isinstance(scripts, dict): + return [] + return [f"npm run {name}" for name in sorted(scripts)[:8]] + + +# ── project-root resolution ─────────────────────────────────────────────── + +#: Overrides the resolved root. Set by a host that knows the workspace. +PROJECT_ROOT_ENV = "GAIA_PROJECT_ROOT" + +#: How far up from the working directory to look for a repository root. +_MAX_ASCEND = 4 + + +def resolve_project_root(explicit: Optional[str] = None) -> Optional[str]: + """The project this task is about, or ``None`` when there isn't one. + + Order: *explicit* argument, then ``GAIA_PROJECT_ROOT``, then the working + directory or the nearest repository above it (at most :data:`_MAX_ASCEND` + levels, never the home directory itself). + + ``None`` is a real answer, not a degraded one — an agent answering questions + from a home directory is not in a project, and inventing a map of ``~`` + would cost tokens to describe nothing. + """ + for candidate in (explicit, os.environ.get(PROJECT_ROOT_ENV)): + if not candidate: + continue + path = Path(candidate).expanduser() + if not path.is_dir(): + raise ValueError( + f"Project root {candidate!r} is not a directory. " + f"Point {PROJECT_ROOT_ENV} (or the agent's project_root config) " + f"at an existing directory, or unset it to use the working " + f"directory." + ) + return str(path.resolve()) + + try: + cwd = Path.cwd().resolve() + home = Path.home().resolve() + except OSError as e: + logger.debug("[project-map] cannot resolve the working directory: %s", e) + return None + + for ancestor in [cwd, *list(cwd.parents)[: _MAX_ASCEND - 1]]: + if ancestor == home or ancestor == ancestor.parent: + break + if is_code_repository(ancestor): + return str(ancestor) + return None + + +# ── rendering, under budget ─────────────────────────────────────────────── + + +def _fit(text: str, token_cap: int) -> str: + """Trim *text* to *token_cap* estimated tokens, whole lines first.""" + if count_tokens(text) <= token_cap: + return text + lines = text.splitlines() + while lines and count_tokens("\n".join(lines))> token_cap: + lines.pop() + return "\n".join(lines) + + +def render_project_map( + pm: ProjectMap, + index_status: Optional[str] = None, + token_budget: int = PROJECT_MAP_TOKEN_BUDGET, +) -> str: + """Render *pm* as a system-prompt block of at most *token_budget* tokens. + + Sections are emitted in priority order and a section that would overflow + stops the render, so what survives truncation is always the highest-value + text rather than whatever happened to come first. + """ + q = pm.quirks + header = [ + "==== PROJECT MAP ====", + f"Root: {pm.root}", + ] + if pm.vcs or pm.manifests: + detail = ", ".join(filter(None, [pm.vcs, ", ".join(pm.manifests[:4])])) + header.append(f"Code repository: yes ({detail})") + else: + header.append("Code repository: no (no VCS directory, no known manifest)") + + quirks = [ + "Platform (these three change the commands you write):", + f"- Path separator: {q.path_separator}", + f"- Paths with spaces: {q.path_quoting}", + f"- Shell dialect: {q.shell_dialect}", + ] + + shape: List[str] = [] + if pm.top_level_dirs: + shape.append("Directories:") + for name in pm.top_level_dirs: + children = pm.subdirs.get(name) + suffix = f" ({', '.join(children)})" if children else "" + shape.append(f"- {name}/{suffix}") + + entries: List[str] = [] + if pm.entry_points: + entries.append(f"Entry points: {', '.join(pm.entry_points)}") + + # Absences first: they are the line that prevents a wasted round trip, so + # they must survive the sub-cap even when the present-list does not. + commands: List[str] = [] + if pm.commands_absent: + commands.append( + f"NOT installed, do not invoke: {', '.join(pm.commands_absent)}" + ) + if pm.commands_present: + commands.append( + "Installed and accepted by run_shell_command: " + f"{', '.join(pm.commands_present)}" + ) + + index: List[str] = [f"Code index: {index_status}"] if index_status else [] + + # The header always ships — a map that says nothing but "you are in + # /x/y, it is a git repo" is still worth more than an empty block — so it + # is clamped to the budget rather than dropped by it. + head = _fit("\n".join(header), token_budget) + out: List[str] = [head] + used = count_tokens(head) + + optional: Sequence[str] = ( + "\n".join(quirks), + _fit("\n".join(shape), int(token_budget * _DIR_SHAPE_SHARE)), + "\n".join(entries), + _fit("\n".join(commands), int(token_budget * _COMMANDS_SHARE)), + "\n".join(index), + ) + for text in optional: + if not text: + continue + cost = count_tokens(text) + 1 # +1 for the blank-line join + if used + cost> token_budget: + logger.debug( + "[project-map] budget of %d tokens reached, dropping the tail", + token_budget, + ) + break + out.append(text) + used += cost + + return "\n\n".join(out) + + +# ── agent mixin ─────────────────────────────────────────────────────────── + +#: Truthy/falsy override for the task-start ``index_codebase`` trigger. +AUTO_INDEX_ENV = "GAIA_PROJECT_MAP_AUTO_INDEX" + + +def auto_index_env_override() -> Optional[bool]: + """``GAIA_PROJECT_MAP_AUTO_INDEX`` as a bool, or ``None`` when unset.""" + raw = os.environ.get(AUTO_INDEX_ENV) + if raw is None or not raw.strip(): + return None + return raw.strip().lower() in ("1", "true", "yes", "on") + + +class ProjectMapMixin: + """Injects a project map into the system prompt and triggers indexing. + + Consumer responsibilities: + + * Compose the mixin on an agent that also has :class:`CodeIndexToolsMixin` + if the ``index_codebase`` trigger is wanted; without it the map still + renders, minus the index line. + * Optionally give the config a ``project_root`` field; otherwise the root + comes from ``GAIA_PROJECT_ROOT`` or the working directory. + """ + + def _project_map_root(self) -> Optional[str]: + explicit = getattr(getattr(self, "config", None), "project_root", None) + return resolve_project_root(explicit) + + def materialize_project_map(self) -> Optional[ProjectMap]: + """This task's map, or ``None`` when the task is not in a project.""" + root = self._project_map_root() + if root is None: + return None + return build_project_map(root) + + def get_project_map_system_prompt(self) -> str: + """Auto-discovered by ``Agent._get_mixin_prompts``.""" + pm = self.materialize_project_map() + if pm is None: + return "" + return render_project_map(pm, index_status=self._code_index_status(pm)) + + # ── code index ──────────────────────────────────────────────────────── + + def _code_index_status(self, pm: ProjectMap) -> Optional[str]: + """One line on the semantic index, or ``None`` when it does not apply.""" + if not pm.is_repository: + return None + indexed = self._code_index_is_built() + if indexed is None: + return None + if indexed: + return "built — use search_code_index before grepping" + if getattr(self, "_project_map_index_started", False): + return "building now in the background; grep until it lands" + return "not built — call index_codebase to enable semantic code search" + + def _code_index_is_built(self) -> Optional[bool]: + """``True``/``False``, or ``None`` when this agent has no code index. + + Reads the code index at *the agent's* configured repo path, and the + trigger below indexes that same path — so the two can never disagree + about which tree they are talking about. A consumer that wants the + index scoped to the project map's root points + ``_init_code_index_state`` at :func:`resolve_project_root`, which is + what ``GaiaAgent`` does. + """ + getter = getattr(self, "_get_code_index_sdk", None) + if getter is None: + return None + sdk = getter() + if sdk is None: + return None + return bool(sdk.get_status().get("indexed")) + + def _auto_index_enabled(self) -> bool: + override = auto_index_env_override() + if override is not None: + return override + return bool(getattr(getattr(self, "config", None), "auto_index", True)) + + def _on_task_start(self, user_input: str) -> None: + """Materialize the map and, if warranted, kick off ``index_codebase``.""" + super()._on_task_start(user_input) + pm = self.materialize_project_map() + if pm is None: + return + self._maybe_start_background_index(pm) + + def _maybe_start_background_index(self, pm: ProjectMap) -> None: + """Start ``index_codebase`` in a background thread, at most once.""" + if getattr(self, "_project_map_index_started", False): + return + if not pm.is_repository or not self._auto_index_enabled(): + return + if self._code_index_is_built() is not False: + return + index_tool = (getattr(self, "_tools_registry", {}) or {}).get("index_codebase") + if index_tool is None: + return + + self._project_map_index_started = True + import threading + + def _run() -> None: + logger.info("[project-map] indexing %s in the background", pm.root) + try: + # No repo_path: the tool's default is the agent's configured + # code-index root, the same one the status above was read from. + index_tool["function"]() + except Exception as e: + # Background work has no caller to raise into; a swallowed + # failure here would show up only as search_code_index + # returning nothing, which is unexplainable from the outside. + logger.error( + "[project-map] background index of %s failed: %s. " + "Call index_codebase directly to see the full error.", + pm.root, + e, + ) + else: + logger.info("[project-map] background index of %s done", pm.root) + + threading.Thread( + target=_run, name="gaia-project-map-index", daemon=True + ).start() + + +__all__ = [ + "AUTO_INDEX_ENV", + "ENTRY_POINT_CANDIDATES", + "IGNORED_DIRS", + "PROJECT_MANIFESTS", + "PROJECT_MAP_TOKEN_BUDGET", + "PROJECT_ROOT_ENV", + "PlatformQuirks", + "ProjectMap", + "ProjectMapMixin", + "VCS_DIRS", + "auto_index_env_override", + "build_project_map", + "clear_project_map_cache", + "detect_platform_quirks", + "is_code_repository", + "render_project_map", + "resolve_project_root", +] diff --git a/src/gaia/agents/base/system_context.py b/src/gaia/agents/base/system_context.py index bcf2933a4..620402f2b 100644 --- a/src/gaia/agents/base/system_context.py +++ b/src/gaia/agents/base/system_context.py @@ -23,7 +23,85 @@ import time from datetime import datetime, timezone from pathlib import Path -from typing import Dict, List +from typing import Dict, Iterable, List, Tuple + +#: CLI tools whose presence also implies a desktop application is installed. +#: Feeds the "Installed applications" fact; labels match the per-platform +#: probes above it so the two never emit the same app twice. +CLI_TOOL_PROBES: Dict[str, str] = { + "git": "git", + "code": "VS Code", + "cursor": "Cursor", + "node": "Node.js", + "docker": "Docker", + "brew": "Homebrew", + "npm": "npm", +} + +#: Developer toolchain binaries — the build, package and VCS commands an agent +#: reaches for while working inside a project. Closed list: a name absent here +#: is simply never reported, and adding one is a one-line change. +#: +#: This is the *extension* over ``CLI_TOOL_PROBES``, which only covers the seven +#: commands that double as desktop-app markers. Knowing ``uv`` is present but +#: ``cargo`` is not is what stops an agent spending a round trip on +#: "command not found". +DEV_TOOL_PROBES: Tuple[str, ...] = ( + "bash", + "cargo", + "cmake", + "curl", + "docker", + "dotnet", + "gcc", + "gh", + "git", + "go", + "gradle", + "java", + "make", + "mvn", + "node", + "npm", + "npx", + "pip", + "pnpm", + "poetry", + "powershell", + "python", + "python3", + "rg", + "ruby", + "rustc", + "tsc", + "uv", + "yarn", +) + +#: ``(PATH value, binary name) -> present``. ``shutil.which`` walks the whole +#: PATH per call, and the project map probes ~80 names; keying on PATH means a +#: shell that prepends a venv still invalidates the answer. +_BINARY_CACHE: Dict[Tuple[str, str], bool] = {} + + +def probe_binaries(names: Iterable[str]) -> Dict[str, bool]: + """Presence on PATH of each binary in *names*. + + The single binary probe in the codebase — ``collect_system_info`` and the + project map both go through it, so "is X installed" has one answer and one + cache rather than drifting per caller. + """ + path_env = os.environ.get("PATH", "") + result: Dict[str, bool] = {} + for name in names: + key = (path_env, name) + if key not in _BINARY_CACHE: + try: + _BINARY_CACHE[key] = shutil.which(name) is not None + except Exception: + _BINARY_CACHE[key] = False + result[name] = _BINARY_CACHE[key] + return result def collect_system_info() -> List[Dict[str, str]]: @@ -342,23 +420,12 @@ def collect_system_info() -> List[Dict[str, str]]: except Exception: pass - # Cross-platform: shutil.which() for CLI tools. + # Cross-platform: PATH probe for CLI tools. # Use the same display names as the platform checks to avoid duplicates. - _cli_tools: dict = { - "git": "git", - "code": "VS Code", - "cursor": "Cursor", - "node": "Node.js", - "docker": "Docker", - "brew": "Homebrew", - "npm": "npm", - } - for cmd, label in _cli_tools.items(): - try: - if shutil.which(cmd) and label not in found_apps: - found_apps.append(label) - except Exception: - pass + present = probe_binaries(CLI_TOOL_PROBES) + for cmd, label in CLI_TOOL_PROBES.items(): + if present.get(cmd) and label not in found_apps: + found_apps.append(label) if found_apps: facts.append( @@ -370,6 +437,22 @@ def collect_system_info() -> List[Dict[str, str]]: except Exception: pass + # 11b. Developer toolchain — same probe, wider list. Build/package/VCS + # commands are what an agent working inside a project actually invokes. + try: + dev_present = [n for n, ok in probe_binaries(DEV_TOOL_PROBES).items() if ok] + if dev_present: + facts.append( + { + "content": ( + f"Developer tools on PATH: {', '.join(sorted(dev_present))}" + ), + "domain": "system:software", + } + ) + except Exception: + pass + # 12. Collection date — regenerated on every refresh, so it reflects the # most recent collection, not a first-ever capture. try: diff --git a/src/gaia/agents/base/turn_metrics.py b/src/gaia/agents/base/turn_metrics.py index b60ba1974..cf86bf6e8 100644 --- a/src/gaia/agents/base/turn_metrics.py +++ b/src/gaia/agents/base/turn_metrics.py @@ -102,6 +102,11 @@ def _tokenizer() -> _Tokenizer: return _TOKENIZER +def count_tokens(text: str) -> int: + """Estimated token count for *text*, via this module's shared estimator.""" + return _tokenizer().count(text) + + def _common_prefix_len(a: str, b: str) -> int: """Length of the longest shared leading substring of *a* and *b*. diff --git a/tests/unit/test_project_map.py b/tests/unit/test_project_map.py new file mode 100644 index 000000000..d69f4337c --- /dev/null +++ b/tests/unit/test_project_map.py @@ -0,0 +1,393 @@ +# Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +# SPDX-License-Identifier: MIT +"""Task-start project map (#3379). + +Pins the four things the feature is only useful if it guarantees: the +"code repository" predicate, the token budget on the 32K NPU profile, cache +invalidation on change, and the once-per-session ``index_codebase`` trigger. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from gaia.agents.base.project_map import ( + PROJECT_MANIFESTS, + PROJECT_MAP_TOKEN_BUDGET, + PROJECT_ROOT_ENV, + PlatformQuirks, + ProjectMapMixin, + build_project_map, + clear_project_map_cache, + detect_platform_quirks, + is_code_repository, + render_project_map, + resolve_project_root, +) +from gaia.agents.base.system_context import ( + CLI_TOOL_PROBES, + DEV_TOOL_PROBES, + probe_binaries, +) +from gaia.agents.base.turn_metrics import count_tokens +from gaia.llm.lemonade_client import NPU_CTX_SIZE + + +@pytest.fixture(autouse=True) +def _clean_cache(): + clear_project_map_cache() + yield + clear_project_map_cache() + + +@pytest.fixture +def repo(tmp_path): + """A minimal but realistic project: VCS dir, manifest, source tree.""" + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "HEAD").write_text("ref: refs/heads/main\n") + (tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n") + (tmp_path / "src").mkdir() + (tmp_path / "src" / "demo").mkdir() + (tmp_path / "src" / "cli.py").write_text("print('hi')\n") + (tmp_path / "tests").mkdir() + (tmp_path / "node_modules").mkdir() + (tmp_path / "Makefile").write_text("all:\n\techo hi\n") + return tmp_path + + +# ── the predicate ───────────────────────────────────────────────────────── + + +def test_predicate_true_for_vcs_directory(tmp_path): + (tmp_path / ".git").mkdir() + assert is_code_repository(tmp_path) + + +@pytest.mark.parametrize("manifest", PROJECT_MANIFESTS) +def test_predicate_true_for_every_declared_manifest(tmp_path, manifest): + (tmp_path / manifest).write_text("") + assert is_code_repository(tmp_path) + + +def test_predicate_false_without_vcs_or_manifest(tmp_path): + (tmp_path / "notes.txt").write_text("hello") + (tmp_path / "photos").mkdir() + assert not is_code_repository(tmp_path) + + +def test_predicate_is_not_recursive(tmp_path): + """A directory of repositories is not itself one.""" + (tmp_path / "proj" / ".git").mkdir(parents=True) + assert not is_code_repository(tmp_path) + + +def test_predicate_false_for_a_file(tmp_path): + f = tmp_path / "pyproject.toml" + f.write_text("") + assert not is_code_repository(f) + + +# ── platform quirks: exactly three, all populated ───────────────────────── + + +def test_platform_quirks_are_a_closed_set_of_three(): + fields = PlatformQuirks.__dataclass_fields__ + assert set(fields) == {"path_separator", "path_quoting", "shell_dialect"} + + +def test_platform_quirks_are_all_populated(): + q = detect_platform_quirks() + assert q.path_separator == os.sep + assert q.path_quoting and q.shell_dialect + + +def test_rendered_map_names_all_three_quirks(repo): + text = render_project_map(build_project_map(repo)) + for label in ("Path separator", "Paths with spaces", "Shell dialect"): + assert label in text + + +# ── shape and entry points ──────────────────────────────────────────────── + + +def test_map_records_directory_shape_and_skips_noise(repo): + pm = build_project_map(repo) + assert "src" in pm.top_level_dirs + assert "tests" in pm.top_level_dirs + assert "node_modules" not in pm.top_level_dirs + assert ".git" not in pm.top_level_dirs + assert pm.subdirs["src"] == ["demo"] + + +def test_map_records_entry_points_from_the_closed_list(repo): + pm = build_project_map(repo) + assert "src/cli.py" in pm.entry_points + assert "Makefile" in pm.entry_points + + +def test_npm_scripts_become_entry_points(tmp_path): + (tmp_path / "package.json").write_text( + json.dumps({"scripts": {"build": "vite build", "dev": "vite"}}) + ) + pm = build_project_map(tmp_path) + assert "npm run build" in pm.entry_points + assert "npm run dev" in pm.entry_points + + +def test_malformed_package_json_is_reported_not_swallowed(tmp_path, caplog): + (tmp_path / "package.json").write_text("{ not json") + with caplog.at_level("WARNING"): + pm = build_project_map(tmp_path) + assert pm.entry_points == [] + assert any("not readable JSON" in r.message for r in caplog.records) + + +# ── binaries: one probe, extended ───────────────────────────────────────── + + +def test_dev_probe_extends_rather_than_duplicates_the_app_probe(): + """The extension is a superset in kind, not a second mechanism.""" + assert set(CLI_TOOL_PROBES) - set(DEV_TOOL_PROBES) <= {"code", "cursor", "brew"} + assert len(DEV_TOOL_PROBES)> len(CLI_TOOL_PROBES) + + +def test_probe_binaries_agrees_with_which(): + import shutil + + probed = probe_binaries(["python", "definitely-not-a-real-binary-xyz"]) + assert probed["python"] == (shutil.which("python") is not None) + assert probed["definitely-not-a-real-binary-xyz"] is False + + +def test_map_names_absent_commands_so_the_agent_does_not_try_them(repo, monkeypatch): + monkeypatch.setenv("PATH", "") + clear_project_map_cache() + pm = build_project_map(repo) + assert pm.commands_present == [] + assert set(pm.commands_absent) == set(DEV_TOOL_PROBES) + assert "NOT installed" in render_project_map(pm) + + +# ── the budget, on the 32K profile ──────────────────────────────────────── + + +def test_budget_is_a_small_fraction_of_the_npu_window(): + assert PROJECT_MAP_TOKEN_BUDGET == 600 + assert PROJECT_MAP_TOKEN_BUDGET < NPU_CTX_SIZE * 0.02 + + +def test_render_stays_within_budget_on_a_realistic_repo(repo): + assert count_tokens(render_project_map(build_project_map(repo))) <= ( + PROJECT_MAP_TOKEN_BUDGET + ) + + +def test_render_stays_within_budget_on_a_pathological_repo(tmp_path): + """200 top-level directories, each with 20 children, must still fit.""" + (tmp_path / ".git").mkdir() + for i in range(200): + d = tmp_path / f"package_with_a_long_name_{i:03d}" + d.mkdir() + for j in range(20): + (d / f"submodule_{j:02d}").mkdir() + pm = build_project_map(tmp_path) + assert count_tokens(render_project_map(pm)) <= PROJECT_MAP_TOKEN_BUDGET + + +def test_high_priority_sections_survive_truncation(tmp_path): + """Quirks outrank the directory listing when the budget bites.""" + (tmp_path / ".git").mkdir() + for i in range(200): + (tmp_path / f"dir_{i:03d}").mkdir() + text = render_project_map(build_project_map(tmp_path)) + assert "Shell dialect" in text + assert "PROJECT MAP" in text + + +def test_budget_is_honoured_when_the_caller_lowers_it(repo): + text = render_project_map(build_project_map(repo), token_budget=40) + assert count_tokens(text) <= 40 + assert "PROJECT MAP" in text + + +# ── caching ─────────────────────────────────────────────────────────────── + + +def test_map_is_cached_between_calls(repo): + assert build_project_map(repo) is build_project_map(repo) + + +def test_cache_invalidates_when_a_directory_appears(repo): + first = build_project_map(repo) + (repo / "docs").mkdir() + second = build_project_map(repo) + assert second is not first + assert "docs" in second.top_level_dirs + + +def test_cache_invalidates_when_a_manifest_changes(repo): + first = build_project_map(repo) + (repo / "pyproject.toml").write_text("[project]\nname='demo'\nversion='2'\n") + assert build_project_map(repo) is not first + + +def test_cache_invalidates_when_path_changes(repo, monkeypatch): + first = build_project_map(repo) + monkeypatch.setenv("PATH", "/some/other/place") + assert build_project_map(repo) is not first + + +# ── root resolution ─────────────────────────────────────────────────────── + + +def test_explicit_root_wins(repo, monkeypatch): + monkeypatch.setenv(PROJECT_ROOT_ENV, str(repo.parent)) + assert resolve_project_root(str(repo)) == str(repo.resolve()) + + +def test_env_root_is_used_when_no_explicit_root(repo, monkeypatch): + monkeypatch.setenv(PROJECT_ROOT_ENV, str(repo)) + assert resolve_project_root() == str(repo.resolve()) + + +def test_a_nonexistent_root_fails_loudly(tmp_path): + missing = tmp_path / "nope" + with pytest.raises(ValueError, match="not a directory"): + resolve_project_root(str(missing)) + + +def test_cwd_resolves_when_it_is_a_repository(repo, monkeypatch): + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + monkeypatch.chdir(repo) + assert resolve_project_root() == str(repo.resolve()) + + +def test_a_subdirectory_resolves_to_the_repository_above_it(repo, monkeypatch): + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + monkeypatch.chdir(repo / "src" / "demo") + assert resolve_project_root() == str(repo.resolve()) + + +def test_no_root_outside_a_repository(tmp_path, monkeypatch): + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + plain = tmp_path / "just" / "files" + plain.mkdir(parents=True) + monkeypatch.chdir(plain) + assert resolve_project_root() is None + + +# ── the mixin: prompt injection and the index trigger ───────────────────── + + +class _FakeSDK: + def __init__(self, indexed: bool): + self._indexed = indexed + + def get_status(self): + return {"indexed": self._indexed} + + +class _Base: + """Stands in for ``Agent``: a no-op ``_on_task_start`` that ends the chain.""" + + def _on_task_start(self, user_input: str) -> None: + pass + + +class _FakeAgent(ProjectMapMixin, _Base): + def __init__(self, root, indexed=False, auto_index=True): + self.config = type( + "C", (), {"project_root": str(root), "auto_index": auto_index} + )() + self._indexed = indexed + self.index_calls = [] + self._tools_registry = { + "index_codebase": {"function": lambda **kw: self.index_calls.append(kw)} + } + + def _get_code_index_sdk(self): + return _FakeSDK(self._indexed) + + +def _join(agent): + """Run the background index thread to completion.""" + import threading + + for t in threading.enumerate(): + if t.name == "gaia-project-map-index": + t.join(timeout=10) + return agent.index_calls + + +def test_prompt_fragment_is_the_rendered_map(repo): + text = _FakeAgent(repo).get_project_map_system_prompt() + assert text.startswith("==== PROJECT MAP ====") + assert str(repo.resolve()) in text + assert count_tokens(text) <= PROJECT_MAP_TOKEN_BUDGET + + +def test_prompt_fragment_is_empty_outside_a_project(tmp_path, monkeypatch): + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + plain = tmp_path / "docs-only" + plain.mkdir() + monkeypatch.chdir(plain) + agent = _FakeAgent(plain) + agent.config.project_root = None + assert agent.get_project_map_system_prompt() == "" + + +def test_index_trigger_fires_for_an_unindexed_repository(repo): + agent = _FakeAgent(repo, indexed=False) + agent._on_task_start("do a thing") + assert _join(agent) == [{}] + assert "building now in the background" in agent.get_project_map_system_prompt() + + +def test_index_trigger_is_skipped_when_already_indexed(repo): + agent = _FakeAgent(repo, indexed=True) + agent._on_task_start("do a thing") + assert _join(agent) == [] + assert "search_code_index" in agent.get_project_map_system_prompt() + + +def test_index_trigger_fires_at_most_once(repo): + agent = _FakeAgent(repo, indexed=False) + for _ in range(3): + agent._on_task_start("again") + assert len(_join(agent)) == 1 + + +def test_index_trigger_respects_the_config_off_switch(repo): + agent = _FakeAgent(repo, indexed=False, auto_index=False) + agent._on_task_start("do a thing") + assert _join(agent) == [] + + +def test_index_trigger_respects_the_env_off_switch(repo, monkeypatch): + monkeypatch.setenv("GAIA_PROJECT_MAP_AUTO_INDEX", "0") + agent = _FakeAgent(repo, indexed=False) + agent._on_task_start("do a thing") + assert _join(agent) == [] + + +def test_index_trigger_skipped_for_a_non_repository(tmp_path): + plain = tmp_path / "plain" + plain.mkdir() + agent = _FakeAgent(plain, indexed=False) + agent._on_task_start("do a thing") + assert _join(agent) == [] + + +def test_background_index_failure_is_logged_not_swallowed(repo, caplog): + def _boom(**_kw): + raise RuntimeError("faiss exploded") + + agent = _FakeAgent(repo, indexed=False) + agent._tools_registry["index_codebase"]["function"] = _boom + with caplog.at_level("ERROR"): + agent._on_task_start("do a thing") + _join(agent) + assert any("faiss exploded" in r.getMessage() for r in caplog.records) From 76d78497802df01952138d55fb08402133e5af26 Mon Sep 17 00:00:00 2001 From: Ovtcharov Date: Sat, 5 Sep 2026 08:11:36 -0700 Subject: [PATCH 2/2] fix(agent): make the project map cheap, honest and unable to point at itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the task-start project map, each a bug a user would have hit: - The index line parsed the whole code-index metadata — every chunk's text — on every prompt composition, several times a turn. `CodeIndexSDK.is_indexed` answers the same question with two `exists()` calls. - A background index that died left the prompt saying "building now" for the rest of the session, so the model kept waiting for something that would never arrive. The trigger is now a four-state machine and inspects the tool's JSON, which is how `index_codebase` reports a refusal rather than raising. - The map told the model `run_shell_command` accepts `uv`, `npm` and `python`. It accepts none of them. Installed-and-allowlisted, installed-but-refused, and not-installed are now three separate lines — and the whole section is omitted for an agent that has no shell tool. - In dev mode the daemon launches the agent sidecar from the GAIA checkout, so the working directory resolved to GAIA's own source and it would have background-indexed itself. `is_agent_own_source` rejects that; an explicit `GAIA_PROJECT_ROOT` is exempt. - Listing `ProjectMapMixin` after the base agent silently disabled the index trigger while the prompt still rendered. `__init_subclass__` now raises. Also: the root resolves once per session so the map and the code index cannot describe two different trees, and the day-0 memory fact that duplicated `git`/`node`/`docker` (and carried an `except Exception: pass`) is gone — `DEV_TOOL_PROBES` feeds the map, which is all #3379 asked for. --- docs/docs.json | 3 +- docs/guides/code-index.mdx | 6 + docs/guides/project-map.mdx | 153 +++++++++++ hub/agents/gaia/npm/CHANGELOG.md | 8 + hub/agents/gaia/npm/SKILL.md | 29 ++- hub/agents/gaia/npm/SPEC.md | 9 + hub/agents/gaia/python/gaia_agent/agent.py | 6 +- src/gaia/agents/base/agent.py | 3 + src/gaia/agents/base/project_map.py | 281 ++++++++++++++------- src/gaia/agents/base/system_context.py | 41 +-- src/gaia/code_index/sdk.py | 12 + tests/unit/test_project_map.py | 177 ++++++++++++- 12 files changed, 592 insertions(+), 136 deletions(-) create mode 100644 docs/guides/project-map.mdx diff --git a/docs/docs.json b/docs/docs.json index 482b852dd..594bce9b1 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -93,7 +93,8 @@ { "group": "Developer Tools", "pages": [ - "guides/code-index" + "guides/code-index", + "guides/project-map" ] }, { diff --git a/docs/guides/code-index.mdx b/docs/guides/code-index.mdx index b360c5b9e..5212d7c6d 100644 --- a/docs/guides/code-index.mdx +++ b/docs/guides/code-index.mdx @@ -66,6 +66,12 @@ The [`coding` skill](https://github.com/amd/gaia/blob/main/hub/skills/coding/SKI | `get_index_status` | Report current index state | | `clear_code_index` | Remove the cached index | +You usually will not have to wait for that first `index_codebase` call. The +[project map](/guides/project-map) starts one in the background at task start +whenever the agent is in a repository that has no index, so by the time a +semantic search matters the index is often already there. Set +`GAIA_PROJECT_MAP_AUTO_INDEX=0` to keep indexing entirely manual. + ### Example interaction ``` diff --git a/docs/guides/project-map.mdx b/docs/guides/project-map.mdx new file mode 100644 index 000000000..4abc9047a --- /dev/null +++ b/docs/guides/project-map.mdx @@ -0,0 +1,153 @@ +--- +title: "Project Map" +description: "The orientation pass the agent runs at task start, so it stops guessing paths and commands." +icon: "map" +--- + + + **Source Code:** [`src/gaia/agents/base/project_map.py`](https://github.com/amd/gaia/blob/main/src/gaia/agents/base/project_map.py) + + +Without a project map the agent opens every task blind. It guesses directory +names, guesses which programs are installed, and guesses which shell it is +talking to — and every wrong guess costs a full round trip to learn something +one orientation pass establishes once. + +The [GAIA agent](/guides/gaia) now runs that pass at the start of every task and +puts the result in its system prompt. + +## What it contains + +```text +==== PROJECT MAP ==== +Root: C:\Users\me\Work\gaia +Code repository: yes (pyproject.toml, setup.py, package.json) + +Platform (these three change the commands you write): +- Path separator: \ +- Paths with spaces: wrap in double quotes: "C:\Program Files\app" +- Shell dialect: cmd.exe + +Directories: +- docs/ (assets, deployment, guides, integrations) +- hub/ (agents, components, skills) +- src/ (gaia, vscode) +- tests/ (electron, fixtures, integration, mcp, unit) + +Entry points: Makefile, src/cli.py, npm run build + +NOT installed, do not invoke: cargo, gradle, java, mvn, rustc +run_shell_command accepts: cat, date, diff, find, git, grep, head, ls, ... +Installed but run_shell_command refuses them — use a tool, not the shell: +cmake, curl, docker, go, node, npm, pip, python, uv + +Code index: not built — call index_codebase to enable semantic code search +``` + +| Section | Where it comes from | +|---------|---------------------| +| Root, repository status | The `is_code_repository` predicate below | +| Platform | `detect_platform_quirks()` — exactly three fields, see [Platform quirks](#platform-quirks) | +| Directories | Two levels deep, build output and vendored dependencies skipped | +| Entry points | A closed list of well-known files, plus `npm run` targets from `package.json` | +| Commands | `probe_binaries()` — the same PATH probe that backs [day-0 memory](/guides/memory), crossed with the shell tool's own allowlist | +| Code index | [`CodeIndexSDK.is_indexed()`](/guides/code-index) — a presence check, so it is cheap enough to run on every render | + +## The token budget + +**600 tokens.** That is 1.8% of the NPU profile's 32,768-token window +(`NPU_CTX_SIZE`) and 0.9% of the GPU profile's 65,536. It is sized against the +smaller window on purpose — a budget that only holds on 64K is not a budget. + +Measured with GAIA's shared estimator (`count_tokens` — cl100k, or a character +ratio when tiktoken is absent), so it bounds the map to within that estimator's +error of the model's own count rather than to the exact token. + +`render_project_map()` enforces it on every render. Sections are emitted in +priority order and the directory listing carries its own sub-cap, so on a +2,000-directory monorepo the platform quirks and the command list still survive +while the tree is what gets trimmed. + +## Which directories count as a code repository + +The predicate is testable, not a judgement call. **`is_code_repository(path)` is +true when a version-control directory or a recognised manifest sits at the +root** — non-recursive, so a home directory full of repositories is not itself +one. + + + + `.git`, `.hg`, `.svn` + + + `pyproject.toml`, `setup.py`, `setup.cfg`, `requirements.txt`, + `package.json`, `Cargo.toml`, `go.mod`, `pom.xml`, `build.gradle`, + `build.gradle.kts`, `CMakeLists.txt`, `Makefile`, `Gemfile`, + `composer.json` + + + +## Platform quirks + +Three, and only three — the differences that change the *text* of a command the +model emits. A contributor can check the list off: + +1. **Path separator** — `\` on Windows, `/` elsewhere. +2. **Path quoting for spaces** — double quotes on Windows, single quotes on POSIX. +3. **Shell dialect** — `cmd.exe` or PowerShell on Windows; `$SHELL`'s basename otherwise. + +## Automatic code indexing + +When the root is a code repository and no [code index](/guides/code-index) +exists, the map calls `index_codebase` in a background thread and says so in the +prompt, so the model knows to grep until it lands. It fires at most once per +session. + +Turn it off on a monorepo where a full embedding pass is not worth it: + +```bash +export GAIA_PROJECT_MAP_AUTO_INDEX=0 +``` + +## Choosing the root + +Resolution order: + +1. `project_root` on `GaiaAgentConfig` +2. the `GAIA_PROJECT_ROOT` environment variable +3. the working directory, or the nearest repository up to four levels above it + +A working directory that resolves to **GAIA's own source tree** is rejected — in +dev mode the agent sidecar is launched from the GAIA checkout, and mapping (and +auto-indexing) its own source is never what the user asked for. An explicitly +configured root is exempt: pointing GAIA at GAIA is fine when you mean it. + +If none of those is a repository the agent gets **no map at all**. That is the +right answer, not a degraded one — an agent answering questions from a home +directory is not in a project, and a map of `~` would spend tokens describing +nothing. + +```bash +export GAIA_PROJECT_ROOT=/home/me/Work/my-service +``` + +A `GAIA_PROJECT_ROOT` that does not exist raises at startup rather than being +quietly ignored. + +## Caching + +The map is built once per root and reused. It is rebuilt when a fingerprint over +four things changes: the top-level directory listing, every manifest's size and +mtime, the VCS head, and `PATH`. Between changes, repeated queries reuse one +walk. + +## Related + + + + The semantic index the map triggers. + + + Day-0 system facts, from the same binary probe. + + diff --git a/hub/agents/gaia/npm/CHANGELOG.md b/hub/agents/gaia/npm/CHANGELOG.md index a1c23bfb1..cef79adc7 100644 --- a/hub/agents/gaia/npm/CHANGELOG.md +++ b/hub/agents/gaia/npm/CHANGELOG.md @@ -14,6 +14,14 @@ the terminal UI meant building it from source. ### Added +- **A project map at task start.** In a code repository the agent now opens + every task knowing the directory shape, the likely entry points, which + commands are installed, and the three platform differences that change + command syntax — instead of discovering each one through a failed tool call. + Capped at 600 prompt tokens. If the repository has no code index the map + starts one in the background; `GAIA_PROJECT_MAP_AUTO_INDEX=0` turns that off, + and `GAIA_PROJECT_ROOT` picks the project when the working directory is not + it. See SKILL §11. - **`503` from `/query` at session capacity.** When every retained session slot is busy and none is idle enough to evict, starting a new session returns `503` with the reason in `detail` — retryable, distinct from a diff --git a/hub/agents/gaia/npm/SKILL.md b/hub/agents/gaia/npm/SKILL.md index 472b1e2c6..2780ac497 100644 --- a/hub/agents/gaia/npm/SKILL.md +++ b/hub/agents/gaia/npm/SKILL.md @@ -413,7 +413,32 @@ packaged sidecar (its CLI accepts only `--host` and `--port`), and an undeclared name raises naming the valid sets rather than falling back to a default. Beyond `gaia-voice`, do not design around a skill being on by default. -## 11. Ports +## 11. The project map — two things it costs you + +When the agent's working directory is a code repository, every task starts with +a **project map** in the system prompt: the root, the directory shape, the +likely entry points, which commands are installed, and the three platform +differences that change command syntax. It exists so the agent stops burning +round trips on "no such file" and "command not found". + +Two consequences an integrator needs to plan for: + +- **Up to 600 prompt tokens, every turn.** That is the enforced ceiling + (1.8% of the NPU profile's 32K window), not a typical value — budget it + alongside `gaia-voice`'s 676. +- **A background embedding pass on first contact with a new repository.** If + the repo has no [code index](https://amd-gaia.ai/docs/guides/code-index), the + map starts one in a background thread so semantic search is ready when it is + needed. On a large monorepo that is minutes of local embedding. + `GAIA_PROJECT_MAP_AUTO_INDEX=0` turns it off. + +The sidecar's CLI accepts only `--host` and `--port`, so pointing the map at a +specific project means `GAIA_PROJECT_ROOT=/path/to/repo` in its environment, or +`GaiaAgentConfig(project_root=...)` when embedding. A directory that is neither +a VCS checkout nor holds a recognised manifest gets **no map** — that is the +designed answer, not a failure. + +## 12. Ports | Service | Port | |---|---| @@ -425,7 +450,7 @@ Port **4001 is reserved repo-wide**: `spawnSidecar` throws a `RangeError` and speaks for the user's documents and memory and has no business on a LAN interface. -## 12. Running in a server or long-lived app +## 13. Running in a server or long-lived app - **`fetchAll` / `fetchBinary` are a build step**, not per request — network plus a full SHA-256 hash of a large artifact. Run once at install time. diff --git a/hub/agents/gaia/npm/SPEC.md b/hub/agents/gaia/npm/SPEC.md index c10f2fe22..97c858808 100644 --- a/hub/agents/gaia/npm/SPEC.md +++ b/hub/agents/gaia/npm/SPEC.md @@ -289,6 +289,15 @@ collapses the rest to a one-line menu entry (re-activated by calling disables it for the session and every body renders. Omitting it is a valid, explicit one-shot: nothing persists past that single turn, and the agent is not told otherwise. +A retained session also carries a **project map** — up to 600 prompt tokens of +directory shape, entry points, installed commands and platform quirks, present +whenever the agent's working directory resolves to a code repository (a VCS +directory or a recognised manifest at its root). `GAIA_PROJECT_ROOT=` +picks the project when the working directory is not it. If that repository has +no code index, the first turn starts one in a background thread; +`GAIA_PROJECT_MAP_AUTO_INDEX=0` disables that. Neither affects the wire +contract — they change what the agent knows and what the first turn costs. + A second `/query` for a `session_id` that already has a turn in flight gets `409 Conflict` — cancel the running turn or wait for it, then retry. A `/query` that needs a **new** session while every retained slot is busy and diff --git a/hub/agents/gaia/python/gaia_agent/agent.py b/hub/agents/gaia/python/gaia_agent/agent.py index 694620c5c..21c21b7c6 100644 --- a/hub/agents/gaia/python/gaia_agent/agent.py +++ b/hub/agents/gaia/python/gaia_agent/agent.py @@ -49,7 +49,7 @@ from gaia_agent_chat.agent import ChatAgent, ChatAgentConfig -from gaia.agents.base.project_map import ProjectMapMixin, resolve_project_root +from gaia.agents.base.project_map import ProjectMapMixin from gaia.agents.base.skill_discovery import ( DISCOVERY_THRESHOLD_ENV, SkillDiscovery, @@ -284,7 +284,9 @@ def _register_tools(self) -> None: # daemon launches this sidecar with cwd = the package directory, so cwd # would sandbox code search to the agent's own source tree. allowed = getattr(self.config, "allowed_paths", None) or [str(Path.home())] - index_root = resolve_project_root(self.config.project_root) or allowed[0] + # Through the mixin, so both read the one cached resolution and can + # never end up describing two different trees. + index_root = self._project_map_root() or allowed[0] self._init_code_index_state(repo_path=index_root) self.register_code_index_tools() super()._register_tools() diff --git a/src/gaia/agents/base/agent.py b/src/gaia/agents/base/agent.py index 628268613..30a9cd1e4 100644 --- a/src/gaia/agents/base/agent.py +++ b/src/gaia/agents/base/agent.py @@ -612,6 +612,9 @@ class Agent(abc.ABC): "get_memory_system_prompt", # changes on any remember()/forget() "get_skills_system_prompt", # per-turn body selection (#2848) "get_recalled_skills_system_prompt", # per-turn procedural recall + # Mostly static, but the index line flips as a background index + # lands and the shape line changes if the project does (#3379). + "get_project_map_system_prompt", } ) diff --git a/src/gaia/agents/base/project_map.py b/src/gaia/agents/base/project_map.py index b9bdba813..541389f47 100644 --- a/src/gaia/agents/base/project_map.py +++ b/src/gaia/agents/base/project_map.py @@ -10,9 +10,9 @@ * **Directory shape and likely entry points** — the genuinely new half. Nothing else in GAIA states them without the model choosing to call ``tree``. -* **Which binaries are present** — from :func:`gaia.agents.base.system_context.probe_binaries`, - the same probe that backs day-0 memory, widened to the developer toolchain and - to the shell tool's own allowlist. Absent commands are named too: "do not run +* **Which binaries are present** — from ``system_context.probe_binaries``, the + same probe that backs day-0 memory, widened to the developer toolchain and to + the shell tool's own allowlist. Absent commands are named too: "do not run ``cargo``" saves the round trip that "command not found" would have cost. * **Three platform quirks** — path separator, quoting for spaces, shell dialect. Exactly three, enumerated in :class:`PlatformQuirks`. @@ -33,7 +33,7 @@ import platform from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple from gaia.agents.base.system_context import DEV_TOOL_PROBES, probe_binaries from gaia.agents.base.turn_metrics import count_tokens @@ -43,7 +43,10 @@ # ── budget ──────────────────────────────────────────────────────────────── -#: Hard ceiling on the rendered map, in estimated tokens. +#: Ceiling on the rendered map, measured with the shared estimator in +#: :mod:`gaia.agents.base.turn_metrics` (cl100k, or a char ratio when tiktoken +#: is absent) — so it bounds the map to within that estimator's error of the +#: model's own count, not to the exact token. #: #: 600 tokens is 1.8% of the NPU profile's 32,768-token window (``NPU_CTX_SIZE``) #: and 0.9% of the GPU profile's 65,536. Sized against the NPU deliberately: the @@ -228,8 +231,11 @@ class ProjectMap: top_level_dirs: List[str] = field(default_factory=list) subdirs: Dict[str, List[str]] = field(default_factory=dict) entry_points: List[str] = field(default_factory=list) - commands_present: List[str] = field(default_factory=list) - commands_absent: List[str] = field(default_factory=list) + #: Toolchain binaries found on PATH, and those looked for and not found. + tools_present: List[str] = field(default_factory=list) + tools_absent: List[str] = field(default_factory=list) + #: The subset of the above that ``run_shell_command`` will actually accept. + shell_commands: List[str] = field(default_factory=list) quirks: PlatformQuirks = field(default_factory=detect_platform_quirks) fingerprint: str = "" @@ -237,8 +243,8 @@ class ProjectMap: def _shell_allowlisted_commands() -> Tuple[str, ...]: """Commands ``run_shell_command`` will accept, as a sorted tuple. - Imported lazily: the shell tool module is heavy and an agent without it - should still get a project map. + Imported lazily to keep ``agents/base`` free of an import-time dependency + on ``agents/tools``. """ from gaia.agents.tools.shell_tools import ALLOWED_COMMANDS @@ -253,13 +259,7 @@ def _fingerprint(root: Path) -> str: and the VCS head (a branch switch rewrites the tree). Reading the whole tree to detect a change would cost as much as rebuilding the map. """ - parts: List[str] = [] - try: - entries = sorted(e.name for e in os.scandir(root)) - except OSError as e: - logger.debug("fingerprint scandir(%s) failed: %s", root, e) - entries = [] - parts.append(",".join(entries)) + parts: List[str] = [",".join(sorted(e.name for e in os.scandir(root)))] for name in PROJECT_MANIFESTS: p = root / name @@ -280,8 +280,8 @@ def _fingerprint(root: Path) -> str: return "|".join(parts) -#: ``absolute root -> (fingerprint, map)``. Module-level so a process that -#: builds several agents over the same project pays for the walk once. +#: ``absolute root -> (fingerprint, map)``. Module-level so several agents over +#: the same project in one process pay for the walk once. _MAP_CACHE: Dict[str, Tuple[str, ProjectMap]] = {} @@ -321,17 +321,14 @@ def _collect(path: Path, fingerprint: str) -> ProjectMap: manifests = [f for f in PROJECT_MANIFESTS if (path / f).is_file()] top_dirs: List[str] = [] - try: - for entry in sorted(os.scandir(path), key=lambda e: e.name): - if len(top_dirs)>= _MAX_TOP_LEVEL_DIRS: - break - if not entry.is_dir(follow_symlinks=False): - continue - if entry.name in IGNORED_DIRS or entry.name.startswith("."): - continue - top_dirs.append(entry.name) - except OSError as e: - logger.debug("[project-map] cannot list %s: %s", path, e) + for entry in sorted(os.scandir(path), key=lambda e: e.name): + if len(top_dirs)>= _MAX_TOP_LEVEL_DIRS: + break + if not entry.is_dir(follow_symlinks=False): + continue + if entry.name in IGNORED_DIRS or entry.name.startswith("."): + continue + top_dirs.append(entry.name) expand_order = sorted( top_dirs, @@ -352,7 +349,10 @@ def _collect(path: Path, fingerprint: str) -> ProjectMap: if entry.name in IGNORED_DIRS or entry.name.startswith("."): continue children.append(entry.name) - except OSError: + except OSError as e: + # One unreadable subdirectory renders with no children rather than + # sinking the whole map. + logger.debug("[project-map] cannot list %s: %s", path / name, e) continue if children: subdirs[name] = children @@ -360,12 +360,17 @@ def _collect(path: Path, fingerprint: str) -> ProjectMap: entry_points = [c for c in ENTRY_POINT_CANDIDATES if (path / c).is_file()] entry_points.extend(_declared_npm_scripts(path)) - probed = probe_binaries(set(DEV_TOOL_PROBES) | set(_shell_allowlisted_commands())) - present = sorted(n for n, ok in probed.items() if ok) + allowlist = _shell_allowlisted_commands() + probed = probe_binaries(set(DEV_TOOL_PROBES) | set(allowlist)) + tools_present = sorted(n for n in DEV_TOOL_PROBES if probed.get(n)) # Only toolchain absences are reported: that list is closed and bounded, # whereas "every allowlisted command not installed" is mostly platform noise # (a Windows box is never going to have ``lspci``). - absent = sorted(n for n in DEV_TOOL_PROBES if not probed.get(n)) + tools_absent = sorted(n for n in DEV_TOOL_PROBES if not probed.get(n)) + # Installed AND allowlisted. Kept separate from ``tools_present`` because + # most of the toolchain is not allowlisted — ``uv`` and ``npm`` are on this + # machine and ``run_shell_command`` refuses both. + shell_commands = sorted(n for n in allowlist if probed.get(n)) return ProjectMap( root=str(path), @@ -375,8 +380,9 @@ def _collect(path: Path, fingerprint: str) -> ProjectMap: top_level_dirs=top_dirs, subdirs=subdirs, entry_points=entry_points, - commands_present=present, - commands_absent=absent, + tools_present=tools_present, + tools_absent=tools_absent, + shell_commands=shell_commands, quirks=detect_platform_quirks(), fingerprint=fingerprint, ) @@ -407,12 +413,29 @@ def _declared_npm_scripts(path: Path) -> List[str]: _MAX_ASCEND = 4 +def is_agent_own_source(root: os.PathLike | str) -> bool: + """Does *root* contain the ``gaia`` package this process is running from? + + The daemon launches the agent sidecar with its working directory set to the + GAIA checkout in dev mode, so a working-directory-derived root there is the + agent's own source tree, not the user's project — and auto-indexing it would + embed thousands of files nobody asked about. An explicitly configured root + is never subject to this check: pointing GAIA at GAIA is legitimate when you + mean it. + """ + import gaia + + package = Path(gaia.__file__).resolve().parent + path = Path(root).resolve() + return path == package or path in package.parents + + def resolve_project_root(explicit: Optional[str] = None) -> Optional[str]: """The project this task is about, or ``None`` when there isn't one. Order: *explicit* argument, then ``GAIA_PROJECT_ROOT``, then the working directory or the nearest repository above it (at most :data:`_MAX_ASCEND` - levels, never the home directory itself). + levels, never the home directory and never :func:`is_agent_own_source`). ``None`` is a real answer, not a degraded one — an agent answering questions from a home directory is not in a project, and inventing a map of ``~`` @@ -431,18 +454,22 @@ def resolve_project_root(explicit: Optional[str] = None) -> Optional[str]: ) return str(path.resolve()) - try: - cwd = Path.cwd().resolve() - home = Path.home().resolve() - except OSError as e: - logger.debug("[project-map] cannot resolve the working directory: %s", e) - return None - + cwd = Path.cwd().resolve() + home = Path.home().resolve() for ancestor in [cwd, *list(cwd.parents)[: _MAX_ASCEND - 1]]: if ancestor == home or ancestor == ancestor.parent: break - if is_code_repository(ancestor): - return str(ancestor) + if not is_code_repository(ancestor): + continue + if is_agent_own_source(ancestor): + logger.info( + "[project-map] %s is GAIA's own source tree — no map. Set %s to " + "the project you want mapped.", + ancestor, + PROJECT_ROOT_ENV, + ) + return None + return str(ancestor) return None @@ -463,12 +490,16 @@ def render_project_map( pm: ProjectMap, index_status: Optional[str] = None, token_budget: int = PROJECT_MAP_TOKEN_BUDGET, + has_shell_tool: bool = True, ) -> str: """Render *pm* as a system-prompt block of at most *token_budget* tokens. Sections are emitted in priority order and a section that would overflow stops the render, so what survives truncation is always the highest-value text rather than whatever happened to come first. + + Pass ``has_shell_tool=False`` for an agent without ``run_shell_command`` — + naming a tool it does not have buys a guaranteed failed call. """ q = pm.quirks header = [ @@ -501,17 +532,24 @@ def render_project_map( entries.append(f"Entry points: {', '.join(pm.entry_points)}") # Absences first: they are the line that prevents a wasted round trip, so - # they must survive the sub-cap even when the present-list does not. + # they must survive the sub-cap even when the longer lists do not. commands: List[str] = [] - if pm.commands_absent: - commands.append( - f"NOT installed, do not invoke: {', '.join(pm.commands_absent)}" - ) - if pm.commands_present: - commands.append( - "Installed and accepted by run_shell_command: " - f"{', '.join(pm.commands_present)}" - ) + if pm.tools_absent: + commands.append(f"NOT installed, do not invoke: {', '.join(pm.tools_absent)}") + if has_shell_tool: + allowed = set(pm.shell_commands) + if pm.shell_commands: + commands.append( + f"run_shell_command accepts: {', '.join(pm.shell_commands)}" + ) + off_limits = [t for t in pm.tools_present if t not in allowed] + if off_limits: + commands.append( + "Installed but run_shell_command refuses them — use a tool, not " + f"the shell: {', '.join(off_limits)}" + ) + elif pm.tools_present: + commands.append(f"Installed: {', '.join(pm.tools_present)}") index: List[str] = [f"Code index: {index_status}"] if index_status else [] @@ -559,35 +597,76 @@ def auto_index_env_override() -> Optional[bool]: return raw.strip().lower() in ("1", "true", "yes", "on") +#: States of the task-start index trigger. All three post-``idle`` states are +#: terminal for the session: the trigger fires at most once, and ``failed`` +#: exists so an index that died does not leave the prompt saying "building". +_IDLE, _RUNNING, _DONE, _FAILED = "idle", "running", "done", "failed" + + class ProjectMapMixin: """Injects a project map into the system prompt and triggers indexing. Consumer responsibilities: - * Compose the mixin on an agent that also has :class:`CodeIndexToolsMixin` - if the ``index_codebase`` trigger is wanted; without it the map still - renders, minus the index line. + * List this mixin **before** the base agent in the bases — ``Agent``'s no-op + ``_on_task_start`` otherwise wins the MRO and the trigger never fires. + ``__init_subclass__`` raises if you get it wrong. + * Compose :class:`CodeIndexToolsMixin` too if the ``index_codebase`` trigger + is wanted; without it the map still renders, minus the index line. * Optionally give the config a ``project_root`` field; otherwise the root comes from ``GAIA_PROJECT_ROOT`` or the working directory. """ + def __init_subclass__(cls, **kwargs) -> None: + """Fail at class definition when the MRO would silence the hook. + + The prompt fragment is found by ``dir()`` and would still render, so a + wrong base order otherwise produces a map with no index trigger and no + symptom at all. + """ + super().__init_subclass__(**kwargs) + from gaia.agents.base.agent import Agent + + mro = cls.__mro__ + if Agent in mro and mro.index(ProjectMapMixin)> mro.index(Agent): + raise TypeError( + f"{cls.__name__} lists ProjectMapMixin after Agent, so " + f"Agent._on_task_start shadows it and the project map never " + f"triggers indexing. Put ProjectMapMixin first in the bases: " + f"class {cls.__name__}(ProjectMapMixin, ...)." + ) + + # ── the map ─────────────────────────────────────────────────────────── + def _project_map_root(self) -> Optional[str]: - explicit = getattr(getattr(self, "config", None), "project_root", None) - return resolve_project_root(explicit) + """This session's project root, resolved once. + + Resolved once rather than per turn so the map and the code index can + never end up describing two different trees after a ``chdir``. + """ + if not hasattr(self, "_project_map_root_cache"): + explicit = getattr(getattr(self, "config", None), "project_root", None) + self._project_map_root_cache = resolve_project_root(explicit) + return self._project_map_root_cache def materialize_project_map(self) -> Optional[ProjectMap]: """This task's map, or ``None`` when the task is not in a project.""" root = self._project_map_root() - if root is None: - return None - return build_project_map(root) + return build_project_map(root) if root else None def get_project_map_system_prompt(self) -> str: """Auto-discovered by ``Agent._get_mixin_prompts``.""" pm = self.materialize_project_map() if pm is None: return "" - return render_project_map(pm, index_status=self._code_index_status(pm)) + return render_project_map( + pm, + index_status=self._code_index_status(pm), + has_shell_tool="run_shell_command" in (self._tool_names()), + ) + + def _tool_names(self) -> Dict[str, Any]: + return getattr(self, "_tools_registry", {}) or {} # ── code index ──────────────────────────────────────────────────────── @@ -600,19 +679,19 @@ def _code_index_status(self, pm: ProjectMap) -> Optional[str]: return None if indexed: return "built — use search_code_index before grepping" - if getattr(self, "_project_map_index_started", False): + state = getattr(self, "_project_map_index_state", _IDLE) + if state == _RUNNING: return "building now in the background; grep until it lands" + if state == _FAILED: + return "build FAILED — grep instead, or call index_codebase to see why" return "not built — call index_codebase to enable semantic code search" def _code_index_is_built(self) -> Optional[bool]: """``True``/``False``, or ``None`` when this agent has no code index. - Reads the code index at *the agent's* configured repo path, and the - trigger below indexes that same path — so the two can never disagree - about which tree they are talking about. A consumer that wants the - index scoped to the project map's root points - ``_init_code_index_state`` at :func:`resolve_project_root`, which is - what ``GaiaAgent`` does. + Reads the index at *the agent's* configured repo path, which the trigger + below also indexes, so the two can never disagree about which tree they + mean. ``GaiaAgent`` points both at :func:`resolve_project_root`. """ getter = getattr(self, "_get_code_index_sdk", None) if getter is None: @@ -620,7 +699,9 @@ def _code_index_is_built(self) -> Optional[bool]: sdk = getter() if sdk is None: return None - return bool(sdk.get_status().get("indexed")) + # Presence check, not get_status(): this runs on every prompt + # composition and get_status parses every indexed chunk. + return bool(sdk.is_indexed()) def _auto_index_enabled(self) -> bool: override = auto_index_env_override() @@ -632,42 +713,51 @@ def _on_task_start(self, user_input: str) -> None: """Materialize the map and, if warranted, kick off ``index_codebase``.""" super()._on_task_start(user_input) pm = self.materialize_project_map() - if pm is None: - return - self._maybe_start_background_index(pm) + if pm is not None: + self._maybe_start_background_index(pm) def _maybe_start_background_index(self, pm: ProjectMap) -> None: """Start ``index_codebase`` in a background thread, at most once.""" - if getattr(self, "_project_map_index_started", False): + if getattr(self, "_project_map_index_state", _IDLE) != _IDLE: return if not pm.is_repository or not self._auto_index_enabled(): return if self._code_index_is_built() is not False: return - index_tool = (getattr(self, "_tools_registry", {}) or {}).get("index_codebase") + index_tool = self._tool_names().get("index_codebase") if index_tool is None: return - self._project_map_index_started = True + self._project_map_index_state = _RUNNING import threading + def _fail(detail: str) -> None: + # Background work has no caller to raise into, and a swallowed + # failure surfaces only as an empty search_code_index later. + self._project_map_index_state = _FAILED + logger.error( + "[project-map] background index of %s failed: %s. " + "Call index_codebase directly to see the full error.", + pm.root, + detail, + ) + def _run() -> None: logger.info("[project-map] indexing %s in the background", pm.root) try: - # No repo_path: the tool's default is the agent's configured - # code-index root, the same one the status above was read from. - index_tool["function"]() + # No repo_path: the tool defaults to the agent's configured + # code-index root, the one the status above was read from. + raw = index_tool["function"]() except Exception as e: - # Background work has no caller to raise into; a swallowed - # failure here would show up only as search_code_index - # returning nothing, which is unexplainable from the outside. - logger.error( - "[project-map] background index of %s failed: %s. " - "Call index_codebase directly to see the full error.", - pm.root, - e, - ) + _fail(str(e)) + return + # The tool reports refusals and internal errors as JSON rather than + # by raising, so "no exception" is not "it worked". + error = _tool_error(raw) + if error: + _fail(error) else: + self._project_map_index_state = _DONE logger.info("[project-map] background index of %s done", pm.root) threading.Thread( @@ -675,6 +765,17 @@ def _run() -> None: ).start() +def _tool_error(raw: Any) -> Optional[str]: + """The ``error`` a code-index tool reported as JSON, if any.""" + if not isinstance(raw, str): + return None + try: + parsed = json.loads(raw) + except ValueError: + return None + return parsed.get("error") if isinstance(parsed, dict) else None + + __all__ = [ "AUTO_INDEX_ENV", "ENTRY_POINT_CANDIDATES", diff --git a/src/gaia/agents/base/system_context.py b/src/gaia/agents/base/system_context.py index 620402f2b..1364e7464 100644 --- a/src/gaia/agents/base/system_context.py +++ b/src/gaia/agents/base/system_context.py @@ -26,8 +26,8 @@ from typing import Dict, Iterable, List, Tuple #: CLI tools whose presence also implies a desktop application is installed. -#: Feeds the "Installed applications" fact; labels match the per-platform -#: probes above it so the two never emit the same app twice. +#: Labels match the per-platform probes in ``collect_system_info`` so the two +#: never emit the same app twice. CLI_TOOL_PROBES: Dict[str, str] = { "git": "git", "code": "VS Code", @@ -38,14 +38,9 @@ "npm": "npm", } -#: Developer toolchain binaries — the build, package and VCS commands an agent -#: reaches for while working inside a project. Closed list: a name absent here -#: is simply never reported, and adding one is a one-line change. -#: -#: This is the *extension* over ``CLI_TOOL_PROBES``, which only covers the seven -#: commands that double as desktop-app markers. Knowing ``uv`` is present but -#: ``cargo`` is not is what stops an agent spending a round trip on -#: "command not found". +#: Developer toolchain binaries, probed by the project map. The extension over +#: ``CLI_TOOL_PROBES``, which covers only the seven commands that double as +#: desktop-app markers. Closed list; adding a name is a one-line change. DEV_TOOL_PROBES: Tuple[str, ...] = ( "bash", "cargo", @@ -78,9 +73,8 @@ "yarn", ) -#: ``(PATH value, binary name) -> present``. ``shutil.which`` walks the whole -#: PATH per call, and the project map probes ~80 names; keying on PATH means a -#: shell that prepends a venv still invalidates the answer. +#: ``(PATH value, binary name) -> present``. Keyed on PATH so a shell that +#: prepends a venv invalidates the answer rather than reusing a stale one. _BINARY_CACHE: Dict[Tuple[str, str], bool] = {} @@ -96,10 +90,7 @@ def probe_binaries(names: Iterable[str]) -> Dict[str, bool]: for name in names: key = (path_env, name) if key not in _BINARY_CACHE: - try: - _BINARY_CACHE[key] = shutil.which(name) is not None - except Exception: - _BINARY_CACHE[key] = False + _BINARY_CACHE[key] = shutil.which(name) is not None result[name] = _BINARY_CACHE[key] return result @@ -437,22 +428,6 @@ def collect_system_info() -> List[Dict[str, str]]: except Exception: pass - # 11b. Developer toolchain — same probe, wider list. Build/package/VCS - # commands are what an agent working inside a project actually invokes. - try: - dev_present = [n for n, ok in probe_binaries(DEV_TOOL_PROBES).items() if ok] - if dev_present: - facts.append( - { - "content": ( - f"Developer tools on PATH: {', '.join(sorted(dev_present))}" - ), - "domain": "system:software", - } - ) - except Exception: - pass - # 12. Collection date — regenerated on every refresh, so it reflects the # most recent collection, not a first-ever capture. try: diff --git a/src/gaia/code_index/sdk.py b/src/gaia/code_index/sdk.py index 9238b3799..14a989a5e 100644 --- a/src/gaia/code_index/sdk.py +++ b/src/gaia/code_index/sdk.py @@ -499,6 +499,18 @@ def search( return results + def is_indexed(self) -> bool: + """Is a usable index on disk for this repository? + + Presence only — it does not read the metadata, and so does not detect a + cache written by an older ``_CACHE_VERSION`` (which ``index_repository`` + rebuilds anyway). Callers that need the version or the chunk counts want + :meth:`get_status`, which parses the whole metadata file: on a large + repo that is tens of megabytes of JSON, far too much to spend on + "is there an index?". + """ + return self._meta_path.exists() and self._index_path.exists() + def get_status(self) -> Dict[str, Any]: """Return index statistics.""" meta = self._load_metadata() diff --git a/tests/unit/test_project_map.py b/tests/unit/test_project_map.py index d69f4337c..3b034b7ba 100644 --- a/tests/unit/test_project_map.py +++ b/tests/unit/test_project_map.py @@ -9,8 +9,12 @@ from __future__ import annotations +import ast import json import os +import pathlib +import re +import threading import pytest @@ -23,6 +27,7 @@ build_project_map, clear_project_map_cache, detect_platform_quirks, + is_agent_own_source, is_code_repository, render_project_map, resolve_project_root, @@ -166,11 +171,27 @@ def test_map_names_absent_commands_so_the_agent_does_not_try_them(repo, monkeypa monkeypatch.setenv("PATH", "") clear_project_map_cache() pm = build_project_map(repo) - assert pm.commands_present == [] - assert set(pm.commands_absent) == set(DEV_TOOL_PROBES) + assert pm.tools_present == [] + assert pm.shell_commands == [] + assert set(pm.tools_absent) == set(DEV_TOOL_PROBES) assert "NOT installed" in render_project_map(pm) +def test_shell_allowlist_is_not_conflated_with_what_is_installed(repo): + """Claiming run_shell_command accepts ``uv`` causes the very refusal + this map exists to prevent.""" + from gaia.agents.tools.shell_tools import ALLOWED_COMMANDS + + pm = build_project_map(repo) + assert set(pm.shell_commands) <= ALLOWED_COMMANDS + off_limits = set(pm.tools_present) - ALLOWED_COMMANDS + accepts = next( + (ln for ln in render_project_map(pm).splitlines() if "accepts:" in ln), "" + ) + named = set(re.findall(r"[\w.-]+", accepts.partition(":")[2])) + assert not (named & off_limits) + + # ── the budget, on the 32K profile ──────────────────────────────────────── @@ -285,8 +306,13 @@ def test_no_root_outside_a_repository(tmp_path, monkeypatch): class _FakeSDK: def __init__(self, indexed: bool): self._indexed = indexed + self.status_calls = 0 + + def is_indexed(self): + return self._indexed def get_status(self): + self.status_calls += 1 return {"indexed": self._indexed} @@ -302,20 +328,19 @@ def __init__(self, root, indexed=False, auto_index=True): self.config = type( "C", (), {"project_root": str(root), "auto_index": auto_index} )() - self._indexed = indexed + self.sdk = _FakeSDK(indexed) self.index_calls = [] self._tools_registry = { - "index_codebase": {"function": lambda **kw: self.index_calls.append(kw)} + "index_codebase": {"function": lambda **kw: self.index_calls.append(kw)}, + "run_shell_command": {"function": lambda **kw: None}, } def _get_code_index_sdk(self): - return _FakeSDK(self._indexed) + return self.sdk def _join(agent): """Run the background index thread to completion.""" - import threading - for t in threading.enumerate(): if t.name == "gaia-project-map-index": t.join(timeout=10) @@ -343,7 +368,53 @@ def test_index_trigger_fires_for_an_unindexed_repository(repo): agent = _FakeAgent(repo, indexed=False) agent._on_task_start("do a thing") assert _join(agent) == [{}] - assert "building now in the background" in agent.get_project_map_system_prompt() + + +def test_prompt_says_building_while_the_index_is_running(repo): + gate = threading.Event() + agent = _FakeAgent(repo, indexed=False) + agent._tools_registry["index_codebase"]["function"] = lambda **kw: gate.wait(10) + agent._on_task_start("do a thing") + try: + text = agent.get_project_map_system_prompt() + finally: + gate.set() + _join(agent) + assert "building now in the background" in text + + +def test_prompt_reports_a_failed_index_instead_of_waiting_forever(repo): + """A dead background index must not read as "still building" all session.""" + + def _boom(**_kw): + raise RuntimeError("faiss exploded") + + agent = _FakeAgent(repo, indexed=False) + agent._tools_registry["index_codebase"]["function"] = _boom + agent._on_task_start("do a thing") + _join(agent) + assert "build FAILED" in agent.get_project_map_system_prompt() + + +def test_a_json_error_from_the_tool_counts_as_a_failure(repo, caplog): + """``index_codebase`` reports refusals as JSON, not by raising.""" + agent = _FakeAgent(repo, indexed=False) + agent._tools_registry["index_codebase"]["function"] = lambda **kw: json.dumps( + {"error": "refused: home directory"} + ) + with caplog.at_level("ERROR"): + agent._on_task_start("do a thing") + _join(agent) + assert "build FAILED" in agent.get_project_map_system_prompt() + assert any("refused: home directory" in r.getMessage() for r in caplog.records) + + +def test_index_status_is_a_presence_check_not_a_metadata_parse(repo): + """``get_status`` parses every indexed chunk; the prompt renders every turn.""" + agent = _FakeAgent(repo, indexed=True) + for _ in range(3): + agent.get_project_map_system_prompt() + assert agent.sdk.status_calls == 0 def test_index_trigger_is_skipped_when_already_indexed(repo): @@ -391,3 +462,93 @@ def _boom(**_kw): agent._on_task_start("do a thing") _join(agent) assert any("faiss exploded" in r.getMessage() for r in caplog.records) + + +# ── wiring, checked against the source ──────────────────────────────────── +# +# ``gaia_agent`` resolves through an editable install that can point at a +# different checkout, so importing ``GaiaAgent`` here would assert against the +# wrong tree. These parse this repository's files instead. + +_REPO = pathlib.Path(__file__).resolve().parents[2] + + +def _bases_of(path: pathlib.Path, class_name: str) -> list: + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and node.name == class_name: + return [b.id for b in node.bases if isinstance(b, ast.Name)] + raise AssertionError(f"class {class_name!r} not found in {path}") + + +def test_project_map_mixin_precedes_the_base_agent_in_gaia_agent(): + """Listed after ChatAgent, ``Agent``'s no-op hook would shadow the override.""" + bases = _bases_of( + _REPO / "hub" / "agents" / "gaia" / "python" / "gaia_agent" / "agent.py", + "GaiaAgent", + ) + assert bases[0] == "ProjectMapMixin" + assert "ChatAgent" in bases[1:] + + +def test_base_agent_calls_the_task_start_hook_before_composing_the_prompt(): + src = (_REPO / "src" / "gaia" / "agents" / "base" / "agent.py").read_text( + encoding="utf-8" + ) + tree = ast.parse(src) + impl = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == "_process_query_impl" + ) + called = [ + n.func.attr + for n in ast.walk(impl) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + ] + assert "_on_task_start" in called + assert called.index("_on_task_start") < called.index("_refresh_active_tool_filter") + + +def test_base_agent_hook_is_a_no_op_so_agents_without_the_mixin_are_unaffected(): + from gaia.agents.base.agent import Agent + + assert Agent._on_task_start(object(), "anything") is None + + +def test_the_agents_own_source_tree_is_never_the_project(monkeypatch): + """The dev-mode sidecar's cwd is the GAIA checkout — not the user's work.""" + import gaia + + gaia_repo = pathlib.Path(gaia.__file__).resolve().parents[2] + assert is_agent_own_source(gaia_repo) + assert is_code_repository(gaia_repo), "precondition: it looks like a project" + + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + monkeypatch.chdir(gaia_repo) + assert resolve_project_root() is None + + +def test_an_explicit_root_may_point_at_gaia_itself(monkeypatch): + """Working on GAIA is legitimate — it just has to be asked for.""" + import gaia + + gaia_repo = pathlib.Path(gaia.__file__).resolve().parents[2] + monkeypatch.delenv(PROJECT_ROOT_ENV, raising=False) + assert resolve_project_root(str(gaia_repo)) == str(gaia_repo) + + +def test_shell_commands_are_omitted_for_an_agent_without_the_shell_tool(repo): + text = render_project_map(build_project_map(repo), has_shell_tool=False) + assert "run_shell_command" not in text + assert "NOT installed" in text + + +def test_a_wrong_base_order_fails_at_class_definition(): + """Silent otherwise: the prompt fragment renders either way.""" + from gaia.agents.base.agent import Agent + + with pytest.raises(TypeError, match="ProjectMapMixin after Agent"): + + class _Wrong(Agent, ProjectMapMixin): + pass

AltStyle によって変換されたページ (->オリジナル) /