diff --git a/haystack/components/agents/agent.py b/haystack/components/agents/agent.py index f792e8251b..974580164d 100644 --- a/haystack/components/agents/agent.py +++ b/haystack/components/agents/agent.py @@ -39,6 +39,7 @@ serialize_tools_or_toolset, warm_up_tools, ) +from haystack.tools.toolset import _ToolsetWrapper from haystack.utils.callable_serialization import deserialize_callable, serialize_callable from haystack.utils.deserialization import deserialize_component_inplace @@ -193,6 +194,63 @@ def _render_prompt_messages( return prompt_messages +def _gather_system_prompt_contributions(item: Any, out: list[str]) -> None: + """ + Recursively collect system prompt contributions from a tool, toolset, or collection thereof. + + A `Toolset` that provides its own contribution takes precedence over its member tools, which are then + not contributed separately (mirroring how an MCP server's `instructions` describe its tools as a whole). + Composed toolsets (created via `toolset_a + toolset_b`) contribute each child toolset independently. + + :param item: A `Tool`, `Toolset`, or list/tuple of them. + :param out: List that collected, non-empty contributions are appended to, in order. + """ + if item is None: + return + if isinstance(item, _ToolsetWrapper): + for toolset in item.toolsets: + _gather_system_prompt_contributions(toolset, out) + elif isinstance(item, Toolset): + contribution = item.system_prompt_contribution() + if contribution: + out.append(contribution) + else: + for member_tool in item: + _gather_system_prompt_contributions(member_tool, out) + elif isinstance(item, Tool): + contribution = item.system_prompt_contribution() + if contribution: + out.append(contribution) + elif isinstance(item, (list, tuple)): + for sub_item in item: + _gather_system_prompt_contributions(sub_item, out) + + +def _apply_system_prompt_contributions(messages: list[ChatMessage], tools: ToolsType | None) -> list[ChatMessage]: + """ + Append tool/toolset system prompt contributions to the agent's system message. + + If the first message is a system message (the rendered `system_prompt`, or a user-supplied system message), + the contributions are appended to it. Otherwise a new system message is prepended. Contributions are merged + into the already-rendered message text (never the Jinja2 template) so arbitrary content is safe. + + :param messages: The messages the agent is about to run with. + :param tools: The tools selected for this run. + :returns: The messages with contributions applied (a new list; the input is not mutated). + """ + contributions: list[str] = [] + _gather_system_prompt_contributions(tools, contributions) + if not contributions: + return messages + + extra = "\n\n".join(contributions) + if messages and messages[0].is_from(ChatRole.SYSTEM): + existing = messages[0].text or "" + combined = f"{existing}\n\n{extra}" if existing else extra + return [ChatMessage.from_system(combined), *messages[1:]] + return [ChatMessage.from_system(extra), *messages] + + @dataclass(kw_only=True) class _ExecutionContext: """ @@ -653,6 +711,10 @@ def _initialize_fresh_execution( selected_tools = self._select_tools(tools) + # Append any system prompt instructions contributed by the selected tools/toolsets (e.g. a SkillToolset + # catalog) to the system message. Based on selected_tools so it respects per-run tool filtering. + messages = _apply_system_prompt_contributions(messages, selected_tools) + state_kwargs: dict[str, Any] = {key: kwargs[key] for key in self.state_schema.keys() if key in kwargs} state = State(schema=self.state_schema, data=state_kwargs) state.set("messages", messages) diff --git a/haystack/tools/__init__.py b/haystack/tools/__init__.py index 2fa7c35cf2..827c484a99 100644 --- a/haystack/tools/__init__.py +++ b/haystack/tools/__init__.py @@ -11,6 +11,7 @@ from haystack.tools.tool import Tool, _check_duplicate_tool_names from haystack.tools.toolset import Toolset from haystack.tools.searchable_toolset import SearchableToolset +from haystack.tools.skills import SkillMeta, SkillToolset from haystack.tools.component_tool import ComponentTool from haystack.tools.pipeline_tool import PipelineTool from haystack.tools.serde_utils import deserialize_tools_or_toolset_inplace, serialize_tools_or_toolset @@ -32,6 +33,8 @@ "serialize_tools_or_toolset", "Tool", "SearchableToolset", + "SkillMeta", + "SkillToolset", "ToolsType", "Toolset", "tool", diff --git a/haystack/tools/skills/__init__.py b/haystack/tools/skills/__init__.py new file mode 100644 index 0000000000..a63816d545 --- /dev/null +++ b/haystack/tools/skills/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +from haystack.tools.skills.skill_toolset import SkillMeta, SkillToolset + +__all__ = ["SkillMeta", "SkillToolset"] diff --git a/haystack/tools/skills/skill_toolset.py b/haystack/tools/skills/skill_toolset.py new file mode 100644 index 0000000000..ee17a2d8bb --- /dev/null +++ b/haystack/tools/skills/skill_toolset.py @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass +from pathlib import Path +from typing import Annotated, Any + +import yaml + +from haystack.core.serialization import generate_qualified_class_name +from haystack.tools.from_function import create_tool_from_function +from haystack.tools.tool import Tool +from haystack.tools.toolset import Toolset + +SKILL_FILE_NAME = "SKILL.md" + + +@dataclass +class SkillMeta: + """ + Metadata describing a single skill discovered on disk. + + :param name: The skill's name, used by the agent to load it. + :param description: A short description of when to use the skill. Shown to the agent up front. + :param path: The skill's directory. + """ + + name: str + description: str + path: Path + + +def _parse_frontmatter(text: str) -> tuple[dict[str, Any], str]: + """ + Split a `SKILL.md` file into its YAML frontmatter and markdown body. + + The frontmatter is the YAML block delimited by leading and trailing `---` lines. If no frontmatter is + present, an empty mapping and the original text are returned. + + :param text: The full contents of a `SKILL.md` file. + :returns: A tuple of (frontmatter mapping, body). + :raises ValueError: If the frontmatter is present but is not a valid YAML mapping. + """ + stripped = text.lstrip() + if not stripped.startswith("---"): + return {}, text + + # Drop the leading '---' line, then split on the closing '---'. + after_open = stripped[len("---") :].lstrip("\n") + parts = after_open.split("\n---", 1) + if len(parts) != 2: + return {}, text + + frontmatter_block, body = parts + loaded = yaml.safe_load(frontmatter_block) or {} + if not isinstance(loaded, dict): + raise ValueError("Skill frontmatter must be a YAML mapping.") # noqa: TRY004 + return loaded, body.lstrip("\n") + + +class SkillToolset(Toolset): + """ + A Toolset that lets an Agent discover and read filesystem "skills" via progressive disclosure. + + A skill is a directory containing a `SKILL.md` file with YAML frontmatter (`name` and `description`) and a + markdown body of instructions. Skills may bundle additional files (reference docs, examples, templates). + This mirrors how Claude Code and Codex expose skills: + + - The name and description of every skill are injected into the Agent's system prompt + (via `system_prompt_contribution`) so the model knows which skills exist. + - `load_skill` returns a skill's full instructions on demand, plus a manifest of its bundled files. + - `read_skill_file` reads a bundled file on demand. + + Expected layout: + + ``` + skills/ + pdf-forms/ + SKILL.md # frontmatter (name, description) + markdown instructions + reference/forms.md + ``` + + ### Usage example + + ```python + from haystack.components.agents import Agent + from haystack.components.generators.chat import OpenAIChatGenerator + from haystack.dataclasses import ChatMessage + from haystack.tools import SkillToolset + + skills = SkillToolset("skills/") + agent = Agent(chat_generator=OpenAIChatGenerator(), tools=skills) + # The skills catalog is appended to the system prompt automatically. + result = agent.run(messages=[ChatMessage.from_user("Fill in this PDF form for me.")]) + ``` + """ + + def __init__(self, skills_dir: str | Path) -> None: + """ + Initialize the SkillToolset by scanning a directory for skills. + + Only the frontmatter of each `SKILL.md` is read at construction time (cheap); bodies and bundled files + are read lazily when the agent calls `load_skill` / `read_skill_file`. + + :param skills_dir: Directory containing one subdirectory per skill, each with a `SKILL.md`. + :raises ValueError: If `skills_dir` does not exist, is not a directory, a skill is missing a required + frontmatter field, or two skills share the same name. + """ + self.skills_dir = Path(skills_dir) + self._skills: dict[str, SkillMeta] = self._scan() + super().__init__(tools=[self._create_load_skill_tool(), self._create_read_skill_file_tool()]) + + @property + def skills(self) -> dict[str, SkillMeta]: + """Mapping of skill name to its metadata.""" + return self._skills + + def _scan(self) -> dict[str, SkillMeta]: + """ + Scan `skills_dir` for skills, reading only the frontmatter of each `SKILL.md`. + + :returns: Mapping of skill name to metadata. + :raises ValueError: On a missing directory, missing required frontmatter, or duplicate skill names. + """ + if not self.skills_dir.is_dir(): + raise ValueError(f"Skills directory '{self.skills_dir}' does not exist or is not a directory.") + + skills: dict[str, SkillMeta] = {} + for skill_file in sorted(self.skills_dir.glob(f"*/{SKILL_FILE_NAME}")): + skill_dir = skill_file.parent + frontmatter, _ = _parse_frontmatter(skill_file.read_text(encoding="utf-8")) + + name = frontmatter.get("name", skill_dir.name) + description = frontmatter.get("description") + if not description: + raise ValueError(f"Skill '{name}' ({skill_file}) is missing a 'description' in its frontmatter.") + if name in skills: + raise ValueError(f"Duplicate skill name '{name}' found in '{self.skills_dir}'.") + + skills[name] = SkillMeta(name=name, description=description, path=skill_dir) + return skills + + def system_prompt_contribution(self) -> str | None: + """ + Render the skills catalog and usage instructions for injection into the Agent's system prompt. + + :returns: The catalog text, or `None` if no skills were found. + """ + if not self._skills: + return None + + lines = [ + "## Available Skills", + "Specialized instruction sets for specific task types. Load one before doing matching work.", + "", + ] + lines += [f"- **{meta.name}**: {meta.description}" for meta in self._skills.values()] + lines += [ + "", + "When a task matches a skill, call `load_skill` with its name BEFORE starting, then follow the loaded " + "instructions exactly (they override your general approach). Load skills only when relevant; if a skill " + "references a file, fetch it with `read_skill_file`. If no skill matches, proceed normally.", + ] + return "\n".join(lines) + + def _create_load_skill_tool(self) -> Tool: + """Create the `load_skill` tool, closed over this toolset's skill registry.""" + + def load_skill(name: Annotated[str, "Exact name of the skill to load, from the Available Skills list."]) -> str: + """Load a skill's full instructions. Call this before doing a task the skill covers.""" + meta = self._skills.get(name) + if meta is None: + available = ", ".join(self._skills) or "none" + return f"Unknown skill '{name}'. Available skills: {available}." + + _, body = _parse_frontmatter((meta.path / SKILL_FILE_NAME).read_text(encoding="utf-8")) + + bundled = sorted( + p.relative_to(meta.path).as_posix() + for p in meta.path.rglob("*") + if p.is_file() and p.name != SKILL_FILE_NAME + ) + if bundled: + manifest = "\n".join(f"- {path}" for path in bundled) + body = f"{body}\n\n---\nBundled files (read with `read_skill_file`):\n{manifest}" + return body + + return create_tool_from_function(function=load_skill, name="load_skill") + + def _create_read_skill_file_tool(self) -> Tool: + """Create the `read_skill_file` tool, closed over this toolset's skill registry.""" + + def read_skill_file( + name: Annotated[str, "Name of the skill that owns the file."], + path: Annotated[str, "Path of the file relative to the skill directory, e.g. 'reference/forms.md'."], + ) -> str: + """Read a file bundled with a skill (reference docs, examples, templates).""" + meta = self._skills.get(name) + if meta is None: + available = ", ".join(self._skills) or "none" + return f"Unknown skill '{name}'. Available skills: {available}." + + skill_dir = meta.path.resolve() + target = (skill_dir / path).resolve() + if skill_dir != target and skill_dir not in target.parents: + return f"Refusing to read '{path}': path escapes the '{name}' skill directory." + if not target.is_file(): + return f"File '{path}' not found in skill '{name}'." + return target.read_text(encoding="utf-8") + + return create_tool_from_function(function=read_skill_file, name="read_skill_file") + + def to_dict(self) -> dict[str, Any]: + """ + Serialize the toolset to a dictionary. + + Only the skills directory is serialized; tools are rebuilt by rescanning on deserialization. + + :returns: Dictionary representation of the toolset. + """ + return {"type": generate_qualified_class_name(type(self)), "data": {"skills_dir": str(self.skills_dir)}} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "SkillToolset": + """ + Deserialize a toolset from a dictionary. + + :param data: Dictionary representation of the toolset. + :returns: A new SkillToolset instance. + """ + return cls(skills_dir=data["data"]["skills_dir"]) diff --git a/haystack/tools/tool.py b/haystack/tools/tool.py index 0a31de086a..9ce36f7734 100644 --- a/haystack/tools/tool.py +++ b/haystack/tools/tool.py @@ -91,6 +91,11 @@ class Tool: "documents": {"handler": custom_handler} } ``` + :param system_prompt_instructions: + Optional system prompt instructions associated with this Tool. When the Tool is used with an `Agent`, + this text is appended to the Agent's system prompt (see `system_prompt_contribution`). Use it to tell + the model how and when to use the Tool. Note that an enclosing `Toolset` that provides its own + `system_prompt_contribution` takes precedence and suppresses the contributions of its member tools. :raises ValueError: If neither `function` nor `async_function` is provided, if `function` is a coroutine function, if `async_function` is not a coroutine function, if `parameters` is not a valid JSON schema, or if the `outputs_to_state`, `outputs_to_string`, or `inputs_from_state` @@ -107,6 +112,7 @@ class Tool: inputs_from_state: dict[str, str] | None = None outputs_to_state: dict[str, dict[str, Any]] | None = None async_function: Callable | None = None + system_prompt_instructions: str | None = None def __post_init__(self) -> None: # noqa: C901, PLR0912 # At least one of function / async_function must be set. @@ -280,6 +286,18 @@ def warm_up(self) -> None: """ pass + def system_prompt_contribution(self) -> str | None: + """ + Return optional system prompt instructions for this Tool. + + When the Tool is used with an `Agent`, the returned text is appended to the Agent's system prompt. + By default this returns the `system_prompt_instructions` attribute (which may be `None`). Subclasses + can override this to generate instructions dynamically. + + :returns: The system prompt contribution, or `None` if the Tool has nothing to contribute. + """ + return self.system_prompt_instructions + def invoke(self, **kwargs: Any) -> Any: """ Invoke the Tool synchronously with the provided keyword arguments. diff --git a/haystack/tools/toolset.py b/haystack/tools/toolset.py index 0941714f7f..f194e2d429 100644 --- a/haystack/tools/toolset.py +++ b/haystack/tools/toolset.py @@ -188,6 +188,22 @@ def __contains__(self, item: str | Tool) -> bool: return item in self.tools return False + def system_prompt_contribution(self) -> str | None: + """ + Return optional system prompt instructions for this Toolset. + + When the Toolset is used with an `Agent`, the returned text is appended to the Agent's system prompt. + This is the Toolset-level analogue of an MCP server's `instructions`: it describes how to use the + Toolset's tools as a whole. The default implementation returns `None`; subclasses (for example + `SkillToolset`) override it to generate instructions dynamically. + + When this returns a non-`None` value, it takes precedence over the `system_prompt_instructions` of the + Toolset's individual member tools, which are then not contributed separately. + + :returns: The system prompt contribution, or `None` if the Toolset has nothing to contribute. + """ + return None + def warm_up(self) -> None: """ Prepare the Toolset for use. diff --git a/releasenotes/notes/add-skill-toolset-7f3c1a9e2b4d6c8a.yaml b/releasenotes/notes/add-skill-toolset-7f3c1a9e2b4d6c8a.yaml new file mode 100644 index 0000000000..818f2ba631 --- /dev/null +++ b/releasenotes/notes/add-skill-toolset-7f3c1a9e2b4d6c8a.yaml @@ -0,0 +1,28 @@ +--- +features: + - | + Added ``SkillToolset``, a ``Toolset`` that lets an ``Agent`` discover and read filesystem "skills" + through progressive disclosure, similar to how Claude Code and Codex expose skills. A skill is a + directory containing a ``SKILL.md`` file with YAML frontmatter (``name`` and ``description``) and a + markdown body of instructions, optionally bundling additional reference files. Point the toolset at a + skills directory and add it to an ``Agent``: + + .. code-block:: python + + from haystack.components.agents import Agent + from haystack.components.generators.chat import OpenAIChatGenerator + from haystack.tools import SkillToolset + + agent = Agent(chat_generator=OpenAIChatGenerator(), tools=SkillToolset("skills/")) + + The names and descriptions of all discovered skills are appended to the Agent's system prompt + automatically. The toolset exposes two tools: ``load_skill`` returns a skill's full instructions on + demand, and ``read_skill_file`` reads a file bundled with a skill (with path-traversal protection). + - | + ``Tool`` and ``Toolset`` now support a ``system_prompt_contribution()`` method. When a tool or toolset + is used with an ``Agent``, the text it returns is appended to the Agent's system prompt. ``Tool`` gains + an optional ``system_prompt_instructions`` field for this; ``Toolset`` subclasses can override the method + to generate instructions dynamically. When a ``Toolset`` provides a contribution, it takes precedence over + the ``system_prompt_instructions`` of its individual member tools, mirroring how an MCP server's + ``instructions`` describe its tools as a whole. Note that a member tool's own ``system_prompt_instructions`` + are therefore suppressed whenever its enclosing ``Toolset`` contributes. diff --git a/test/components/agents/test_agent.py b/test/components/agents/test_agent.py index 3c8fff1023..94afb0bca4 100644 --- a/test/components/agents/test_agent.py +++ b/test/components/agents/test_agent.py @@ -264,6 +264,7 @@ def test_to_dict(self, weather_tool, component_tool, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, }, { @@ -348,6 +349,7 @@ def test_to_dict_with_toolset(self, monkeypatch, weather_tool): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] @@ -419,6 +421,7 @@ def test_from_dict(self, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, }, { @@ -510,6 +513,7 @@ def test_from_dict_with_toolset(self, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] @@ -1272,7 +1276,7 @@ def test_agent_tracing_span_run(self, caplog, monkeypatch, weather_tool): assert set(llm_tags) == {"haystack.agent.step.llm.input", "haystack.agent.step.llm.output"} assert ( llm_tags["haystack.agent.step.llm.input"] - == '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "tools": [{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null}}]}' # noqa: E501 + == '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "tools": [{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null, "system_prompt_instructions": null}}]}' # noqa: E501 ) assert ( llm_tags["haystack.agent.step.llm.output"] @@ -1287,7 +1291,7 @@ def test_agent_tracing_span_run(self, caplog, monkeypatch, weather_tool): _, run_tags = agent_spans[2] assert run_tags == { "haystack.agent.max_steps": 100, - "haystack.agent.tools": '[{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null}}]', # noqa: E501 + "haystack.agent.tools": '[{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null, "system_prompt_instructions": null}}]', # noqa: E501 "haystack.agent.exit_conditions": '["text"]', "haystack.agent.state_schema": '{"messages": {"type": "list[haystack.dataclasses.chat_message.ChatMessage]", "handler": "haystack.components.agents.state.state_utils.merge_lists"}, "step_count": {"type": "int", "handler": "haystack.components.agents.state.state_utils.replace_values"}, "token_usage": {"type": "dict[str, typing.Any]", "handler": "haystack.components.agents.state.state_utils.replace_values"}, "tool_call_counts": {"type": "dict[str, int]", "handler": "haystack.components.agents.state.state_utils.replace_values"}}', # noqa: E501 "haystack.agent.input": '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "streaming_callback": null}', # noqa: E501 @@ -1380,7 +1384,7 @@ async def test_agent_tracing_span_async_run(self, caplog, monkeypatch, weather_t assert set(llm_tags) == {"haystack.agent.step.llm.input", "haystack.agent.step.llm.output"} assert ( llm_tags["haystack.agent.step.llm.input"] - == '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "tools": [{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null}}]}' # noqa: E501 + == '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "tools": [{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null, "system_prompt_instructions": null}}]}' # noqa: E501 ) assert ( llm_tags["haystack.agent.step.llm.output"] @@ -1393,7 +1397,7 @@ async def test_agent_tracing_span_async_run(self, caplog, monkeypatch, weather_t _, run_tags = agent_spans[2] assert run_tags == { "haystack.agent.max_steps": 100, - "haystack.agent.tools": '[{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null}}]', # noqa: E501 + "haystack.agent.tools": '[{"type": "haystack.tools.tool.Tool", "data": {"name": "weather_tool", "description": "Provides weather information for a given location.", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}, "function": "test_agent.weather_function", "outputs_to_string": null, "inputs_from_state": null, "outputs_to_state": null, "async_function": null, "system_prompt_instructions": null}}]', # noqa: E501 "haystack.agent.exit_conditions": '["text"]', "haystack.agent.state_schema": '{"messages": {"type": "list[haystack.dataclasses.chat_message.ChatMessage]", "handler": "haystack.components.agents.state.state_utils.merge_lists"}, "step_count": {"type": "int", "handler": "haystack.components.agents.state.state_utils.replace_values"}, "token_usage": {"type": "dict[str, typing.Any]", "handler": "haystack.components.agents.state.state_utils.replace_values"}, "tool_call_counts": {"type": "dict[str, int]", "handler": "haystack.components.agents.state.state_utils.replace_values"}}', # noqa: E501 "haystack.agent.input": '{"messages": [{"role": "user", "meta": {}, "name": null, "content": [{"text": "What\'s the weather in Paris?"}]}], "streaming_callback": null}', # noqa: E501 @@ -2224,3 +2228,84 @@ def run(self) -> dict: assert "agent" not in result chat_generator.run.assert_not_called() + + +@component +class CapturingChatGenerator: + """Records the messages it last received so tests can inspect the system prompt sent to the model.""" + + def __init__(self): + self.last_messages = None + + @component.output_types(replies=list[ChatMessage]) + def run(self, messages: list[ChatMessage], tools=None, **kwargs) -> dict[str, Any]: + self.last_messages = messages + return {"replies": [ChatMessage.from_assistant("done")]} + + +class _ContributingToolset(Toolset): + def __init__(self, tools, contribution): + super().__init__(tools) + self._contribution = contribution + + def system_prompt_contribution(self): + return self._contribution + + +class TestSystemPromptContributions: + def _tool(self, name, system_prompt=None): + return Tool( + name=name, + description="d", + parameters={"type": "object", "properties": {}}, + function=lambda: None, + system_prompt_instructions=system_prompt, + ) + + def test_toolset_contribution_appended_to_system_prompt(self): + generator = CapturingChatGenerator() + toolset = _ContributingToolset([self._tool("a")], "TOOLSET INSTRUCTIONS") + agent = Agent(chat_generator=generator, tools=toolset, system_prompt="You are helpful.") + agent.warm_up() + agent.run(messages=[ChatMessage.from_user("hi")]) + + system_message = generator.last_messages[0] + assert system_message.is_from(ChatRole.SYSTEM) + assert system_message.text == "You are helpful.\n\nTOOLSET INSTRUCTIONS" + + def test_contribution_prepended_when_no_system_prompt(self): + generator = CapturingChatGenerator() + toolset = _ContributingToolset([self._tool("a")], "TOOLSET INSTRUCTIONS") + agent = Agent(chat_generator=generator, tools=toolset) + agent.warm_up() + agent.run(messages=[ChatMessage.from_user("hi")]) + + assert generator.last_messages[0].is_from(ChatRole.SYSTEM) + assert generator.last_messages[0].text == "TOOLSET INSTRUCTIONS" + assert generator.last_messages[1].is_from(ChatRole.USER) + + def test_toolset_contribution_suppresses_member_tool_contributions(self): + generator = CapturingChatGenerator() + toolset = _ContributingToolset([self._tool("a", system_prompt="MEMBER")], "TOOLSET") + agent = Agent(chat_generator=generator, tools=toolset) + agent.warm_up() + agent.run(messages=[ChatMessage.from_user("hi")]) + + assert generator.last_messages[0].text == "TOOLSET" + + def test_bare_tool_system_prompt_is_contributed(self): + generator = CapturingChatGenerator() + agent = Agent(chat_generator=generator, tools=[self._tool("a", system_prompt="USE TOOL A")]) + agent.warm_up() + agent.run(messages=[ChatMessage.from_user("hi")]) + + assert generator.last_messages[0].text == "USE TOOL A" + + def test_no_contribution_leaves_messages_unchanged(self): + generator = CapturingChatGenerator() + agent = Agent(chat_generator=generator, tools=[self._tool("a")]) + agent.warm_up() + agent.run(messages=[ChatMessage.from_user("hi")]) + + assert len(generator.last_messages) == 1 + assert generator.last_messages[0].is_from(ChatRole.USER) diff --git a/test/components/agents/test_agent_hitl.py b/test/components/agents/test_agent_hitl.py index e6fb6db3d8..6fc10096b5 100644 --- a/test/components/agents/test_agent_hitl.py +++ b/test/components/agents/test_agent_hitl.py @@ -94,6 +94,7 @@ def test_to_dict(self, tools, confirmation_strategies, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ], diff --git a/test/components/generators/chat/test_azure.py b/test/components/generators/chat/test_azure.py index edb2bdd452..f746083752 100644 --- a/test/components/generators/chat/test_azure.py +++ b/test/components/generators/chat/test_azure.py @@ -430,6 +430,7 @@ def test_to_dict_with_toolset(self, tools, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] diff --git a/test/components/generators/chat/test_azure_responses.py b/test/components/generators/chat/test_azure_responses.py index 636a99356c..a0e185ed3a 100644 --- a/test/components/generators/chat/test_azure_responses.py +++ b/test/components/generators/chat/test_azure_responses.py @@ -250,6 +250,7 @@ def test_to_dict_with_toolset(self, tools, monkeypatch): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] diff --git a/test/components/generators/chat/test_hugging_face_api.py b/test/components/generators/chat/test_hugging_face_api.py index 50a0fcb368..4665696db6 100644 --- a/test/components/generators/chat/test_hugging_face_api.py +++ b/test/components/generators/chat/test_hugging_face_api.py @@ -279,6 +279,7 @@ def test_to_dict(self, mock_check_valid_model): "inputs_from_state": None, "name": "name", "outputs_to_state": None, + "system_prompt_instructions": None, "outputs_to_string": None, "parameters": {"x": {"type": "string"}}, }, @@ -343,6 +344,7 @@ def test_serde_in_pipeline(self, mock_check_valid_model): "inputs_from_state": None, "name": "name", "outputs_to_state": None, + "system_prompt_instructions": None, "outputs_to_string": None, "description": "description", "parameters": {"x": {"type": "string"}}, @@ -1218,6 +1220,7 @@ def test_to_dict_with_toolset(self, mock_check_valid_model, tools): "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] diff --git a/test/components/generators/chat/test_hugging_face_local.py b/test/components/generators/chat/test_hugging_face_local.py index 3f134abd7e..c7176c587b 100644 --- a/test/components/generators/chat/test_hugging_face_local.py +++ b/test/components/generators/chat/test_hugging_face_local.py @@ -206,6 +206,7 @@ def test_to_dict(self, model_info_mock, tools): "inputs_from_state": None, "name": "weather", "outputs_to_state": None, + "system_prompt_instructions": None, "outputs_to_string": None, "description": "useful to determine the weather in a given location", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, @@ -788,6 +789,7 @@ def test_to_dict_with_toolset(self, model_info_mock, mock_pipeline_with_tokenize "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, + "system_prompt_instructions": None, }, } ] diff --git a/test/components/generators/chat/test_openai.py b/test/components/generators/chat/test_openai.py index af9ac47cbd..1c0842bbce 100644 --- a/test/components/generators/chat/test_openai.py +++ b/test/components/generators/chat/test_openai.py @@ -347,6 +347,7 @@ def test_to_dict_with_parameters(self, monkeypatch, calendar_event_model): "inputs_from_state": None, "name": "name", "outputs_to_state": None, + "system_prompt_instructions": None, "outputs_to_string": None, "parameters": {"x": {"type": "string"}}, }, diff --git a/test/components/generators/chat/test_openai_responses.py b/test/components/generators/chat/test_openai_responses.py index 08800a1fbe..85745f4dfa 100644 --- a/test/components/generators/chat/test_openai_responses.py +++ b/test/components/generators/chat/test_openai_responses.py @@ -261,6 +261,7 @@ def test_to_dict_with_parameters(self, monkeypatch, calendar_event_model): "inputs_from_state": None, "name": "name", "outputs_to_state": None, + "system_prompt_instructions": None, "outputs_to_string": None, "parameters": {"x": {"type": "string"}}, }, diff --git a/test/tools/skills/test_skill_toolset.py b/test/tools/skills/test_skill_toolset.py new file mode 100644 index 0000000000..093e86ec04 --- /dev/null +++ b/test/tools/skills/test_skill_toolset.py @@ -0,0 +1,119 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from haystack.tools import SkillToolset +from haystack.tools.skills.skill_toolset import _parse_frontmatter + + +def _write_skill(skills_dir, name, description=None, body="Instructions.", files=None): + skill_dir = skills_dir / name + skill_dir.mkdir(parents=True) + frontmatter = f"---\nname: {name}\n" + if description is not None: + frontmatter += f"description: {description}\n" + frontmatter += "---\n" + (skill_dir / "SKILL.md").write_text(frontmatter + body, encoding="utf-8") + for rel_path, content in (files or {}).items(): + target = skill_dir / rel_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return skill_dir + + +class TestParseFrontmatter: + def test_parses_frontmatter_and_body(self): + frontmatter, body = _parse_frontmatter("---\nname: a\ndescription: d\n---\nThe body.") + assert frontmatter == {"name": "a", "description": "d"} + assert body == "The body." + + def test_no_frontmatter_returns_empty_mapping(self): + frontmatter, body = _parse_frontmatter("Just a body, no frontmatter.") + assert frontmatter == {} + assert body == "Just a body, no frontmatter." + + def test_non_mapping_frontmatter_raises(self): + with pytest.raises(ValueError): + _parse_frontmatter("---\n- just\n- a\n- list\n---\nbody") + + +class TestSkillToolset: + def test_scans_skills(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="Use to fill PDF forms.") + _write_skill(tmp_path, "excel", description="Use to edit spreadsheets.") + + toolset = SkillToolset(tmp_path) + + assert set(toolset.skills) == {"pdf-forms", "excel"} + assert toolset.skills["pdf-forms"].description == "Use to fill PDF forms." + assert {t.name for t in toolset} == {"load_skill", "read_skill_file"} + + def test_missing_directory_raises(self, tmp_path): + with pytest.raises(ValueError, match="does not exist"): + SkillToolset(tmp_path / "nope") + + def test_missing_description_raises(self, tmp_path): + _write_skill(tmp_path, "broken", description=None) + with pytest.raises(ValueError, match="missing a 'description'"): + SkillToolset(tmp_path) + + def test_system_prompt_contribution_lists_skills(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="Use to fill PDF forms.") + contribution = SkillToolset(tmp_path).system_prompt_contribution() + assert "## Available Skills" in contribution + assert "**pdf-forms**: Use to fill PDF forms." in contribution + assert "load_skill" in contribution and "read_skill_file" in contribution + + def test_system_prompt_contribution_none_when_empty(self, tmp_path): + assert SkillToolset(tmp_path).system_prompt_contribution() is None + + def test_load_skill_returns_body_and_manifest(self, tmp_path): + _write_skill( + tmp_path, + "pdf-forms", + description="Use to fill PDF forms.", + body="Step 1. Do the thing.", + files={"reference/forms.md": "details"}, + ) + load_skill = next(t for t in SkillToolset(tmp_path) if t.name == "load_skill") + result = load_skill.invoke(name="pdf-forms") + assert "Step 1. Do the thing." in result + assert "reference/forms.md" in result + + def test_load_skill_unknown(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="Use to fill PDF forms.") + load_skill = next(t for t in SkillToolset(tmp_path) if t.name == "load_skill") + assert "Unknown skill 'nope'" in load_skill.invoke(name="nope") + + def test_read_skill_file(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="d", files={"reference/forms.md": "form details"}) + read = next(t for t in SkillToolset(tmp_path) if t.name == "read_skill_file") + assert read.invoke(name="pdf-forms", path="reference/forms.md") == "form details" + + def test_read_skill_file_blocks_traversal(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="d") + (tmp_path / "secret.txt").write_text("top secret") + read = next(t for t in SkillToolset(tmp_path) if t.name == "read_skill_file") + result = read.invoke(name="pdf-forms", path="../secret.txt") + assert "escapes" in result + assert "top secret" not in result + + def test_read_skill_file_missing(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="d") + read = next(t for t in SkillToolset(tmp_path) if t.name == "read_skill_file") + assert "not found" in read.invoke(name="pdf-forms", path="nope.md") + + def test_to_dict_and_from_dict(self, tmp_path): + _write_skill(tmp_path, "pdf-forms", description="Use to fill PDF forms.") + toolset = SkillToolset(tmp_path) + + data = toolset.to_dict() + assert data == { + "type": "haystack.tools.skills.skill_toolset.SkillToolset", + "data": {"skills_dir": str(tmp_path)}, + } + + restored = SkillToolset.from_dict(data) + assert set(restored.skills) == {"pdf-forms"} diff --git a/test/tools/test_tool.py b/test/tools/test_tool.py index 5148cf7060..7d3d84d0f8 100644 --- a/test/tools/test_tool.py +++ b/test/tools/test_tool.py @@ -175,6 +175,7 @@ def test_to_dict(self): "outputs_to_string": {"handler": "test_tool.format_string"}, "inputs_from_state": {"location": "city"}, "outputs_to_state": {"documents": {"source": "docs", "handler": "test_tool.get_weather_report"}}, + "system_prompt_instructions": None, }, } @@ -202,6 +203,23 @@ def test_from_dict(self): assert tool.inputs_from_state == {"location": "city"} assert tool.outputs_to_state == {"documents": {"source": "docs", "handler": get_weather_report}} + def test_system_prompt_contribution(self): + tool = Tool( + name="weather", + description="Get weather report", + parameters=parameters, + function=get_weather_report, + system_prompt_instructions="Always call weather before answering about the weather.", + ) + assert tool.system_prompt_contribution() == "Always call weather before answering about the weather." + + def test_system_prompt_contribution_defaults_to_none(self): + tool = Tool( + name="weather", description="Get weather report", parameters=parameters, function=get_weather_report + ) + assert tool.system_prompt_instructions is None + assert tool.system_prompt_contribution() is None + def test_serialize_outputs_to_string(self): config = {"handler": format_string, "source": "result", "raw_result": False} serialized = _serialize_outputs_to_string(config) diff --git a/test/tools/test_toolset.py b/test/tools/test_toolset.py index 214bceceb3..336d0a2fc0 100644 --- a/test/tools/test_toolset.py +++ b/test/tools/test_toolset.py @@ -157,6 +157,11 @@ def faulty_tool_func(location): class TestToolset: + def test_system_prompt_contribution_defaults_to_none(self): + """A plain Toolset contributes nothing to the system prompt by default.""" + toolset = Toolset([]) + assert toolset.system_prompt_contribution() is None + def test_toolset_with_multiple_tools(self): """Test that a Toolset with multiple tools works properly.""" add_tool = Tool(