feat(agent): Add opt-in flag to include tool specs in traces for evaluation

Author: Ratish1

src/strands/agent/agent.py

@@ -660,6 +660,27 @@ async def stream_async(
 
         self.trace_span = self._start_agent_trace_span(messages)
 
+        if self.tracer.include_tool_definitions:
+            try:
+                all_tools_config = self.tool_registry.get_all_tools_config() or {}
+
+                tool_details = [
+                    {
+                        "name": name,
+                        "description": spec.get("description"),
+                        "inputSchema": spec.get("inputSchema"),
+                        "outputSchema": spec.get("outputSchema"),
+                    }
+                    for name, spec in all_tools_config.items()
+                ]
+
+                serialized_tools = serialize(tool_details)
+                self.trace_span.set_attribute("gen_ai.tool.definitions", serialized_tools)
+
+            except Exception:
+                # A failure in telemetry should not crash the agent.
+                logger.exception("failed to attach tool metadata to agent span")
+
         with trace_api.use_span(self.trace_span):
             try:
                 events = self._run_loop(messages, merged_state, structured_output_model)

src/strands/telemetry/config.py

@@ -20,6 +20,8 @@
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
+from . import tracer
+
 logger = logging.getLogger(__name__)
 
 
@@ -83,12 +85,15 @@ class StrandsTelemetry:
     def __init__(
         self,
         tracer_provider: SDKTracerProvider | None = None,
+        include_tool_definitions: bool = False,
     ) -> None:
         """Initialize the StrandsTelemetry instance.
 
         Args:
             tracer_provider: Optional pre-configured tracer provider.
                 If None, a new one will be created and set as global.
+            include_tool_definitions: Whether to include tool definitions in traces.
+                Defaults to False.
 
         The instance is ready to use immediately after initialization, though
         trace exporters must be configured separately using the setup methods.
@@ -99,6 +104,9 @@ def __init__(
         else:
             self._initialize_tracer()
 
+        # Overwrite the singleton tracer instance to apply the new configuration.
+        tracer._tracer_instance = tracer.Tracer(include_tool_definitions=include_tool_definitions)
+
     def _initialize_tracer(self) -> None:
         """Initialize the OpenTelemetry tracer."""
         logger.info("Initializing tracer")

src/strands/telemetry/tracer.py

@@ -81,11 +81,10 @@ class Tracer:
     are sent to the OTLP endpoint.
     """
 
-    def __init__(
-        self,
-    ) -> None:
+    def __init__(self, include_tool_definitions: bool = False) -> None:
         """Initialize the tracer."""
         self.service_name = __name__
+        self.include_tool_definitions = include_tool_definitions
         self.tracer_provider: Optional[trace_api.TracerProvider] = None
         self.tracer_provider = trace_api.get_tracer_provider()
         self.tracer = self.tracer_provider.get_tracer(self.service_name)

tests/strands/agent/test_agent.py

@@ -2162,6 +2162,54 @@ def shell(command: str):
     assert agent.messages[-1] == {"content": [{"text": "I invoked a tool!"}], "role": "assistant"}
 
 
+def test_agent_does_not_include_tools_in_trace_by_default(tool_decorated):
+    """Verify that by default, the agent does not add tool specs to the trace."""
+    with unittest.mock.patch("strands.agent.agent.get_tracer") as mock_get_tracer:
+        mock_tracer_instance = unittest.mock.MagicMock()
+        mock_span = unittest.mock.MagicMock()
+        mock_tracer_instance.start_agent_span.return_value = mock_span
+        mock_tracer_instance.include_tool_definitions = False  # Default behavior
+        mock_get_tracer.return_value = mock_tracer_instance
+
+        mock_model = MockedModelProvider([{"role": "assistant", "content": [{"text": "hello!"}]}])
+
+        agent = Agent(tools=[tool_decorated], model=mock_model)
+        agent("test prompt")
+
+        # Check that set_attribute was not called for our specific key
+        called_attributes = [call.args[0] for call in mock_span.set_attribute.call_args_list]
+        assert "gen_ai.tool.definitions" not in called_attributes
+
+
+def test_agent_includes_tools_in_trace_when_enabled(tool_decorated):
+    """Verify that the agent adds tool specs to the trace when the flag is enabled."""
+    with unittest.mock.patch("strands.agent.agent.get_tracer") as mock_get_tracer:
+        mock_tracer_instance = unittest.mock.MagicMock()
+        mock_span = unittest.mock.MagicMock()
+        mock_tracer_instance.start_agent_span.return_value = mock_span
+        mock_tracer_instance.include_tool_definitions = True  # Enable the feature
+        mock_get_tracer.return_value = mock_tracer_instance
+
+        mock_model = MockedModelProvider([{"role": "assistant", "content": [{"text": "hello!"}]}])
+
+        agent = Agent(tools=[tool_decorated], model=mock_model)
+        agent("test prompt")
+
+        # Verify the correct data is serialized and set as an attribute
+        tool_spec = tool_decorated.tool_spec
+        expected_tool_details = [
+            {
+                "name": tool_spec.get("name"),
+                "description": tool_spec.get("description"),
+                "inputSchema": tool_spec.get("inputSchema"),
+                "outputSchema": tool_spec.get("outputSchema"),
+            }
+        ]
+        expected_json = serialize(expected_tool_details)
+
+        mock_span.set_attribute.assert_called_with("gen_ai.tool.definitions", expected_json)
+
+
 @pytest.mark.parametrize(
     "content, expected",
     [