feat(tools): add TraceIndex for progressive trace disclosure in judge-based evaluators

README.md

@@ -201,6 +201,37 @@ evaluator = TrajectoryEvaluator(
 )
 ```
 
+### Evaluating Large Traces with Progressive Disclosure
+
+When a session is too large to inline into a judge prompt (large tool results,
+many turns), inlining the whole trajectory overflows the judge's context window
+and the case is scored as a failure even when the agent was correct. The judge
+evaluators handle this automatically: before each call they preflight the
+rendered prompt against the judge model's context window, and when it would
+overflow they hand the judge a compact overview plus discovery tools instead of
+the full trajectory, so the judge loads only the spans the rubric requires.
+
+```python
+from strands_evals.evaluators import TrajectoryEvaluator
+
+# disclosure="auto" (the default): inline the trajectory when it fits, fall back
+# to overview + tools only when it would overflow the judge's context window.
+evaluator = TrajectoryEvaluator(
+    rubric=(
+        "Every factual claim must be supported by tool-result evidence in the trace. "
+        "Verify each claim against the trace before scoring."
+    ),
+    disclosure="auto",
+)
+```
+
+`disclosure` accepts `"auto"` (default), `"always"` (always use the overview +
+tools), or `"never"` (always inline, restoring the prior behavior where a
+genuine overflow surfaces as a judge error). On the disclosure path the judge
+gets a one-line-per-span overview and three tools — `list_spans`, `get_span`,
+and `search_spans` — that page or cap their output at `max_read_chars` so no
+single tool return can overflow the judge's context.
+
 ### Trace-based Helpfulness Evaluation
 
 Evaluate agent helpfulness using OpenTelemetry traces with seven-level scoring:

src/strands_evals/evaluators/chaos/failure_communication_evaluator.py

@@ -7,7 +7,7 @@
 
 from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ...types.trace import EvaluationLevel
-from ..evaluator import Evaluator
+from ..evaluator import DisclosureMode, Evaluator
 from .prompt_templates.failure_communication import get_template
 
 
@@ -47,12 +47,14 @@ def __init__(
         model: Model | str | None = None,
         system_prompt: str | None = None,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.version = version
         default_prompt = get_template(version).SYSTEM_PROMPT
         self.system_prompt = system_prompt if system_prompt is not None else default_prompt
         self.model = model
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def _build_output(self, rating: FailureCommunicationRating) -> list[EvaluationOutput]:
         normalized_score = self._score_mapping[rating.score]
@@ -67,16 +69,20 @@ def _build_output(self, rating: FailureCommunicationRating) -> list[EvaluationOu
 
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=FailureCommunicationRating)
         rating = cast(FailureCommunicationRating, result.structured_output)
         return self._build_output(rating)
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = await evaluator_agent.invoke_async(prompt, structured_output_model=FailureCommunicationRating)
         rating = cast(FailureCommunicationRating, result.structured_output)
         return self._build_output(rating)

src/strands_evals/evaluators/chaos/partial_completion_evaluator.py

@@ -6,7 +6,7 @@
 
 from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ...types.trace import EvaluationLevel
-from ..evaluator import Evaluator
+from ..evaluator import DisclosureMode, Evaluator
 from .prompt_templates.partial_completion import get_template
 
 
@@ -28,12 +28,14 @@ def __init__(
         model: Model | str | None = None,
         system_prompt: str | None = None,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.version = version
         default_prompt = get_template(version).SYSTEM_PROMPT
         self.system_prompt = system_prompt if system_prompt is not None else default_prompt
         self.model = model
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def _build_output(self, rating: PartialCompletionRating) -> list[EvaluationOutput]:
         return [
@@ -47,16 +49,20 @@ def _build_output(self, rating: PartialCompletionRating) -> list[EvaluationOutpu
 
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=PartialCompletionRating)
         rating = cast(PartialCompletionRating, result.structured_output)
         return self._build_output(rating)
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = await evaluator_agent.invoke_async(prompt, structured_output_model=PartialCompletionRating)
         rating = cast(PartialCompletionRating, result.structured_output)
         return self._build_output(rating)

src/strands_evals/evaluators/chaos/recovery_strategy_evaluator.py

@@ -7,7 +7,7 @@
 
 from ...types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ...types.trace import EvaluationLevel
-from ..evaluator import Evaluator
+from ..evaluator import DisclosureMode, Evaluator
 from .prompt_templates.recovery_strategy import get_template
 
 
@@ -47,12 +47,14 @@ def __init__(
         model: Model | str | None = None,
         system_prompt: str | None = None,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.version = version
         default_prompt = get_template(version).SYSTEM_PROMPT
         self.system_prompt = system_prompt if system_prompt is not None else default_prompt
         self.model = model
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def _build_output(self, rating: RecoveryStrategyRating) -> list[EvaluationOutput]:
         normalized_score = self._score_mapping[rating.score]
@@ -67,16 +69,20 @@ def _build_output(self, rating: RecoveryStrategyRating) -> list[EvaluationOutput
 
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=RecoveryStrategyRating)
         rating = cast(RecoveryStrategyRating, result.structured_output)
         return self._build_output(rating)
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = await evaluator_agent.invoke_async(prompt, structured_output_model=RecoveryStrategyRating)
         rating = cast(RecoveryStrategyRating, result.structured_output)
         return self._build_output(rating)

src/strands_evals/evaluators/coherence_evaluator.py

@@ -7,7 +7,8 @@
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ..types.trace import EvaluationLevel, TextContent, ToolExecution, TraceLevelInput
-from .evaluator import Evaluator
+from ._trace_index import TraceIndex
+from .evaluator import DisclosureMode, Evaluator
 from .prompt_templates.coherence import get_template
 
 
@@ -60,17 +61,21 @@ def __init__(
         system_prompt: str | None = None,
         include_inputs: bool = True,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT
         self.version = version
         self.model = model
         self.include_inputs = include_inputs
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=CoherenceRating)
         return self._create_evaluation_output(result)
 
@@ -86,18 +91,23 @@ def _create_evaluation_output(self, result) -> list[EvaluationOutput]:
             )
         ]
 
-    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+    def _format_prompt(self, parsed_input: TraceLevelInput, trace_index: TraceIndex | None = None) -> str:
         """Format evaluation prompt from parsed trace data.
 
         Args:
             parsed_input: Trace-level input containing agent response and session history
+            trace_index: When set, the previous turns are too large to inline, so the
+                paged trace-overview block replaces them and the judge reads spans
+                through the trace tools.
 
         Returns:
             Formatted prompt string with conversation history and target turn
         """
         parts = []
 
-        if parsed_input.session_history:
+        if trace_index is not None:
+            parts.append(f"# Previous turns:\n{self._disclosed_trace_section(trace_index)}")
+        elif parsed_input.session_history:
             history_lines = []
             for msg in parsed_input.session_history:
                 if isinstance(msg, list) and msg and isinstance(msg[0], ToolExecution):

src/strands_evals/evaluators/conciseness_evaluator.py

@@ -7,7 +7,7 @@
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ..types.trace import EvaluationLevel
-from .evaluator import Evaluator
+from .evaluator import DisclosureMode, Evaluator
 from .prompt_templates.conciseness import get_template
 
 
@@ -44,24 +44,30 @@ def __init__(
         system_prompt: str | None = None,
         include_inputs: bool = True,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.system_prompt = system_prompt or get_template(version).SYSTEM_PROMPT
         self.version = version
         self.model = model
         self.include_inputs = include_inputs
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=ConcisenessRating)
         return self._create_evaluation_output(result)
 
     async def evaluate_async(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[EvaluationOutput]:
         parsed_input = self._get_last_turn(evaluation_case)
-        prompt = self._format_trace_level_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_trace_level_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = await evaluator_agent.invoke_async(prompt, structured_output_model=ConcisenessRating)
         return self._create_evaluation_output(result)
 

src/strands_evals/evaluators/correctness_evaluator.py

@@ -7,7 +7,8 @@
 
 from ..types.evaluation import EvaluationData, EvaluationOutput, InputT, OutputT
 from ..types.trace import EvaluationLevel, TraceLevelInput
-from .evaluator import Evaluator
+from ._trace_index import TraceIndex
+from .evaluator import DisclosureMode, Evaluator
 from .prompt_templates.correctness import get_reference_template, get_template
 
 
@@ -76,6 +77,7 @@ def __init__(
         system_prompt: str | None = None,
         reference_system_prompt: str | None = None,
         name: str | None = None,
+        disclosure: DisclosureMode = "auto",
     ):
         super().__init__(name=name)
         self.system_prompt = system_prompt if system_prompt is not None else get_template(version).SYSTEM_PROMPT
@@ -86,6 +88,7 @@ def __init__(
         )
         self.version = version
         self.model = model
+        self.disclosure = self._validate_disclosure(disclosure)
 
     def _has_reference(self, evaluation_case: EvaluationData[InputT, OutputT]) -> bool:
         """Check if the evaluation case contains an expected_assertion for reference-based evaluation."""
@@ -97,12 +100,16 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva
         if self._has_reference(evaluation_case):
             return self._evaluate_with_reference(parsed_input, evaluation_case)
 
-        return self._evaluate_basic(parsed_input)
+        return self._evaluate_basic(parsed_input, evaluation_case)
 
-    def _evaluate_basic(self, parsed_input: TraceLevelInput) -> list[EvaluationOutput]:
+    def _evaluate_basic(
+        self, parsed_input: TraceLevelInput, evaluation_case: EvaluationData[InputT, OutputT]
+    ) -> list[EvaluationOutput]:
         """Evaluate correctness using the basic 3-level prompt."""
-        prompt = self._format_prompt(parsed_input)
-        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, callback_handler=None)
+        prompt, tools = self._render_with_disclosure(
+            evaluation_case, lambda idx: self._format_prompt(parsed_input, idx)
+        )
+        evaluator_agent = Agent(model=self.model, system_prompt=self.system_prompt, tools=tools, callback_handler=None)
         result = evaluator_agent(prompt, structured_output_model=CorrectnessRating)
         rating = cast(CorrectnessRating, result.structured_output)
         normalized_score = self._score_mapping[rating.score]
@@ -135,12 +142,12 @@ def _evaluate_with_reference(
             )
         ]
 
-    def _format_prompt(self, parsed_input: TraceLevelInput) -> str:
+    def _format_prompt(self, parsed_input: TraceLevelInput, trace_index: TraceIndex | None = None) -> str:
         """Format evaluation prompt for basic correctness evaluation."""
         parts = []
 
         # Format conversation context
-        parts.append(f"Context: {self._format_trace_level_prompt(parsed_input)}")
+        parts.append(f"Context: {self._format_trace_level_prompt(parsed_input, trace_index)}")
 
         # Format the candidate response (the assistant's last response)
         parts.append(f"Candidate Response: {parsed_input.agent_response.text}")