From f1585179ef51fa1bf223dbeaeb8d7412aeb997fa Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Mon, 13 Jul 2026 22:30:16 -0700 Subject: [PATCH] feat: Resolve Interactions API data for agent trace display in eval results. PiperOrigin-RevId: 947443975 --- agentplatform/_genai/_evals_common.py | 157 ++++++++++ .../genai/replays/test_evaluate.py | 51 +++ tests/unit/agentplatform/genai/test_evals.py | 291 ++++++++++++++++++ 3 files changed, 499 insertions(+) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index 249a81c2b2..c5d14ac6c3 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -64,6 +64,7 @@ MAX_WORKERS = 100 AGENT_MAX_WORKERS = 20 +_MAX_INTERACTION_CHAIN_DEPTH = 10 CONTENT = _evals_constant.CONTENT PARTS = _evals_constant.PARTS USER_AUTHOR = _evals_constant.USER_AUTHOR @@ -1007,6 +1008,157 @@ def _build_interaction_id_dataset( return types.EvaluationDataset(eval_cases=eval_cases) +def _has_interactions_data_source( + eval_cases: list[types.EvalCase], +) -> bool: + """Returns True if any EvalCase has interactions_data_source set.""" + return any(case.interactions_data_source is not None for case in eval_cases) + + +def _resolve_interactions_to_eval_cases( + api_client: BaseApiClient, + eval_cases: list[types.EvalCase], +) -> list[types.EvalCase]: + """Resolves EvalCases with interactions_data_source to agent_data. + + For each EvalCase that has interactions_data_source set, fetches the + Interaction via the SDK's interactions.get() API, converts the steps + to AgentData, and returns a new EvalCase with agent_data populated. + + Args: + api_client: The API client (must have an interactions module). + eval_cases: EvalCases with interactions_data_source set. + + Returns: + New list of EvalCases with agent_data populated from resolved + interactions. + + Raises: + ValueError: If eval_cases have missing interaction references. + """ + # Validate all cases up front before making any API calls. + for case in eval_cases: + ids = case.interactions_data_source + if ids is None: + raise ValueError( + "All eval_cases must have interactions_data_source set when" + " using interaction resolution. Found a case without it. Do" + " not mix interaction-based and prompt-based eval cases." + ) + if not ids.interaction: + raise ValueError( + "interactions_data_source.interaction is required. Each" + " EvalCase must reference an existing Interaction resource." + ) + + resolved_cases = [] + + for case in eval_cases: + if case.agent_data: + resolved_cases.append(case) + continue + ids = case.interactions_data_source + + # Extract the interaction short ID from the resource name. + # Handles both full resource names (projects/.../interactions/{id}) + # and bare IDs by always taking the last path component. + interaction_id = ids.interaction.split("/")[-1] + + logger.info("Fetching interaction: %s", ids.interaction) + + current_interaction_id = interaction_id + interactions = [] + seen_ids = set() + for _ in range(_MAX_INTERACTION_CHAIN_DEPTH): + if current_interaction_id in seen_ids: + break + seen_ids.add(current_interaction_id) + path = f"interactions/{current_interaction_id}" + response = api_client.request("get", path, {}, None) + if not response.body: + if not interactions: + logger.warning( + "Empty response fetching interaction %s.", + ids.interaction, + ) + break + interaction_dict = json.loads(response.body) + try: + typed_interaction = interaction_types.Interaction.model_validate( + interaction_dict + ) + except Exception as e: + logger.warning("Failed to validate interaction model: %s", e) + break + + interactions.append(typed_interaction) + if not typed_interaction.previous_interaction_id: + break + current_interaction_id = typed_interaction.previous_interaction_id.split( + "/" + )[-1] + + if not interactions: + agent_data = types.evals.AgentData(turns=[]) # Fallback + else: + interactions.reverse() # chronological order + all_steps = [] + for i_typed in interactions: + all_steps.extend(i_typed.steps or []) + + combined_interaction = interactions[-1].model_dump() + combined_interaction["steps"] = all_steps + agent_data = _interaction_dict_to_agent_data(combined_interaction) + + # Best-effort: fetch the agent config (instruction, tools, + # description) from the Agent API so the display can render + # the System Topology section. + gemini_cfg = ids.gemini_agent_config + agent_name = gemini_cfg.gemini_agent if gemini_cfg else None + agent_config = _fetch_agent_config_dict(api_client, agent_name or "") + agent_data.agents = {agent_config.agent_id: agent_config} + + # Merge consecutive text events and parts so multi-paragraph + # responses render as a single block in the trace display. + _merge_text_parts_in_agent_data(agent_data) + + # Preserve all original EvalCase fields; only update agent_data + # and clear the now-resolved interactions_data_source. + resolved_cases.append( + case.model_copy( + update={ + "agent_data": agent_data, + "interactions_data_source": None, + } + ) + ) + + return resolved_cases + + +def _resolve_interactions_for_display( + api_client: BaseApiClient, + dataset_list: list[types.EvaluationDataset], +) -> list[types.EvaluationDataset]: + """Resolves Interaction traces for visualization.""" + resolved_datasets = [] + for dataset in dataset_list: + if dataset.eval_cases and _has_interactions_data_source(dataset.eval_cases): + try: + resolved_cases = _resolve_interactions_to_eval_cases( + api_client, dataset.eval_cases + ) + resolved_datasets.append( + dataset.model_copy(update={"eval_cases": resolved_cases}) + ) + except Exception as e: + logger.warning("Failed to resolve interactions for display: %s", e) + resolved_datasets.append(dataset) + else: + resolved_datasets.append(dataset) + return resolved_datasets + + def _add_evaluation_run_labels( labels: Optional[dict[str, str]] = None, agent: Optional[str] = None, @@ -2349,6 +2501,11 @@ def _execute_evaluation( # type: ignore[no-untyped-def] t2 = time.perf_counter() logger.info("Evaluation took: %f seconds", t2 - t1) + # Resolve interactions_data_source to agent_data for display. + # This fetches Interaction trace data client-side so that show() can + # render the System Topology and Conversation Trace sections. + dataset_list = _resolve_interactions_for_display(api_client, dataset_list) + evaluation_result.evaluation_dataset = dataset_list evaluation_result.agent_info = validated_agent_info diff --git a/tests/unit/agentplatform/genai/replays/test_evaluate.py b/tests/unit/agentplatform/genai/replays/test_evaluate.py index fe25760b06..66e908076c 100644 --- a/tests/unit/agentplatform/genai/replays/test_evaluate.py +++ b/tests/unit/agentplatform/genai/replays/test_evaluate.py @@ -609,6 +609,57 @@ def test_evaluation_with_interaction_id(client): assert len(evaluation_result.eval_case_results) == 1 +def test_evaluation_with_interaction_resolution(client): + """Tests evaluate() resolves interaction trace + agent config for display.""" + interaction_id = "ChAxMDE5YTgzOGU3Mjg4NDE4EAgaATAqBG1haW4" + agent = ( + "projects/model-evaluation-dev/locations/global/agents/" + "async-scrape-test-fe111cc6" + ) + eval_dataset = types.EvaluationDataset( + eval_cases=[ + types.EvalCase( + interactions_data_source=types.InteractionsDataSource( + interaction=( + "projects/model-evaluation-dev/locations/global" + f"/interactions/{interaction_id}" + ), + gemini_agent_config=types.GeminiAgentConfig( + gemini_agent=agent, + ), + ), + ) + ] + ) + + evaluation_result = client.evals.evaluate( + dataset=eval_dataset, + metrics=[types.RubricMetric.MULTI_TURN_TASK_SUCCESS], + ) + + assert isinstance(evaluation_result, types.EvaluationResult) + assert evaluation_result.summary_metrics is not None + assert len(evaluation_result.summary_metrics) > 0 + + assert evaluation_result.eval_case_results is not None + assert len(evaluation_result.eval_case_results) == 1 + + # Verify the interaction was resolved for display: agent_data should + # be populated with conversation trace and agent topology. + eval_dataset_list = evaluation_result.evaluation_dataset + assert eval_dataset_list is not None + assert len(eval_dataset_list) > 0 + resolved_case = eval_dataset_list[0].eval_cases[0] + assert resolved_case.agent_data is not None + # Conversation trace: at least one turn with events. + assert resolved_case.agent_data.turns is not None + assert len(resolved_case.agent_data.turns) > 0 + assert len(resolved_case.agent_data.turns[0].events) > 0 + # Agent topology: agents map populated with agent config. + assert resolved_case.agent_data.agents is not None + assert len(resolved_case.agent_data.agents) > 0 + + pytestmark = pytest_helper.setup( file=__file__, globals_for_file=globals(), diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 803a03fde0..95ad563599 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -10086,6 +10086,297 @@ def test_create_evaluation_run_agent_engine_does_not_set_gemini(self): assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE +class TestResolveInteractionsForDisplay: + """Tests for _resolve_interactions_for_display.""" + + def _make_api_response(self, body_dict): + """Create a mock API response with a JSON body.""" + resp = mock.MagicMock() + resp.body = json.dumps(body_dict) + return resp + + def test_no_interactions_data_source_is_noop(self): + """Cases without interactions_data_source are returned as-is.""" + case = agentplatform_genai_types.EvalCase( + prompt=genai_types.Content( + parts=[genai_types.Part(text="hello")] + ), + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + mock_api_client = mock.MagicMock() + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + assert result[0].eval_cases[0] is case + mock_api_client.request.assert_not_called() + + def test_resolves_interaction_to_agent_data(self): + """When interactions_data_source is present, agent_data is populated.""" + case = agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction=( + "projects/p/locations/l/interactions/test-id" + ), + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/my-agent", + ), + ) + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + + interaction_json = { + "status": "completed", + "steps": [ + { + "type": "user_input", + "content": [{"type": "text", "text": "hello"}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "hi there"}], + }, + ] + } + agent_json = {"system_instruction": "Be helpful"} + + def mock_request(method, path, *args, **kwargs): + if "interactions/" in path: + return self._make_api_response(interaction_json) + if "agents/" in path: + return self._make_api_response(agent_json) + return self._make_api_response({}) + + mock_api_client = mock.MagicMock() + mock_api_client.request.side_effect = mock_request + + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + resolved_case = result[0].eval_cases[0] + assert resolved_case.agent_data is not None + assert "my-agent" in resolved_case.agent_data.agents + + # Verify the interaction and agent were fetched. + assert mock_api_client.request.call_count == 2 + mock_api_client.request.assert_any_call( + "get", "interactions/test-id", {}, None + ) + mock_api_client.request.assert_any_call( + "get", "agents/my-agent", {}, None + ) + + def test_agents_map_populated_with_full_config(self): + """The agents dict includes instruction and tools from the Agent API.""" + case = agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/i1", + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/weather-bot", + ), + ) + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + interaction_json = {"status": "completed", "steps": []} + agent_json = { + "system_instruction": "You are a weather assistant.", + "description": "Helps with weather queries.", + "base_agent": "gemini-2.0-flash", + "tools": [ + {"type": "code_execution"}, + {"type": "google_search"}, + { + "type": "function", + "function_declarations": [ + {"name": "get_weather", "description": "Get weather"} + ], + }, + ], + } + + def mock_request(method, path, *args, **kwargs): + if "interactions/" in path: + return self._make_api_response(interaction_json) + if "agents/" in path: + return self._make_api_response(agent_json) + return self._make_api_response({}) + + mock_api_client = mock.MagicMock() + mock_api_client.request.side_effect = mock_request + + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + resolved_case = result[0].eval_cases[0] + agent_cfg = resolved_case.agent_data.agents["weather-bot"] + assert agent_cfg.agent_id == "weather-bot" + assert agent_cfg.instruction == "You are a weather assistant." + assert agent_cfg.description == "Helps with weather queries." + assert agent_cfg.agent_type == "gemini-2.0-flash" + # Built-in tools are mapped to typed Tool objects; function tool also included. + assert agent_cfg.tools is not None + assert len(agent_cfg.tools) == 3 + assert any(t.code_execution is not None for t in agent_cfg.tools) + assert any(t.google_search is not None for t in agent_cfg.tools) + assert any(t.function_declarations is not None for t in agent_cfg.tools) + + def test_consecutive_model_output_steps_merged(self): + """Multiple consecutive model_output steps are merged into one event.""" + case = agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/i1", + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/a", + ), + ) + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + # Simulate multiple model_output steps (one per paragraph). + interaction_json = { + "status": "completed", + "steps": [ + { + "type": "model_output", + "content": [{"type": "text", "text": "Paragraph 1."}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Paragraph 2."}], + }, + { + "type": "model_output", + "content": [{"type": "text", "text": "Paragraph 3."}], + }, + ] + } + + def mock_request(method, path, *args, **kwargs): + if "interactions/" in path: + return self._make_api_response(interaction_json) + return self._make_api_response({}) + + mock_api_client = mock.MagicMock() + mock_api_client.request.side_effect = mock_request + + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + resolved_case = result[0].eval_cases[0] + events = resolved_case.agent_data.turns[0].events + # All three model_output steps should be merged into one event. + assert len(events) == 1 + text_parts = [ + p for p in events[0].content.parts if hasattr(p, "text") and p.text + ] + assert len(text_parts) == 1 + assert "Paragraph 1." in text_parts[0].text + assert "Paragraph 2." in text_parts[0].text + assert "Paragraph 3." in text_parts[0].text + + def test_existing_agent_data_not_overwritten(self): + """Cases that already have agent_data are not resolved again.""" + existing_ad = agentplatform_genai_types.evals.AgentData( + turns=[agentplatform_genai_types.evals.ConversationTurn( + turn_index=0, events=[] + )], + agents={"existing": agentplatform_genai_types.evals.AgentConfig( + agent_id="existing" + )}, + ) + case = agentplatform_genai_types.EvalCase( + agent_data=existing_ad, + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/test-id", + ), + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + + mock_api_client = mock.MagicMock() + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + # Should not have called request since agent_data already exists. + mock_api_client.request.assert_not_called() + assert result[0].eval_cases[0].agent_data is existing_ad + + def test_interaction_fetch_failure_returns_original(self): + """If fetching the interaction fails, the original case is returned.""" + case = agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/bad-id", + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/my-agent", + ), + ) + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + + mock_api_client = mock.MagicMock() + mock_api_client.request.side_effect = RuntimeError("API error") + + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + # Original case returned (no agent_data). + resolved_case = result[0].eval_cases[0] + assert resolved_case.agent_data is None + + def test_agent_fetch_failure_still_shows_trace(self): + """If fetching the agent config fails, trace is still populated.""" + case = agentplatform_genai_types.EvalCase( + interactions_data_source=agentplatform_genai_types.InteractionsDataSource( + interaction="projects/p/locations/l/interactions/i1", + gemini_agent_config=agentplatform_genai_types.GeminiAgentConfig( + gemini_agent="projects/p/locations/l/agents/bad-agent", + ), + ) + ) + dataset = agentplatform_genai_types.EvaluationDataset( + eval_cases=[case] + ) + interaction_json = { + "status": "completed", + "steps": [ + { + "type": "user_input", + "content": [{"type": "text", "text": "hi"}], + }, + ] + } + + def mock_request(method, path, *args, **kwargs): + if "interactions/" in path: + return self._make_api_response(interaction_json) + if "agents/" in path: + raise RuntimeError("Agent not found") + return self._make_api_response({}) + + mock_api_client = mock.MagicMock() + mock_api_client.request.side_effect = mock_request + + result = _evals_common._resolve_interactions_for_display( + mock_api_client, [dataset] + ) + resolved_case = result[0].eval_cases[0] + # Trace should still be populated even though agent config failed. + assert resolved_case.agent_data is not None + assert len(resolved_case.agent_data.turns) == 1 + # Agent config is minimal (just agent_id, no instruction/tools). + agent_cfg = resolved_case.agent_data.agents["bad-agent"] + assert agent_cfg.agent_id == "bad-agent" + assert agent_cfg.instruction is None + class TestStepToAgentEvent: """Tests for _step_to_agent_event using typed GenAI SDK step objects."""