feat(agent): add executor evidence output contract

2026-07-07 19:02:02 +08:00
parent 7f2e47ca38
commit 04eb50e2b4
19 changed files with 986 additions and 25 deletions
@@ -73,6 +73,41 @@ class DiagnosisTraceEvaluatorTest {
        assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
    }

+    @Test
+    void evaluateFailsWhenStructuredConfirmedClaimHasNoEvidenceBindings() {
+        DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
+                .id("structured-claim-case")
+                .title("Structured claim case")
+                .expectedRootCauseKeywords(List.of())
+                .requiredEvidenceTools(List.of())
+                .allowedVerdicts(List.of("LOW_CONFID", "PASS"))
+                .build();
+        DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
+                .session(DiagnosisTraceResponse.SessionTrace.builder()
+                        .answer("以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。")
+                        .selfEvaluation(java.util.Map.of(
+                                "verifier_evaluation", java.util.Map.of(
+                                        "verdict", "LOW_CONFID",
+                                        "executor_structured_output", java.util.Map.of(
+                                                "claims", java.util.List.of(java.util.Map.of(
+                                                        "claim_id", "claim-unsupported",
+                                                        "claim_text", "OOM 导致连接泄漏",
+                                                        "support_level", "direct",
+                                                        "evidence_bindings", java.util.List.of()
+                                                ))
+                                        )
+                                )))
+                        .build())
+                .toolInvocations(List.of())
+                .build();
+
+        DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
+
+        assertFalse(result.isPassed());
+        assertTrue(result.getFailedChecks().contains(
+                "executor confirmed claim missing evidence bindings: claim-unsupported"));
+    }
+
    @Test
    void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
        DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
@@ -0,0 +1,181 @@
+package com.superbiz.agent.hook;
+
+import com.alibaba.cloud.ai.graph.RunnableConfig;
+import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.superbiz.agent.service.ToolTraceSummaryService;
+import com.superbiz.agent.util.VerifierContextHolder;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.messages.Message;
+import org.springframework.ai.chat.messages.UserMessage;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class VerifierInputHookTest {
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    @AfterEach
+    void tearDown() {
+        VerifierContextHolder.clear();
+    }
+
+    @Test
+    void beforeModelAddsStructuredExecutorOutputWhenJsonContractIsValid() throws Exception {
+        ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
+        when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
+                Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
+        ));
+        VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
+        VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
+
+        String executorOutput = """
+                {
+                  "answer_version": "executor_evidence_v1",
+                  "diagnosis_summary": "连接池已满，但缺少泄漏证据。",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "source_id": "trace-1",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [101],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": ["缺少泄漏检测日志"],
+                  "user_facing_answer": "已确认连接池 active 达到上限。"
+                }
+                """;
+
+        AgentCommand command = hook.beforeModel(
+                List.of(new AssistantMessage(executorOutput)),
+                RunnableConfig.builder().addMetadata("sessionId", "structured-session").build()
+        );
+
+        JsonNode payload = readPayload(command);
+        assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
+        assertEquals("executor_evidence_v1",
+                payload.path("executor_structured_output").path("answer_version").asText());
+        assertEquals("连接池 active 达到上限",
+                payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
+        assertNotNull(VerifierContextHolder.getExecutorStructuredOutput());
+        assertEquals("valid", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
+    }
+
+    @Test
+    void beforeModelExtractsStructuredOutputFromPrefixedJsonFence() throws Exception {
+        ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
+        when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
+        VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
+
+        String executorOutput = """
+                现在我已经收集了足够的数据，最终输出如下。
+
+                ```json
+                {
+                  "answer_version": "executor_evidence_v1",
+                  "diagnosis_summary": "已确认连接池 active 达到上限。",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "source_id": "trace-1",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [101],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": [],
+                  "user_facing_answer": "已确认连接池 active 达到上限。"
+                }
+                ```
+                """;
+
+        AgentCommand command = hook.beforeModel(
+                List.of(new AssistantMessage(executorOutput)),
+                RunnableConfig.builder().addMetadata("sessionId", "fenced-session").build()
+        );
+
+        JsonNode payload = readPayload(command);
+        assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
+        assertEquals("claim-1",
+                payload.path("executor_structured_output").path("claims").get(0).path("claim_id").asText());
+    }
+
+    @Test
+    void beforeModelMarksMalformedJsonAndKeepsRawAnswerFallback() throws Exception {
+        ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
+        when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
+        VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
+
+        AgentCommand command = hook.beforeModel(
+                List.of(new AssistantMessage("{\"diagnosis_summary\":\"缺少 claims\"}")),
+                RunnableConfig.builder().addMetadata("sessionId", "malformed-session").build()
+        );
+
+        JsonNode payload = readPayload(command);
+        assertEquals("malformed", payload.path("executor_output_parse_status").path("status").asText());
+        assertTrue(payload.path("executor_structured_output").isNull());
+        assertEquals("{\"diagnosis_summary\":\"缺少 claims\"}", payload.path("executor_final_answer").asText());
+        assertEquals("malformed", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
+    }
+
+    @Test
+    void beforeModelMarksPlainTextAsMissingStructuredOutput() throws Exception {
+        ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
+        when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
+        VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
+
+        AgentCommand command = hook.beforeModel(
+                List.of(new AssistantMessage("普通自然语言答案")),
+                RunnableConfig.builder().addMetadata("sessionId", "plain-session").build()
+        );
+
+        JsonNode payload = readPayload(command);
+        assertEquals("missing", payload.path("executor_output_parse_status").path("status").asText());
+        assertTrue(payload.path("executor_structured_output").isNull());
+        assertFalse(payload.path("executor_final_answer").asText().isBlank());
+    }
+
+    private JsonNode readPayload(AgentCommand command) throws Exception {
+        var field = AgentCommand.class.getDeclaredField("messages");
+        field.setAccessible(true);
+        @SuppressWarnings("unchecked")
+        List<Message> messages = (List<Message>) field.get(command);
+        assertEquals(1, messages.size());
+        Message message = messages.get(0);
+        assertTrue(message instanceof UserMessage);
+        return objectMapper.readTree(((UserMessage) message).getText());
+    }
+}
@@ -226,6 +226,53 @@ class ChatServiceSequentialAgentTest {
        assertTrue(chatModel.sawVerifierPrompt);
    }

+    @Test
+    void verifierReceivesStructuredExecutorPayloadFields() throws Exception {
+        ChatService chatService = createChatService();
+        ScriptedChatModel chatModel = new ScriptedChatModel();
+        chatModel.executorOutput = """
+                {
+                  "answer_version": "executor_evidence_v1",
+                  "diagnosis_summary": "已确认连接池 active 达到上限。",
+                  "claims": [
+                    {
+                      "claim_id": "claim-1",
+                      "claim_type": "symptom",
+                      "claim_text": "连接池 active 达到上限",
+                      "support_level": "direct",
+                      "evidence_bindings": [
+                        {
+                          "source_type": "tool_trace",
+                          "source_id": "trace-1",
+                          "tool_name": "query_metrics",
+                          "source_invocation_ids": [101],
+                          "evidence_excerpt": "active=50 max=50"
+                        }
+                      ]
+                    }
+                  ],
+                  "hypotheses": [],
+                  "recommended_actions": [],
+                  "missing_info": [],
+                  "user_facing_answer": "已确认连接池 active 达到上限。"
+                }
+                """;
+
+        ChatService.ChatResult result = chatService.executeChatComplex(
+                chatModel,
+                new ToolCallback[0],
+                "请分析 MySQL 连接池耗尽",
+                List.of(),
+                "sequential-structured-executor-session"
+        );
+
+        assertEquals("已确认连接池 active 达到上限。", result.answer());
+        assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\""));
+        assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\""));
+        assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\""));
+        assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限"));
+    }
+
    @Test
    void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
        ChatService chatService = new ChatService();
@@ -353,6 +400,7 @@ class ChatServiceSequentialAgentTest {
        private String plannerPromptText = "";
        private String executorPromptText = "";
        private String verifierPromptText = "";
+        private String executorOutput = "EXECUTOR_FINAL_ANSWER";
        private boolean sawVerifierPrompt;
        private final java.util.List<String> verifierOutputs;
        private int verifierOutputIndex;
@@ -396,7 +444,7 @@ class ChatServiceSequentialAgentTest {
            } else if (promptText.contains("EXECUTOR_TEST_PROMPT")) {
                agentCalls.add("chat_executor");
                executorPromptText = promptText;
-                text = "EXECUTOR_FINAL_ANSWER";
+                text = executorOutput;
            } else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
                agentCalls.add("chat_verifier");
                verifierPromptText = promptText;