Harden evidence trace semantics

This commit is contained in:
aruo
2026-07-04 22:36:30 +08:00
parent 246c99b954
commit dc6cd32a67
24 changed files with 1096 additions and 141 deletions
@@ -24,6 +24,7 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -85,6 +86,73 @@ class ChatServiceSequentialAgentTest {
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-missing-verifier-session"
);
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-invalid-verifier-session"
);
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
}
@Test
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel("""
{
"verdict": "REJECT",
"groundedness_score": 0.0,
"critical_fact_count": 1,
"facts_checked": [
{
"fact": "payment timeout root cause",
"is_critical": true,
"verification": "contradicted",
"detail": "scripted contradiction",
"evidence_refs": []
}
],
"rationale": "scripted reject"
}
""");
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析订单支付超时的原因,并给出修复建议",
List.of(),
"sequential-reject-session"
);
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
}
@Test
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
ChatService chatService = createChatService();
@@ -174,7 +242,8 @@ class ChatServiceSequentialAgentTest {
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
private String promptText = "";
private boolean sawVerifierPrompt;
private final String verifierOutput;
private final java.util.List<String> verifierOutputs;
private int verifierOutputIndex;
private ScriptedChatModel() {
this("""
@@ -197,7 +266,11 @@ class ChatServiceSequentialAgentTest {
}
private ScriptedChatModel(String verifierOutput) {
this.verifierOutput = verifierOutput;
this.verifierOutputs = java.util.List.of(verifierOutput);
}
private ScriptedChatModel(String... verifierOutputs) {
this.verifierOutputs = java.util.List.of(verifierOutputs);
}
@Override
@@ -213,7 +286,9 @@ class ChatServiceSequentialAgentTest {
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
agentCalls.add("chat_verifier");
sawVerifierPrompt = true;
text = verifierOutput;
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
text = verifierOutputs.get(index);
verifierOutputIndex++;
} else {
text = "UNEXPECTED_PROMPT";
}