Harden evidence trace semantics
This commit is contained in:
@@ -24,6 +24,7 @@ import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -85,6 +86,73 @@ class ChatServiceSequentialAgentTest {
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因,并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-missing-verifier-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因,并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-invalid-verifier-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "REJECT",
|
||||
"groundedness_score": 0.0,
|
||||
"critical_fact_count": 1,
|
||||
"facts_checked": [
|
||||
{
|
||||
"fact": "payment timeout root cause",
|
||||
"is_critical": true,
|
||||
"verification": "contradicted",
|
||||
"detail": "scripted contradiction",
|
||||
"evidence_refs": []
|
||||
}
|
||||
],
|
||||
"rationale": "scripted reject"
|
||||
}
|
||||
""");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因,并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-reject-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
||||
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
@@ -174,7 +242,8 @@ class ChatServiceSequentialAgentTest {
|
||||
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
|
||||
private String promptText = "";
|
||||
private boolean sawVerifierPrompt;
|
||||
private final String verifierOutput;
|
||||
private final java.util.List<String> verifierOutputs;
|
||||
private int verifierOutputIndex;
|
||||
|
||||
private ScriptedChatModel() {
|
||||
this("""
|
||||
@@ -197,7 +266,11 @@ class ChatServiceSequentialAgentTest {
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String verifierOutput) {
|
||||
this.verifierOutput = verifierOutput;
|
||||
this.verifierOutputs = java.util.List.of(verifierOutput);
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String... verifierOutputs) {
|
||||
this.verifierOutputs = java.util.List.of(verifierOutputs);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -213,7 +286,9 @@ class ChatServiceSequentialAgentTest {
|
||||
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_verifier");
|
||||
sawVerifierPrompt = true;
|
||||
text = verifierOutput;
|
||||
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
|
||||
text = verifierOutputs.get(index);
|
||||
verifierOutputIndex++;
|
||||
} else {
|
||||
text = "UNEXPECTED_PROMPT";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user