Merge branch 'emdash/shy-items-fry-f4zze' into refactor/mvp1.0
# Conflicts: # mvp/issues/README.md
This commit is contained in:
aruo
@@ -0,0 +1,140 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisEvalBaselineDiffTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final DiagnosisEvalBaselineDiffer differ = new DiagnosisEvalBaselineDiffer();
|
||||
|
||||
@Test
|
||||
void compareReportsDetectsAggregateAndCaseRegressions() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
degradeRedisCase(current);
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
|
||||
assertTrue(diff.isHasRegression());
|
||||
assertEquals(6, diff.getRegressionCount());
|
||||
assertEquals(2, diff.getChangedCount());
|
||||
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "passRate"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "averageToolCallCount"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "passed"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "verdict"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "matchedKeywordCount"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "evidenceCoverage.query_logs"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.LOW_CONFID"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.REJECT"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void compareReportsDetectsMissingAndNewCases() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
DiagnosisEvalResult removed = current.getResults().remove(0);
|
||||
current.getResults().add(DiagnosisEvalResult.builder()
|
||||
.caseId("new-case")
|
||||
.title("New case")
|
||||
.passed(true)
|
||||
.failedChecks(List.of())
|
||||
.verdict("PASS")
|
||||
.matchedKeywordCount(1)
|
||||
.requiredKeywordCount(1)
|
||||
.evidenceCoverage(new LinkedHashMap<>())
|
||||
.toolCallCount(1)
|
||||
.durationMs(1000)
|
||||
.build());
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", removed.getCaseId(), "casePresence"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "case", "new-case", "casePresence"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void compareSameReportHasNoDiff() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, readBaselineReport());
|
||||
|
||||
assertFalse(diff.isHasRegression());
|
||||
assertEquals(0, diff.getRegressionCount());
|
||||
assertTrue(diff.getItems().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void writerOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
degradeRedisCase(current);
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
DiagnosisEvalDiffReportWriter writer = new DiagnosisEvalDiffReportWriter(objectMapper);
|
||||
|
||||
Path json = tempDir.resolve("baseline-diff.json");
|
||||
Path markdown = tempDir.resolve("baseline-diff.md");
|
||||
writer.writeJson(diff, json);
|
||||
writer.writeMarkdown(diff, markdown);
|
||||
|
||||
assertTrue(Files.exists(json));
|
||||
assertTrue(Files.readString(json).contains("\"hasRegression\" : true"));
|
||||
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Baseline Diff"));
|
||||
assertTrue(Files.readString(markdown).contains("redis-timeout"));
|
||||
}
|
||||
|
||||
private DiagnosisEvalReport readBaselineReport() throws Exception {
|
||||
return objectMapper.readValue(Path.of("mvp/eval/reports/baseline-report.json").toFile(),
|
||||
DiagnosisEvalReport.class);
|
||||
}
|
||||
|
||||
private void degradeRedisCase(DiagnosisEvalReport report) {
|
||||
report.setPassedCases(4);
|
||||
report.setPassRate(0.8);
|
||||
report.setAverageToolCallCount(3.0);
|
||||
report.setAverageDurationMs(45800.0);
|
||||
report.setVerdictDistribution(new LinkedHashMap<>());
|
||||
report.getVerdictDistribution().put("PASS", 2L);
|
||||
report.getVerdictDistribution().put("LOW_CONFID", 2L);
|
||||
report.getVerdictDistribution().put("REJECT", 1L);
|
||||
|
||||
DiagnosisEvalResult redis = result(report, "redis-timeout");
|
||||
redis.setPassed(false);
|
||||
redis.setFailedChecks(new ArrayList<>(List.of("missing required evidence tool: query_logs")));
|
||||
redis.setVerdict("REJECT");
|
||||
redis.setMatchedKeywordCount(1);
|
||||
redis.getEvidenceCoverage().put("query_logs", false);
|
||||
redis.setToolCallCount(1);
|
||||
redis.setDurationMs(36000);
|
||||
}
|
||||
|
||||
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
|
||||
return report.getResults().stream()
|
||||
.filter(item -> caseId.equals(item.getCaseId()))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
private boolean hasItem(DiagnosisEvalDiffReport diff,
|
||||
String type,
|
||||
String scope,
|
||||
String caseId,
|
||||
String metric) {
|
||||
return diff.getItems().stream().anyMatch(item ->
|
||||
type.equals(item.getType())
|
||||
&& scope.equals(item.getScope())
|
||||
&& java.util.Objects.equals(caseId, item.getCaseId())
|
||||
&& metric.equals(item.getMetric()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisTraceEvaluatorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
|
||||
|
||||
@Test
|
||||
void evaluateFixtureReportsFullBaseline() {
|
||||
List<DiagnosisEvalCase> cases = readCases();
|
||||
|
||||
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
|
||||
|
||||
assertEquals(5, report.getTotalCases());
|
||||
assertEquals(5, report.getPassedCases());
|
||||
assertEquals(1.0, report.getPassRate(), 0.001);
|
||||
assertEquals(2L, report.getVerdictDistribution().get("PASS"));
|
||||
assertEquals(3L, report.getVerdictDistribution().get("LOW_CONFID"));
|
||||
|
||||
DiagnosisEvalResult payment = result(report, "payment-timeout");
|
||||
assertTrue(payment.isPassed());
|
||||
assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
|
||||
assertTrue(payment.getEvidenceCoverage().get("query_logs"));
|
||||
assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
|
||||
|
||||
DiagnosisEvalResult redis = result(report, "redis-timeout");
|
||||
assertTrue(redis.isPassed());
|
||||
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyFixedCaseReferencesExistingFixture() {
|
||||
for (DiagnosisEvalCase evalCase : readCases()) {
|
||||
Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture());
|
||||
assertTrue(Files.exists(fixture), "missing fixture: " + fixture);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateRejectRequiresDegradedOutput() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("reject-case")
|
||||
.title("Reject case")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("REJECT"))
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("EXECUTOR_FINAL_ANSWER")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of("verdict", "REJECT")))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
|
||||
DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
|
||||
DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper);
|
||||
|
||||
Path json = tempDir.resolve("eval-report.json");
|
||||
Path markdown = tempDir.resolve("eval-report.md");
|
||||
writer.writeJson(report, json);
|
||||
writer.writeMarkdown(report, markdown);
|
||||
|
||||
assertTrue(Files.exists(json));
|
||||
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
|
||||
assertTrue(Files.readString(markdown).contains("payment-timeout"));
|
||||
assertEquals(
|
||||
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))),
|
||||
comparableReportText(Files.readString(json)));
|
||||
assertEquals(
|
||||
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))),
|
||||
comparableReportText(Files.readString(markdown)));
|
||||
}
|
||||
|
||||
private List<DiagnosisEvalCase> readCases() {
|
||||
try {
|
||||
return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json"));
|
||||
} catch (Exception e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
|
||||
return report.getResults().stream()
|
||||
.filter(item -> caseId.equals(item.getCaseId()))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
private String comparableReportText(String value) {
|
||||
return value.replace("\r\n", "\n").stripTrailing();
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
@@ -86,6 +87,73 @@ class ChatServiceSequentialAgentTest {
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-missing-verifier-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-invalid-verifier-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "REJECT",
|
||||
"groundedness_score": 0.0,
|
||||
"critical_fact_count": 1,
|
||||
"facts_checked": [
|
||||
{
|
||||
"fact": "payment timeout root cause",
|
||||
"is_critical": true,
|
||||
"verification": "contradicted",
|
||||
"detail": "scripted contradiction",
|
||||
"evidence_refs": []
|
||||
}
|
||||
],
|
||||
"rationale": "scripted reject"
|
||||
}
|
||||
""");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-reject-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
||||
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
@@ -178,7 +246,8 @@ class ChatServiceSequentialAgentTest {
|
||||
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
|
||||
private String promptText = "";
|
||||
private boolean sawVerifierPrompt;
|
||||
private final String verifierOutput;
|
||||
private final java.util.List<String> verifierOutputs;
|
||||
private int verifierOutputIndex;
|
||||
|
||||
private ScriptedChatModel() {
|
||||
this("""
|
||||
@@ -201,7 +270,11 @@ class ChatServiceSequentialAgentTest {
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String verifierOutput) {
|
||||
this.verifierOutput = verifierOutput;
|
||||
this.verifierOutputs = java.util.List.of(verifierOutput);
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String... verifierOutputs) {
|
||||
this.verifierOutputs = java.util.List.of(verifierOutputs);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -217,7 +290,9 @@ class ChatServiceSequentialAgentTest {
|
||||
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_verifier");
|
||||
sawVerifierPrompt = true;
|
||||
text = verifierOutput;
|
||||
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
|
||||
text = verifierOutputs.get(index);
|
||||
verifierOutputIndex++;
|
||||
} else {
|
||||
text = "UNEXPECTED_PROMPT";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ToolInvocationRecorderTest {
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolPreservesNoEvidenceSemantics() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("recorder-test-session");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_logs",
|
||||
Map.of("query", "timeout"),
|
||||
"{\"success\":false,\"message\":\"未找到匹配的日志\"}",
|
||||
true,
|
||||
System.currentTimeMillis() - 10,
|
||||
null,
|
||||
"application-logs",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
|
||||
Map.of("log_topic", "application-logs")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
|
||||
assertEquals("query_logs", saved.getToolName());
|
||||
assertEquals(Boolean.TRUE, saved.getSuccess());
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"no_evidence\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("lookup-recorder-session");
|
||||
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
|
||||
.query("ERR_TIMEOUT")
|
||||
.outputPreview("matched payment doc")
|
||||
.outputLength(18)
|
||||
.retrievalLayer("L0")
|
||||
.l0MatchCount(1)
|
||||
.l1MatchCount(null)
|
||||
.truncated(false)
|
||||
.relevanceLevel("PRECISE")
|
||||
.completenessHint("already precise")
|
||||
.domain("payment")
|
||||
.dedupReason("doc_retrieved")
|
||||
.durationMs(42)
|
||||
.success(true)
|
||||
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED)
|
||||
.l0Titles(List.of("payment/errors.md"))
|
||||
.build();
|
||||
|
||||
try {
|
||||
recorder.recordLookupKnowledge(record);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
|
||||
assertEquals("lookup_knowledge", saved.getToolName());
|
||||
assertEquals("PRECISE", saved.getRelevanceLevel());
|
||||
assertEquals("doc_retrieved", saved.getDedupReason());
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"deduped\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"payment\"]"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ToolTraceSummaryServiceTest {
|
||||
|
||||
@Test
|
||||
void buildVerifierTraceSummaryTreatsNoEvidenceAsGapWithoutLosingSuccessfulEvidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
ToolInvocation.builder()
|
||||
.id(1L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"timeout\"}")
|
||||
.outputPreview("payment timeout stack trace")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
|
||||
.success(true)
|
||||
.build(),
|
||||
ToolInvocation.builder()
|
||||
.id(2L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"timeout\"}")
|
||||
.outputPreview("{\"success\":false,\"message\":\"未找到匹配的日志\"}")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"no_evidence\"}")
|
||||
.success(true)
|
||||
.build(),
|
||||
ToolInvocation.builder()
|
||||
.id(3L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_metrics")
|
||||
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
|
||||
.errorMessage("prometheus timeout")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"failed\"}")
|
||||
.success(false)
|
||||
.build()
|
||||
));
|
||||
|
||||
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
|
||||
|
||||
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-1", "application-logs point to timeout");
|
||||
|
||||
assertEquals(2, summaries.size());
|
||||
|
||||
Map<String, Object> logsSummary = summaries.stream()
|
||||
.filter(item -> "query_logs".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertEquals(Boolean.TRUE, logsSummary.get("success"));
|
||||
assertEquals("direct", logsSummary.get("evidence_level"));
|
||||
assertEquals(2, logsSummary.get("invocation_count"));
|
||||
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
|
||||
|
||||
Map<String, Object> metricsSummary = summaries.stream()
|
||||
.filter(item -> "query_metrics".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertEquals(Boolean.FALSE, metricsSummary.get("success"));
|
||||
assertEquals("none", metricsSummary.get("evidence_level"));
|
||||
assertEquals(1, metricsSummary.get("failed_invocation_count"));
|
||||
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package com.superbiz.agent.tool;
|
||||
import com.superbiz.agent.dto.KnowledgeEntry;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
import com.superbiz.agent.service.KnowledgeIndexService;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InjectMocks;
|
||||
@@ -28,6 +30,15 @@ class LookupKnowledgeToolTest {
|
||||
@Mock
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
@Mock
|
||||
private ToolInvocationRecorder toolInvocationRecorder;
|
||||
|
||||
@Mock
|
||||
private RetrievedDocTracker retrievedDocTracker;
|
||||
|
||||
@Mock
|
||||
private ObjectMapper objectMapper;
|
||||
|
||||
@InjectMocks
|
||||
private LookupKnowledgeTool tool;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user