refactor(harness): remove legacy agent architecture
This commit is contained in:
zhuyongxin
@@ -2,14 +2,12 @@ package com.superbiz.agent.agent.tool;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class QueryLogsToolsTest {
|
||||
|
||||
@@ -17,7 +15,7 @@ class QueryLogsToolsTest {
|
||||
|
||||
@Test
|
||||
void queryLogsReturnsHikariPositiveMockForOrderService() throws Exception {
|
||||
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
|
||||
QueryLogsTools tools = new QueryLogsTools();
|
||||
ReflectionTestUtils.setField(tools, "mockEnabled", true);
|
||||
|
||||
String output = tools.queryLogs("ap-guangzhou", "application-logs",
|
||||
@@ -33,14 +31,14 @@ class QueryLogsToolsTest {
|
||||
|
||||
@Test
|
||||
void queryLogsReturnsEmptyNoHitForOtherServiceHikariQuery() throws Exception {
|
||||
QueryLogsTools tools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
|
||||
QueryLogsTools tools = new QueryLogsTools();
|
||||
ReflectionTestUtils.setField(tools, "mockEnabled", true);
|
||||
|
||||
String output = tools.queryLogs("ap-guangzhou", "application-logs",
|
||||
"inventory-service HikariCP connection pool active=50/50 waiting", 10);
|
||||
|
||||
JsonNode root = objectMapper.readTree(output);
|
||||
assertFalse(root.path("success").asBoolean());
|
||||
assertTrue(root.path("success").asBoolean());
|
||||
assertEquals(0, root.path("logs").size());
|
||||
assertEquals(0, root.path("total").asInt());
|
||||
assertFalse(root.toString().contains("generic-service"));
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
package com.superbiz.agent.config;
|
||||
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
|
||||
import com.superbiz.agent.harness.application.persistence.ChatRunStore;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.mysql.MysqlSqlValidator;
|
||||
import com.superbiz.agent.harness.tool.mysql.MysqlSecurityException;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.tool.RetrievedDocTracker;
|
||||
import com.superbiz.agent.service.KnowledgeContextPacker;
|
||||
import com.superbiz.agent.service.KnowledgeDocumentRetriever;
|
||||
import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
@@ -75,8 +74,7 @@ class HarnessChatConfigurationTest {
|
||||
.withBean(KnowledgeEvidencePostProcessor.class, () -> mock(KnowledgeEvidencePostProcessor.class))
|
||||
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
|
||||
.withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class))
|
||||
.withBean(ToolInvocationRecorder.class, () -> mock(ToolInvocationRecorder.class))
|
||||
.withBean(RetrievedDocTracker.class, () -> mock(RetrievedDocTracker.class))
|
||||
.withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop)
|
||||
.withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class))
|
||||
.withBean(ChatRunStore.class, () -> mock(ChatRunStore.class))
|
||||
.withBean(MysqlToolProperties.class, MysqlToolProperties::new)
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
package com.superbiz.agent.controller;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class AiOpsControllerTest {
|
||||
|
||||
@Test
|
||||
void metadataMessageCarriesSessionAndRunId() {
|
||||
AiOpsController.SseMessage message = AiOpsController.SseMessage.metadata("session-1", "run-1");
|
||||
|
||||
assertEquals("metadata", message.getType());
|
||||
assertEquals("session-1", message.getSessionId());
|
||||
assertEquals("run-1", message.getRunId());
|
||||
}
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisEvalBaselineDiffTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final DiagnosisEvalBaselineDiffer differ = new DiagnosisEvalBaselineDiffer();
|
||||
|
||||
@Test
|
||||
void compareReportsDetectsAggregateAndCaseRegressions() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
degradeRedisCase(current);
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
|
||||
assertTrue(diff.isHasRegression());
|
||||
assertEquals(6, diff.getRegressionCount());
|
||||
assertEquals(2, diff.getChangedCount());
|
||||
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "passRate"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "averageToolCallCount"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "passed"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "verdict"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "matchedKeywordCount"));
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "evidenceCoverage.query_logs"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.LOW_CONFID"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.REJECT"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void compareReportsDetectsMissingAndNewCases() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
DiagnosisEvalResult removed = current.getResults().remove(0);
|
||||
current.getResults().add(DiagnosisEvalResult.builder()
|
||||
.caseId("new-case")
|
||||
.title("New case")
|
||||
.passed(true)
|
||||
.failedChecks(List.of())
|
||||
.verdict("PASS")
|
||||
.matchedKeywordCount(1)
|
||||
.requiredKeywordCount(1)
|
||||
.evidenceCoverage(new LinkedHashMap<>())
|
||||
.toolCallCount(1)
|
||||
.durationMs(1000)
|
||||
.build());
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
|
||||
assertTrue(hasItem(diff, "REGRESSION", "case", removed.getCaseId(), "casePresence"));
|
||||
assertTrue(hasItem(diff, "CHANGED", "case", "new-case", "casePresence"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void compareSameReportHasNoDiff() throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, readBaselineReport());
|
||||
|
||||
assertFalse(diff.isHasRegression());
|
||||
assertEquals(0, diff.getRegressionCount());
|
||||
assertTrue(diff.getItems().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void writerOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
|
||||
DiagnosisEvalReport baseline = readBaselineReport();
|
||||
DiagnosisEvalReport current = readBaselineReport();
|
||||
degradeRedisCase(current);
|
||||
DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
|
||||
DiagnosisEvalDiffReportWriter writer = new DiagnosisEvalDiffReportWriter(objectMapper);
|
||||
|
||||
Path json = tempDir.resolve("baseline-diff.json");
|
||||
Path markdown = tempDir.resolve("baseline-diff.md");
|
||||
writer.writeJson(diff, json);
|
||||
writer.writeMarkdown(diff, markdown);
|
||||
|
||||
assertTrue(Files.exists(json));
|
||||
assertTrue(Files.readString(json).contains("\"hasRegression\" : true"));
|
||||
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Baseline Diff"));
|
||||
assertTrue(Files.readString(markdown).contains("redis-timeout"));
|
||||
}
|
||||
|
||||
private DiagnosisEvalReport readBaselineReport() throws Exception {
|
||||
return objectMapper.readValue(Path.of("mvp/eval/reports/baseline-report.json").toFile(),
|
||||
DiagnosisEvalReport.class);
|
||||
}
|
||||
|
||||
private void degradeRedisCase(DiagnosisEvalReport report) {
|
||||
report.setPassedCases(11);
|
||||
report.setPassRate(11.0 / 12.0);
|
||||
report.setAverageToolCallCount(3.0);
|
||||
report.setAverageDurationMs(38500.0);
|
||||
report.setVerdictDistribution(new LinkedHashMap<>());
|
||||
report.getVerdictDistribution().put("PASS", 5L);
|
||||
report.getVerdictDistribution().put("LOW_CONFID", 5L);
|
||||
report.getVerdictDistribution().put("REJECT", 2L);
|
||||
|
||||
DiagnosisEvalResult redis = result(report, "redis-timeout");
|
||||
redis.setPassed(false);
|
||||
redis.setFailedChecks(new ArrayList<>(List.of("missing required evidence tool: query_logs")));
|
||||
redis.setVerdict("REJECT");
|
||||
redis.setMatchedKeywordCount(1);
|
||||
redis.getEvidenceCoverage().put("query_logs", false);
|
||||
redis.setToolCallCount(1);
|
||||
redis.setDurationMs(36000);
|
||||
}
|
||||
|
||||
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
|
||||
return report.getResults().stream()
|
||||
.filter(item -> caseId.equals(item.getCaseId()))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
private boolean hasItem(DiagnosisEvalDiffReport diff,
|
||||
String type,
|
||||
String scope,
|
||||
String caseId,
|
||||
String metric) {
|
||||
return diff.getItems().stream().anyMatch(item ->
|
||||
type.equals(item.getType())
|
||||
&& scope.equals(item.getScope())
|
||||
&& java.util.Objects.equals(caseId, item.getCaseId())
|
||||
&& metric.equals(item.getMetric()));
|
||||
}
|
||||
}
|
||||
@@ -1,391 +0,0 @@
|
||||
package com.superbiz.agent.eval;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisTraceEvaluatorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
|
||||
|
||||
@Test
|
||||
void evaluateFixtureReportsFullBaseline() {
|
||||
List<DiagnosisEvalCase> cases = readCases();
|
||||
|
||||
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
|
||||
|
||||
assertEquals(12, report.getTotalCases());
|
||||
assertEquals(12, report.getPassedCases());
|
||||
assertEquals(1.0, report.getPassRate(), 0.001);
|
||||
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
|
||||
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
|
||||
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
|
||||
|
||||
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
|
||||
assertTrue(narrowHighCpu.isPassed());
|
||||
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
|
||||
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
|
||||
|
||||
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
|
||||
assertTrue(promptGatekeeperAudit.isPassed());
|
||||
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
|
||||
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
|
||||
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
|
||||
|
||||
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
|
||||
assertTrue(hikariNoEvidence.isPassed());
|
||||
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
|
||||
assertEquals("pass", hikariNoEvidence.getGatekeeperStatus());
|
||||
|
||||
DiagnosisEvalResult payment = result(report, "payment-timeout");
|
||||
assertTrue(payment.isPassed());
|
||||
assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
|
||||
assertTrue(payment.getEvidenceCoverage().get("query_logs"));
|
||||
assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
|
||||
|
||||
DiagnosisEvalResult redis = result(report, "redis-timeout");
|
||||
assertTrue(redis.isPassed());
|
||||
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
|
||||
|
||||
DiagnosisEvalResult fabricatedInvocation = result(report, "gatekeeper-fabricated-invocation");
|
||||
assertTrue(fabricatedInvocation.isPassed());
|
||||
assertEquals("fail", fabricatedInvocation.getGatekeeperStatus());
|
||||
assertEquals("valid", fabricatedInvocation.getComposerStatus());
|
||||
assertEquals(1, fabricatedInvocation.getClaimCheckCount());
|
||||
|
||||
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
|
||||
assertTrue(composerFallback.isPassed());
|
||||
assertEquals("composer_malformed", composerFallback.getComposerStatus());
|
||||
|
||||
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
|
||||
assertTrue(auditMetadataLowConfid.isPassed());
|
||||
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
|
||||
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
|
||||
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
void everyFixedCaseReferencesExistingFixture() {
|
||||
for (DiagnosisEvalCase evalCase : readCases()) {
|
||||
Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture());
|
||||
assertTrue(Files.exists(fixture), "missing fixture: " + fixture);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateRejectRequiresDegradedOutput() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("reject-case")
|
||||
.title("Reject case")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("REJECT"))
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("EXECUTOR_FINAL_ANSWER")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of("verdict", "REJECT")))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenStructuredConfirmedClaimHasNoEvidenceBindings() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("structured-claim-case")
|
||||
.title("Structured claim case")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("LOW_CONFID", "PASS"))
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "LOW_CONFID",
|
||||
"executor_structured_output", java.util.Map.of(
|
||||
"claims", java.util.List.of(java.util.Map.of(
|
||||
"claim_id", "claim-unsupported",
|
||||
"claim_text", "OOM 导致连接泄漏",
|
||||
"support_level", "direct",
|
||||
"evidence_bindings", java.util.List.of()
|
||||
))
|
||||
)
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains(
|
||||
"executor confirmed claim missing evidence bindings: claim-unsupported"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenGatekeeperFailStillPassesVerifier() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("gatekeeper-pass-leak")
|
||||
.title("Gatekeeper pass leak")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("PASS", "LOW_CONFID", "REJECT"))
|
||||
.requireV2AuditClosure(true)
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("安全回答")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "PASS",
|
||||
"gatekeeper_result", java.util.Map.of("status", "fail"),
|
||||
"claim_checks", java.util.List.of(java.util.Map.of(
|
||||
"claim_id", "claim-1",
|
||||
"verification", "unsupported",
|
||||
"detail", "evidence ref invalid"
|
||||
)),
|
||||
"composer_output", java.util.Map.of("status", "valid")
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenGatekeeperRuleSetVersionMismatches() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("rule-version")
|
||||
.title("Rule version")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("PASS"))
|
||||
.expectedGatekeeperRuleSetVersion("gatekeeper-rules-v1")
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("安全回答")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "PASS",
|
||||
"gatekeeper_result", java.util.Map.of(
|
||||
"status", "pass",
|
||||
"rule_set_version", "old-rules"
|
||||
)
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains(
|
||||
"gatekeeper rule set version not expected: old-rules"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("prompt-audit")
|
||||
.title("Prompt audit")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("PASS"))
|
||||
.requirePromptAudit(true)
|
||||
.expectedPromptAuditVersion("chat-prompts-v1")
|
||||
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("安全回答")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "PASS",
|
||||
"prompt_audit", java.util.Map.of(
|
||||
"version", "old-prompts",
|
||||
"prompts", java.util.List.of(java.util.Map.of(
|
||||
"name", "chat_executor",
|
||||
"version", "chat-executor-v1"
|
||||
))
|
||||
)
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains(
|
||||
"prompt audit version not expected: old-prompts"));
|
||||
assertTrue(result.getFailedChecks().contains(
|
||||
"prompt version not expected: chat_executor=chat-executor-v1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenGatekeeperRulesAreMissing() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("gatekeeper-rules")
|
||||
.title("Gatekeeper rules")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("PASS"))
|
||||
.requireGatekeeperRules(true)
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("安全回答")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "PASS",
|
||||
"gatekeeper_result", java.util.Map.of(
|
||||
"status", "pass",
|
||||
"rule_set_version", "gatekeeper-rules-v1"
|
||||
)
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("unsupported-leak")
|
||||
.title("Unsupported leak")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("LOW_CONFID"))
|
||||
.requireV2AuditClosure(true)
|
||||
.forbiddenConfirmedClaimKeywords(List.of("主库故障"))
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("已经确认主库故障。")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "LOW_CONFID",
|
||||
"gatekeeper_result", java.util.Map.of("status", "pass"),
|
||||
"claim_checks", java.util.List.of(java.util.Map.of(
|
||||
"claim_id", "claim-1",
|
||||
"verification", "unsupported",
|
||||
"detail", "missing database evidence"
|
||||
)),
|
||||
"composer_output", java.util.Map.of("status", "valid")
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains(
|
||||
"answer contains forbidden confirmed claim keyword: 主库故障"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenFinalAnswerLeaksRawExecutorMarker() {
|
||||
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
||||
.id("raw-json-leak")
|
||||
.title("Raw json leak")
|
||||
.expectedRootCauseKeywords(List.of())
|
||||
.requiredEvidenceTools(List.of())
|
||||
.allowedVerdicts(List.of("LOW_CONFID"))
|
||||
.requireV2AuditClosure(true)
|
||||
.build();
|
||||
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
||||
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
||||
.answer("answer_version=executor_evidence_v2")
|
||||
.selfEvaluation(java.util.Map.of(
|
||||
"verifier_evaluation", java.util.Map.of(
|
||||
"verdict", "LOW_CONFID",
|
||||
"gatekeeper_result", java.util.Map.of("status", "pass"),
|
||||
"claim_checks", java.util.List.of(java.util.Map.of(
|
||||
"claim_id", "claim-1",
|
||||
"verification", "direct_observation",
|
||||
"detail", "log evidence"
|
||||
)),
|
||||
"composer_output", java.util.Map.of("status", "composer_malformed")
|
||||
)))
|
||||
.build())
|
||||
.toolInvocations(List.of())
|
||||
.build();
|
||||
|
||||
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
||||
|
||||
assertFalse(result.isPassed());
|
||||
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: executor_evidence_v2"));
|
||||
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: answer_version"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
|
||||
DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
|
||||
DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper);
|
||||
|
||||
Path json = tempDir.resolve("eval-report.json");
|
||||
Path markdown = tempDir.resolve("eval-report.md");
|
||||
writer.writeJson(report, json);
|
||||
writer.writeMarkdown(report, markdown);
|
||||
|
||||
assertTrue(Files.exists(json));
|
||||
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
|
||||
assertTrue(Files.readString(markdown).contains("payment-timeout"));
|
||||
assertEquals(
|
||||
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))),
|
||||
comparableReportText(Files.readString(json)));
|
||||
assertEquals(
|
||||
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))),
|
||||
comparableReportText(Files.readString(markdown)));
|
||||
}
|
||||
|
||||
private List<DiagnosisEvalCase> readCases() {
|
||||
try {
|
||||
return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json"));
|
||||
} catch (Exception e) {
|
||||
throw new AssertionError(e);
|
||||
}
|
||||
}
|
||||
|
||||
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
|
||||
return report.getResults().stream()
|
||||
.filter(item -> caseId.equals(item.getCaseId()))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
}
|
||||
|
||||
private String comparableReportText(String value) {
|
||||
return value.replace("\r\n", "\n").stripTrailing();
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.superbiz.agent.Main;
|
||||
import com.superbiz.agent.dto.LookupResult;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
@@ -56,13 +55,7 @@ class RagLookupSnapshotGeneratorTest {
|
||||
String caseId = requiredText(testCase, "caseId");
|
||||
String query = requiredText(testCase, "query");
|
||||
|
||||
SessionContextHolder.clear();
|
||||
LookupResult lookupResult;
|
||||
try {
|
||||
lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
LookupResult lookupResult = lookupKnowledgeTool.lookupKnowledge(query);
|
||||
|
||||
ObjectNode fixture = objectMapper.createObjectNode();
|
||||
fixture.put("caseId", caseId);
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentFactory;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class HarnessAgentAuditHookTest {
|
||||
|
||||
@Test
|
||||
void persistsExactIdentityAndMetadataWithoutContentOrArguments() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(7L).build();
|
||||
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
|
||||
when(repository.findById(7L)).thenReturn(Optional.of(persisted));
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", "session-audit")
|
||||
.addMetadata("runId", "run-audit")
|
||||
.build();
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("secret-query")), config);
|
||||
AssistantMessage response = AssistantMessage.builder()
|
||||
.content("secret-model-output")
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
"call-1", "function", "query_logs", "{\"query\":\"secret-argument\"}")))
|
||||
.build();
|
||||
hook.afterModel(List.of(response), config);
|
||||
|
||||
ArgumentCaptor<AgentStep> captor = ArgumentCaptor.forClass(AgentStep.class);
|
||||
verify(repository, org.mockito.Mockito.times(2)).save(captor.capture());
|
||||
AgentStep started = captor.getAllValues().get(0);
|
||||
AgentStep completed = captor.getAllValues().get(1);
|
||||
assertEquals("session-audit", started.getSessionId());
|
||||
assertEquals("run-audit", started.getRunId());
|
||||
assertFalse(started.getModelInput().contains("secret-query"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-model-output"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-argument"));
|
||||
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"]}", completed.getModelOutput());
|
||||
assertNull(completed.getThought());
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingIdentitySkipsPersistence() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME);
|
||||
|
||||
hook.beforeModel(List.of(new UserMessage("secret")), RunnableConfig.builder().build());
|
||||
|
||||
verify(repository, never()).save(any());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
class JpaToolInvocationAuditSinkTest {
|
||||
|
||||
@Test
|
||||
void persistsOnlyBoundedStableMetadata() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(repository, new ObjectMapper());
|
||||
|
||||
sink.record(new ToolInvocationAuditEvent(
|
||||
"session-1", "run-1", "call-1", "query_logs",
|
||||
InvocationStatus.ERROR, EvidenceStatus.ERROR, "TOOL_EXECUTION_ERROR",
|
||||
17, 83, 0));
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
assertEquals("session-1", saved.getSessionId());
|
||||
assertEquals("run-1", saved.getRunId());
|
||||
assertEquals("query_logs", saved.getToolName());
|
||||
assertEquals("{\"tool_call_id\":\"call-1\",\"request_bytes\":83}", saved.getInputParams());
|
||||
assertEquals("status=ERROR,evidence_status=ERROR", saved.getOutputPreview());
|
||||
assertEquals("TOOL_EXECUTION_ERROR", saved.getErrorMessage());
|
||||
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
|
||||
assertFalse(serialized.contains("query"));
|
||||
assertFalse(serialized.contains("raw_response"));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.tool.boundary;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditEvent;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
@@ -21,6 +22,7 @@ import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
@@ -157,6 +159,42 @@ class ToolBoundaryTest {
|
||||
assertFalse(store.find("superbiz:harness:tool-call:run-1:call-9").orElseThrow().isReferencableBy("run-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void emitsExactSafeAuditAndAuditFailureDoesNotChangeResult() {
|
||||
MutableClock clock = new MutableClock(Instant.parse("2026-07-21T10:00:00Z"));
|
||||
FakeStore store = new FakeStore(new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
|
||||
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
|
||||
AtomicReference<ToolInvocationAuditEvent> audit = new AtomicReference<>();
|
||||
ToolBoundary boundary = new ToolBoundary(core,
|
||||
new ToolCallKeyFactory("superbiz:harness:tool-call"),
|
||||
store, objectMapper, clock, audit::set);
|
||||
RunContext context = core.startRun("session-audit", "run-audit");
|
||||
|
||||
ToolBoundaryResult result = boundary.execute(
|
||||
context, request("run-audit", "call-audit", true, true),
|
||||
ignored -> "secret-raw-response",
|
||||
ignored -> new ProjectedToolResult("secret-agent-result", EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
assertEquals(InvocationStatus.READY, result.status());
|
||||
assertEquals("session-audit", audit.get().sessionId());
|
||||
assertEquals("run-audit", audit.get().runId());
|
||||
assertEquals("call-audit", audit.get().toolCallId());
|
||||
|
||||
FakeStore failingAuditStore = new FakeStore(
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 4096, 1024));
|
||||
ToolBoundary failingAuditBoundary = new ToolBoundary(core,
|
||||
new ToolCallKeyFactory("superbiz:harness:tool-call"),
|
||||
failingAuditStore, objectMapper, clock,
|
||||
ignored -> { throw new IllegalStateException("audit database secret"); });
|
||||
ToolBoundaryResult unchanged = failingAuditBoundary.execute(
|
||||
context, request("run-audit", "call-audit-failure", true, true),
|
||||
ignored -> "raw",
|
||||
ignored -> new ProjectedToolResult("agent", EvidenceStatus.NO_EVIDENCE));
|
||||
|
||||
assertEquals(InvocationStatus.READY, unchanged.status());
|
||||
assertEquals(EvidenceStatus.NO_EVIDENCE, unchanged.evidenceStatus());
|
||||
}
|
||||
|
||||
private ToolBoundary boundary(MutableClock clock, FakeStore store) {
|
||||
DiagnosisHarnessCore core = HarnessCoreFixtures.core(clock);
|
||||
return new ToolBoundary(core, new ToolCallKeyFactory("superbiz:harness:tool-call"),
|
||||
|
||||
+2
-1
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.tool.store;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.config.WebConfig;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -28,7 +29,7 @@ class CanonicalInvocationStoreTest {
|
||||
private ValueOperations<String, Object> values;
|
||||
private RedisCanonicalInvocationStore store;
|
||||
private CanonicalToolInvocation projecting;
|
||||
private final ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
|
||||
private final ObjectMapper objectMapper = new WebConfig().objectMapper();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
|
||||
@@ -1,380 +0,0 @@
|
||||
package com.superbiz.agent.hook;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.messages.AgentCommand;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.service.ExecutorGatekeeperService;
|
||||
import com.superbiz.agent.service.ToolTraceSummaryService;
|
||||
import com.superbiz.agent.util.VerifierContextHolder;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class VerifierInputHookTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
VerifierContextHolder.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelAddsStructuredExecutorOutputWhenJsonContractIsValid() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
|
||||
Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
|
||||
));
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
|
||||
VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
|
||||
|
||||
String executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v1",
|
||||
"diagnosis_summary": "连接池已满，但缺少泄漏证据。",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_ids": [101],
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": ["缺少泄漏检测日志"],
|
||||
"user_facing_answer": "已确认连接池 active 达到上限。"
|
||||
}
|
||||
""";
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage(executorOutput)),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "structured-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
|
||||
assertEquals("executor_evidence_v1",
|
||||
payload.path("executor_structured_output").path("answer_version").asText());
|
||||
assertEquals("连接池 active 达到上限",
|
||||
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
|
||||
assertNotNull(VerifierContextHolder.getExecutorStructuredOutput());
|
||||
assertEquals("valid", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelAddsStructuredExecutorOutputWhenV2ContractHasNoUserFacingAnswer() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
|
||||
Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
|
||||
));
|
||||
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("structured-v2-session")).thenReturn(List.of(
|
||||
invocation(101L, "structured-v2-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
|
||||
));
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
|
||||
new ExecutorGatekeeperService(invocationRepository));
|
||||
VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
|
||||
|
||||
String executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage(executorOutput)),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "structured-v2-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
|
||||
assertEquals("executor_evidence_v2",
|
||||
payload.path("executor_structured_output").path("answer_version").asText());
|
||||
assertFalse(payload.path("executor_structured_output").has("user_facing_answer"));
|
||||
assertEquals("连接池 active 达到上限",
|
||||
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
|
||||
assertEquals("pass", payload.path("gatekeeper_result").path("status").asText());
|
||||
assertEquals("none", payload.path("gatekeeper_result").path("severity").asText());
|
||||
assertEquals("pass", VerifierContextHolder.getGatekeeperResult().get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelBackfillsOnlyUniqueInvocationIdAndDoesNotPassWithoutRawPath() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
|
||||
Map.of(
|
||||
"trace_ref", "metrics-1",
|
||||
"tool_name", "query_metrics",
|
||||
"source_invocation_ids", List.of(101L)
|
||||
)
|
||||
));
|
||||
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("backfill-session")).thenReturn(List.of(
|
||||
invocation(101L, "backfill-session", "query_metrics", "$.alerts[0]",
|
||||
"CPU 使用率持续超过 80%，当前值为 92%")
|
||||
));
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
|
||||
new ExecutorGatekeeperService(invocationRepository));
|
||||
|
||||
String executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "payment-service CPU 使用率超过 92%",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "prometheus-alert-HighCPUUsage",
|
||||
"tool_name": "queryPrometheusAlerts",
|
||||
"evidence_excerpt": "CPU 使用率持续超过 80%，当前值为 92%"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [
|
||||
{
|
||||
"action_text": "restart payment-service",
|
||||
"reason": "cpu alert is firing",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "prometheus-alert-HighCPUUsage",
|
||||
"tool_name": "queryPrometheusAlerts",
|
||||
"evidence_excerpt": "CPU usage is 92%"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage(executorOutput)),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "backfill-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
JsonNode binding = payload.path("executor_structured_output")
|
||||
.path("claims").get(0)
|
||||
.path("evidence_bindings").get(0);
|
||||
assertEquals("query_metrics", binding.path("tool_name").asText());
|
||||
assertEquals(101L, binding.path("source_invocation_id").asLong());
|
||||
JsonNode actionBinding = payload.path("executor_structured_output")
|
||||
.path("recommended_actions").get(0)
|
||||
.path("evidence_bindings").get(0);
|
||||
assertEquals("query_metrics", actionBinding.path("tool_name").asText());
|
||||
assertEquals(101L, actionBinding.path("source_invocation_id").asLong());
|
||||
assertFalse(binding.has("raw_path"));
|
||||
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
|
||||
assertEquals("low_confid", payload.path("gatekeeper_result").path("severity").asText());
|
||||
assertEquals("evidence.invocation_auto_backfill",
|
||||
payload.path("gatekeeper_result").path("warnings").get(0).path("rule").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelAddsFailingGatekeeperResultForFabricatedInvocationId() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
||||
ToolInvocationRepository invocationRepository = mock(ToolInvocationRepository.class);
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("fabricated-invocation-session")).thenReturn(List.of(
|
||||
invocation(101L, "fabricated-invocation-session", "query_metrics", "$.alerts[0]", "active=50 max=50")
|
||||
));
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService,
|
||||
new ExecutorGatekeeperService(invocationRepository));
|
||||
|
||||
String executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 999,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage(executorOutput)),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "fabricated-invocation-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("fail", payload.path("gatekeeper_result").path("status").asText());
|
||||
assertEquals("reject", payload.path("gatekeeper_result").path("severity").asText());
|
||||
assertEquals("evidence.invocation_ref",
|
||||
payload.path("gatekeeper_result").path("failed_rules").get(0).asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelExtractsStructuredOutputFromPrefixedJsonFence() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
|
||||
|
||||
String executorOutput = """
|
||||
现在我已经收集了足够的数据，最终输出如下。
|
||||
|
||||
```json
|
||||
{
|
||||
"answer_version": "executor_evidence_v1",
|
||||
"diagnosis_summary": "已确认连接池 active 达到上限。",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_ids": [101],
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": [],
|
||||
"user_facing_answer": "已确认连接池 active 达到上限。"
|
||||
}
|
||||
```
|
||||
""";
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage(executorOutput)),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "fenced-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
|
||||
assertEquals("claim-1",
|
||||
payload.path("executor_structured_output").path("claims").get(0).path("claim_id").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelMarksMalformedJsonAndKeepsRawAnswerFallback() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage("{\"diagnosis_summary\":\"缺少 claims\"}")),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "malformed-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("malformed", payload.path("executor_output_parse_status").path("status").asText());
|
||||
assertTrue(payload.path("executor_structured_output").isNull());
|
||||
assertEquals("{\"diagnosis_summary\":\"缺少 claims\"}", payload.path("executor_final_answer").asText());
|
||||
assertEquals("malformed", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void beforeModelMarksPlainTextAsMissingStructuredOutput() throws Exception {
|
||||
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
||||
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
|
||||
|
||||
AgentCommand command = hook.beforeModel(
|
||||
List.of(new AssistantMessage("普通自然语言答案")),
|
||||
RunnableConfig.builder().addMetadata("sessionId", "plain-session").build()
|
||||
);
|
||||
|
||||
JsonNode payload = readPayload(command);
|
||||
assertEquals("missing", payload.path("executor_output_parse_status").path("status").asText());
|
||||
assertTrue(payload.path("executor_structured_output").isNull());
|
||||
assertFalse(payload.path("executor_final_answer").asText().isBlank());
|
||||
}
|
||||
|
||||
private JsonNode readPayload(AgentCommand command) throws Exception {
|
||||
var field = AgentCommand.class.getDeclaredField("messages");
|
||||
field.setAccessible(true);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Message> messages = (List<Message>) field.get(command);
|
||||
assertEquals(1, messages.size());
|
||||
Message message = messages.get(0);
|
||||
assertTrue(message instanceof UserMessage);
|
||||
return objectMapper.readTree(((UserMessage) message).getText());
|
||||
}
|
||||
|
||||
private ToolInvocation invocation(Long id, String sessionId, String toolName, String rawPath, String text) {
|
||||
return ToolInvocation.builder()
|
||||
.id(id)
|
||||
.sessionId(sessionId)
|
||||
.toolName(toolName)
|
||||
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
|
||||
+ "\",\"text\":\"" + text + "\"}]}")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.AIOpsRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class AiOpsRuleEvaluationServiceTest {
|
||||
|
||||
private final AiOpsRuleEvaluationService service = new AiOpsRuleEvaluationService();
|
||||
|
||||
@Test
|
||||
void evaluatePassesWhenReportFocusesPayloadAndHasEvidenceTools() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("HighCPUUsage");
|
||||
request.setService("payment-service");
|
||||
|
||||
ToolInvocation invocation = ToolInvocation.builder()
|
||||
.toolName("lookup_knowledge")
|
||||
.build();
|
||||
|
||||
Map<String, Object> evaluation = service.evaluate(
|
||||
request,
|
||||
"HighCPUUsage alert on payment-service was diagnosed using metrics and knowledge evidence.",
|
||||
List.of(invocation)
|
||||
);
|
||||
|
||||
assertEquals("PASS", evaluation.get("verdict"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateWarnsWhenEvidenceToolsAreMissing() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("HighCPUUsage");
|
||||
request.setService("payment-service");
|
||||
|
||||
Map<String, Object> evaluation = service.evaluate(
|
||||
request,
|
||||
"HighCPUUsage alert on payment-service has a likely resource saturation issue.",
|
||||
List.of()
|
||||
);
|
||||
|
||||
assertEquals("WARN", evaluation.get("verdict"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evaluateFailsWhenReportIsMissing() {
|
||||
Map<String, Object> evaluation = service.evaluate(null, "too short", List.of());
|
||||
|
||||
assertEquals("FAIL", evaluation.get("verdict"));
|
||||
}
|
||||
}
|
||||
@@ -1,268 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
||||
import com.alibaba.cloud.ai.graph.agent.flow.agent.SupervisorAgent;
|
||||
import com.superbiz.agent.config.AiOpsPromptProperties;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisRun;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisSession;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.AIOpsRequest;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisSessionRepository;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class AiOpsServiceTest {
|
||||
|
||||
private final DiagnosisSessionRepository diagnosisSessionRepository = mock(DiagnosisSessionRepository.class);
|
||||
private final DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class);
|
||||
private final AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
|
||||
private final ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
|
||||
private final AiOpsPromptProperties promptProperties = mock(AiOpsPromptProperties.class);
|
||||
private final AiOpsService service = new AiOpsService();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
ReflectionTestUtils.setField(service, "diagnosisSessionRepository", diagnosisSessionRepository);
|
||||
ReflectionTestUtils.setField(service, "diagnosisRunRepository", diagnosisRunRepository);
|
||||
ReflectionTestUtils.setField(service, "agentStepRepository", agentStepRepository);
|
||||
ReflectionTestUtils.setField(service, "toolInvocationRepository", toolInvocationRepository);
|
||||
ReflectionTestUtils.setField(service, "aiOpsRuleEvaluationService", new AiOpsRuleEvaluationService());
|
||||
ReflectionTestUtils.setField(service, "selfEvaluationMergeService", new SelfEvaluationMergeService());
|
||||
ReflectionTestUtils.setField(service, "promptProperties", promptProperties);
|
||||
when(promptProperties.getSupervisor()).thenReturn("supervisor prompt");
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolveSessionIdUsesRequestValueWhenPresent() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setSessionId(" aiops-demo-session ");
|
||||
|
||||
assertEquals("aiops-demo-session", service.resolveSessionId(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolveSessionIdGeneratesWhenMissing() {
|
||||
String sessionId = service.resolveSessionId(null);
|
||||
|
||||
assertNotNull(sessionId);
|
||||
assertFalse(sessionId.isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildQuerySummaryUsesAlertFieldsAndUserRequestFallback() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("payment-service-latency-high");
|
||||
request.setService("payment-service");
|
||||
request.setSeverity("P1");
|
||||
request.setTimeRange("last_15m");
|
||||
request.setDescription("P95 latency is high");
|
||||
request.setUserRequest("check logs and metrics for payment timeout");
|
||||
|
||||
String summary = service.buildQuerySummary(request);
|
||||
|
||||
assertTrue(summary.contains("AI Ops alert analysis"));
|
||||
assertTrue(summary.contains("alert: payment-service-latency-high"));
|
||||
assertTrue(summary.contains("service: payment-service"));
|
||||
assertTrue(summary.contains("severity: P1"));
|
||||
assertTrue(summary.contains("timeRange: last_15m"));
|
||||
assertTrue(summary.contains("description: P95 latency is high"));
|
||||
assertTrue(summary.contains("request: "));
|
||||
}
|
||||
|
||||
@Test
|
||||
void hasAlertPayloadIgnoresUserRequestOnly() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setUserRequest("please discover active alerts");
|
||||
|
||||
assertFalse(service.hasAlertPayload(request));
|
||||
|
||||
request.setAlertName("HighCPUUsage");
|
||||
|
||||
assertTrue(service.hasAlertPayload(request));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildTaskPromptUsesPayloadTargetedModeWhenAlertFieldsExist() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("HighCPUUsage");
|
||||
request.setService("payment-service");
|
||||
request.setSeverity("P1");
|
||||
request.setTimeRange("last_15m");
|
||||
request.setDescription("CPU usage is above 80%");
|
||||
|
||||
String prompt = service.buildTaskPrompt(request);
|
||||
|
||||
assertTrue(prompt.contains("AIOps scope mode: PAYLOAD_TARGETED"));
|
||||
assertTrue(prompt.contains("primary and only main diagnosis target"));
|
||||
assertTrue(prompt.contains("queryPrometheusAlerts only to verify"));
|
||||
assertTrue(prompt.contains("do not create full root-cause or remediation sections"));
|
||||
assertTrue(prompt.contains("Related Risk"));
|
||||
assertTrue(prompt.contains("Recommended lookup_knowledge query: HighCPUUsage payment-service P1 CPU usage is above 80% last_15m"));
|
||||
assertTrue(prompt.contains("preserves alertName and service"));
|
||||
assertTrue(prompt.contains("alert: HighCPUUsage"));
|
||||
assertTrue(prompt.contains("service: payment-service"));
|
||||
assertFalse(prompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildKnowledgeRetrievalQueryUsesPayloadFieldsAndSkipsBlankValues() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("HighLatency");
|
||||
request.setService(" payment-service ");
|
||||
request.setSeverity(" ");
|
||||
request.setDescription("P95 latency above threshold");
|
||||
request.setTimeRange("last_10m");
|
||||
request.setUserRequest("check logs and metrics");
|
||||
|
||||
String query = service.buildKnowledgeRetrievalQuery(request);
|
||||
|
||||
assertEquals("HighLatency payment-service P95 latency above threshold last_10m check logs and metrics", query);
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildTaskPromptUsesAutoDiscoveryModeWhenAlertPayloadIsMissing() {
|
||||
String nullRequestPrompt = service.buildTaskPrompt(null);
|
||||
|
||||
assertTrue(nullRequestPrompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
|
||||
assertTrue(nullRequestPrompt.contains("First call queryPrometheusAlerts"));
|
||||
assertTrue(nullRequestPrompt.contains("current active/firing alerts"));
|
||||
assertFalse(nullRequestPrompt.contains("AIOps scope mode: PAYLOAD_TARGETED"));
|
||||
|
||||
AIOpsRequest userRequestOnly = new AIOpsRequest();
|
||||
userRequestOnly.setUserRequest("check what is firing now");
|
||||
|
||||
String userRequestOnlyPrompt = service.buildTaskPrompt(userRequestOnly);
|
||||
|
||||
assertTrue(userRequestOnlyPrompt.contains("AIOps scope mode: AUTO_DISCOVERY"));
|
||||
assertTrue(userRequestOnlyPrompt.contains("First call queryPrometheusAlerts"));
|
||||
assertFalse(userRequestOnlyPrompt.contains("Recommended lookup_knowledge query"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistFinalReportUpdatesDiagnosisRunAnswerAndEvaluationByRun() {
|
||||
DiagnosisRun run = DiagnosisRun.builder()
|
||||
.sessionId("aiops-session-001")
|
||||
.runId("run-aiops-001")
|
||||
.query("AI Ops alert analysis")
|
||||
.status("SUCCESS")
|
||||
.agentFlow("AI_OPS")
|
||||
.build();
|
||||
ToolInvocation invocation = ToolInvocation.builder()
|
||||
.sessionId("aiops-session-001")
|
||||
.runId("run-aiops-001")
|
||||
.toolName("query_logs")
|
||||
.success(true)
|
||||
.build();
|
||||
when(diagnosisRunRepository.findBySessionIdAndRunId("aiops-session-001", "run-aiops-001"))
|
||||
.thenReturn(Optional.of(run));
|
||||
when(toolInvocationRepository.findByRunIdOrderByIdAsc("run-aiops-001")).thenReturn(List.of(invocation));
|
||||
|
||||
service.persistFinalReport("aiops-session-001", "run-aiops-001",
|
||||
"# 告警分析报告\nHighCPUUsage payment-service analysis with evidence summary.", null);
|
||||
|
||||
assertEquals("# 告警分析报告\nHighCPUUsage payment-service analysis with evidence summary.", run.getAnswer());
|
||||
assertTrue(run.getSelfEvaluation().contains("aiops_rule_evaluation"));
|
||||
verify(toolInvocationRepository).findByRunIdOrderByIdAsc("run-aiops-001");
|
||||
verify(diagnosisRunRepository).save(run);
|
||||
verify(diagnosisSessionRepository, never()).save(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyPersistFinalReportStillUpdatesHistoricalDiagnosisSession() {
|
||||
DiagnosisSession session = DiagnosisSession.builder()
|
||||
.sessionId("legacy-aiops-session")
|
||||
.query("AI Ops alert analysis")
|
||||
.status("SUCCESS")
|
||||
.agentFlow("AI_OPS")
|
||||
.build();
|
||||
when(diagnosisSessionRepository.findBySessionId("legacy-aiops-session")).thenReturn(Optional.of(session));
|
||||
when(toolInvocationRepository.findBySessionIdOrderByIdAsc("legacy-aiops-session")).thenReturn(List.of());
|
||||
|
||||
service.persistFinalReport("legacy-aiops-session", "# 告警分析报告\nLegacy analysis with evidence summary.");
|
||||
|
||||
assertEquals("# 告警分析报告\nLegacy analysis with evidence summary.", session.getAnswer());
|
||||
assertTrue(session.getSelfEvaluation().contains("aiops_rule_evaluation"));
|
||||
verify(diagnosisSessionRepository).save(session);
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistFinalReportSkipsBlankInput() {
|
||||
service.persistFinalReport("aiops-session-001", " ");
|
||||
|
||||
verifyNoInteractions(diagnosisSessionRepository, diagnosisRunRepository);
|
||||
}
|
||||
|
||||
@Test
|
||||
void backfillRunMetricsUsesRunScopedRows() {
|
||||
DiagnosisRun run = DiagnosisRun.builder()
|
||||
.sessionId("aiops-session-002")
|
||||
.runId("run-aiops-002")
|
||||
.build();
|
||||
AgentStep stepWithTool = AgentStep.builder()
|
||||
.sessionId("aiops-session-002")
|
||||
.runId("run-aiops-002")
|
||||
.hasToolCall(true)
|
||||
.tokenCount(10)
|
||||
.build();
|
||||
AgentStep stepWithoutTool = AgentStep.builder()
|
||||
.sessionId("aiops-session-002")
|
||||
.runId("run-aiops-002")
|
||||
.hasToolCall(false)
|
||||
.tokenCount(20)
|
||||
.build();
|
||||
when(agentStepRepository.findByRunIdOrderByStepIndex("run-aiops-002"))
|
||||
.thenReturn(List.of(stepWithTool, stepWithoutTool));
|
||||
when(toolInvocationRepository.countByRunId("run-aiops-002")).thenReturn(11L);
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "backfillRunMetrics", run);
|
||||
|
||||
assertEquals(2, run.getStepCount());
|
||||
assertEquals(30, run.getTotalTokenCount());
|
||||
assertEquals(11, run.getToolCallCount());
|
||||
verify(agentStepRepository).findByRunIdOrderByStepIndex("run-aiops-002");
|
||||
verify(toolInvocationRepository).countByRunId("run-aiops-002");
|
||||
}
|
||||
|
||||
@Test
|
||||
void sameAiOpsSessionCanStartDistinctRuns() {
|
||||
AIOpsRequest request = new AIOpsRequest();
|
||||
request.setAlertName("HighCPUUsage");
|
||||
|
||||
ReflectionTestUtils.invokeMethod(service, "startDiagnosisRun", "same-session", "run-aiops-a", request);
|
||||
ReflectionTestUtils.invokeMethod(service, "startDiagnosisRun", "same-session", "run-aiops-b", request);
|
||||
|
||||
verify(diagnosisRunRepository).save(argThat(run ->
|
||||
"same-session".equals(run.getSessionId())
|
||||
&& "run-aiops-a".equals(run.getRunId())
|
||||
&& "AI_OPS".equals(run.getAgentFlow())
|
||||
&& "RUNNING".equals(run.getStatus())));
|
||||
verify(diagnosisRunRepository).save(argThat(run ->
|
||||
"same-session".equals(run.getSessionId())
|
||||
&& "run-aiops-b".equals(run.getRunId())
|
||||
&& "AI_OPS".equals(run.getAgentFlow())
|
||||
&& "RUNNING".equals(run.getStatus())));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildSupervisorAgentSetsPlannerAsMainAgent() {
|
||||
ChatModel chatModel = mock(ChatModel.class);
|
||||
ReactAgent planner = mock(ReactAgent.class);
|
||||
ReactAgent executor = mock(ReactAgent.class);
|
||||
|
||||
SupervisorAgent supervisor = service.buildSupervisorAgent(chatModel, planner, executor);
|
||||
|
||||
assertSame(planner, supervisor.getMainAgent());
|
||||
}
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.ToolCallbackProvider;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.context.TestConfiguration;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Chat + Embedding 解耦验证测试
|
||||
* <p>
|
||||
* 验证 ChatModel Bean 注入、ModelRoutingConfig 路由、ChatService 接口兼容。
|
||||
* Chat: DeepSeek via OpenAI-compatible API
|
||||
* Embedding: Ollama BGE-M3 (需要本地 ollama 运行)
|
||||
*/
|
||||
@SpringBootTest
|
||||
@DisplayName("Chat + Embedding 解耦验证")
|
||||
class ChatAndEmbeddingSmokeTest {
|
||||
|
||||
@Autowired
|
||||
private ApplicationContext context;
|
||||
|
||||
@Autowired
|
||||
private ChatModel chatModel;
|
||||
|
||||
@Autowired
|
||||
private ChatService chatService;
|
||||
|
||||
/**
|
||||
* 提供 mock ToolCallbackProvider（MCP 已禁用时需要）
|
||||
*/
|
||||
@TestConfiguration
|
||||
static class MockToolConfig {
|
||||
@Bean
|
||||
public ToolCallbackProvider toolCallbackProvider() {
|
||||
return () -> new ToolCallback[0];
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Spring 容器启动成功")
|
||||
void contextLoads() {
|
||||
assertNotNull(context, "Spring 容器应为非空");
|
||||
assertNotNull(chatModel, "ChatModel Bean 应注入成功");
|
||||
assertNotNull(chatService, "ChatService Bean 应注入成功");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ModelRoutingConfig @Primary ChatModel 生效")
|
||||
void chatModelPrimaryBeanWorks() {
|
||||
assertNotNull(chatModel, "@Primary ChatModel 应被自动注入");
|
||||
System.out.println("✓ ChatModel 类型: " + chatModel.getClass().getName());
|
||||
|
||||
// 验证路由到 openAiChatModel (DeepSeek)
|
||||
assertTrue(context.containsBean("openAiChatModel"), "openAiChatModel 应存在");
|
||||
System.out.println(" Chat → openAiChatModel (DeepSeek) ✓");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ChatService.createReactAgent 接受 ChatModel 接口")
|
||||
void chatServiceAcceptsChatModelInterface() {
|
||||
var agent = chatService.createReactAgent(chatModel, "测试系统提示词");
|
||||
assertNotNull(agent, "ReactAgent 应创建成功");
|
||||
assertEquals("intelligent_assistant", agent.name());
|
||||
System.out.println("✓ ReactAgent 创建成功: " + agent.name());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ChatModel 实现 ChatModel 接口（类型安全验证）")
|
||||
void chatModelIsProperType() {
|
||||
assertNotNull(chatModel, "注入的 Bean 应为 ChatModel 实例");
|
||||
System.out.println("✓ ChatModel 接口实现: " + chatModel.getClass().getSimpleName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("EmbeddingModel 状态")
|
||||
void embeddingModelStatus() {
|
||||
boolean hasEmbedding = context.containsBean("embeddingModel");
|
||||
if (hasEmbedding) {
|
||||
System.out.println("✓ EmbeddingModel 已配置");
|
||||
} else {
|
||||
System.out.println("⚠ EmbeddingModel 未找到 — 检查 Ollama 是否运行");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,918 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
||||
import com.alibaba.cloud.ai.graph.skills.registry.SkillRegistry;
|
||||
import com.alibaba.cloud.ai.graph.skills.registry.classpath.ClasspathSkillRegistry;
|
||||
import com.superbiz.agent.agent.tool.DateTimeTools;
|
||||
import com.superbiz.agent.agent.tool.QueryLogsTools;
|
||||
import com.superbiz.agent.agent.tool.QueryMetricsTools;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.ChatSession;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisRun;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.ChatSessionRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.tool.LookupKnowledgeTool;
|
||||
import com.superbiz.agent.tool.RetrievedDocTracker;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.test.util.ReflectionTestUtils;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.atLeast;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ChatServiceSequentialAgentTest {
|
||||
|
||||
@Test
|
||||
void executeChatComplexInvokesSequentialWorkflow() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-test-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
||||
assertFalse(result.answer().contains("\"answer_version\""));
|
||||
assertEquals("sequential-test-session", result.sessionId());
|
||||
assertTrue(result.runId().startsWith("run-"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
||||
assertTrue(chatModel.sawVerifierPrompt);
|
||||
|
||||
ChatSessionRepository chatSessionRepository =
|
||||
(ChatSessionRepository) ReflectionTestUtils.getField(chatService, "chatSessionRepository");
|
||||
DiagnosisRunRepository diagnosisRunRepository =
|
||||
(DiagnosisRunRepository) ReflectionTestUtils.getField(chatService, "diagnosisRunRepository");
|
||||
EvaluationService evaluationService =
|
||||
(EvaluationService) ReflectionTestUtils.getField(chatService, "evaluationService");
|
||||
|
||||
ArgumentCaptor<ChatSession> chatSessionCaptor = ArgumentCaptor.forClass(ChatSession.class);
|
||||
verify(chatSessionRepository, atLeastOnce()).save(chatSessionCaptor.capture());
|
||||
assertEquals("sequential-test-session", chatSessionCaptor.getValue().getSessionId());
|
||||
|
||||
ArgumentCaptor<DiagnosisRun> runCaptor = ArgumentCaptor.forClass(DiagnosisRun.class);
|
||||
verify(diagnosisRunRepository, atLeastOnce()).save(runCaptor.capture());
|
||||
DiagnosisRun savedRun = runCaptor.getValue();
|
||||
assertEquals(result.runId(), savedRun.getRunId());
|
||||
assertEquals("sequential-test-session", savedRun.getSessionId());
|
||||
assertEquals("SUCCESS", savedRun.getStatus());
|
||||
assertEquals(result.answer(), savedRun.getAnswer());
|
||||
verify(evaluationService).evaluateRun(eq(result.runId()), eq(result.answer()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexCreatesDistinctRunsForSameSessionAcrossTurns() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel firstRoundModel = new ScriptedChatModel();
|
||||
ScriptedChatModel secondRoundModel = new ScriptedChatModel();
|
||||
String sessionId = "sequential-same-session";
|
||||
|
||||
ChatService.ChatResult first = chatService.executeChatComplex(
|
||||
firstRoundModel,
|
||||
new ToolCallback[0],
|
||||
"第一轮：请分析支付超时",
|
||||
List.of(),
|
||||
sessionId
|
||||
);
|
||||
ChatService.ChatResult second = chatService.executeChatComplex(
|
||||
secondRoundModel,
|
||||
new ToolCallback[0],
|
||||
"第二轮：基于上一轮结论列出缺失证据",
|
||||
List.of(
|
||||
Map.of("role", "user", "content", "第一轮：请分析支付超时"),
|
||||
Map.of("role", "assistant", "content", first.answer())
|
||||
),
|
||||
sessionId
|
||||
);
|
||||
|
||||
assertEquals(sessionId, first.sessionId());
|
||||
assertEquals(sessionId, second.sessionId());
|
||||
assertNotEquals(first.runId(), second.runId());
|
||||
|
||||
DiagnosisRunRepository diagnosisRunRepository =
|
||||
(DiagnosisRunRepository) ReflectionTestUtils.getField(chatService, "diagnosisRunRepository");
|
||||
ArgumentCaptor<DiagnosisRun> runCaptor = ArgumentCaptor.forClass(DiagnosisRun.class);
|
||||
verify(diagnosisRunRepository, atLeast(2)).save(runCaptor.capture());
|
||||
|
||||
List<String> savedRunIds = runCaptor.getAllValues().stream()
|
||||
.filter(run -> sessionId.equals(run.getSessionId()))
|
||||
.map(DiagnosisRun::getRunId)
|
||||
.distinct()
|
||||
.toList();
|
||||
assertEquals(2, savedRunIds.size());
|
||||
assertTrue(savedRunIds.contains(first.runId()));
|
||||
assertTrue(savedRunIds.contains(second.runId()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexDoesNotRetryLowConfidenceByDefault() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.1,
|
||||
"critical_fact_count": 1,
|
||||
"facts_checked": [
|
||||
{
|
||||
"fact": "missing direct evidence",
|
||||
"is_critical": true,
|
||||
"verification": "no_evidence",
|
||||
"detail": "scripted evidence gap",
|
||||
"evidence_refs": []
|
||||
}
|
||||
],
|
||||
"rationale": "scripted low confidence"
|
||||
}
|
||||
""");
|
||||
chatModel.composerOutput = "not-json";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-low-confidence-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
||||
assertTrue(result.answer().contains("当前缺口"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexLowConfidenceConfirmedFactsOnlyUseDirectEvidence() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.37,
|
||||
"critical_fact_count": 3,
|
||||
"facts_checked": [
|
||||
{
|
||||
"fact": "连接池耗尽 active=50/50",
|
||||
"is_critical": true,
|
||||
"verification": "direct_evidence",
|
||||
"detail": "log evidence",
|
||||
"evidence_refs": []
|
||||
},
|
||||
{
|
||||
"fact": "临时扩容连接池到 80",
|
||||
"is_critical": true,
|
||||
"verification": "indirect_support",
|
||||
"detail": "suggestion inferred from evidence",
|
||||
"evidence_refs": []
|
||||
},
|
||||
{
|
||||
"fact": "OOM 导致连接泄漏",
|
||||
"is_critical": true,
|
||||
"verification": "no_evidence",
|
||||
"detail": "missing OOM log",
|
||||
"evidence_refs": []
|
||||
}
|
||||
],
|
||||
"rationale": "scripted low confidence"
|
||||
}
|
||||
""");
|
||||
chatModel.composerOutput = "not-json";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-low-confid-direct-only-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().contains("已确认信息：\n- 连接池耗尽 active=50/50"));
|
||||
assertTrue(result.answer().contains("80"));
|
||||
assertTrue(result.answer().contains("suggestion inferred from evidence"));
|
||||
assertTrue(result.answer().contains("missing OOM log"));
|
||||
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierOutputMissing() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("", "");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-missing-verifier-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexFallsBackToLowConfidenceWhenVerifierJsonInvalid() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("not-json");
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-invalid-verifier-session"
|
||||
);
|
||||
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier"), chatModel.agentCalls);
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRejectOutputDoesNotLeakExecutorAnswer() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "REJECT",
|
||||
"groundedness_score": 0.0,
|
||||
"critical_fact_count": 1,
|
||||
"facts_checked": [
|
||||
{
|
||||
"fact": "payment timeout root cause",
|
||||
"is_critical": true,
|
||||
"verification": "contradicted",
|
||||
"detail": "scripted contradiction",
|
||||
"evidence_refs": []
|
||||
}
|
||||
],
|
||||
"rationale": "scripted reject"
|
||||
}
|
||||
""");
|
||||
chatModel.composerOutput = "not-json";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-reject-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
||||
assertFalse(result.answer().contains("EXECUTOR_FINAL_ANSWER"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRunsPlannerExecutorVerifierInFixedOrder() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析订单支付超时的原因，并给出修复建议",
|
||||
List.of(),
|
||||
"sequential-workflow-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
||||
assertFalse(result.answer().contains("\"answer_version\""));
|
||||
assertEquals(List.of("chat_planner", "chat_executor", "chat_verifier", "chat_composer"), chatModel.agentCalls);
|
||||
assertTrue(chatModel.sawVerifierPrompt);
|
||||
}
|
||||
|
||||
@Test
|
||||
void verifierReceivesStructuredExecutorPayloadFields() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
chatModel.executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-structured-executor-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
||||
assertFalse(result.answer().contains("\"answer_version\""));
|
||||
assertTrue(chatModel.verifierPromptText.contains("\"executor_structured_output\""));
|
||||
assertTrue(chatModel.verifierPromptText.contains("\"executor_output_parse_status\""));
|
||||
assertTrue(chatModel.verifierPromptText.contains("\"status\" : \"valid\""));
|
||||
assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexRendersExecutorEvidenceV2InsteadOfRawJsonOnPass() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
chatModel.composerOutput = "not-json";
|
||||
chatModel.executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [
|
||||
{
|
||||
"hypothesis_text": "连接泄漏可能参与了连接池耗尽",
|
||||
"basis": "已有连接池满载证据，但缺少泄漏检测日志",
|
||||
"needed_evidence": ["连接泄漏检测日志"]
|
||||
}
|
||||
],
|
||||
"recommended_actions": [
|
||||
{
|
||||
"action_text": "补充查询连接池泄漏检测日志",
|
||||
"reason": "用于确认是否存在连接未释放"
|
||||
}
|
||||
],
|
||||
"missing_info": ["缺少连接泄漏检测日志"]
|
||||
}
|
||||
""";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-v2-render-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().contains("已确认信息"));
|
||||
assertTrue(result.answer().contains("连接池 active 达到上限"));
|
||||
assertTrue(result.answer().contains("建议下一步"));
|
||||
assertFalse(result.answer().contains("\"answer_version\""));
|
||||
assertFalse(result.answer().contains("executor_evidence_v2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexPersistsGatekeeperResultInVerifierEvaluation() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
SelfEvaluationMergeService mergeService =
|
||||
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
||||
ToolInvocationRepository invocationRepository =
|
||||
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-persist-session"))
|
||||
.thenReturn(List.of(ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.sessionId("sequential-gatekeeper-persist-session")
|
||||
.toolName("query_metrics")
|
||||
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
||||
.build()));
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
chatModel.executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-gatekeeper-persist-session"
|
||||
);
|
||||
|
||||
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
||||
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
||||
Map<String, Object> verifierEvaluation = captor.getValue();
|
||||
assertTrue(verifierEvaluation.containsKey("gatekeeper_result"));
|
||||
assertTrue(verifierEvaluation.containsKey("prompt_audit"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> gatekeeperResult = (Map<String, Object>) verifierEvaluation.get("gatekeeper_result");
|
||||
assertEquals("pass", gatekeeperResult.get("status"));
|
||||
assertEquals("none", gatekeeperResult.get("severity"));
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> promptAudit = (Map<String, Object>) verifierEvaluation.get("prompt_audit");
|
||||
assertEquals("chat-prompts-v1", promptAudit.get("version"));
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Map<String, Object>> prompts = (List<Map<String, Object>>) promptAudit.get("prompts");
|
||||
assertEquals(4, prompts.size());
|
||||
assertTrue(prompts.stream().anyMatch(prompt ->
|
||||
"chat_executor".equals(prompt.get("name"))
|
||||
&& "chat-executor-v2".equals(prompt.get("version"))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexMapsClaimChecksToFactsCheckedAndPersistsBoth() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
SelfEvaluationMergeService mergeService =
|
||||
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
||||
ToolInvocationRepository invocationRepository =
|
||||
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-claim-check-session"))
|
||||
.thenReturn(List.of(ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.sessionId("sequential-claim-check-session")
|
||||
.toolName("query_metrics")
|
||||
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
||||
.build()));
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "LOW_CONFID",
|
||||
"groundedness_score": 0.32,
|
||||
"critical_fact_count": 6,
|
||||
"claim_checks": [
|
||||
{"claim_id":"claim-1","claim_text":"CPU 使用率 92%","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[{"trace_ref":"trace-1","tool_name":"query_metrics","source_invocation_ids":[101],"note":"cpu"}]},
|
||||
{"claim_id":"claim-2","claim_text":"CPU 过高可能导致超时","claim_type":"risk","verification":"reasonable_inference","detail":"inference","evidence_refs":[]},
|
||||
{"claim_id":"claim-3","claim_text":"CPU 是唯一根因","claim_type":"root_cause","verification":"overstated","detail":"too strong","evidence_refs":[]},
|
||||
{"claim_id":"claim-4","claim_text":"缺少线程池证据","claim_type":"symptom","verification":"unsupported","detail":"missing","evidence_refs":[]},
|
||||
{"claim_id":"claim-5","claim_text":"出现证据外错误码 ERR_FAKE","claim_type":"symptom","verification":"external_unknown","detail":"external","evidence_refs":[]},
|
||||
{"claim_id":"claim-6","claim_text":"证据显示 CPU 很低","claim_type":"symptom","verification":"contradicted","detail":"conflict","evidence_refs":[]}
|
||||
],
|
||||
"facts_checked": [],
|
||||
"rationale": "claim checks drive compatibility"
|
||||
}
|
||||
""");
|
||||
chatModel.executorOutput = validExecutorV2Output();
|
||||
|
||||
chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-claim-check-session"
|
||||
);
|
||||
|
||||
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
||||
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
||||
Map<String, Object> verifierEvaluation = captor.getValue();
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Map<String, Object>> claimChecks = (List<Map<String, Object>>) verifierEvaluation.get("claim_checks");
|
||||
@SuppressWarnings("unchecked")
|
||||
List<Map<String, Object>> factsChecked = (List<Map<String, Object>>) verifierEvaluation.get("facts_checked");
|
||||
|
||||
assertEquals(6, claimChecks.size());
|
||||
assertEquals(6, factsChecked.size());
|
||||
assertEquals("direct_evidence", factsChecked.get(0).get("verification"));
|
||||
assertEquals("indirect_support", factsChecked.get(1).get("verification"));
|
||||
assertEquals("indirect_support", factsChecked.get(2).get("verification"));
|
||||
assertEquals("no_evidence", factsChecked.get(3).get("verification"));
|
||||
assertEquals("no_evidence", factsChecked.get(4).get("verification"));
|
||||
assertEquals("contradicted", factsChecked.get(5).get("verification"));
|
||||
assertTrue(String.valueOf(factsChecked.get(0).get("fact")).startsWith("claim-1:"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexDowngradesPassToRejectWhenGatekeeperInvocationRefFails() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
SelfEvaluationMergeService mergeService =
|
||||
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
||||
ToolInvocationRepository invocationRepository =
|
||||
(ToolInvocationRepository) ReflectionTestUtils.getField(chatService, "toolInvocationRepository");
|
||||
when(invocationRepository.findBySessionIdOrderByIdAsc("sequential-gatekeeper-fail-session"))
|
||||
.thenReturn(List.of(ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.sessionId("sequential-gatekeeper-fail-session")
|
||||
.toolName("query_metrics")
|
||||
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
||||
.build()));
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "PASS",
|
||||
"groundedness_score": 1.0,
|
||||
"critical_fact_count": 1,
|
||||
"claim_checks": [
|
||||
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"direct","evidence_refs":[]}
|
||||
],
|
||||
"facts_checked": [],
|
||||
"rationale": "model tried pass"
|
||||
}
|
||||
""");
|
||||
chatModel.composerOutput = "not-json";
|
||||
chatModel.executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 999,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-gatekeeper-fail-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("当前无法基于已获取证据生成可靠结论"));
|
||||
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
||||
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
||||
assertEquals("REJECT", captor.getValue().get("verdict"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeChatComplexDowngradesPassToLowConfidenceWhenExecutorOutputMalformed() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
SelfEvaluationMergeService mergeService =
|
||||
(SelfEvaluationMergeService) ReflectionTestUtils.getField(chatService, "selfEvaluationMergeService");
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel("""
|
||||
{
|
||||
"verdict": "PASS",
|
||||
"groundedness_score": 1.0,
|
||||
"critical_fact_count": 0,
|
||||
"claim_checks": [],
|
||||
"facts_checked": [],
|
||||
"rationale": "model tried pass"
|
||||
}
|
||||
""");
|
||||
chatModel.composerOutput = "not-json";
|
||||
chatModel.executorOutput = "{ not-json";
|
||||
|
||||
ChatService.ChatResult result = chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"请分析 MySQL 连接池耗尽",
|
||||
List.of(),
|
||||
"sequential-malformed-pass-session"
|
||||
);
|
||||
|
||||
assertTrue(result.answer().startsWith("以下结论基于当前已获取证据"));
|
||||
ArgumentCaptor<Map<String, Object>> captor = ArgumentCaptor.forClass(Map.class);
|
||||
verify(mergeService).mergeVerifierEvaluation(isNull(), captor.capture());
|
||||
assertEquals("LOW_CONFID", captor.getValue().get("verdict"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
|
||||
ChatService chatService = new ChatService();
|
||||
DateTimeTools dateTimeTools = new DateTimeTools();
|
||||
LookupKnowledgeTool lookupKnowledgeTool = new LookupKnowledgeTool();
|
||||
QueryLogsTools queryLogsTools = new QueryLogsTools(mock(ToolInvocationRecorder.class));
|
||||
QueryMetricsTools queryMetricsTools = new QueryMetricsTools(mock(ToolInvocationRecorder.class));
|
||||
|
||||
ReflectionTestUtils.setField(chatService, "dateTimeTools", dateTimeTools);
|
||||
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", lookupKnowledgeTool);
|
||||
ReflectionTestUtils.setField(chatService, "queryLogsTools", queryLogsTools);
|
||||
ReflectionTestUtils.setField(chatService, "queryMetricsTools", queryMetricsTools);
|
||||
|
||||
Object[] methodTools = chatService.buildMethodToolsArray();
|
||||
|
||||
assertEquals(4, methodTools.length);
|
||||
assertSame(dateTimeTools, methodTools[0]);
|
||||
assertSame(lookupKnowledgeTool, methodTools[1]);
|
||||
assertSame(queryLogsTools, methodTools[2]);
|
||||
assertSame(queryMetricsTools, methodTools[3]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void createReactAgentInjectsSkillCatalogThroughAlibabaHook() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
|
||||
.classpathPath("skills")
|
||||
.basePath("target/test-skills-cache")
|
||||
.build();
|
||||
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
|
||||
|
||||
ReactAgent agent = chatService.createReactAgent(chatModel, "BASE_TEST_PROMPT");
|
||||
agent.call("diagnose mysql connection pool exhaustion");
|
||||
|
||||
assertTrue(chatModel.promptText.contains("BASE_TEST_PROMPT"));
|
||||
assertTrue(chatModel.promptText.contains("## Skills System"));
|
||||
assertTrue(chatModel.promptText.contains("diagnose-mysql-connection-pool"));
|
||||
assertTrue(chatModel.promptText.contains("read_skill"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void plannerGetsSkillMetadataAndExecutorGetsReadSkillTool() throws Exception {
|
||||
ChatService chatService = createChatService();
|
||||
ScriptedChatModel chatModel = new ScriptedChatModel();
|
||||
SkillRegistry skillRegistry = ClasspathSkillRegistry.builder()
|
||||
.classpathPath("skills")
|
||||
.basePath("target/test-skills-cache")
|
||||
.build();
|
||||
ReflectionTestUtils.setField(chatService, "skillRegistry", skillRegistry);
|
||||
|
||||
chatService.executeChatComplex(
|
||||
chatModel,
|
||||
new ToolCallback[0],
|
||||
"diagnose mysql connection pool exhaustion",
|
||||
List.of(),
|
||||
"planner-skill-metadata-session"
|
||||
);
|
||||
|
||||
assertTrue(chatModel.plannerPromptText.contains("\"skill_catalog\""));
|
||||
assertTrue(chatModel.plannerPromptText.contains("diagnose-mysql-connection-pool"));
|
||||
assertTrue(chatModel.plannerPromptText.contains("\"selected_skill\""));
|
||||
assertFalse(chatModel.plannerPromptText.contains("## Skills System"));
|
||||
assertFalse(chatModel.plannerPromptText.contains("read_skill"));
|
||||
|
||||
assertTrue(chatModel.executorPromptText.contains("## Skills System"));
|
||||
assertTrue(chatModel.executorPromptText.contains("diagnose-mysql-connection-pool"));
|
||||
assertTrue(chatModel.executorPromptText.contains("read_skill"));
|
||||
assertTrue(chatModel.executorPromptText.contains("只允许对该 skill 调用一次 read_skill"));
|
||||
assertFalse(chatModel.verifierPromptText.contains("diagnose-mysql-connection-pool"));
|
||||
assertFalse(chatModel.verifierPromptText.contains("read_skill"));
|
||||
}
|
||||
|
||||
private ChatService createChatService() {
|
||||
ChatService chatService = new ChatService();
|
||||
|
||||
ChatSessionRepository chatSessionRepository = mock(ChatSessionRepository.class);
|
||||
when(chatSessionRepository.findBySessionId(anyString())).thenReturn(Optional.empty());
|
||||
when(chatSessionRepository.save(any(ChatSession.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class);
|
||||
when(diagnosisRunRepository.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
|
||||
AtomicInteger stepId = new AtomicInteger(1);
|
||||
AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
|
||||
when(agentStepRepository.save(any(AgentStep.class))).thenAnswer(invocation -> {
|
||||
AgentStep step = invocation.getArgument(0);
|
||||
if (step.getId() == null) {
|
||||
step.setId((long) stepId.getAndIncrement());
|
||||
}
|
||||
return step;
|
||||
});
|
||||
when(agentStepRepository.findById(any())).thenReturn(Optional.of(new AgentStep()));
|
||||
when(agentStepRepository.findBySessionIdOrderByStepIndex(anyString())).thenReturn(List.of());
|
||||
when(agentStepRepository.findByRunIdOrderByStepIndex(anyString())).thenReturn(List.of());
|
||||
ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
|
||||
when(toolInvocationRepository.countBySessionId(anyString())).thenReturn(0L);
|
||||
when(toolInvocationRepository.countByRunId(anyString())).thenReturn(0L);
|
||||
when(toolInvocationRepository.findBySessionIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.toolName("query_metrics")
|
||||
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
||||
.build()));
|
||||
when(toolInvocationRepository.findByRunIdOrderByIdAsc(anyString())).thenReturn(List.of(ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.toolName("query_metrics")
|
||||
.retrievalDetails(evidenceRefs("$.alerts[0]", "active=50 max=50"))
|
||||
.build()));
|
||||
|
||||
EvaluationService evaluationService = mock(EvaluationService.class);
|
||||
RetrievedDocTracker retrievedDocTracker = mock(RetrievedDocTracker.class);
|
||||
KnowledgeDomainService knowledgeDomainService = mock(KnowledgeDomainService.class);
|
||||
when(knowledgeDomainService.buildKnowledgeMap()).thenReturn("");
|
||||
ToolTraceSummaryService toolTraceSummaryService = mock(ToolTraceSummaryService.class);
|
||||
when(toolTraceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of());
|
||||
when(toolTraceSummaryService.buildVerifierTraceSummaryForRun(anyString(), anyString())).thenReturn(List.of());
|
||||
SelfEvaluationMergeService selfEvaluationMergeService = mock(SelfEvaluationMergeService.class);
|
||||
when(selfEvaluationMergeService.mergeVerifierEvaluation(any(), any())).thenReturn("{}");
|
||||
ExecutorGatekeeperService executorGatekeeperService = new ExecutorGatekeeperService(toolInvocationRepository);
|
||||
|
||||
ReflectionTestUtils.setField(chatService, "dateTimeTools", new DateTimeTools());
|
||||
ReflectionTestUtils.setField(chatService, "lookupKnowledgeTool", new LookupKnowledgeTool());
|
||||
ReflectionTestUtils.setField(chatService, "queryLogsTools", new QueryLogsTools(mock(ToolInvocationRecorder.class)));
|
||||
ReflectionTestUtils.setField(chatService, "chatSessionRepository", chatSessionRepository);
|
||||
ReflectionTestUtils.setField(chatService, "diagnosisRunRepository", diagnosisRunRepository);
|
||||
ReflectionTestUtils.setField(chatService, "agentStepRepository", agentStepRepository);
|
||||
ReflectionTestUtils.setField(chatService, "toolInvocationRepository", toolInvocationRepository);
|
||||
ReflectionTestUtils.setField(chatService, "evaluationService", evaluationService);
|
||||
ReflectionTestUtils.setField(chatService, "retrievedDocTracker", retrievedDocTracker);
|
||||
ReflectionTestUtils.setField(chatService, "knowledgeDomainService", knowledgeDomainService);
|
||||
ReflectionTestUtils.setField(chatService, "toolTraceSummaryService", toolTraceSummaryService);
|
||||
ReflectionTestUtils.setField(chatService, "selfEvaluationMergeService", selfEvaluationMergeService);
|
||||
ReflectionTestUtils.setField(chatService, "executorGatekeeperService", executorGatekeeperService);
|
||||
ReflectionTestUtils.setField(chatService, "verifierLowConfidenceThreshold", 0.5d);
|
||||
ReflectionTestUtils.setField(chatService, "chatPlannerPrompt", "PLANNER_TEST_PROMPT");
|
||||
ReflectionTestUtils.setField(chatService, "chatExecutorPrompt", "EXECUTOR_TEST_PROMPT");
|
||||
ReflectionTestUtils.setField(chatService, "chatVerifierPrompt", "VERIFIER_TEST_PROMPT");
|
||||
ReflectionTestUtils.setField(chatService, "chatComposerPrompt", "COMPOSER_TEST_PROMPT");
|
||||
return chatService;
|
||||
}
|
||||
|
||||
private String validExecutorV2Output() {
|
||||
return """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
}
|
||||
|
||||
private String evidenceRefs(String rawPath, String text) {
|
||||
return "{\"evidence_refs\":[{\"raw_path\":\"" + rawPath + "\",\"text\":\"" + text + "\"}]}";
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
private final java.util.ArrayList<String> agentCalls = new java.util.ArrayList<>();
|
||||
private String promptText = "";
|
||||
private String plannerPromptText = "";
|
||||
private String executorPromptText = "";
|
||||
private String verifierPromptText = "";
|
||||
private String composerPromptText = "";
|
||||
private String executorOutput = """
|
||||
{
|
||||
"answer_version": "executor_evidence_v2",
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "claim-1",
|
||||
"claim_type": "symptom",
|
||||
"claim_text": "连接池 active 达到上限",
|
||||
"support_level": "direct",
|
||||
"evidence_bindings": [
|
||||
{
|
||||
"source_type": "tool_trace",
|
||||
"source_id": "trace-1",
|
||||
"tool_name": "query_metrics",
|
||||
"source_invocation_id": 101,
|
||||
"raw_path": "$.alerts[0]",
|
||||
"evidence_excerpt": "active=50 max=50"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"hypotheses": [],
|
||||
"recommended_actions": [],
|
||||
"missing_info": []
|
||||
}
|
||||
""";
|
||||
private String composerOutput = """
|
||||
{
|
||||
"answer_summary": "已确认连接池 active 达到上限。",
|
||||
"recommended_actions": [
|
||||
{
|
||||
"action_text": "补充查询连接池泄漏检测日志",
|
||||
"reason": "用于确认是否存在连接未释放"
|
||||
}
|
||||
],
|
||||
"user_facing_answer": "已确认连接池 active 达到上限。建议补充查询连接池泄漏检测日志。"
|
||||
}
|
||||
""";
|
||||
private boolean sawVerifierPrompt;
|
||||
private final java.util.List<String> verifierOutputs;
|
||||
private int verifierOutputIndex;
|
||||
|
||||
private ScriptedChatModel() {
|
||||
this("""
|
||||
{
|
||||
"verdict": "PASS",
|
||||
"groundedness_score": 1.0,
|
||||
"critical_fact_count": 1,
|
||||
"claim_checks": [
|
||||
{"claim_id":"claim-1","claim_text":"连接池 active 达到上限","claim_type":"symptom","verification":"direct_observation","detail":"covered by scripted verifier","evidence_refs":[]}
|
||||
],
|
||||
"facts_checked": [],
|
||||
"rationale": "scripted pass"
|
||||
}
|
||||
""");
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String verifierOutput) {
|
||||
this.verifierOutputs = java.util.List.of(verifierOutput);
|
||||
}
|
||||
|
||||
private ScriptedChatModel(String... verifierOutputs) {
|
||||
this.verifierOutputs = java.util.List.of(verifierOutputs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
promptText = prompt.getContents();
|
||||
String text;
|
||||
if (promptText.contains("PLANNER_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_planner");
|
||||
plannerPromptText = promptText;
|
||||
text = "PLANNER_PLAN";
|
||||
} else if (promptText.contains("EXECUTOR_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_executor");
|
||||
executorPromptText = promptText;
|
||||
text = executorOutput;
|
||||
} else if (promptText.contains("VERIFIER_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_verifier");
|
||||
verifierPromptText = promptText;
|
||||
sawVerifierPrompt = true;
|
||||
int index = Math.min(verifierOutputIndex, verifierOutputs.size() - 1);
|
||||
text = verifierOutputs.get(index);
|
||||
verifierOutputIndex++;
|
||||
} else if (promptText.contains("COMPOSER_TEST_PROMPT")) {
|
||||
agentCalls.add("chat_composer");
|
||||
composerPromptText = promptText;
|
||||
text = composerOutput;
|
||||
} else {
|
||||
text = "UNEXPECTED_PROMPT";
|
||||
}
|
||||
return new ChatResponse(List.of(new Generation(new AssistantMessage(text))));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,389 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ExecutorGatekeeperServiceTest {
|
||||
|
||||
@Test
|
||||
void validateRunUsesRunScopedToolRows() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findByRunIdOrderByIdAsc("run-gatekeeper-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]",
|
||||
"HighCPUUsage firing, service=payment-service, current=92%")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validateRun("run-gatekeeper-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[0]",
|
||||
"HighCPUUsage firing, service=payment-service, current=92%"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
verify(repository).findByRunIdOrderByIdAsc("run-gatekeeper-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void ruleCatalogLoadsDefaultMetadata() {
|
||||
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.loadDefault(new com.fasterxml.jackson.databind.ObjectMapper());
|
||||
|
||||
assertEquals("gatekeeper-rules-v1", catalog.version());
|
||||
assertFalse(catalog.auditRules().isEmpty());
|
||||
assertTrue(catalog.auditRules().stream()
|
||||
.anyMatch(rule -> "evidence.raw_path".equals(rule.get("id"))));
|
||||
assertEquals(0.5, catalog.doubleParameter("evidence.excerpt_mismatch",
|
||||
"min_token_overlap", 0.0), 0.001);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]",
|
||||
"HighCPUUsage firing, service=payment-service, current=92%, duration=25m")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[0]",
|
||||
"HighCPUUsage firing, service=payment-service, current=92%"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePassesForNoEvidenceReference() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; topic=application-logs; total=0; message=未找到匹配的日志")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateBackfillsNoEvidenceInvocationByRawPathWhenToolHasMultipleCalls() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志"),
|
||||
invocation(102L, "query_logs", "$.logs[0]",
|
||||
"order-service HikariCP active=50/50 waiting=32")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(null, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; query=inventory-service HikariCP; total=0; evidence_status=no_evidence",
|
||||
"negative_observation"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
|
||||
assertEquals("evidence.invocation_auto_backfill_by_raw_path",
|
||||
((Map<?, ?>) ((List<?>) result.get("warnings")).get(0)).get("rule"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateBackfillsNoEvidenceInvocationByExcerptWhenRawPathIsRepeated() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; evidence_status=no_evidence; query=service:inventory-service AND HikariCP; total=0; message=未找到匹配的日志"),
|
||||
invocation(102L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service; total=0; message=未找到匹配的日志")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(null, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; query=service:inventory-service AND HikariCP; total=0; evidence_status=no_evidence",
|
||||
"negative_observation"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertEquals(101L,
|
||||
((Map<?, ?>) ((List<?>) result.get("checked_bindings")).get(0)).get("source_invocation_id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsNoEvidenceReferenceWhenExcerptClaimsPositiveEvidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.no_evidence",
|
||||
"query_logs returned no evidence; evidence_status=no_evidence; query=inventory-service HikariCP; total=0; message=未找到匹配的日志")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_logs", "$.no_evidence",
|
||||
"HikariCP active=50/50 waiting=32"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsPositiveBindingOnNegativeObservation() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.logs[0]",
|
||||
"order-service HikariCP active=50/50 waiting=32")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_logs", "$.logs[0]",
|
||||
"order-service HikariCP active=50/50 waiting=32",
|
||||
"negative_observation"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateFailsWhenRemovedFieldsArePresent() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of());
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
|
||||
output.put("user_facing_answer", "旧版最终答案");
|
||||
|
||||
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("schema.executor_v2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateFailsForFabricatedInvocationId() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(999L, "query_metrics", "$.alerts[0]", "cpu=92"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failResultIncludesRuleAuditMetadata() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.fail("gatekeeper.internal_error", "gatekeeper", "boom");
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateFailsForToolNameMismatch() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_logs", "$.logs[0]", "cpu=92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateDowngradesMissingRawPathToLowConfidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
Map<String, Object> output = validOutput(101L, "query_metrics", null, "cpu=92");
|
||||
|
||||
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("low_confid", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateRejectsUnknownRawPath() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[99]", "cpu=92"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.raw_path"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateDowngradesOldInvocationWithoutEvidenceRefsToLowConfidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
ToolInvocation.builder().id(101L).sessionId("session-1").toolName("query_metrics")
|
||||
.retrievalDetails("{\"evidence_status\":\"supported\"}")
|
||||
.build()
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("low_confid", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.missing"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void validateRejectsMismatchedExcerpt() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]", "HighCPUUsage firing service payment-service current 92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.validate("session-1",
|
||||
validOutput(101L, "query_metrics", "$.alerts[0]", "HikariCP active=50/50 waiting=32"),
|
||||
Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateFailsForRecommendedActionFabricatedInvocationId() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
invocation(101L, "query_metrics", "$.alerts[0]", "cpu=92")
|
||||
));
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
Map<String, Object> output = validOutput(101L, "query_metrics", "$.alerts[0]", "cpu=92");
|
||||
output.put("recommended_actions", List.of(Map.of(
|
||||
"action_text", "restart service",
|
||||
"reason", "alert is firing",
|
||||
"evidence_bindings", List.of(Map.of(
|
||||
"source_type", "tool_trace",
|
||||
"source_id", "trace-1",
|
||||
"tool_name", "query_metrics",
|
||||
"source_invocation_id", 999L,
|
||||
"raw_path", "$.alerts[0]",
|
||||
"evidence_excerpt", "cpu=92"
|
||||
))
|
||||
)));
|
||||
|
||||
Map<String, Object> result = service.validate("session-1", output, Map.of("status", "valid"));
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
|
||||
}
|
||||
|
||||
private ToolInvocation invocation(Long id, String toolName, String rawPath, String text) {
|
||||
return ToolInvocation.builder()
|
||||
.id(id)
|
||||
.sessionId("session-1")
|
||||
.toolName(toolName)
|
||||
.retrievalDetails("{\"evidence_refs\":[{\"raw_path\":\"" + rawPath
|
||||
+ "\",\"text\":\"" + text + "\"}]}")
|
||||
.build();
|
||||
}
|
||||
|
||||
private Map<String, Object> validOutput(Long invocationId, String toolName, String rawPath, String excerpt) {
|
||||
return validOutput(invocationId, toolName, rawPath, excerpt, "symptom");
|
||||
}
|
||||
|
||||
private Map<String, Object> validOutput(Long invocationId,
|
||||
String toolName,
|
||||
String rawPath,
|
||||
String excerpt,
|
||||
String claimType) {
|
||||
Map<String, Object> binding = new java.util.LinkedHashMap<>();
|
||||
binding.put("source_type", "tool_trace");
|
||||
binding.put("source_id", "trace-1");
|
||||
binding.put("tool_name", toolName);
|
||||
if (invocationId != null) {
|
||||
binding.put("source_invocation_id", invocationId);
|
||||
}
|
||||
if (rawPath != null) {
|
||||
binding.put("raw_path", rawPath);
|
||||
}
|
||||
binding.put("evidence_excerpt", excerpt);
|
||||
return new java.util.LinkedHashMap<>(Map.of(
|
||||
"answer_version", "executor_evidence_v2",
|
||||
"claims", List.of(Map.of(
|
||||
"claim_id", "claim-1",
|
||||
"claim_type", claimType,
|
||||
"claim_text", "连接池 active 达到上限",
|
||||
"support_level", "direct",
|
||||
"evidence_bindings", List.of(binding)
|
||||
)),
|
||||
"hypotheses", List.of(),
|
||||
"recommended_actions", List.of(),
|
||||
"missing_info", List.of()
|
||||
));
|
||||
}
|
||||
|
||||
private void assertRuleAudit(Map<String, Object> result) {
|
||||
assertEquals("gatekeeper-rules-v1", result.get("rule_set_version"));
|
||||
assertTrue(result.get("rules") instanceof List<?>);
|
||||
List<?> rules = (List<?>) result.get("rules");
|
||||
assertFalse(rules.isEmpty());
|
||||
assertTrue(rules.stream().anyMatch(rule ->
|
||||
rule instanceof Map<?, ?> map && "evidence.raw_path".equals(map.get("id"))));
|
||||
}
|
||||
}
|
||||
@@ -1,363 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.ContextPack;
|
||||
import com.superbiz.agent.dto.EvidenceBlock;
|
||||
import com.superbiz.agent.dto.KnowledgeQuery;
|
||||
import com.superbiz.agent.dto.RetrievalTrace;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ToolInvocationRecorderTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolWritesRunIdFromExecutionContext() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setContext("recorder-run-session", "run-recorder-1");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_metrics",
|
||||
Map.of("query", "active_prometheus_alerts"),
|
||||
"{\"success\":true,\"alerts\":[]}",
|
||||
true,
|
||||
12,
|
||||
null,
|
||||
"prometheus_alerts",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
|
||||
Map.of("metric_family", "prometheus_alerts")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
assertEquals("recorder-run-session", saved.getSessionId());
|
||||
assertEquals("run-recorder-1", saved.getRunId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolPreservesNoEvidenceSemantics() throws Exception {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("recorder-test-session");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_logs",
|
||||
Map.of("query", "inventory-service HikariCP", "log_topic", "application-logs"),
|
||||
"{\"success\":false,\"query\":\"inventory-service HikariCP\",\"log_topic\":\"application-logs\",\"logs\":[],\"total\":0,\"message\":\"未找到匹配的日志\"}",
|
||||
true,
|
||||
System.currentTimeMillis() - 10,
|
||||
null,
|
||||
"application-logs",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
|
||||
Map.of("log_topic", "application-logs")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
|
||||
assertEquals("query_logs", saved.getToolName());
|
||||
assertEquals(Boolean.TRUE, saved.getSuccess());
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"no_evidence\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"application-logs\"]"));
|
||||
|
||||
JsonNode details = objectMapper.readTree(saved.getRetrievalDetails());
|
||||
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
|
||||
String text = details.path("evidence_refs").get(0).path("text").asText();
|
||||
assertTrue(text.contains("query_logs returned no evidence"));
|
||||
assertTrue(text.contains("inventory-service HikariCP"));
|
||||
assertTrue(text.contains("total=0"));
|
||||
assertTrue(text.contains("evidence_status=no_evidence"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolExtractsLogEvidenceRefs() throws Exception {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("log-ref-session");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_logs",
|
||||
Map.of("query", "HikariCP order-service"),
|
||||
"""
|
||||
{"success":true,"logs":[{"timestamp":"2026-07-08 10:00:00","level":"ERROR","service":"order-service","message":"HikariPool-1 - Connection is not available, request timed out after 30000ms","metrics":{"waiting":"32"}}]}
|
||||
""",
|
||||
true,
|
||||
System.currentTimeMillis() - 10,
|
||||
null,
|
||||
"application-logs",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
|
||||
Map.of("log_topic", "application-logs")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
|
||||
|
||||
assertEquals("$.logs[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HikariPool-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolExtractsMetricEvidenceRefs() throws Exception {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("metric-ref-session");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_metrics",
|
||||
Map.of("query", "active_prometheus_alerts"),
|
||||
"""
|
||||
{"success":true,"alerts":[{"alert_name":"HighMemoryUsage","state":"firing","description":"服务 order-service 当前值为 91%","active_at":"2026-07-08T10:00:00Z","duration":"15m"}]}
|
||||
""",
|
||||
true,
|
||||
System.currentTimeMillis() - 10,
|
||||
null,
|
||||
"prometheus_alerts",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED,
|
||||
Map.of("metric_family", "prometheus_alerts")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
|
||||
|
||||
assertEquals("$.alerts[0]", details.path("evidence_refs").get(0).path("raw_path").asText());
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("HighMemoryUsage"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordEvidenceToolExtractsMetricNoEvidenceRef() throws Exception {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("metric-no-evidence-session");
|
||||
|
||||
try {
|
||||
recorder.recordEvidenceTool(
|
||||
"query_metrics",
|
||||
Map.of("query", "active_prometheus_alerts"),
|
||||
"""
|
||||
{"success":true,"alerts":[],"message":"成功检索到 0 个活动告警"}
|
||||
""",
|
||||
true,
|
||||
System.currentTimeMillis() - 10,
|
||||
null,
|
||||
"prometheus_alerts",
|
||||
ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE,
|
||||
Map.of("metric_family", "prometheus_alerts")
|
||||
);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
|
||||
|
||||
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("query_metrics returned no evidence"));
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("prometheus_alerts"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordLookupKnowledgePreservesRetrievalSpecificFields() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("lookup-recorder-session");
|
||||
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
|
||||
.query("ERR_TIMEOUT")
|
||||
.outputPreview("matched payment doc")
|
||||
.outputLength(18)
|
||||
.retrievalLayer("L0")
|
||||
.l0MatchCount(1)
|
||||
.l1MatchCount(null)
|
||||
.truncated(false)
|
||||
.relevanceLevel("PRECISE")
|
||||
.completenessHint("already precise")
|
||||
.domain("payment")
|
||||
.dedupReason("doc_retrieved")
|
||||
.durationMs(42)
|
||||
.success(true)
|
||||
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_DEDUPED)
|
||||
.l0Titles(List.of("payment/errors.md"))
|
||||
.l0MatchedKeywords(List.of("ERR_TIMEOUT"))
|
||||
.l0Domains(List.of("payment"))
|
||||
.l0Entities(List.of("ERR_TIMEOUT"))
|
||||
.evidenceCandidateCount(2)
|
||||
.evidenceBlockCount(1)
|
||||
.evidenceBlocks(List.of(Map.of(
|
||||
"source", "payment/errors.md",
|
||||
"title", "payment/errors.md",
|
||||
"retrieval_layer", "L0+L1",
|
||||
"hit_reasons", List.of("l0_keywords:ERR_TIMEOUT", "semantic_rank:1")
|
||||
)))
|
||||
.build();
|
||||
|
||||
try {
|
||||
recorder.recordLookupKnowledge(record);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
ToolInvocation saved = captor.getValue();
|
||||
|
||||
assertEquals("lookup_knowledge", saved.getToolName());
|
||||
assertEquals("PRECISE", saved.getRelevanceLevel());
|
||||
assertEquals("doc_retrieved", saved.getDedupReason());
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_status\":\"deduped\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"retrieved_domains\":[\"payment\"]"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"l0_matched_keywords\":[\"ERR_TIMEOUT\"]"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"l0_domains\":[\"payment\"]"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"l0_entities\":[\"ERR_TIMEOUT\"]"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_candidate_count\":2"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_block_count\":1"));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_blocks\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"evidence_refs\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"raw_path\":\"$.evidence_blocks[0]\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"query_transform\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"retrieval_trace\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"context_pack_summary\""));
|
||||
assertTrue(saved.getRetrievalDetails().contains("\"rerank_trace\""));
|
||||
}
|
||||
|
||||
@Test
|
||||
void recordLookupKnowledgeAddsNoEvidenceRefWhenNoBlocks() throws Exception {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.save(any(ToolInvocation.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
ToolInvocationRecorder recorder = new ToolInvocationRecorder(repository, new ObjectMapper());
|
||||
SessionContextHolder.setSessionId("lookup-no-evidence-session");
|
||||
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.builder()
|
||||
.query("inventory-service HikariCP")
|
||||
.outputPreview("")
|
||||
.outputLength(0)
|
||||
.durationMs(12)
|
||||
.success(true)
|
||||
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_NO_EVIDENCE)
|
||||
.evidenceBlocks(List.of())
|
||||
.build();
|
||||
|
||||
try {
|
||||
recorder.recordLookupKnowledge(record);
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
|
||||
ArgumentCaptor<ToolInvocation> captor = ArgumentCaptor.forClass(ToolInvocation.class);
|
||||
verify(repository).save(captor.capture());
|
||||
JsonNode details = objectMapper.readTree(captor.getValue().getRetrievalDetails());
|
||||
|
||||
assertEquals("$.no_evidence", details.path("evidence_refs").get(0).path("raw_path").asText());
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("lookup_knowledge returned no evidence"));
|
||||
assertTrue(details.path("evidence_refs").get(0).path("text").asText().contains("inventory-service HikariCP"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void lookupKnowledgeRecordFromSummarizesEvidenceBlocks() {
|
||||
EvidenceBlock block = EvidenceBlock.builder()
|
||||
.source("doc.md")
|
||||
.title("Doc")
|
||||
.breadcrumb("A > B")
|
||||
.retrievalLayer("L1")
|
||||
.score(0.42)
|
||||
.hitReasons(List.of("semantic_rank:1"))
|
||||
.content("x".repeat(220))
|
||||
.build();
|
||||
|
||||
com.superbiz.agent.dto.LookupResult result = com.superbiz.agent.dto.LookupResult.builder()
|
||||
.found(true)
|
||||
.evidenceCandidateCount(3)
|
||||
.evidenceBlockCount(1)
|
||||
.evidenceBlocks(List.of(block))
|
||||
.contextPack(ContextPack.builder()
|
||||
.packedText("packed evidence")
|
||||
.strategy("ranked_evidence_char_budget")
|
||||
.charBudget(4000)
|
||||
.usedChars(15)
|
||||
.includedSources(List.of("doc.md"))
|
||||
.omittedSources(List.of())
|
||||
.build())
|
||||
.retrievalTrace(RetrievalTrace.builder()
|
||||
.originalQuery("query")
|
||||
.rewrittenQuery("query")
|
||||
.selectedAttempt("UNFILTERED_VECTOR")
|
||||
.evidenceStatus(ToolInvocationRecorder.EVIDENCE_STATUS_SUPPORTED)
|
||||
.attempts(List.of(RetrievalTrace.Attempt.builder()
|
||||
.name("UNFILTERED_VECTOR")
|
||||
.candidateCount(3)
|
||||
.topScore(0.42)
|
||||
.topSimilarity(0.79)
|
||||
.usable(true)
|
||||
.build()))
|
||||
.build())
|
||||
.build();
|
||||
KnowledgeQuery query = KnowledgeQuery.builder()
|
||||
.originalQuery("query")
|
||||
.rewrittenQuery("query")
|
||||
.domainHints(List.of())
|
||||
.matchedKeywords(List.of())
|
||||
.entities(List.of())
|
||||
.l0Titles(List.of())
|
||||
.l0MatchCount(0)
|
||||
.build();
|
||||
|
||||
ToolInvocationRecorder.LookupKnowledgeRecord record = ToolInvocationRecorder.LookupKnowledgeRecord.from(
|
||||
query,
|
||||
result,
|
||||
null,
|
||||
null,
|
||||
10
|
||||
);
|
||||
|
||||
assertEquals(3, record.evidenceCandidateCount());
|
||||
assertEquals(1, record.evidenceBlockCount());
|
||||
assertEquals(1, record.evidenceBlocks().size());
|
||||
assertTrue(String.valueOf(record.evidenceBlocks().get(0).get("content_preview")).endsWith("..."));
|
||||
assertEquals("L1", record.retrievalLayer());
|
||||
assertEquals(3, record.l1MatchCount());
|
||||
assertTrue(record.retrievalTrace().containsKey("selected_attempt"));
|
||||
assertTrue(record.contextPack().containsKey("included_sources"));
|
||||
}
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
package com.superbiz.agent.service;
|
||||
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ToolTraceSummaryServiceTest {
|
||||
|
||||
@Test
|
||||
void buildVerifierTraceSummaryForRunUsesRunScopedToolRows() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findByRunIdOrderByIdAsc("run-summary-1")).thenReturn(List.of(
|
||||
ToolInvocation.builder()
|
||||
.id(101L)
|
||||
.sessionId("session-1")
|
||||
.runId("run-summary-1")
|
||||
.toolName("query_metrics")
|
||||
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
|
||||
.outputPreview("active=50 max=50")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
|
||||
.success(true)
|
||||
.build()
|
||||
));
|
||||
|
||||
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
|
||||
|
||||
List<Map<String, Object>> summaries = service.buildVerifierTraceSummaryForRun(
|
||||
"run-summary-1", "active=50 max=50");
|
||||
|
||||
assertEquals(1, summaries.size());
|
||||
assertEquals("query_metrics", summaries.get(0).get("tool_name"));
|
||||
assertEquals(List.of(101L), summaries.get(0).get("source_invocation_ids"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildVerifierTraceSummaryTreatsNoEvidenceAsGapWithoutLosingSuccessfulEvidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-1")).thenReturn(List.of(
|
||||
ToolInvocation.builder()
|
||||
.id(1L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"timeout\"}")
|
||||
.outputPreview("payment timeout stack trace")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
|
||||
.success(true)
|
||||
.build(),
|
||||
ToolInvocation.builder()
|
||||
.id(2L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"timeout\"}")
|
||||
.outputPreview("{\"success\":false,\"message\":\"未找到匹配的日志\"}")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"no_evidence\"}")
|
||||
.success(true)
|
||||
.build(),
|
||||
ToolInvocation.builder()
|
||||
.id(3L)
|
||||
.sessionId("session-1")
|
||||
.toolName("query_metrics")
|
||||
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
|
||||
.errorMessage("prometheus timeout")
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"failed\"}")
|
||||
.success(false)
|
||||
.build()
|
||||
));
|
||||
|
||||
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
|
||||
|
||||
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-1", "application-logs point to timeout");
|
||||
|
||||
assertEquals(2, summaries.size());
|
||||
|
||||
Map<String, Object> logsSummary = summaries.stream()
|
||||
.filter(item -> "query_logs".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertEquals(Boolean.TRUE, logsSummary.get("success"));
|
||||
assertEquals("direct", logsSummary.get("evidence_level"));
|
||||
assertEquals(2, logsSummary.get("invocation_count"));
|
||||
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
|
||||
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("payment timeout stack trace"));
|
||||
|
||||
Map<String, Object> metricsSummary = summaries.stream()
|
||||
.filter(item -> "query_metrics".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertEquals(Boolean.FALSE, metricsSummary.get("success"));
|
||||
assertEquals("none", metricsSummary.get("evidence_level"));
|
||||
assertEquals(1, metricsSummary.get("failed_invocation_count"));
|
||||
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("call failed"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildVerifierTraceSummaryPreservesConcreteFactsFromTruncatedLogAndMetricRows() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
String logPreview = """
|
||||
{
|
||||
"success" : true,
|
||||
"logs" : [ {
|
||||
"timestamp" : "2026-07-06 22:15:45",
|
||||
"level" : "ERROR",
|
||||
"service" : "order-service",
|
||||
"message" : "数据库连接池耗尽: Cannot acquire connection from pool, active: 50/50, waiting: 23, timeout: 30000ms"
|
||||
} ]
|
||||
}
|
||||
""";
|
||||
String metricPreview = """
|
||||
{
|
||||
"success" : true,
|
||||
"alerts" : [ {
|
||||
"alert_name" : "HighCPUUsage",
|
||||
"service" : "payment-service",
|
||||
"description" : "服务 payment-service 的 CPU 使用率持续超过 80%，当前值为 92%。"
|
||||
} ]
|
||||
}
|
||||
""";
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-2")).thenReturn(List.of(
|
||||
ToolInvocation.builder()
|
||||
.id(10L)
|
||||
.sessionId("session-2")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"pool\"}")
|
||||
.outputPreview(logPreview)
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"application-logs\"],\"evidence_status\":\"supported\"}")
|
||||
.isTruncated(true)
|
||||
.success(true)
|
||||
.build(),
|
||||
ToolInvocation.builder()
|
||||
.id(11L)
|
||||
.sessionId("session-2")
|
||||
.toolName("query_metrics")
|
||||
.inputParams("{\"query\":\"active_prometheus_alerts\"}")
|
||||
.outputPreview(metricPreview)
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"prometheus_alerts\"],\"evidence_status\":\"supported\"}")
|
||||
.isTruncated(true)
|
||||
.success(true)
|
||||
.build()
|
||||
));
|
||||
|
||||
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
|
||||
|
||||
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-2", "连接池耗尽 HighCPUUsage");
|
||||
|
||||
Map<String, Object> logsSummary = summaries.stream()
|
||||
.filter(item -> "query_logs".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("连接池耗尽"));
|
||||
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("active: 50/50"));
|
||||
assertEquals(List.of(10L), logsSummary.get("source_invocation_ids"));
|
||||
|
||||
Map<String, Object> metricsSummary = summaries.stream()
|
||||
.filter(item -> "query_metrics".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("HighCPUUsage"));
|
||||
assertTrue(String.valueOf(metricsSummary.get("output_summary")).contains("payment-service"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildVerifierTraceSummaryDoesNotTreatGenericMockLogsAsDirectEvidence() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
String genericLogPreview = """
|
||||
{
|
||||
"success" : true,
|
||||
"logs" : [ {
|
||||
"timestamp" : "2026-07-06 23:44:41",
|
||||
"level" : "ERROR",
|
||||
"service" : "generic-service",
|
||||
"message" : "日志消息 #0, 查询条件: service:payment-service"
|
||||
} ]
|
||||
}
|
||||
""";
|
||||
when(repository.findBySessionIdOrderByIdAsc("session-3")).thenReturn(List.of(
|
||||
ToolInvocation.builder()
|
||||
.id(20L)
|
||||
.sessionId("session-3")
|
||||
.toolName("query_logs")
|
||||
.inputParams("{\"query\":\"service:payment-service\"}")
|
||||
.outputPreview(genericLogPreview)
|
||||
.retrievalDetails("{\"retrieved_domains\":[\"system-metrics\"],\"evidence_status\":\"supported\"}")
|
||||
.success(true)
|
||||
.build()
|
||||
));
|
||||
|
||||
ToolTraceSummaryService service = new ToolTraceSummaryService(repository);
|
||||
|
||||
List<Map<String, Object>> summaries = service.buildVerifierTraceSummary("session-3", "payment-service timeout");
|
||||
|
||||
Map<String, Object> logsSummary = summaries.stream()
|
||||
.filter(item -> "query_logs".equals(item.get("tool_name")))
|
||||
.findFirst()
|
||||
.orElseThrow();
|
||||
assertEquals(Boolean.FALSE, logsSummary.get("success"));
|
||||
assertEquals("none", logsSummary.get("evidence_level"));
|
||||
assertEquals(1, logsSummary.get("no_hit_invocation_count"));
|
||||
assertTrue(String.valueOf(logsSummary.get("output_summary")).contains("日志消息 #0"));
|
||||
}
|
||||
}
|
||||
@@ -1,246 +0,0 @@
|
||||
package com.superbiz.agent.service.session;
|
||||
|
||||
import com.superbiz.agent.domain.model.SessionContext;
|
||||
import com.superbiz.agent.domain.model.ToolCall;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* RedisSessionManager 单元测试
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE)
|
||||
@EnabledIfEnvironmentVariable(named = "SUPERBIZ_REDIS_PASSWORD", matches = ".+")
|
||||
@TestPropertySource(properties = {
|
||||
"spring.data.redis.host=${SUPERBIZ_REDIS_HOST:119.29.78.52}",
|
||||
"spring.data.redis.port=${SUPERBIZ_REDIS_PORT:33308}",
|
||||
"spring.data.redis.password=${SUPERBIZ_REDIS_PASSWORD}"
|
||||
})
|
||||
class RedisSessionManagerTest {
|
||||
|
||||
@Autowired
|
||||
private SessionManager sessionManager;
|
||||
|
||||
private String testSessionId;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
testSessionId = "test-session-" + UUID.randomUUID().toString();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreateAndGetSession() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-123")
|
||||
.businessId("order-456")
|
||||
.traceId("trace-789")
|
||||
.build();
|
||||
|
||||
String sessionId = sessionManager.createSession(context, 300); // 5分钟
|
||||
assertNotNull(sessionId);
|
||||
assertEquals(testSessionId, sessionId);
|
||||
System.out.println("✓ 创建会话成功: " + sessionId);
|
||||
|
||||
// 获取会话
|
||||
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
|
||||
assertTrue(retrieved.isPresent());
|
||||
assertEquals("user-123", retrieved.get().getUserId());
|
||||
assertEquals("ACTIVE", retrieved.get().getStatus());
|
||||
assertNotNull(retrieved.get().getCreatedAt());
|
||||
System.out.println("✓ 获取会话成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testUpdateSession() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-update")
|
||||
.status("ACTIVE")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
|
||||
// 获取并更新
|
||||
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
|
||||
assertTrue(retrieved.isPresent());
|
||||
|
||||
SessionContext toUpdate = retrieved.get();
|
||||
toUpdate.setStatus("COMPLETED");
|
||||
toUpdate.setBusinessId("updated-business-id");
|
||||
|
||||
sessionManager.updateSession(toUpdate);
|
||||
|
||||
// 验证更新
|
||||
Optional<SessionContext> updated = sessionManager.getSession(testSessionId);
|
||||
assertTrue(updated.isPresent());
|
||||
assertEquals("COMPLETED", updated.get().getStatus());
|
||||
assertEquals("updated-business-id", updated.get().getBusinessId());
|
||||
System.out.println("✓ 更新会话成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDeleteSession() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-delete")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
assertTrue(sessionManager.exists(testSessionId));
|
||||
|
||||
// 删除会话
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
assertFalse(sessionManager.exists(testSessionId));
|
||||
System.out.println("✓ 删除会话成功");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExists() {
|
||||
assertFalse(sessionManager.exists(testSessionId));
|
||||
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-exists")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
assertTrue(sessionManager.exists(testSessionId));
|
||||
System.out.println("✓ 会话存在性检查成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRefreshSession() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-refresh")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 60); // 1分钟
|
||||
|
||||
// 刷新过期时间
|
||||
boolean refreshed = sessionManager.refreshSession(testSessionId, 600); // 延长到10分钟
|
||||
assertTrue(refreshed);
|
||||
assertTrue(sessionManager.exists(testSessionId));
|
||||
System.out.println("✓ 刷新会话过期时间成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testAddToolCall() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-toolcall")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
|
||||
// 添加工具调用记录
|
||||
Map<String, Object> args = new HashMap<>();
|
||||
args.put("query", "test query");
|
||||
args.put("limit", 10);
|
||||
|
||||
ToolCall toolCall = ToolCall.builder()
|
||||
.toolName("search_documents")
|
||||
.arguments(args)
|
||||
.result("found 5 documents")
|
||||
.status("SUCCESS")
|
||||
.duration(150L)
|
||||
.calledAt(LocalDateTime.now())
|
||||
.build();
|
||||
|
||||
sessionManager.addToolCall(testSessionId, toolCall);
|
||||
|
||||
// 验证工具调用已添加
|
||||
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
|
||||
assertTrue(retrieved.isPresent());
|
||||
assertFalse(retrieved.get().getToolCalls().isEmpty());
|
||||
assertEquals(1, retrieved.get().getToolCalls().size());
|
||||
assertEquals("search_documents", retrieved.get().getToolCalls().get(0).getToolName());
|
||||
System.out.println("✓ 添加工具调用记录成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testUpdateStatus() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-status")
|
||||
.status("ACTIVE")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
|
||||
// 更新状态
|
||||
sessionManager.updateStatus(testSessionId, "COMPLETED");
|
||||
|
||||
// 验证状态已更新
|
||||
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
|
||||
assertTrue(retrieved.isPresent());
|
||||
assertEquals("COMPLETED", retrieved.get().getStatus());
|
||||
System.out.println("✓ 更新会话状态成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMultipleToolCalls() {
|
||||
// 创建会话
|
||||
SessionContext context = SessionContext.builder()
|
||||
.sessionId(testSessionId)
|
||||
.userId("user-multi-tools")
|
||||
.build();
|
||||
|
||||
sessionManager.createSession(context, 300);
|
||||
|
||||
// 添加多个工具调用
|
||||
for (int i = 0; i < 3; i++) {
|
||||
ToolCall toolCall = ToolCall.builder()
|
||||
.toolName("tool_" + i)
|
||||
.status("SUCCESS")
|
||||
.calledAt(LocalDateTime.now())
|
||||
.build();
|
||||
sessionManager.addToolCall(testSessionId, toolCall);
|
||||
}
|
||||
|
||||
// 验证所有工具调用
|
||||
Optional<SessionContext> retrieved = sessionManager.getSession(testSessionId);
|
||||
assertTrue(retrieved.isPresent());
|
||||
assertEquals(3, retrieved.get().getToolCalls().size());
|
||||
System.out.println("✓ 添加多个工具调用记录成功");
|
||||
|
||||
// 清理
|
||||
sessionManager.deleteSession(testSessionId);
|
||||
}
|
||||
}
|
||||
@@ -9,9 +9,7 @@ import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeIndexService;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.service.ToolInvocationRecorder;
|
||||
import com.superbiz.agent.service.VectorSearchService;
|
||||
import com.superbiz.agent.util.SessionContextHolder;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
@@ -40,12 +38,6 @@ class LookupKnowledgeToolTest {
|
||||
@Mock
|
||||
private VectorSearchService vectorSearchService;
|
||||
|
||||
@Mock
|
||||
private ToolInvocationRecorder toolInvocationRecorder;
|
||||
|
||||
@Mock
|
||||
private RetrievedDocTracker retrievedDocTracker;
|
||||
|
||||
private LookupKnowledgeTool tool;
|
||||
|
||||
@BeforeEach
|
||||
@@ -61,8 +53,6 @@ class LookupKnowledgeToolTest {
|
||||
ReflectionTestUtils.setField(tool, "evidencePostProcessor", postProcessor);
|
||||
ReflectionTestUtils.setField(tool, "contextPacker", contextPacker);
|
||||
ReflectionTestUtils.setField(tool, "resultAssembler", new LookupResultAssembler());
|
||||
ReflectionTestUtils.setField(tool, "toolInvocationRecorder", toolInvocationRecorder);
|
||||
ReflectionTestUtils.setField(tool, "retrievedDocTracker", retrievedDocTracker);
|
||||
ReflectionTestUtils.setField(tool, "topK", 3);
|
||||
}
|
||||
|
||||
@@ -250,36 +240,6 @@ class LookupKnowledgeToolTest {
|
||||
assertEquals("shared.md", lookup.getEvidenceBlocks().get(0).getSource());
|
||||
}
|
||||
|
||||
@Test
|
||||
void sessionDedupDoesNotReturnConsumableEvidenceAgain() {
|
||||
KnowledgeEntry entry = entry("payment.md", "Payment", "ERR_TIMEOUT", "payment");
|
||||
VectorSearchService.SearchResult result = searchResult(
|
||||
"vec-1",
|
||||
"payment.md",
|
||||
"{\"_source\":\"payment.md\",\"title\":\"Payment\",\"category\":\"payment\"}",
|
||||
"payment timeout runbook",
|
||||
0.2f);
|
||||
|
||||
when(knowledgeIndexService.analyzeQuery("ERR_TIMEOUT")).thenReturn(hint(entry));
|
||||
when(vectorSearchService.searchSimilarDocuments("ERR_TIMEOUT", 3, "payment"))
|
||||
.thenReturn(List.of(result));
|
||||
when(retrievedDocTracker.isAlreadyRetrieved("session-1", "payment.md")).thenReturn(true);
|
||||
when(retrievedDocTracker.getRetrievedDomains("session-1")).thenReturn(List.of("payment"));
|
||||
|
||||
SessionContextHolder.setSessionId("session-1");
|
||||
try {
|
||||
LookupResult lookup = tool.lookupKnowledge("ERR_TIMEOUT");
|
||||
|
||||
assertFalse(lookup.isFound());
|
||||
assertEquals(0, lookup.getEvidenceBlockCount());
|
||||
assertTrue(lookup.getEvidenceBlocks().isEmpty());
|
||||
assertTrue(lookup.getMessage().contains("无需重复召回"));
|
||||
assertEquals(List.of("payment"), lookup.getRetrievedDomainsThisSession());
|
||||
} finally {
|
||||
SessionContextHolder.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private KnowledgeEntry entry(String filePath, String title, String keyword, String category) {
|
||||
return KnowledgeEntry.builder()
|
||||
.filePath(filePath)
|
||||
|
||||
Reference in New Issue
Block a user