392 lines
19 KiB
Java
392 lines
19 KiB
Java
package com.superbiz.agent.eval;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.junit.jupiter.api.io.TempDir;
|
|
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.util.List;
|
|
|
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
|
|
class DiagnosisTraceEvaluatorTest {
|
|
|
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
|
private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
|
|
|
|
@Test
|
|
void evaluateFixtureReportsFullBaseline() {
|
|
List<DiagnosisEvalCase> cases = readCases();
|
|
|
|
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
|
|
|
|
assertEquals(12, report.getTotalCases());
|
|
assertEquals(12, report.getPassedCases());
|
|
assertEquals(1.0, report.getPassRate(), 0.001);
|
|
assertEquals(5L, report.getVerdictDistribution().get("PASS"));
|
|
assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID"));
|
|
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
|
|
|
|
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
|
|
assertTrue(narrowHighCpu.isPassed());
|
|
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
|
|
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
|
|
|
|
DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure");
|
|
assertTrue(promptGatekeeperAudit.isPassed());
|
|
assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion());
|
|
assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion());
|
|
assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount());
|
|
|
|
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
|
|
assertTrue(hikariNoEvidence.isPassed());
|
|
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
|
|
assertEquals("pass", hikariNoEvidence.getGatekeeperStatus());
|
|
|
|
DiagnosisEvalResult payment = result(report, "payment-timeout");
|
|
assertTrue(payment.isPassed());
|
|
assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
|
|
assertTrue(payment.getEvidenceCoverage().get("query_logs"));
|
|
assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
|
|
|
|
DiagnosisEvalResult redis = result(report, "redis-timeout");
|
|
assertTrue(redis.isPassed());
|
|
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
|
|
|
|
DiagnosisEvalResult fabricatedInvocation = result(report, "gatekeeper-fabricated-invocation");
|
|
assertTrue(fabricatedInvocation.isPassed());
|
|
assertEquals("fail", fabricatedInvocation.getGatekeeperStatus());
|
|
assertEquals("valid", fabricatedInvocation.getComposerStatus());
|
|
assertEquals(1, fabricatedInvocation.getClaimCheckCount());
|
|
|
|
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
|
|
assertTrue(composerFallback.isPassed());
|
|
assertEquals("composer_malformed", composerFallback.getComposerStatus());
|
|
|
|
DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid");
|
|
assertTrue(auditMetadataLowConfid.isPassed());
|
|
assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict());
|
|
assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion());
|
|
assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount());
|
|
}
|
|
|
|
@Test
|
|
void everyFixedCaseReferencesExistingFixture() {
|
|
for (DiagnosisEvalCase evalCase : readCases()) {
|
|
Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture());
|
|
assertTrue(Files.exists(fixture), "missing fixture: " + fixture);
|
|
}
|
|
}
|
|
|
|
@Test
|
|
void evaluateRejectRequiresDegradedOutput() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("reject-case")
|
|
.title("Reject case")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("REJECT"))
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("EXECUTOR_FINAL_ANSWER")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of("verdict", "REJECT")))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenStructuredConfirmedClaimHasNoEvidenceBindings() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("structured-claim-case")
|
|
.title("Structured claim case")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("LOW_CONFID", "PASS"))
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "LOW_CONFID",
|
|
"executor_structured_output", java.util.Map.of(
|
|
"claims", java.util.List.of(java.util.Map.of(
|
|
"claim_id", "claim-unsupported",
|
|
"claim_text", "OOM 导致连接泄漏",
|
|
"support_level", "direct",
|
|
"evidence_bindings", java.util.List.of()
|
|
))
|
|
)
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains(
|
|
"executor confirmed claim missing evidence bindings: claim-unsupported"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenGatekeeperFailStillPassesVerifier() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("gatekeeper-pass-leak")
|
|
.title("Gatekeeper pass leak")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("PASS", "LOW_CONFID", "REJECT"))
|
|
.requireV2AuditClosure(true)
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("安全回答")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "PASS",
|
|
"gatekeeper_result", java.util.Map.of("status", "fail"),
|
|
"claim_checks", java.util.List.of(java.util.Map.of(
|
|
"claim_id", "claim-1",
|
|
"verification", "unsupported",
|
|
"detail", "evidence ref invalid"
|
|
)),
|
|
"composer_output", java.util.Map.of("status", "valid")
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenGatekeeperRuleSetVersionMismatches() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("rule-version")
|
|
.title("Rule version")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("PASS"))
|
|
.expectedGatekeeperRuleSetVersion("gatekeeper-rules-v1")
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("安全回答")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "PASS",
|
|
"gatekeeper_result", java.util.Map.of(
|
|
"status", "pass",
|
|
"rule_set_version", "old-rules"
|
|
)
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains(
|
|
"gatekeeper rule set version not expected: old-rules"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenPromptAuditMissingOrMismatches() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("prompt-audit")
|
|
.title("Prompt audit")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("PASS"))
|
|
.requirePromptAudit(true)
|
|
.expectedPromptAuditVersion("chat-prompts-v1")
|
|
.expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2"))
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("安全回答")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "PASS",
|
|
"prompt_audit", java.util.Map.of(
|
|
"version", "old-prompts",
|
|
"prompts", java.util.List.of(java.util.Map.of(
|
|
"name", "chat_executor",
|
|
"version", "chat-executor-v1"
|
|
))
|
|
)
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains(
|
|
"prompt audit version not expected: old-prompts"));
|
|
assertTrue(result.getFailedChecks().contains(
|
|
"prompt version not expected: chat_executor=chat-executor-v1"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenGatekeeperRulesAreMissing() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("gatekeeper-rules")
|
|
.title("Gatekeeper rules")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("PASS"))
|
|
.requireGatekeeperRules(true)
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("安全回答")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "PASS",
|
|
"gatekeeper_result", java.util.Map.of(
|
|
"status", "pass",
|
|
"rule_set_version", "gatekeeper-rules-v1"
|
|
)
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules"));
|
|
}
|
|
|
|
|
|
@Test
|
|
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("unsupported-leak")
|
|
.title("Unsupported leak")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("LOW_CONFID"))
|
|
.requireV2AuditClosure(true)
|
|
.forbiddenConfirmedClaimKeywords(List.of("主库故障"))
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("已经确认主库故障。")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "LOW_CONFID",
|
|
"gatekeeper_result", java.util.Map.of("status", "pass"),
|
|
"claim_checks", java.util.List.of(java.util.Map.of(
|
|
"claim_id", "claim-1",
|
|
"verification", "unsupported",
|
|
"detail", "missing database evidence"
|
|
)),
|
|
"composer_output", java.util.Map.of("status", "valid")
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains(
|
|
"answer contains forbidden confirmed claim keyword: 主库故障"));
|
|
}
|
|
|
|
@Test
|
|
void evaluateFailsWhenFinalAnswerLeaksRawExecutorMarker() {
|
|
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
|
|
.id("raw-json-leak")
|
|
.title("Raw json leak")
|
|
.expectedRootCauseKeywords(List.of())
|
|
.requiredEvidenceTools(List.of())
|
|
.allowedVerdicts(List.of("LOW_CONFID"))
|
|
.requireV2AuditClosure(true)
|
|
.build();
|
|
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
|
|
.session(DiagnosisTraceResponse.SessionTrace.builder()
|
|
.answer("answer_version=executor_evidence_v2")
|
|
.selfEvaluation(java.util.Map.of(
|
|
"verifier_evaluation", java.util.Map.of(
|
|
"verdict", "LOW_CONFID",
|
|
"gatekeeper_result", java.util.Map.of("status", "pass"),
|
|
"claim_checks", java.util.List.of(java.util.Map.of(
|
|
"claim_id", "claim-1",
|
|
"verification", "direct_observation",
|
|
"detail", "log evidence"
|
|
)),
|
|
"composer_output", java.util.Map.of("status", "composer_malformed")
|
|
)))
|
|
.build())
|
|
.toolInvocations(List.of())
|
|
.build();
|
|
|
|
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
|
|
|
|
assertFalse(result.isPassed());
|
|
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: executor_evidence_v2"));
|
|
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: answer_version"));
|
|
}
|
|
|
|
@Test
|
|
void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
|
|
DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
|
|
DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper);
|
|
|
|
Path json = tempDir.resolve("eval-report.json");
|
|
Path markdown = tempDir.resolve("eval-report.md");
|
|
writer.writeJson(report, json);
|
|
writer.writeMarkdown(report, markdown);
|
|
|
|
assertTrue(Files.exists(json));
|
|
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
|
|
assertTrue(Files.readString(markdown).contains("payment-timeout"));
|
|
assertEquals(
|
|
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))),
|
|
comparableReportText(Files.readString(json)));
|
|
assertEquals(
|
|
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))),
|
|
comparableReportText(Files.readString(markdown)));
|
|
}
|
|
|
|
private List<DiagnosisEvalCase> readCases() {
|
|
try {
|
|
return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json"));
|
|
} catch (Exception e) {
|
|
throw new AssertionError(e);
|
|
}
|
|
}
|
|
|
|
private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
|
|
return report.getResults().stream()
|
|
.filter(item -> caseId.equals(item.getCaseId()))
|
|
.findFirst()
|
|
.orElseThrow();
|
|
}
|
|
|
|
private String comparableReportText(String value) {
|
|
return value.replace("\r\n", "\n").stripTrailing();
|
|
}
|
|
}
|