package com.superbiz.agent.eval; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.dto.DiagnosisTraceResponse; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; class DiagnosisTraceEvaluatorTest { private final ObjectMapper objectMapper = new ObjectMapper(); private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper); @Test void evaluateFixtureReportsFullBaseline() { List cases = readCases(); DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures")); assertEquals(12, report.getTotalCases()); assertEquals(12, report.getPassedCases()); assertEquals(1.0, report.getPassRate(), 0.001); assertEquals(5L, report.getVerdictDistribution().get("PASS")); assertEquals(6L, report.getVerdictDistribution().get("LOW_CONFID")); assertEquals(1L, report.getVerdictDistribution().get("REJECT")); DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation"); assertTrue(narrowHighCpu.isPassed()); assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion()); assertEquals("pass", narrowHighCpu.getGatekeeperStatus()); DiagnosisEvalResult promptGatekeeperAudit = result(report, "prompt-gatekeeper-audit-closure"); assertTrue(promptGatekeeperAudit.isPassed()); assertEquals("chat-prompts-v1", promptGatekeeperAudit.getPromptAuditVersion()); assertEquals("gatekeeper-rules-v1", promptGatekeeperAudit.getGatekeeperRuleSetVersion()); assertEquals(2, promptGatekeeperAudit.getGatekeeperRuleCount()); DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation"); assertTrue(hikariNoEvidence.isPassed()); assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion()); assertEquals("pass", hikariNoEvidence.getGatekeeperStatus()); DiagnosisEvalResult payment = result(report, "payment-timeout"); assertTrue(payment.isPassed()); assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge")); assertTrue(payment.getEvidenceCoverage().get("query_logs")); assertTrue(payment.getEvidenceCoverage().get("query_metrics")); DiagnosisEvalResult redis = result(report, "redis-timeout"); assertTrue(redis.isPassed()); assertTrue(redis.getEvidenceCoverage().get("query_logs")); DiagnosisEvalResult fabricatedInvocation = result(report, "gatekeeper-fabricated-invocation"); assertTrue(fabricatedInvocation.isPassed()); assertEquals("fail", fabricatedInvocation.getGatekeeperStatus()); assertEquals("valid", fabricatedInvocation.getComposerStatus()); assertEquals(1, fabricatedInvocation.getClaimCheckCount()); DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json"); assertTrue(composerFallback.isPassed()); assertEquals("composer_malformed", composerFallback.getComposerStatus()); DiagnosisEvalResult auditMetadataLowConfid = result(report, "audit-metadata-low-confid"); assertTrue(auditMetadataLowConfid.isPassed()); assertEquals("LOW_CONFID", auditMetadataLowConfid.getVerdict()); assertEquals("chat-prompts-v1", auditMetadataLowConfid.getPromptAuditVersion()); assertEquals(2, auditMetadataLowConfid.getGatekeeperRuleCount()); } @Test void everyFixedCaseReferencesExistingFixture() { for (DiagnosisEvalCase evalCase : readCases()) { Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture()); assertTrue(Files.exists(fixture), "missing fixture: " + fixture); } } @Test void evaluateRejectRequiresDegradedOutput() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("reject-case") .title("Reject case") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("REJECT")) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("EXECUTOR_FINAL_ANSWER") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of("verdict", "REJECT"))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains("reject output does not use degraded template")); } @Test void evaluateFailsWhenStructuredConfirmedClaimHasNoEvidenceBindings() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("structured-claim-case") .title("Structured claim case") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("LOW_CONFID", "PASS")) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("以下结论基于当前已获取证据，仍存在部分证据缺口，请谨慎参考。") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "LOW_CONFID", "executor_structured_output", java.util.Map.of( "claims", java.util.List.of(java.util.Map.of( "claim_id", "claim-unsupported", "claim_text", "OOM 导致连接泄漏", "support_level", "direct", "evidence_bindings", java.util.List.of() )) ) ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains( "executor confirmed claim missing evidence bindings: claim-unsupported")); } @Test void evaluateFailsWhenGatekeeperFailStillPassesVerifier() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("gatekeeper-pass-leak") .title("Gatekeeper pass leak") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("PASS", "LOW_CONFID", "REJECT")) .requireV2AuditClosure(true) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("安全回答") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "PASS", "gatekeeper_result", java.util.Map.of("status", "fail"), "claim_checks", java.util.List.of(java.util.Map.of( "claim_id", "claim-1", "verification", "unsupported", "detail", "evidence ref invalid" )), "composer_output", java.util.Map.of("status", "valid") ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict")); } @Test void evaluateFailsWhenGatekeeperRuleSetVersionMismatches() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("rule-version") .title("Rule version") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("PASS")) .expectedGatekeeperRuleSetVersion("gatekeeper-rules-v1") .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("安全回答") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "PASS", "gatekeeper_result", java.util.Map.of( "status", "pass", "rule_set_version", "old-rules" ) ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains( "gatekeeper rule set version not expected: old-rules")); } @Test void evaluateFailsWhenPromptAuditMissingOrMismatches() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("prompt-audit") .title("Prompt audit") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("PASS")) .requirePromptAudit(true) .expectedPromptAuditVersion("chat-prompts-v1") .expectedPromptVersions(java.util.Map.of("chat_executor", "chat-executor-v2")) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("安全回答") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "PASS", "prompt_audit", java.util.Map.of( "version", "old-prompts", "prompts", java.util.List.of(java.util.Map.of( "name", "chat_executor", "version", "chat-executor-v1" )) ) ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains( "prompt audit version not expected: old-prompts")); assertTrue(result.getFailedChecks().contains( "prompt version not expected: chat_executor=chat-executor-v1")); } @Test void evaluateFailsWhenGatekeeperRulesAreMissing() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("gatekeeper-rules") .title("Gatekeeper rules") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("PASS")) .requireGatekeeperRules(true) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("安全回答") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "PASS", "gatekeeper_result", java.util.Map.of( "status", "pass", "rule_set_version", "gatekeeper-rules-v1" ) ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains("gatekeeper_result missing rules")); } @Test void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("unsupported-leak") .title("Unsupported leak") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("LOW_CONFID")) .requireV2AuditClosure(true) .forbiddenConfirmedClaimKeywords(List.of("主库故障")) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("已经确认主库故障。") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "LOW_CONFID", "gatekeeper_result", java.util.Map.of("status", "pass"), "claim_checks", java.util.List.of(java.util.Map.of( "claim_id", "claim-1", "verification", "unsupported", "detail", "missing database evidence" )), "composer_output", java.util.Map.of("status", "valid") ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains( "answer contains forbidden confirmed claim keyword: 主库故障")); } @Test void evaluateFailsWhenFinalAnswerLeaksRawExecutorMarker() { DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder() .id("raw-json-leak") .title("Raw json leak") .expectedRootCauseKeywords(List.of()) .requiredEvidenceTools(List.of()) .allowedVerdicts(List.of("LOW_CONFID")) .requireV2AuditClosure(true) .build(); DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder() .session(DiagnosisTraceResponse.SessionTrace.builder() .answer("answer_version=executor_evidence_v2") .selfEvaluation(java.util.Map.of( "verifier_evaluation", java.util.Map.of( "verdict", "LOW_CONFID", "gatekeeper_result", java.util.Map.of("status", "pass"), "claim_checks", java.util.List.of(java.util.Map.of( "claim_id", "claim-1", "verification", "direct_observation", "detail", "log evidence" )), "composer_output", java.util.Map.of("status", "composer_malformed") ))) .build()) .toolInvocations(List.of()) .build(); DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace); assertFalse(result.isPassed()); assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: executor_evidence_v2")); assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: answer_version")); } @Test void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception { DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures")); DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper); Path json = tempDir.resolve("eval-report.json"); Path markdown = tempDir.resolve("eval-report.md"); writer.writeJson(report, json); writer.writeMarkdown(report, markdown); assertTrue(Files.exists(json)); assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report")); assertTrue(Files.readString(markdown).contains("payment-timeout")); assertEquals( comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))), comparableReportText(Files.readString(json))); assertEquals( comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))), comparableReportText(Files.readString(markdown))); } private List readCases() { try { return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json")); } catch (Exception e) { throw new AssertionError(e); } } private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) { return report.getResults().stream() .filter(item -> caseId.equals(item.getCaseId())) .findFirst() .orElseThrow(); } private String comparableReportText(String value) { return value.replace("\r\n", "\n").stripTrailing(); } }