Add diagnosis eval harness

2026-07-04 23:51:43 +08:00
parent dc6cd32a67
commit ca5c61fabf
24 changed files with 1251 additions and 2 deletions
@@ -0,0 +1,97 @@
+package com.superbiz.agent.eval;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.superbiz.agent.dto.DiagnosisTraceResponse;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DiagnosisTraceEvaluatorTest {
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
+
+    @Test
+    void evaluateFixtureReportsPassingAndMissingCases() {
+        List<DiagnosisEvalCase> cases = readCases();
+
+        DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
+
+        assertEquals(5, report.getTotalCases());
+        assertEquals(2, report.getPassedCases());
+        assertEquals(0.4, report.getPassRate(), 0.001);
+        assertEquals(1L, report.getVerdictDistribution().get("PASS"));
+        assertEquals(1L, report.getVerdictDistribution().get("LOW_CONFID"));
+
+        DiagnosisEvalResult payment = result(report, "payment-timeout");
+        assertTrue(payment.isPassed());
+        assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
+        assertTrue(payment.getEvidenceCoverage().get("query_logs"));
+        assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
+
+        DiagnosisEvalResult missing = result(report, "redis-timeout");
+        assertFalse(missing.isPassed());
+        assertTrue(missing.getFailedChecks().get(0).contains("trace fixture unavailable"));
+    }
+
+    @Test
+    void evaluateRejectRequiresDegradedOutput() {
+        DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
+                .id("reject-case")
+                .title("Reject case")
+                .expectedRootCauseKeywords(List.of())
+                .requiredEvidenceTools(List.of())
+                .allowedVerdicts(List.of("REJECT"))
+                .build();
+        DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
+                .session(DiagnosisTraceResponse.SessionTrace.builder()
+                        .answer("EXECUTOR_FINAL_ANSWER")
+                        .selfEvaluation(java.util.Map.of(
+                                "verifier_evaluation", java.util.Map.of("verdict", "REJECT")))
+                        .build())
+                .toolInvocations(List.of())
+                .build();
+
+        DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
+
+        assertFalse(result.isPassed());
+        assertTrue(result.getFailedChecks().contains("reject output does not use degraded template"));
+    }
+
+    @Test
+    void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
+        DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));
+        DiagnosisEvalReportWriter writer = new DiagnosisEvalReportWriter(objectMapper);
+
+        Path json = tempDir.resolve("eval-report.json");
+        Path markdown = tempDir.resolve("eval-report.md");
+        writer.writeJson(report, json);
+        writer.writeMarkdown(report, markdown);
+
+        assertTrue(Files.exists(json));
+        assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
+        assertTrue(Files.readString(markdown).contains("payment-timeout"));
+    }
+
+    private List<DiagnosisEvalCase> readCases() {
+        try {
+            return evaluator.loadCases(Path.of("mvp/eval/cases/diagnosis-cases.json"));
+        } catch (Exception e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
+        return report.getResults().stream()
+                .filter(item -> caseId.equals(item.getCaseId()))
+                .findFirst()
+                .orElseThrow();
+    }
+}