Add diagnosis eval baseline diff

2026-07-05 00:59:53 +08:00
parent 4c7c53b024
commit 69deb15330
22 changed files with 1069 additions and 0 deletions
@@ -0,0 +1,140 @@
+package com.superbiz.agent.eval;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DiagnosisEvalBaselineDiffTest {
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+    private final DiagnosisEvalBaselineDiffer differ = new DiagnosisEvalBaselineDiffer();
+
+    @Test
+    void compareReportsDetectsAggregateAndCaseRegressions() throws Exception {
+        DiagnosisEvalReport baseline = readBaselineReport();
+        DiagnosisEvalReport current = readBaselineReport();
+        degradeRedisCase(current);
+
+        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
+
+        assertTrue(diff.isHasRegression());
+        assertEquals(6, diff.getRegressionCount());
+        assertEquals(2, diff.getChangedCount());
+        assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "passRate"));
+        assertTrue(hasItem(diff, "REGRESSION", "aggregate", null, "averageToolCallCount"));
+        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "passed"));
+        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "verdict"));
+        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "matchedKeywordCount"));
+        assertTrue(hasItem(diff, "REGRESSION", "case", "redis-timeout", "evidenceCoverage.query_logs"));
+        assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.LOW_CONFID"));
+        assertTrue(hasItem(diff, "CHANGED", "aggregate", null, "verdictDistribution.REJECT"));
+    }
+
+    @Test
+    void compareReportsDetectsMissingAndNewCases() throws Exception {
+        DiagnosisEvalReport baseline = readBaselineReport();
+        DiagnosisEvalReport current = readBaselineReport();
+        DiagnosisEvalResult removed = current.getResults().remove(0);
+        current.getResults().add(DiagnosisEvalResult.builder()
+                .caseId("new-case")
+                .title("New case")
+                .passed(true)
+                .failedChecks(List.of())
+                .verdict("PASS")
+                .matchedKeywordCount(1)
+                .requiredKeywordCount(1)
+                .evidenceCoverage(new LinkedHashMap<>())
+                .toolCallCount(1)
+                .durationMs(1000)
+                .build());
+
+        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
+
+        assertTrue(hasItem(diff, "REGRESSION", "case", removed.getCaseId(), "casePresence"));
+        assertTrue(hasItem(diff, "CHANGED", "case", "new-case", "casePresence"));
+    }
+
+    @Test
+    void compareSameReportHasNoDiff() throws Exception {
+        DiagnosisEvalReport baseline = readBaselineReport();
+
+        DiagnosisEvalDiffReport diff = differ.compare(baseline, readBaselineReport());
+
+        assertFalse(diff.isHasRegression());
+        assertEquals(0, diff.getRegressionCount());
+        assertTrue(diff.getItems().isEmpty());
+    }
+
+    @Test
+    void writerOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
+        DiagnosisEvalReport baseline = readBaselineReport();
+        DiagnosisEvalReport current = readBaselineReport();
+        degradeRedisCase(current);
+        DiagnosisEvalDiffReport diff = differ.compare(baseline, current);
+        DiagnosisEvalDiffReportWriter writer = new DiagnosisEvalDiffReportWriter(objectMapper);
+
+        Path json = tempDir.resolve("baseline-diff.json");
+        Path markdown = tempDir.resolve("baseline-diff.md");
+        writer.writeJson(diff, json);
+        writer.writeMarkdown(diff, markdown);
+
+        assertTrue(Files.exists(json));
+        assertTrue(Files.readString(json).contains("\"hasRegression\" : true"));
+        assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Baseline Diff"));
+        assertTrue(Files.readString(markdown).contains("redis-timeout"));
+    }
+
+    private DiagnosisEvalReport readBaselineReport() throws Exception {
+        return objectMapper.readValue(Path.of("mvp/eval/reports/baseline-report.json").toFile(),
+                DiagnosisEvalReport.class);
+    }
+
+    private void degradeRedisCase(DiagnosisEvalReport report) {
+        report.setPassedCases(4);
+        report.setPassRate(0.8);
+        report.setAverageToolCallCount(3.0);
+        report.setAverageDurationMs(45800.0);
+        report.setVerdictDistribution(new LinkedHashMap<>());
+        report.getVerdictDistribution().put("PASS", 2L);
+        report.getVerdictDistribution().put("LOW_CONFID", 2L);
+        report.getVerdictDistribution().put("REJECT", 1L);
+
+        DiagnosisEvalResult redis = result(report, "redis-timeout");
+        redis.setPassed(false);
+        redis.setFailedChecks(new ArrayList<>(List.of("missing required evidence tool: query_logs")));
+        redis.setVerdict("REJECT");
+        redis.setMatchedKeywordCount(1);
+        redis.getEvidenceCoverage().put("query_logs", false);
+        redis.setToolCallCount(1);
+        redis.setDurationMs(36000);
+    }
+
+    private DiagnosisEvalResult result(DiagnosisEvalReport report, String caseId) {
+        return report.getResults().stream()
+                .filter(item -> caseId.equals(item.getCaseId()))
+                .findFirst()
+                .orElseThrow();
+    }
+
+    private boolean hasItem(DiagnosisEvalDiffReport diff,
+                            String type,
+                            String scope,
+                            String caseId,
+                            String metric) {
+        return diff.getItems().stream().anyMatch(item ->
+                type.equals(item.getType())
+                        && scope.equals(item.getScope())
+                        && java.util.Objects.equals(caseId, item.getCaseId())
+                        && metric.equals(item.getMetric()));
+    }
+}