Add diagnosis eval harness

2026-07-04 23:51:43 +08:00
parent dc6cd32a67
commit ca5c61fabf
24 changed files with 1251 additions and 2 deletions
@@ -0,0 +1,76 @@
+package com.superbiz.agent.eval;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+
+public class DiagnosisEvalReportWriter {
+
+    private final ObjectMapper objectMapper;
+
+    public DiagnosisEvalReportWriter(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+    }
+
+    public void writeJson(DiagnosisEvalReport report, Path outputFile) throws IOException {
+        Files.createDirectories(outputFile.getParent());
+        objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
+    }
+
+    public void writeMarkdown(DiagnosisEvalReport report, Path outputFile) throws IOException {
+        Files.createDirectories(outputFile.getParent());
+        Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
+    }
+
+    public String toMarkdown(DiagnosisEvalReport report) {
+        StringBuilder builder = new StringBuilder();
+        builder.append("# Diagnosis Eval Report\n\n");
+        builder.append("- Total cases: ").append(report.getTotalCases()).append("\n");
+        builder.append("- Passed cases: ").append(report.getPassedCases()).append("\n");
+        builder.append("- Pass rate: ").append(String.format("%.2f%%", report.getPassRate() * 100)).append("\n");
+        builder.append("- Average tool calls: ").append(String.format("%.2f", report.getAverageToolCallCount())).append("\n");
+        builder.append("- Average duration ms: ").append(String.format("%.2f", report.getAverageDurationMs())).append("\n\n");
+
+        builder.append("## Verdict Distribution\n\n");
+        if (report.getVerdictDistribution() == null || report.getVerdictDistribution().isEmpty()) {
+            builder.append("- None\n\n");
+        } else {
+            for (Map.Entry<String, Long> entry : report.getVerdictDistribution().entrySet()) {
+                builder.append("- ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
+            }
+            builder.append("\n");
+        }
+
+        builder.append("## Cases\n\n");
+        builder.append("| Case | Result | Verdict | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
+        builder.append("| --- | --- | --- | --- | ---: | ---: | --- |\n");
+        for (DiagnosisEvalResult result : report.getResults()) {
+            builder.append("| ")
+                    .append(result.getCaseId())
+                    .append(" | ")
+                    .append(result.isPassed() ? "PASS" : "FAIL")
+                    .append(" | ")
+                    .append(valueOrDash(result.getVerdict()))
+                    .append(" | ")
+                    .append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
+                    .append(" | ")
+                    .append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
+                    .append(" | ")
+                    .append(result.getDurationMs() == null ? "-" : result.getDurationMs())
+                    .append(" | ")
+                    .append(result.getFailedChecks() == null || result.getFailedChecks().isEmpty()
+                            ? "-"
+                            : String.join("; ", result.getFailedChecks()))
+                    .append(" |\n");
+        }
+        return builder.toString();
+    }
+
+    private String valueOrDash(String value) {
+        return value == null || value.isBlank() ? "-" : value;
+    }
+}