89 lines
4.3 KiB
Java
89 lines
4.3 KiB
Java
package com.superbiz.agent.eval;
|
|
|
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
|
|
import java.io.IOException;
|
|
import java.nio.charset.StandardCharsets;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Path;
|
|
import java.util.Map;
|
|
|
|
public class DiagnosisEvalReportWriter {
|
|
|
|
private final ObjectMapper objectMapper;
|
|
|
|
public DiagnosisEvalReportWriter(ObjectMapper objectMapper) {
|
|
this.objectMapper = objectMapper;
|
|
}
|
|
|
|
public void writeJson(DiagnosisEvalReport report, Path outputFile) throws IOException {
|
|
Files.createDirectories(outputFile.getParent());
|
|
objectMapper.writerWithDefaultPrettyPrinter().writeValue(outputFile.toFile(), report);
|
|
}
|
|
|
|
public void writeMarkdown(DiagnosisEvalReport report, Path outputFile) throws IOException {
|
|
Files.createDirectories(outputFile.getParent());
|
|
Files.writeString(outputFile, toMarkdown(report), StandardCharsets.UTF_8);
|
|
}
|
|
|
|
public String toMarkdown(DiagnosisEvalReport report) {
|
|
StringBuilder builder = new StringBuilder();
|
|
builder.append("# Diagnosis Eval Report\n\n");
|
|
builder.append("- Total cases: ").append(report.getTotalCases()).append("\n");
|
|
builder.append("- Passed cases: ").append(report.getPassedCases()).append("\n");
|
|
builder.append("- Pass rate: ").append(String.format("%.2f%%", report.getPassRate() * 100)).append("\n");
|
|
builder.append("- Average tool calls: ").append(String.format("%.2f", report.getAverageToolCallCount())).append("\n");
|
|
builder.append("- Average duration ms: ").append(String.format("%.2f", report.getAverageDurationMs())).append("\n\n");
|
|
|
|
builder.append("## Verdict Distribution\n\n");
|
|
if (report.getVerdictDistribution() == null || report.getVerdictDistribution().isEmpty()) {
|
|
builder.append("- None\n\n");
|
|
} else {
|
|
for (Map.Entry<String, Long> entry : report.getVerdictDistribution().entrySet()) {
|
|
builder.append("- ").append(entry.getKey()).append(": ").append(entry.getValue()).append("\n");
|
|
}
|
|
builder.append("\n");
|
|
}
|
|
|
|
builder.append("## Cases\n\n");
|
|
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Prompt Audit | Composer | Claim Checks | Rules | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
|
|
builder.append("| --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- | ---: | ---: | --- |\n");
|
|
for (DiagnosisEvalResult result : report.getResults()) {
|
|
builder.append("| ")
|
|
.append(result.getCaseId())
|
|
.append(" | ")
|
|
.append(result.isPassed() ? "PASS" : "FAIL")
|
|
.append(" | ")
|
|
.append(valueOrDash(result.getVerdict()))
|
|
.append(" | ")
|
|
.append(valueOrDash(result.getGatekeeperStatus()))
|
|
.append(" | ")
|
|
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
|
|
.append(" | ")
|
|
.append(valueOrDash(result.getPromptAuditVersion()))
|
|
.append(" | ")
|
|
.append(valueOrDash(result.getComposerStatus()))
|
|
.append(" | ")
|
|
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
|
|
.append(" | ")
|
|
.append(result.getGatekeeperRuleCount() == null ? "-" : result.getGatekeeperRuleCount())
|
|
.append(" | ")
|
|
.append(result.getMatchedKeywordCount()).append("/").append(result.getRequiredKeywordCount())
|
|
.append(" | ")
|
|
.append(result.getToolCallCount() == null ? "-" : result.getToolCallCount())
|
|
.append(" | ")
|
|
.append(result.getDurationMs() == null ? "-" : result.getDurationMs())
|
|
.append(" | ")
|
|
.append(result.getFailedChecks() == null || result.getFailedChecks().isEmpty()
|
|
? "-"
|
|
: String.join("; ", result.getFailedChecks()))
|
|
.append(" |\n");
|
|
}
|
|
return builder.toString();
|
|
}
|
|
|
|
private String valueOrDash(String value) {
|
|
return value == null || value.isBlank() ? "-" : value;
|
|
}
|
|
}
|