osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand diagnosis eval fixtures

Browse Source
This commit is contained in:
aruo
2026-07-05 00:27:57 +08:00
parent ca5c61fabf
commit 4c7c53b024
21 changed files with 622 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+26 -8
View File
@@ -19,16 +19,16 @@ class DiagnosisTraceEvaluatorTest {
private final DiagnosisTraceEvaluator evaluator = new DiagnosisTraceEvaluator(objectMapper);
@Test
void evaluateFixtureReportsPassingAndMissingCases() {
void evaluateFixtureReportsFullBaseline() {
List<DiagnosisEvalCase> cases = readCases();
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(5, report.getTotalCases());
assertEquals(2, report.getPassedCases());
assertEquals(0.4, report.getPassRate(), 0.001);
assertEquals(1L, report.getVerdictDistribution().get("PASS"));
assertEquals(1L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(5, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(2L, report.getVerdictDistribution().get("PASS"));
assertEquals(3L, report.getVerdictDistribution().get("LOW_CONFID"));
DiagnosisEvalResult payment = result(report, "payment-timeout");
assertTrue(payment.isPassed());
@@ -36,9 +36,17 @@ class DiagnosisTraceEvaluatorTest {
assertTrue(payment.getEvidenceCoverage().get("query_logs"));
assertTrue(payment.getEvidenceCoverage().get("query_metrics"));
DiagnosisEvalResult missing = result(report, "redis-timeout");
assertFalse(missing.isPassed());
assertTrue(missing.getFailedChecks().get(0).contains("trace fixture unavailable"));
DiagnosisEvalResult redis = result(report, "redis-timeout");
assertTrue(redis.isPassed());
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
}
@Test
void everyFixedCaseReferencesExistingFixture() {
for (DiagnosisEvalCase evalCase : readCases()) {
Path fixture = Path.of("mvp/eval/fixtures").resolve(evalCase.getTraceFixture());
assertTrue(Files.exists(fixture), "missing fixture: " + fixture);
}
}
@Test
@@ -78,6 +86,12 @@ class DiagnosisTraceEvaluatorTest {
assertTrue(Files.exists(json));
assertTrue(Files.readString(markdown).contains("# Diagnosis Eval Report"));
assertTrue(Files.readString(markdown).contains("payment-timeout"));
assertEquals(
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.json"))),
comparableReportText(Files.readString(json)));
assertEquals(
comparableReportText(Files.readString(Path.of("mvp/eval/reports/baseline-report.md"))),
comparableReportText(Files.readString(markdown)));
}
private List<DiagnosisEvalCase> readCases() {
@@ -94,4 +108,8 @@ class DiagnosisTraceEvaluatorTest {
.findFirst()
.orElseThrow();
}
private String comparableReportText(String value) {
return value.replace("\r\n", "\n").stripTrailing();
}
}