osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(eval): add evidence pipeline acceptance closure

Browse Source
This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+46 -3
View File
@@ -24,13 +24,23 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(8, report.getTotalCases());
assertEquals(8, report.getPassedCases());
assertEquals(10, report.getTotalCases());
assertEquals(10, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(2L, report.getVerdictDistribution().get("PASS"));
assertEquals(4L, report.getVerdictDistribution().get("PASS"));
assertEquals(5L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult narrowHighCpu = result(report, "narrow-highcpu-observation");
assertTrue(narrowHighCpu.isPassed());
assertEquals("gatekeeper-rules-v1", narrowHighCpu.getGatekeeperRuleSetVersion());
assertEquals("pass", narrowHighCpu.getGatekeeperStatus());
DiagnosisEvalResult hikariNoEvidence = result(report, "hikari-no-evidence-negative-observation");
assertTrue(hikariNoEvidence.isPassed());
assertEquals("gatekeeper-rules-v1", hikariNoEvidence.getGatekeeperRuleSetVersion());
assertEquals("pass", hikariNoEvidence.getGatekeeperStatus());
DiagnosisEvalResult payment = result(report, "payment-timeout");
assertTrue(payment.isPassed());
assertTrue(payment.getEvidenceCoverage().get("lookup_knowledge"));
@@ -153,6 +163,39 @@ class DiagnosisTraceEvaluatorTest {
assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict"));
}
@Test
void evaluateFailsWhenGatekeeperRuleSetVersionMismatches() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("rule-version")
.title("Rule version")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS"))
.expectedGatekeeperRuleSetVersion("gatekeeper-rules-v1")
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of(
"status", "pass",
"rule_set_version", "old-rules"
)
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"gatekeeper rule set version not expected: old-rules"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()