osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(eval): add executor audit closure checks

Browse Source
This commit is contained in:
aruo
2026-07-08 10:21:39 +08:00
parent 6015bcbf6f
commit a08672b31e
20 changed files with 1142 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/java/com/superbiz/agent/eval/DiagnosisTraceEvaluatorTest.java
+119 -3
View File
@@ -24,11 +24,12 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalReport report = evaluator.evaluate(cases, Path.of("mvp/eval/fixtures"));
assertEquals(5, report.getTotalCases());
assertEquals(5, report.getPassedCases());
assertEquals(8, report.getTotalCases());
assertEquals(8, report.getPassedCases());
assertEquals(1.0, report.getPassRate(), 0.001);
assertEquals(2L, report.getVerdictDistribution().get("PASS"));
assertEquals(3L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(5L, report.getVerdictDistribution().get("LOW_CONFID"));
assertEquals(1L, report.getVerdictDistribution().get("REJECT"));
DiagnosisEvalResult payment = result(report, "payment-timeout");
assertTrue(payment.isPassed());
@@ -39,6 +40,16 @@ class DiagnosisTraceEvaluatorTest {
DiagnosisEvalResult redis = result(report, "redis-timeout");
assertTrue(redis.isPassed());
assertTrue(redis.getEvidenceCoverage().get("query_logs"));
DiagnosisEvalResult fabricatedInvocation = result(report, "gatekeeper-fabricated-invocation");
assertTrue(fabricatedInvocation.isPassed());
assertEquals("fail", fabricatedInvocation.getGatekeeperStatus());
assertEquals("valid", fabricatedInvocation.getComposerStatus());
assertEquals(1, fabricatedInvocation.getClaimCheckCount());
DiagnosisEvalResult composerFallback = result(report, "composer-fallback-no-raw-json");
assertTrue(composerFallback.isPassed());
assertEquals("composer_malformed", composerFallback.getComposerStatus());
}
@Test
@@ -108,6 +119,111 @@ class DiagnosisTraceEvaluatorTest {
"executor confirmed claim missing evidence bindings: claim-unsupported"));
}
@Test
void evaluateFailsWhenGatekeeperFailStillPassesVerifier() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("gatekeeper-pass-leak")
.title("Gatekeeper pass leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("PASS", "LOW_CONFID", "REJECT"))
.requireV2AuditClosure(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("安全回答")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "PASS",
"gatekeeper_result", java.util.Map.of("status", "fail"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "unsupported",
"detail", "evidence ref invalid"
)),
"composer_output", java.util.Map.of("status", "valid")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("gatekeeper fail cannot have PASS verdict"));
}
@Test
void evaluateFailsWhenUnsupportedClaimLeaksIntoFinalAnswer() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("unsupported-leak")
.title("Unsupported leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("LOW_CONFID"))
.requireV2AuditClosure(true)
.forbiddenConfirmedClaimKeywords(List.of("主库故障"))
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("已经确认主库故障。")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "LOW_CONFID",
"gatekeeper_result", java.util.Map.of("status", "pass"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "unsupported",
"detail", "missing database evidence"
)),
"composer_output", java.util.Map.of("status", "valid")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains(
"answer contains forbidden confirmed claim keyword: 主库故障"));
}
@Test
void evaluateFailsWhenFinalAnswerLeaksRawExecutorMarker() {
DiagnosisEvalCase evalCase = DiagnosisEvalCase.builder()
.id("raw-json-leak")
.title("Raw json leak")
.expectedRootCauseKeywords(List.of())
.requiredEvidenceTools(List.of())
.allowedVerdicts(List.of("LOW_CONFID"))
.requireV2AuditClosure(true)
.build();
DiagnosisTraceResponse trace = DiagnosisTraceResponse.builder()
.session(DiagnosisTraceResponse.SessionTrace.builder()
.answer("answer_version=executor_evidence_v2")
.selfEvaluation(java.util.Map.of(
"verifier_evaluation", java.util.Map.of(
"verdict", "LOW_CONFID",
"gatekeeper_result", java.util.Map.of("status", "pass"),
"claim_checks", java.util.List.of(java.util.Map.of(
"claim_id", "claim-1",
"verification", "direct_observation",
"detail", "log evidence"
)),
"composer_output", java.util.Map.of("status", "composer_malformed")
)))
.build())
.toolInvocations(List.of())
.build();
DiagnosisEvalResult result = evaluator.evaluate(evalCase, trace);
assertFalse(result.isPassed());
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: executor_evidence_v2"));
assertTrue(result.getFailedChecks().contains("answer leaks raw executor marker: answer_version"));
}
@Test
void reportWriterOutputsJsonAndMarkdown(@TempDir Path tempDir) throws Exception {
DiagnosisEvalReport report = evaluator.evaluate(readCases(), Path.of("mvp/eval/fixtures"));