feat(eval): add evidence pipeline acceptance closure

This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
@@ -8,12 +8,25 @@ import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
class ExecutorGatekeeperServiceTest {
@Test
void ruleCatalogLoadsDefaultMetadata() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.loadDefault(new com.fasterxml.jackson.databind.ObjectMapper());
assertEquals("gatekeeper-rules-v1", catalog.version());
assertFalse(catalog.auditRules().isEmpty());
assertTrue(catalog.auditRules().stream()
.anyMatch(rule -> "evidence.raw_path".equals(rule.get("id"))));
assertEquals(0.5, catalog.doubleParameter("evidence.excerpt_mismatch",
"min_token_overlap", 0.0), 0.001);
}
@Test
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -30,6 +43,7 @@ class ExecutorGatekeeperServiceTest {
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@@ -49,6 +63,7 @@ class ExecutorGatekeeperServiceTest {
assertEquals("pass", result.get("status"));
assertEquals("none", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
}
@@ -115,6 +130,7 @@ class ExecutorGatekeeperServiceTest {
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
}
@@ -167,9 +183,22 @@ class ExecutorGatekeeperServiceTest {
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
}
@Test
void failResultIncludesRuleAuditMetadata() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
Map<String, Object> result = service.fail("gatekeeper.internal_error", "gatekeeper", "boom");
assertEquals("fail", result.get("status"));
assertEquals("reject", result.get("severity"));
assertRuleAudit(result);
}
@Test
void validateFailsForToolNameMismatch() {
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
@@ -329,4 +358,13 @@ class ExecutorGatekeeperServiceTest {
"missing_info", List.of()
));
}
private void assertRuleAudit(Map<String, Object> result) {
assertEquals("gatekeeper-rules-v1", result.get("rule_set_version"));
assertTrue(result.get("rules") instanceof List<?>);
List<?> rules = (List<?>) result.get("rules");
assertFalse(rules.isEmpty());
assertTrue(rules.stream().anyMatch(rule ->
rule instanceof Map<?, ?> map && "evidence.raw_path".equals(map.get("id"))));
}
}