feat(eval): add evidence pipeline acceptance closure
This commit is contained in:
@@ -8,12 +8,25 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ExecutorGatekeeperServiceTest {
|
||||
|
||||
@Test
|
||||
void ruleCatalogLoadsDefaultMetadata() {
|
||||
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.loadDefault(new com.fasterxml.jackson.databind.ObjectMapper());
|
||||
|
||||
assertEquals("gatekeeper-rules-v1", catalog.version());
|
||||
assertFalse(catalog.auditRules().isEmpty());
|
||||
assertTrue(catalog.auditRules().stream()
|
||||
.anyMatch(rule -> "evidence.raw_path".equals(rule.get("id"))));
|
||||
assertEquals(0.5, catalog.doubleParameter("evidence.excerpt_mismatch",
|
||||
"min_token_overlap", 0.0), 0.001);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validatePassesForExecutorEvidenceV2WithMatchingInvocation() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
@@ -30,6 +43,7 @@ class ExecutorGatekeeperServiceTest {
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
|
||||
}
|
||||
|
||||
@@ -49,6 +63,7 @@ class ExecutorGatekeeperServiceTest {
|
||||
|
||||
assertEquals("pass", result.get("status"));
|
||||
assertEquals("none", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).isEmpty());
|
||||
}
|
||||
|
||||
@@ -115,6 +130,7 @@ class ExecutorGatekeeperServiceTest {
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.excerpt_mismatch"));
|
||||
}
|
||||
|
||||
@@ -167,9 +183,22 @@ class ExecutorGatekeeperServiceTest {
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
assertTrue(((List<?>) result.get("failed_rules")).contains("evidence.invocation_ref"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void failResultIncludesRuleAuditMetadata() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
ExecutorGatekeeperService service = new ExecutorGatekeeperService(repository);
|
||||
|
||||
Map<String, Object> result = service.fail("gatekeeper.internal_error", "gatekeeper", "boom");
|
||||
|
||||
assertEquals("fail", result.get("status"));
|
||||
assertEquals("reject", result.get("severity"));
|
||||
assertRuleAudit(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateFailsForToolNameMismatch() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
@@ -329,4 +358,13 @@ class ExecutorGatekeeperServiceTest {
|
||||
"missing_info", List.of()
|
||||
));
|
||||
}
|
||||
|
||||
private void assertRuleAudit(Map<String, Object> result) {
|
||||
assertEquals("gatekeeper-rules-v1", result.get("rule_set_version"));
|
||||
assertTrue(result.get("rules") instanceof List<?>);
|
||||
List<?> rules = (List<?>) result.get("rules");
|
||||
assertFalse(rules.isEmpty());
|
||||
assertTrue(rules.stream().anyMatch(rule ->
|
||||
rule instanceof Map<?, ?> map && "evidence.raw_path".equals(map.get("id"))));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user