osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(eval): add evidence pipeline acceptance closure

Browse Source
This commit is contained in:
aruo
2026-07-09 00:47:48 +08:00
parent a77c947cd4
commit db0f229285
46 changed files with 1434 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/eval/DiagnosisEvalCase.java
+1
View File
@@ -26,6 +26,7 @@ public class DiagnosisEvalCase {
private Boolean requireClaimChecks;
private Boolean requireComposerOutput;
private List<String> expectedGatekeeperStatuses;
private String expectedGatekeeperRuleSetVersion;
private List<String> expectedComposerStatuses;
private List<String> forbiddenConfirmedClaimKeywords;
}
src/main/java/com/superbiz/agent/eval/DiagnosisEvalReportWriter.java
+4 -2
View File
@@ -46,8 +46,8 @@ public class DiagnosisEvalReportWriter {
}
builder.append("## Cases\n\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |\n");
builder.append("| Case | Result | Verdict | Gatekeeper | Rule Set | Composer | Claim Checks | Keywords | Tool Calls | Duration ms | Failed Checks |\n");
builder.append("| --- | --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- |\n");
for (DiagnosisEvalResult result : report.getResults()) {
builder.append("| ")
.append(result.getCaseId())
@@ -58,6 +58,8 @@ public class DiagnosisEvalReportWriter {
.append(" | ")
.append(valueOrDash(result.getGatekeeperStatus()))
.append(" | ")
.append(valueOrDash(result.getGatekeeperRuleSetVersion()))
.append(" | ")
.append(valueOrDash(result.getComposerStatus()))
.append(" | ")
.append(result.getClaimCheckCount() == null ? "-" : result.getClaimCheckCount())
src/main/java/com/superbiz/agent/eval/DiagnosisEvalResult.java
+1
View File
@@ -23,6 +23,7 @@ public class DiagnosisEvalResult {
private int requiredKeywordCount;
private Map<String, Boolean> evidenceCoverage;
private String gatekeeperStatus;
private String gatekeeperRuleSetVersion;
private String composerStatus;
private Integer claimCheckCount;
private Integer toolCallCount;
src/main/java/com/superbiz/agent/eval/DiagnosisTraceEvaluator.java
+11 -1
View File
@@ -66,6 +66,7 @@ public class DiagnosisTraceEvaluator {
.requiredKeywordCount(size(evalCase.getExpectedRootCauseKeywords()))
.evidenceCoverage(emptyCoverage(evalCase.getRequiredEvidenceTools()))
.gatekeeperStatus(null)
.gatekeeperRuleSetVersion(null)
.composerStatus(null)
.claimCheckCount(null)
.toolCallCount(null)
@@ -120,10 +121,12 @@ public class DiagnosisTraceEvaluator {
failedChecks.addAll(validateExecutorStructuredOutput(trace));
String gatekeeperStatus = extractNestedString(trace, "verifier_evaluation", "gatekeeper_result", "status");
String gatekeeperRuleSetVersion = extractNestedString(trace, "verifier_evaluation",
"gatekeeper_result", "rule_set_version");
String composerStatus = extractNestedString(trace, "verifier_evaluation", "composer_output", "status");
Integer claimCheckCount = countList(trace, "verifier_evaluation", "claim_checks");
failedChecks.addAll(validateV2AuditClosure(evalCase, trace, normalizedAnswer, verdict,
gatekeeperStatus, composerStatus));
gatekeeperStatus, gatekeeperRuleSetVersion, composerStatus));
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
@@ -138,6 +141,7 @@ public class DiagnosisTraceEvaluator {
.requiredKeywordCount(requiredKeywordCount)
.evidenceCoverage(evidenceCoverage)
.gatekeeperStatus(gatekeeperStatus)
.gatekeeperRuleSetVersion(gatekeeperRuleSetVersion)
.composerStatus(composerStatus)
.claimCheckCount(claimCheckCount)
.toolCallCount(toolCallCount)
@@ -232,6 +236,7 @@ public class DiagnosisTraceEvaluator {
String normalizedAnswer,
String verdict,
String gatekeeperStatus,
String gatekeeperRuleSetVersion,
String composerStatus) {
List<String> failedChecks = new ArrayList<>();
boolean requireV2AuditClosure = Boolean.TRUE.equals(evalCase.getRequireV2AuditClosure());
@@ -249,6 +254,11 @@ public class DiagnosisTraceEvaluator {
&& !safeList(evalCase.getExpectedGatekeeperStatuses()).contains(gatekeeperStatus)) {
failedChecks.add("gatekeeper status not expected: " + valueOrMissing(gatekeeperStatus));
}
if (!isBlank(evalCase.getExpectedGatekeeperRuleSetVersion())
&& !evalCase.getExpectedGatekeeperRuleSetVersion().equals(gatekeeperRuleSetVersion)) {
failedChecks.add("gatekeeper rule set version not expected: "
+ valueOrMissing(gatekeeperRuleSetVersion));
}
failedChecks.addAll(validateClaimChecks(trace, requireClaimChecks));
src/main/java/com/superbiz/agent/hook/VerifierInputHook.java
+4
View File
@@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.service.ExecutorGatekeeperService;
import com.superbiz.agent.service.GatekeeperRuleCatalog;
import com.superbiz.agent.service.ToolTraceSummaryService;
import com.superbiz.agent.util.SessionContextHolder;
import com.superbiz.agent.util.VerifierContextHolder;
@@ -110,9 +111,12 @@ public class VerifierInputHook extends MessagesModelHook {
}
private Map<String, Object> passGatekeeperResult() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", "pass");
result.put("severity", "none");
result.put("rule_set_version", catalog.version());
result.put("rules", catalog.auditRules());
result.put("checked_bindings", List.of());
result.put("failed_rules", List.of());
result.put("warnings", List.of());
src/main/java/com/superbiz/agent/service/ChatService.java
+15 -2
View File
@@ -892,8 +892,7 @@ public class ChatService {
Optional.ofNullable(VerifierContextHolder.getToolTraceSummary()).orElse(List.of()));
verifierEvaluation.put("gatekeeper_result",
Optional.ofNullable(VerifierContextHolder.getGatekeeperResult())
.orElse(Map.of("status", "pass", "severity", "none", "checked_bindings", List.of(),
"failed_rules", List.of(), "warnings", List.of(), "errors", List.of())));
.orElse(defaultGatekeeperPass()));
if (composerOutput != null) {
verifierEvaluation.put("composer_output", composerOutput);
}
@@ -903,6 +902,20 @@ public class ChatService {
diagnosisSessionRepository.save(session);
}
private Map<String, Object> defaultGatekeeperPass() {
GatekeeperRuleCatalog catalog = GatekeeperRuleCatalog.fallback();
return Map.of(
"status", "pass",
"severity", "none",
"rule_set_version", catalog.version(),
"rules", catalog.auditRules(),
"checked_bindings", List.of(),
"failed_rules", List.of(),
"warnings", List.of(),
"errors", List.of()
);
}
private ComposerRenderResult composeFinalAnswer(ChatModel chatModel, String originalQuery,
VerifierDecision decision, RunnableConfig config) {
Map<String, Object> composerInput = buildComposerInput(originalQuery, decision);
src/main/java/com/superbiz/agent/service/ExecutorGatekeeperService.java
+25 -5
View File
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
@@ -35,19 +36,29 @@ public class ExecutorGatekeeperService {
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
private static final double MIN_TOKEN_OVERLAP = 0.5;
private static final double DEFAULT_MIN_TOKEN_OVERLAP = 0.5;
private final ToolInvocationRepository toolInvocationRepository;
private final GatekeeperRuleCatalog ruleCatalog;
private final ObjectMapper objectMapper = new ObjectMapper();
@Autowired
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository) {
this(toolInvocationRepository, null);
}
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository,
GatekeeperRuleCatalog ruleCatalog) {
this.toolInvocationRepository = toolInvocationRepository;
this.ruleCatalog = ruleCatalog == null
? GatekeeperRuleCatalog.loadDefault(objectMapper)
: ruleCatalog;
}
public Map<String, Object> validate(String sessionId,
Map<String, Object> structuredOutput,
Map<String, Object> parseStatus) {
GatekeeperResult result = new GatekeeperResult();
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
validateSchema(structuredOutput, parseStatus, result);
if (structuredOutput != null) {
validateInvocationRefs(sessionId, structuredOutput, result);
@@ -57,11 +68,11 @@ public class ExecutorGatekeeperService {
}
public Map<String, Object> pass() {
return new GatekeeperResult().toMap();
return new GatekeeperResult(ruleCatalog).toMap();
}
public Map<String, Object> fail(String ruleId, String target, String message) {
GatekeeperResult result = new GatekeeperResult();
GatekeeperResult result = new GatekeeperResult(ruleCatalog);
result.fail(ruleId, target, message, SEVERITY_REJECT);
return result.toMap();
}
@@ -474,7 +485,9 @@ public class ExecutorGatekeeperService {
overlap++;
}
}
return (double) overlap / excerptTokens.size() >= MIN_TOKEN_OVERLAP;
double minTokenOverlap = ruleCatalog.doubleParameter(RULE_EXCERPT_MISMATCH,
"min_token_overlap", DEFAULT_MIN_TOKEN_OVERLAP);
return (double) overlap / excerptTokens.size() >= minTokenOverlap;
}
private String normalized(String value) {
@@ -527,12 +540,17 @@ public class ExecutorGatekeeperService {
}
private static final class GatekeeperResult {
private final GatekeeperRuleCatalog ruleCatalog;
private final List<String> failedRules = new ArrayList<>();
private final List<Map<String, Object>> checkedBindings = new ArrayList<>();
private final List<Map<String, Object>> warnings = new ArrayList<>();
private final List<Map<String, Object>> errors = new ArrayList<>();
private String severity = SEVERITY_NONE;
GatekeeperResult(GatekeeperRuleCatalog ruleCatalog) {
this.ruleCatalog = ruleCatalog == null ? GatekeeperRuleCatalog.fallback() : ruleCatalog;
}
void fail(String ruleId, String target, String message, String failureSeverity) {
if (!failedRules.contains(ruleId)) {
failedRules.add(ruleId);
@@ -563,6 +581,8 @@ public class ExecutorGatekeeperService {
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", failedRules.isEmpty() ? STATUS_PASS : STATUS_FAIL);
result.put("severity", failedRules.isEmpty() ? SEVERITY_NONE : severity);
result.put("rule_set_version", ruleCatalog.version());
result.put("rules", ruleCatalog.auditRules());
result.put("checked_bindings", checkedBindings);
result.put("failed_rules", failedRules);
result.put("warnings", warnings);
src/main/java/com/superbiz/agent/service/GatekeeperRuleCatalog.java
+142
View File
@@ -0,0 +1,142 @@
package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/**
* Lightweight metadata catalog for deterministic Gatekeeper rules.
*/
public class GatekeeperRuleCatalog {
public static final String DEFAULT_RESOURCE = "gatekeeper/gatekeeper-rules.json";
public static final String FALLBACK_VERSION = "gatekeeper-rules-v1";
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
private final String version;
private final List<RuleMetadata> rules;
public GatekeeperRuleCatalog(String version, List<RuleMetadata> rules) {
this.version = version == null || version.isBlank() ? FALLBACK_VERSION : version;
this.rules = List.copyOf(rules == null ? List.of() : rules);
}
public static GatekeeperRuleCatalog loadDefault(ObjectMapper objectMapper) {
try (InputStream input = GatekeeperRuleCatalog.class.getClassLoader()
.getResourceAsStream(DEFAULT_RESOURCE)) {
if (input == null) {
return fallback();
}
Map<String, Object> root = objectMapper.readValue(input, MAP_TYPE);
String version = stringValue(root.get("version"));
List<RuleMetadata> rules = new ArrayList<>();
Object rulesValue = root.get("rules");
if (rulesValue instanceof List<?> ruleList) {
for (Object ruleValue : ruleList) {
if (ruleValue instanceof Map<?, ?> ruleMap) {
rules.add(RuleMetadata.from(ruleMap));
}
}
}
return new GatekeeperRuleCatalog(version, rules);
} catch (Exception ignored) {
return fallback();
}
}
public static GatekeeperRuleCatalog fallback() {
return new GatekeeperRuleCatalog(FALLBACK_VERSION, List.of(
new RuleMetadata("schema.executor_v2", "Executor output must match executor_evidence_v2 schema",
true, "low_confid", Map.of()),
new RuleMetadata("evidence.invocation_ref", "source_invocation_id must refer to a real current-session tool invocation",
true, "reject", Map.of()),
new RuleMetadata("evidence.raw_path", "raw_path must exist in retrieval_details.evidence_refs",
true, "reject", Map.of()),
new RuleMetadata("evidence.excerpt_mismatch", "evidence_excerpt must be supported by the matched evidence ref text",
true, "reject", Map.of("min_token_overlap", 0.5)),
new RuleMetadata("evidence.missing", "claims must include usable evidence bindings",
true, "low_confid", Map.of())
));
}
public String version() {
return version;
}
public List<Map<String, Object>> auditRules() {
List<Map<String, Object>> result = new ArrayList<>();
for (RuleMetadata rule : rules) {
if (rule.enabled()) {
result.add(rule.toAuditMap());
}
}
return result;
}
public double doubleParameter(String ruleId, String parameterName, double fallback) {
for (RuleMetadata rule : rules) {
if (!rule.id().equals(ruleId) || !rule.enabled()) {
continue;
}
Object value = rule.parameters().get(parameterName);
if (value instanceof Number number) {
return number.doubleValue();
}
if (value instanceof String text) {
try {
return Double.parseDouble(text);
} catch (NumberFormatException ignored) {
return fallback;
}
}
}
return fallback;
}
private static String stringValue(Object value) {
return value == null ? "" : String.valueOf(value);
}
public record RuleMetadata(String id,
String description,
boolean enabled,
String defaultSeverity,
Map<String, Object> parameters) {
static RuleMetadata from(Map<?, ?> raw) {
String id = stringValue(raw.get("id"));
String description = stringValue(raw.get("description"));
boolean enabled = !(raw.get("enabled") instanceof Boolean value) || value;
String defaultSeverity = stringValue(raw.get("default_severity"));
Map<String, Object> parameters = new LinkedHashMap<>();
Object parametersValue = raw.get("parameters");
if (parametersValue instanceof Map<?, ?> parameterMap) {
for (Map.Entry<?, ?> entry : parameterMap.entrySet()) {
if (entry.getKey() != null) {
parameters.put(String.valueOf(entry.getKey()), entry.getValue());
}
}
}
return new RuleMetadata(id, description, enabled, defaultSeverity, parameters);
}
Map<String, Object> toAuditMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("id", id);
result.put("description", description);
result.put("enabled", enabled);
result.put("default_severity", defaultSeverity);
if (!parameters.isEmpty()) {
result.put("parameters", parameters);
}
return result;
}
}
}
src/main/resources/gatekeeper/gatekeeper-rules.json
+38
View File
@@ -0,0 +1,38 @@
{
"version": "gatekeeper-rules-v1",
"rules": [
{
"id": "schema.executor_v2",
"description": "Executor output must match executor_evidence_v2 schema",
"enabled": true,
"default_severity": "low_confid"
},
{
"id": "evidence.invocation_ref",
"description": "source_invocation_id must refer to a real current-session tool invocation",
"enabled": true,
"default_severity": "reject"
},
{
"id": "evidence.raw_path",
"description": "raw_path must exist in retrieval_details.evidence_refs",
"enabled": true,
"default_severity": "reject"
},
{
"id": "evidence.excerpt_mismatch",
"description": "evidence_excerpt must be supported by the matched evidence ref text",
"enabled": true,
"default_severity": "reject",
"parameters": {
"min_token_overlap": 0.5
}
},
{
"id": "evidence.missing",
"description": "claims must include usable evidence bindings",
"enabled": true,
"default_severity": "low_confid"
}
]
}