Files
SuperBizAgent-java/src/main/java/com/superbiz/agent/service/ExecutorGatekeeperService.java
T

574 lines
25 KiB
Java

package com.superbiz.agent.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.domain.entity.ToolInvocation;
import com.superbiz.agent.repository.ToolInvocationRepository;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Deterministic checks for Executor structured output before verifier reasoning.
*/
@Service
public class ExecutorGatekeeperService {
public static final String STATUS_PASS = "pass";
public static final String STATUS_FAIL = "fail";
public static final String SEVERITY_NONE = "none";
public static final String SEVERITY_LOW_CONFID = "low_confid";
public static final String SEVERITY_REJECT = "reject";
public static final String RULE_SCHEMA = "schema.executor_v2";
public static final String RULE_INVOCATION_REF = "evidence.invocation_ref";
public static final String RULE_RAW_PATH = "evidence.raw_path";
public static final String RULE_EXCERPT_MISMATCH = "evidence.excerpt_mismatch";
public static final String RULE_EVIDENCE_MISSING = "evidence.missing";
private static final TypeReference<Map<String, Object>> MAP_TYPE = new TypeReference<>() {
};
private static final double MIN_TOKEN_OVERLAP = 0.5;
private final ToolInvocationRepository toolInvocationRepository;
private final ObjectMapper objectMapper = new ObjectMapper();
public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository) {
this.toolInvocationRepository = toolInvocationRepository;
}
public Map<String, Object> validate(String sessionId,
Map<String, Object> structuredOutput,
Map<String, Object> parseStatus) {
GatekeeperResult result = new GatekeeperResult();
validateSchema(structuredOutput, parseStatus, result);
if (structuredOutput != null) {
validateInvocationRefs(sessionId, structuredOutput, result);
importWarnings(structuredOutput, result);
}
return result.toMap();
}
public Map<String, Object> pass() {
return new GatekeeperResult().toMap();
}
public Map<String, Object> fail(String ruleId, String target, String message) {
GatekeeperResult result = new GatekeeperResult();
result.fail(ruleId, target, message, SEVERITY_REJECT);
return result.toMap();
}
private void validateSchema(Map<String, Object> structuredOutput,
Map<String, Object> parseStatus,
GatekeeperResult result) {
String status = parseStatus == null ? "" : String.valueOf(parseStatus.getOrDefault("status", ""));
if (structuredOutput == null) {
if ("valid".equals(status)) {
result.fail(RULE_SCHEMA, "executor_structured_output",
"structured output is missing after valid parse", SEVERITY_LOW_CONFID);
}
return;
}
if (!"executor_evidence_v2".equals(String.valueOf(structuredOutput.get("answer_version")))) {
result.fail(RULE_SCHEMA, "answer_version", "answer_version must be executor_evidence_v2",
SEVERITY_LOW_CONFID);
}
if (structuredOutput.containsKey("diagnosis_summary")) {
result.fail(RULE_SCHEMA, "diagnosis_summary", "diagnosis_summary is removed from executor_evidence_v2",
SEVERITY_REJECT);
}
if (structuredOutput.containsKey("user_facing_answer")) {
result.fail(RULE_SCHEMA, "user_facing_answer", "user_facing_answer is removed from executor_evidence_v2",
SEVERITY_REJECT);
}
Object claimsValue = structuredOutput.get("claims");
if (!(claimsValue instanceof List<?> claims)) {
result.fail(RULE_SCHEMA, "claims", "claims must be an array", SEVERITY_LOW_CONFID);
return;
}
for (int i = 0; i < claims.size(); i++) {
String target = "claims[" + i + "]";
Object claimValue = claims.get(i);
if (!(claimValue instanceof Map<?, ?> claim)) {
result.fail(RULE_SCHEMA, target, "claim must be an object", SEVERITY_LOW_CONFID);
continue;
}
requireString(claim, "claim_id", target, result);
requireString(claim, "claim_type", target, result);
requireString(claim, "claim_text", target, result);
String supportLevel = stringValue(claim.get("support_level"));
if (!"direct".equals(supportLevel) && !"indirect".equals(supportLevel)) {
result.fail(RULE_SCHEMA, target + ".support_level", "support_level must be direct or indirect",
SEVERITY_LOW_CONFID);
}
Object bindings = claim.get("evidence_bindings");
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
result.fail(RULE_EVIDENCE_MISSING, target + ".evidence_bindings",
"claims must include non-empty evidence_bindings", SEVERITY_LOW_CONFID);
}
}
requireArray(structuredOutput, "hypotheses", result);
requireArray(structuredOutput, "recommended_actions", result);
requireArray(structuredOutput, "missing_info", result);
}
private void validateInvocationRefs(String sessionId, Map<String, Object> structuredOutput, GatekeeperResult result) {
if (sessionId == null || sessionId.isBlank()) {
result.fail(RULE_INVOCATION_REF, "session_id", "session id is required to validate source_invocation_id",
SEVERITY_LOW_CONFID);
return;
}
Map<Long, ToolInvocation> validInvocations = toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId)
.stream()
.filter(invocation -> invocation.getId() != null)
.collect(Collectors.toMap(ToolInvocation::getId, Function.identity(), (left, right) -> left));
Object claimsValue = structuredOutput.get("claims");
if (!(claimsValue instanceof List<?> claims)) {
return;
}
for (int claimIndex = 0; claimIndex < claims.size(); claimIndex++) {
Object claimValue = claims.get(claimIndex);
if (!(claimValue instanceof Map<?, ?> claim)) {
continue;
}
Object bindingsValue = claim.get("evidence_bindings");
if (!(bindingsValue instanceof List<?> bindings)) {
continue;
}
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
String target = "claims[" + claimIndex + "].evidence_bindings[" + bindingIndex + "]";
Object bindingValue = bindings.get(bindingIndex);
if (!(bindingValue instanceof Map<?, ?> binding)) {
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
SEVERITY_LOW_CONFID);
continue;
}
validateEvidenceBinding(binding, validInvocations, target,
claim.get("claim_id"), claim.get("claim_type"), result);
}
}
Object actionsValue = structuredOutput.get("recommended_actions");
if (!(actionsValue instanceof List<?> actions)) {
return;
}
for (int actionIndex = 0; actionIndex < actions.size(); actionIndex++) {
Object actionValue = actions.get(actionIndex);
if (!(actionValue instanceof Map<?, ?> action)) {
continue;
}
Object bindingsValue = action.get("evidence_bindings");
if (!(bindingsValue instanceof List<?> bindings)) {
continue;
}
for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) {
String target = "recommended_actions[" + actionIndex + "].evidence_bindings[" + bindingIndex + "]";
Object bindingValue = bindings.get(bindingIndex);
if (!(bindingValue instanceof Map<?, ?> binding)) {
result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object",
SEVERITY_LOW_CONFID);
continue;
}
validateEvidenceBinding(binding, validInvocations, target, action.get("action_id"), null, result);
}
}
}
private void validateEvidenceBinding(Map<?, ?> binding,
Map<Long, ToolInvocation> validInvocations,
String target,
Object ownerId,
Object ownerType,
GatekeeperResult result) {
Map<String, Object> checked = new LinkedHashMap<>();
checked.put("claim_id", ownerId == null ? "" : String.valueOf(ownerId));
checked.put("tool_name", stringValue(binding.get("tool_name")));
checked.put("source_invocation_id", binding.get("source_invocation_id"));
checked.put("raw_path", stringValue(binding.get("raw_path")));
String claimedToolName = stringValue(binding.get("tool_name"));
if (claimedToolName.isBlank()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "tool_name is required");
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".tool_name", "tool_name is required", SEVERITY_LOW_CONFID);
return;
}
String rawPath = stringValue(binding.get("raw_path"));
if ("negative_observation".equals(stringValue(ownerType)) && !rawPath.isBlank()
&& !"$.no_evidence".equals(rawPath)) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "negative_observation must only bind $.no_evidence references");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"negative_observation must only bind $.no_evidence references", SEVERITY_REJECT);
return;
}
Long id = singleInvocationId(binding);
if (id == null) {
id = uniqueInvocationIdByToolRawPathAndExcerpt(validInvocations, claimedToolName, rawPath,
stringValue(binding.get("evidence_excerpt")));
if (id == null) {
id = uniqueInvocationIdByToolAndRawPath(validInvocations, claimedToolName, rawPath);
}
if (id != null) {
result.warn(Map.of(
"rule", "evidence.invocation_auto_backfill_by_raw_path",
"message", "source_invocation_id was auto-filled from the unique evidence reference candidate",
"tool_name", claimedToolName,
"raw_path", rawPath,
"source_invocation_id", id
));
}
}
if (id == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "source_invocation_id is required");
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".source_invocation_id",
"source_invocation_id is required", SEVERITY_LOW_CONFID);
return;
}
checked.put("source_invocation_id", id);
ToolInvocation invocation = validInvocations.get(id);
if (invocation == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "source_invocation_id not found in current session: " + id);
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target,
"source_invocation_id not found in current session: " + id, SEVERITY_REJECT);
return;
}
if (!Objects.equals(claimedToolName, invocation.getToolName())) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_INVOCATION_REF);
checked.put("message", "tool_name does not match invocation " + id + ": expected " + invocation.getToolName());
result.checked(checked);
result.fail(RULE_INVOCATION_REF, target + ".tool_name",
"tool_name does not match invocation " + id + ": expected " + invocation.getToolName(),
SEVERITY_REJECT);
return;
}
if (rawPath.isBlank()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "raw_path is required for precise evidence reference");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"raw_path is required for precise evidence reference", SEVERITY_LOW_CONFID);
return;
}
Map<String, String> refs = evidenceRefsByRawPath(invocation.getRetrievalDetails());
if (refs.isEmpty()) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EVIDENCE_MISSING);
checked.put("message", "invocation has no retrieval_details.evidence_refs");
result.checked(checked);
result.fail(RULE_EVIDENCE_MISSING, target,
"invocation has no retrieval_details.evidence_refs", SEVERITY_LOW_CONFID);
return;
}
String matchedText = refs.get(rawPath);
if (matchedText == null) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_RAW_PATH);
checked.put("message", "raw_path not found in retrieval_details.evidence_refs");
result.checked(checked);
result.fail(RULE_RAW_PATH, target + ".raw_path",
"raw_path not found in retrieval_details.evidence_refs", SEVERITY_REJECT);
return;
}
checked.put("matched_text", matchedText);
String excerpt = stringValue(binding.get("evidence_excerpt"));
if (normalized(excerpt).length() < 8) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EXCERPT_MISMATCH);
checked.put("message", "evidence_excerpt is too short to compare");
result.checked(checked);
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
"evidence_excerpt is too short to compare", SEVERITY_LOW_CONFID);
return;
}
if (!isExcerptSupported(excerpt, matchedText)) {
checked.put("status", STATUS_FAIL);
checked.put("rule", RULE_EXCERPT_MISMATCH);
checked.put("message", "evidence_excerpt is not supported by matched evidence ref text");
result.checked(checked);
result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt",
"evidence_excerpt is not supported by matched evidence ref text", SEVERITY_REJECT);
return;
}
checked.put("status", STATUS_PASS);
result.checked(checked);
}
private Long uniqueInvocationIdByToolAndRawPath(Map<Long, ToolInvocation> validInvocations,
String toolName,
String rawPath) {
if (toolName == null || toolName.isBlank() || rawPath == null || rawPath.isBlank()) {
return null;
}
Long matchedId = null;
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
ToolInvocation invocation = entry.getValue();
if (!Objects.equals(toolName, invocation.getToolName())) {
continue;
}
if (!evidenceRefsByRawPath(invocation.getRetrievalDetails()).containsKey(rawPath)) {
continue;
}
if (matchedId != null) {
return null;
}
matchedId = entry.getKey();
}
return matchedId;
}
private Long uniqueInvocationIdByToolRawPathAndExcerpt(Map<Long, ToolInvocation> validInvocations,
String toolName,
String rawPath,
String excerpt) {
if (toolName == null || toolName.isBlank()
|| rawPath == null || rawPath.isBlank()
|| excerpt == null || excerpt.isBlank()) {
return null;
}
Long matchedId = null;
for (Map.Entry<Long, ToolInvocation> entry : validInvocations.entrySet()) {
ToolInvocation invocation = entry.getValue();
if (!Objects.equals(toolName, invocation.getToolName())) {
continue;
}
String matchedText = evidenceRefsByRawPath(invocation.getRetrievalDetails()).get(rawPath);
if (matchedText == null || !isBackfillCandidateSupported(rawPath, excerpt, matchedText)) {
continue;
}
if (matchedId != null) {
return null;
}
matchedId = entry.getKey();
}
return matchedId;
}
private boolean isBackfillCandidateSupported(String rawPath, String excerpt, String matchedText) {
if ("$.no_evidence".equals(rawPath)) {
String excerptQuery = semicolonField(excerpt, "query");
String matchedQuery = semicolonField(matchedText, "query");
if (!excerptQuery.isBlank() && !matchedQuery.isBlank()
&& !normalized(excerptQuery).equals(normalized(matchedQuery))) {
return false;
}
}
return isExcerptSupported(excerpt, matchedText);
}
private String semicolonField(String text, String field) {
if (text == null || text.isBlank() || field == null || field.isBlank()) {
return "";
}
String prefix = field + "=";
for (String part : text.split(";")) {
String trimmed = part.trim();
if (trimmed.regionMatches(true, 0, prefix, 0, prefix.length())) {
return trimmed.substring(prefix.length()).trim();
}
}
return "";
}
private Long singleInvocationId(Map<?, ?> binding) {
Long singular = asLong(binding.get("source_invocation_id"));
if (singular != null) {
return singular;
}
Object idsValue = binding.get("source_invocation_ids");
if (!(idsValue instanceof List<?> ids) || ids.size() != 1) {
return null;
}
return asLong(ids.get(0));
}
@SuppressWarnings("unchecked")
private void importWarnings(Map<String, Object> structuredOutput, GatekeeperResult result) {
Object warningsValue = structuredOutput.remove("_gatekeeper_warnings");
if (!(warningsValue instanceof List<?> warnings)) {
return;
}
for (Object warning : warnings) {
if (warning instanceof Map<?, ?> map) {
result.warn((Map<String, Object>) map);
} else if (warning != null) {
result.warn(Map.of("message", String.valueOf(warning)));
}
}
}
private Map<String, String> evidenceRefsByRawPath(String retrievalDetails) {
if (retrievalDetails == null || retrievalDetails.isBlank()) {
return Map.of();
}
try {
Map<String, Object> details = objectMapper.readValue(retrievalDetails, MAP_TYPE);
Object refsValue = details.get("evidence_refs");
if (!(refsValue instanceof List<?> refs)) {
return Map.of();
}
Map<String, String> result = new LinkedHashMap<>();
for (Object refValue : refs) {
if (!(refValue instanceof Map<?, ?> ref)) {
continue;
}
String rawPath = stringValue(ref.get("raw_path"));
String text = stringValue(ref.get("text"));
if (!rawPath.isBlank() && !text.isBlank()) {
result.put(rawPath, text);
}
}
return result;
} catch (Exception ignored) {
return Map.of();
}
}
private boolean isExcerptSupported(String excerpt, String matchedText) {
String normalizedExcerpt = normalized(excerpt);
String normalizedMatched = normalized(matchedText);
if (normalizedMatched.contains(normalizedExcerpt) || normalizedExcerpt.contains(normalizedMatched)) {
return true;
}
Set<String> excerptTokens = tokens(normalizedExcerpt);
if (excerptTokens.isEmpty()) {
return false;
}
Set<String> matchedTokens = tokens(normalizedMatched);
int overlap = 0;
for (String token : excerptTokens) {
if (matchedTokens.contains(token)) {
overlap++;
}
}
return (double) overlap / excerptTokens.size() >= MIN_TOKEN_OVERLAP;
}
private String normalized(String value) {
return value == null ? "" : value.toLowerCase()
.replaceAll("[\\p{Punct}\\s,。;:、()【】《》“”‘’]+", " ")
.trim();
}
private Set<String> tokens(String text) {
if (text == null || text.isBlank()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (String token : text.split("\\s+")) {
if (token.length() >= 2) {
result.add(token);
}
}
return result;
}
private void requireArray(Map<String, Object> output, String field, GatekeeperResult result) {
if (!(output.get(field) instanceof List<?>)) {
result.fail(RULE_SCHEMA, field, field + " must be an array", SEVERITY_LOW_CONFID);
}
}
private void requireString(Map<?, ?> object, String field, String target, GatekeeperResult result) {
if (stringValue(object.get(field)).isBlank()) {
result.fail(RULE_SCHEMA, target + "." + field, field + " is required", SEVERITY_LOW_CONFID);
}
}
private String stringValue(Object value) {
return value == null ? "" : String.valueOf(value);
}
private Long asLong(Object value) {
if (value instanceof Number number) {
return number.longValue();
}
if (value instanceof String text) {
try {
return Long.parseLong(text);
} catch (NumberFormatException ignored) {
return null;
}
}
return null;
}
private static final class GatekeeperResult {
private final List<String> failedRules = new ArrayList<>();
private final List<Map<String, Object>> checkedBindings = new ArrayList<>();
private final List<Map<String, Object>> warnings = new ArrayList<>();
private final List<Map<String, Object>> errors = new ArrayList<>();
private String severity = SEVERITY_NONE;
void fail(String ruleId, String target, String message, String failureSeverity) {
if (!failedRules.contains(ruleId)) {
failedRules.add(ruleId);
}
if (SEVERITY_REJECT.equals(failureSeverity)) {
severity = SEVERITY_REJECT;
} else if (!SEVERITY_REJECT.equals(severity)) {
severity = SEVERITY_LOW_CONFID;
}
Map<String, Object> error = new LinkedHashMap<>();
error.put("rule_id", ruleId);
error.put("rule", ruleId);
error.put("target", target);
error.put("message", message);
error.put("severity", failureSeverity);
errors.add(error);
}
void checked(Map<String, Object> checked) {
checkedBindings.add(checked);
}
void warn(Map<String, Object> warning) {
warnings.add(new LinkedHashMap<>(warning));
}
Map<String, Object> toMap() {
Map<String, Object> result = new LinkedHashMap<>();
result.put("status", failedRules.isEmpty() ? STATUS_PASS : STATUS_FAIL);
result.put("severity", failedRules.isEmpty() ? SEVERITY_NONE : severity);
result.put("checked_bindings", checkedBindings);
result.put("failed_rules", failedRules);
result.put("warnings", warnings);
result.put("errors", errors);
return result;
}
}
}