package com.superbiz.agent.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.ToolInvocation; import com.superbiz.agent.repository.ToolInvocationRepository; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; /** * Deterministic checks for Executor structured output before verifier reasoning. */ @Service public class ExecutorGatekeeperService { public static final String STATUS_PASS = "pass"; public static final String STATUS_FAIL = "fail"; public static final String SEVERITY_NONE = "none"; public static final String SEVERITY_LOW_CONFID = "low_confid"; public static final String SEVERITY_REJECT = "reject"; public static final String RULE_SCHEMA = "schema.executor_v2"; public static final String RULE_INVOCATION_REF = "evidence.invocation_ref"; public static final String RULE_RAW_PATH = "evidence.raw_path"; public static final String RULE_EXCERPT_MISMATCH = "evidence.excerpt_mismatch"; public static final String RULE_EVIDENCE_MISSING = "evidence.missing"; private static final TypeReference> MAP_TYPE = new TypeReference<>() { }; private static final double MIN_TOKEN_OVERLAP = 0.5; private final ToolInvocationRepository toolInvocationRepository; private final ObjectMapper objectMapper = new ObjectMapper(); public ExecutorGatekeeperService(ToolInvocationRepository toolInvocationRepository) { this.toolInvocationRepository = toolInvocationRepository; } public Map validate(String sessionId, Map structuredOutput, Map parseStatus) { GatekeeperResult result = new GatekeeperResult(); validateSchema(structuredOutput, parseStatus, result); if (structuredOutput != null) { validateInvocationRefs(sessionId, structuredOutput, result); importWarnings(structuredOutput, result); } return result.toMap(); } public Map pass() { return new GatekeeperResult().toMap(); } public Map fail(String ruleId, String target, String message) { GatekeeperResult result = new GatekeeperResult(); result.fail(ruleId, target, message, SEVERITY_REJECT); return result.toMap(); } private void validateSchema(Map structuredOutput, Map parseStatus, GatekeeperResult result) { String status = parseStatus == null ? "" : String.valueOf(parseStatus.getOrDefault("status", "")); if (structuredOutput == null) { if ("valid".equals(status)) { result.fail(RULE_SCHEMA, "executor_structured_output", "structured output is missing after valid parse", SEVERITY_LOW_CONFID); } return; } if (!"executor_evidence_v2".equals(String.valueOf(structuredOutput.get("answer_version")))) { result.fail(RULE_SCHEMA, "answer_version", "answer_version must be executor_evidence_v2", SEVERITY_LOW_CONFID); } if (structuredOutput.containsKey("diagnosis_summary")) { result.fail(RULE_SCHEMA, "diagnosis_summary", "diagnosis_summary is removed from executor_evidence_v2", SEVERITY_REJECT); } if (structuredOutput.containsKey("user_facing_answer")) { result.fail(RULE_SCHEMA, "user_facing_answer", "user_facing_answer is removed from executor_evidence_v2", SEVERITY_REJECT); } Object claimsValue = structuredOutput.get("claims"); if (!(claimsValue instanceof List claims)) { result.fail(RULE_SCHEMA, "claims", "claims must be an array", SEVERITY_LOW_CONFID); return; } for (int i = 0; i < claims.size(); i++) { String target = "claims[" + i + "]"; Object claimValue = claims.get(i); if (!(claimValue instanceof Map claim)) { result.fail(RULE_SCHEMA, target, "claim must be an object", SEVERITY_LOW_CONFID); continue; } requireString(claim, "claim_id", target, result); requireString(claim, "claim_type", target, result); requireString(claim, "claim_text", target, result); String supportLevel = stringValue(claim.get("support_level")); if (!"direct".equals(supportLevel) && !"indirect".equals(supportLevel)) { result.fail(RULE_SCHEMA, target + ".support_level", "support_level must be direct or indirect", SEVERITY_LOW_CONFID); } Object bindings = claim.get("evidence_bindings"); if (!(bindings instanceof List bindingList) || bindingList.isEmpty()) { result.fail(RULE_EVIDENCE_MISSING, target + ".evidence_bindings", "claims must include non-empty evidence_bindings", SEVERITY_LOW_CONFID); } } requireArray(structuredOutput, "hypotheses", result); requireArray(structuredOutput, "recommended_actions", result); requireArray(structuredOutput, "missing_info", result); } private void validateInvocationRefs(String sessionId, Map structuredOutput, GatekeeperResult result) { if (sessionId == null || sessionId.isBlank()) { result.fail(RULE_INVOCATION_REF, "session_id", "session id is required to validate source_invocation_id", SEVERITY_LOW_CONFID); return; } Map validInvocations = toolInvocationRepository.findBySessionIdOrderByIdAsc(sessionId) .stream() .filter(invocation -> invocation.getId() != null) .collect(Collectors.toMap(ToolInvocation::getId, Function.identity(), (left, right) -> left)); Object claimsValue = structuredOutput.get("claims"); if (!(claimsValue instanceof List claims)) { return; } for (int claimIndex = 0; claimIndex < claims.size(); claimIndex++) { Object claimValue = claims.get(claimIndex); if (!(claimValue instanceof Map claim)) { continue; } Object bindingsValue = claim.get("evidence_bindings"); if (!(bindingsValue instanceof List bindings)) { continue; } for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) { String target = "claims[" + claimIndex + "].evidence_bindings[" + bindingIndex + "]"; Object bindingValue = bindings.get(bindingIndex); if (!(bindingValue instanceof Map binding)) { result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object", SEVERITY_LOW_CONFID); continue; } validateEvidenceBinding(binding, validInvocations, target, claim.get("claim_id"), claim.get("claim_type"), result); } } Object actionsValue = structuredOutput.get("recommended_actions"); if (!(actionsValue instanceof List actions)) { return; } for (int actionIndex = 0; actionIndex < actions.size(); actionIndex++) { Object actionValue = actions.get(actionIndex); if (!(actionValue instanceof Map action)) { continue; } Object bindingsValue = action.get("evidence_bindings"); if (!(bindingsValue instanceof List bindings)) { continue; } for (int bindingIndex = 0; bindingIndex < bindings.size(); bindingIndex++) { String target = "recommended_actions[" + actionIndex + "].evidence_bindings[" + bindingIndex + "]"; Object bindingValue = bindings.get(bindingIndex); if (!(bindingValue instanceof Map binding)) { result.fail(RULE_INVOCATION_REF, target, "evidence binding must be an object", SEVERITY_LOW_CONFID); continue; } validateEvidenceBinding(binding, validInvocations, target, action.get("action_id"), null, result); } } } private void validateEvidenceBinding(Map binding, Map validInvocations, String target, Object ownerId, Object ownerType, GatekeeperResult result) { Map checked = new LinkedHashMap<>(); checked.put("claim_id", ownerId == null ? "" : String.valueOf(ownerId)); checked.put("tool_name", stringValue(binding.get("tool_name"))); checked.put("source_invocation_id", binding.get("source_invocation_id")); checked.put("raw_path", stringValue(binding.get("raw_path"))); String claimedToolName = stringValue(binding.get("tool_name")); if (claimedToolName.isBlank()) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_INVOCATION_REF); checked.put("message", "tool_name is required"); result.checked(checked); result.fail(RULE_INVOCATION_REF, target + ".tool_name", "tool_name is required", SEVERITY_LOW_CONFID); return; } String rawPath = stringValue(binding.get("raw_path")); if ("negative_observation".equals(stringValue(ownerType)) && !rawPath.isBlank() && !"$.no_evidence".equals(rawPath)) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_RAW_PATH); checked.put("message", "negative_observation must only bind $.no_evidence references"); result.checked(checked); result.fail(RULE_RAW_PATH, target + ".raw_path", "negative_observation must only bind $.no_evidence references", SEVERITY_REJECT); return; } Long id = singleInvocationId(binding); if (id == null) { id = uniqueInvocationIdByToolRawPathAndExcerpt(validInvocations, claimedToolName, rawPath, stringValue(binding.get("evidence_excerpt"))); if (id == null) { id = uniqueInvocationIdByToolAndRawPath(validInvocations, claimedToolName, rawPath); } if (id != null) { result.warn(Map.of( "rule", "evidence.invocation_auto_backfill_by_raw_path", "message", "source_invocation_id was auto-filled from the unique evidence reference candidate", "tool_name", claimedToolName, "raw_path", rawPath, "source_invocation_id", id )); } } if (id == null) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_INVOCATION_REF); checked.put("message", "source_invocation_id is required"); result.checked(checked); result.fail(RULE_INVOCATION_REF, target + ".source_invocation_id", "source_invocation_id is required", SEVERITY_LOW_CONFID); return; } checked.put("source_invocation_id", id); ToolInvocation invocation = validInvocations.get(id); if (invocation == null) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_INVOCATION_REF); checked.put("message", "source_invocation_id not found in current session: " + id); result.checked(checked); result.fail(RULE_INVOCATION_REF, target, "source_invocation_id not found in current session: " + id, SEVERITY_REJECT); return; } if (!Objects.equals(claimedToolName, invocation.getToolName())) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_INVOCATION_REF); checked.put("message", "tool_name does not match invocation " + id + ": expected " + invocation.getToolName()); result.checked(checked); result.fail(RULE_INVOCATION_REF, target + ".tool_name", "tool_name does not match invocation " + id + ": expected " + invocation.getToolName(), SEVERITY_REJECT); return; } if (rawPath.isBlank()) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_RAW_PATH); checked.put("message", "raw_path is required for precise evidence reference"); result.checked(checked); result.fail(RULE_RAW_PATH, target + ".raw_path", "raw_path is required for precise evidence reference", SEVERITY_LOW_CONFID); return; } Map refs = evidenceRefsByRawPath(invocation.getRetrievalDetails()); if (refs.isEmpty()) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_EVIDENCE_MISSING); checked.put("message", "invocation has no retrieval_details.evidence_refs"); result.checked(checked); result.fail(RULE_EVIDENCE_MISSING, target, "invocation has no retrieval_details.evidence_refs", SEVERITY_LOW_CONFID); return; } String matchedText = refs.get(rawPath); if (matchedText == null) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_RAW_PATH); checked.put("message", "raw_path not found in retrieval_details.evidence_refs"); result.checked(checked); result.fail(RULE_RAW_PATH, target + ".raw_path", "raw_path not found in retrieval_details.evidence_refs", SEVERITY_REJECT); return; } checked.put("matched_text", matchedText); String excerpt = stringValue(binding.get("evidence_excerpt")); if (normalized(excerpt).length() < 8) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_EXCERPT_MISMATCH); checked.put("message", "evidence_excerpt is too short to compare"); result.checked(checked); result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt", "evidence_excerpt is too short to compare", SEVERITY_LOW_CONFID); return; } if (!isExcerptSupported(excerpt, matchedText)) { checked.put("status", STATUS_FAIL); checked.put("rule", RULE_EXCERPT_MISMATCH); checked.put("message", "evidence_excerpt is not supported by matched evidence ref text"); result.checked(checked); result.fail(RULE_EXCERPT_MISMATCH, target + ".evidence_excerpt", "evidence_excerpt is not supported by matched evidence ref text", SEVERITY_REJECT); return; } checked.put("status", STATUS_PASS); result.checked(checked); } private Long uniqueInvocationIdByToolAndRawPath(Map validInvocations, String toolName, String rawPath) { if (toolName == null || toolName.isBlank() || rawPath == null || rawPath.isBlank()) { return null; } Long matchedId = null; for (Map.Entry entry : validInvocations.entrySet()) { ToolInvocation invocation = entry.getValue(); if (!Objects.equals(toolName, invocation.getToolName())) { continue; } if (!evidenceRefsByRawPath(invocation.getRetrievalDetails()).containsKey(rawPath)) { continue; } if (matchedId != null) { return null; } matchedId = entry.getKey(); } return matchedId; } private Long uniqueInvocationIdByToolRawPathAndExcerpt(Map validInvocations, String toolName, String rawPath, String excerpt) { if (toolName == null || toolName.isBlank() || rawPath == null || rawPath.isBlank() || excerpt == null || excerpt.isBlank()) { return null; } Long matchedId = null; for (Map.Entry entry : validInvocations.entrySet()) { ToolInvocation invocation = entry.getValue(); if (!Objects.equals(toolName, invocation.getToolName())) { continue; } String matchedText = evidenceRefsByRawPath(invocation.getRetrievalDetails()).get(rawPath); if (matchedText == null || !isBackfillCandidateSupported(rawPath, excerpt, matchedText)) { continue; } if (matchedId != null) { return null; } matchedId = entry.getKey(); } return matchedId; } private boolean isBackfillCandidateSupported(String rawPath, String excerpt, String matchedText) { if ("$.no_evidence".equals(rawPath)) { String excerptQuery = semicolonField(excerpt, "query"); String matchedQuery = semicolonField(matchedText, "query"); if (!excerptQuery.isBlank() && !matchedQuery.isBlank() && !normalized(excerptQuery).equals(normalized(matchedQuery))) { return false; } } return isExcerptSupported(excerpt, matchedText); } private String semicolonField(String text, String field) { if (text == null || text.isBlank() || field == null || field.isBlank()) { return ""; } String prefix = field + "="; for (String part : text.split(";")) { String trimmed = part.trim(); if (trimmed.regionMatches(true, 0, prefix, 0, prefix.length())) { return trimmed.substring(prefix.length()).trim(); } } return ""; } private Long singleInvocationId(Map binding) { Long singular = asLong(binding.get("source_invocation_id")); if (singular != null) { return singular; } Object idsValue = binding.get("source_invocation_ids"); if (!(idsValue instanceof List ids) || ids.size() != 1) { return null; } return asLong(ids.get(0)); } @SuppressWarnings("unchecked") private void importWarnings(Map structuredOutput, GatekeeperResult result) { Object warningsValue = structuredOutput.remove("_gatekeeper_warnings"); if (!(warningsValue instanceof List warnings)) { return; } for (Object warning : warnings) { if (warning instanceof Map map) { result.warn((Map) map); } else if (warning != null) { result.warn(Map.of("message", String.valueOf(warning))); } } } private Map evidenceRefsByRawPath(String retrievalDetails) { if (retrievalDetails == null || retrievalDetails.isBlank()) { return Map.of(); } try { Map details = objectMapper.readValue(retrievalDetails, MAP_TYPE); Object refsValue = details.get("evidence_refs"); if (!(refsValue instanceof List refs)) { return Map.of(); } Map result = new LinkedHashMap<>(); for (Object refValue : refs) { if (!(refValue instanceof Map ref)) { continue; } String rawPath = stringValue(ref.get("raw_path")); String text = stringValue(ref.get("text")); if (!rawPath.isBlank() && !text.isBlank()) { result.put(rawPath, text); } } return result; } catch (Exception ignored) { return Map.of(); } } private boolean isExcerptSupported(String excerpt, String matchedText) { String normalizedExcerpt = normalized(excerpt); String normalizedMatched = normalized(matchedText); if (normalizedMatched.contains(normalizedExcerpt) || normalizedExcerpt.contains(normalizedMatched)) { return true; } Set excerptTokens = tokens(normalizedExcerpt); if (excerptTokens.isEmpty()) { return false; } Set matchedTokens = tokens(normalizedMatched); int overlap = 0; for (String token : excerptTokens) { if (matchedTokens.contains(token)) { overlap++; } } return (double) overlap / excerptTokens.size() >= MIN_TOKEN_OVERLAP; } private String normalized(String value) { return value == null ? "" : value.toLowerCase() .replaceAll("[\\p{Punct}\\s,。;:、()【】《》“”‘’]+", " ") .trim(); } private Set tokens(String text) { if (text == null || text.isBlank()) { return Set.of(); } Set result = new HashSet<>(); for (String token : text.split("\\s+")) { if (token.length() >= 2) { result.add(token); } } return result; } private void requireArray(Map output, String field, GatekeeperResult result) { if (!(output.get(field) instanceof List)) { result.fail(RULE_SCHEMA, field, field + " must be an array", SEVERITY_LOW_CONFID); } } private void requireString(Map object, String field, String target, GatekeeperResult result) { if (stringValue(object.get(field)).isBlank()) { result.fail(RULE_SCHEMA, target + "." + field, field + " is required", SEVERITY_LOW_CONFID); } } private String stringValue(Object value) { return value == null ? "" : String.valueOf(value); } private Long asLong(Object value) { if (value instanceof Number number) { return number.longValue(); } if (value instanceof String text) { try { return Long.parseLong(text); } catch (NumberFormatException ignored) { return null; } } return null; } private static final class GatekeeperResult { private final List failedRules = new ArrayList<>(); private final List> checkedBindings = new ArrayList<>(); private final List> warnings = new ArrayList<>(); private final List> errors = new ArrayList<>(); private String severity = SEVERITY_NONE; void fail(String ruleId, String target, String message, String failureSeverity) { if (!failedRules.contains(ruleId)) { failedRules.add(ruleId); } if (SEVERITY_REJECT.equals(failureSeverity)) { severity = SEVERITY_REJECT; } else if (!SEVERITY_REJECT.equals(severity)) { severity = SEVERITY_LOW_CONFID; } Map error = new LinkedHashMap<>(); error.put("rule_id", ruleId); error.put("rule", ruleId); error.put("target", target); error.put("message", message); error.put("severity", failureSeverity); errors.add(error); } void checked(Map checked) { checkedBindings.add(checked); } void warn(Map warning) { warnings.add(new LinkedHashMap<>(warning)); } Map toMap() { Map result = new LinkedHashMap<>(); result.put("status", failedRules.isEmpty() ? STATUS_PASS : STATUS_FAIL); result.put("severity", failedRules.isEmpty() ? SEVERITY_NONE : severity); result.put("checked_bindings", checkedBindings); result.put("failed_rules", failedRules); result.put("warnings", warnings); result.put("errors", errors); return result; } } }