feat(harness): add evidence and semantic guards
This commit is contained in:
zhuyongxin
@@ -0,0 +1,435 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.LogEvent;
|
||||
import com.superbiz.agent.harness.tool.contract.LogPattern;
|
||||
import com.superbiz.agent.harness.tool.contract.LogQueryScope;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolResult;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsToolResult;
|
||||
import com.superbiz.agent.harness.tool.contract.RagEvidence;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolResult;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public final class EvidenceGuard {
|
||||
|
||||
private final CanonicalInvocationStore store;
|
||||
private final ToolCallKeyFactory keyFactory;
|
||||
private final ObjectReader ragReader;
|
||||
private final ObjectReader logsReader;
|
||||
private final ObjectReader mysqlRequestReader;
|
||||
private final ObjectReader mysqlResultReader;
|
||||
|
||||
public EvidenceGuard(CanonicalInvocationStore store,
|
||||
ToolCallKeyFactory keyFactory,
|
||||
ObjectMapper objectMapper) {
|
||||
this.store = Objects.requireNonNull(store, "store must not be null");
|
||||
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
|
||||
ObjectMapper mapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.ragReader = mapper.readerFor(RagToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.logsReader = mapper.readerFor(QueryLogsToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.mysqlRequestReader = mapper.readerFor(MysqlToolRequest.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.mysqlResultReader = mapper.readerFor(MysqlToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
}
|
||||
|
||||
public EvidenceGuardResult validate(RunContext context, DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
List<EvidenceViolation> violations = validateDraft(draft);
|
||||
if (!violations.isEmpty()) {
|
||||
return EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
List<VerifiedAnalysisEvidence> verifiedAnalyses = new ArrayList<>();
|
||||
for (int index = 0; index < draft.analysis().size(); index++) {
|
||||
DiagnosisDraft.AnalysisItem analysis = draft.analysis().get(index);
|
||||
List<VerifiedEvidence> evidence = new ArrayList<>();
|
||||
for (String toolCallId : analysis.toolCallIds()) {
|
||||
verifyInvocation(context, analysis, index, toolCallId, evidence, violations);
|
||||
}
|
||||
if (!evidence.isEmpty()) {
|
||||
verifiedAnalyses.add(new VerifiedAnalysisEvidence(
|
||||
analysis.analysisId(), analysis.text(), analysis.kind(), evidence));
|
||||
}
|
||||
}
|
||||
return violations.isEmpty()
|
||||
? EvidenceGuardResult.valid(new VerifiedEvidenceSnapshot(verifiedAnalyses))
|
||||
: EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
private List<EvidenceViolation> validateDraft(DiagnosisDraft draft) {
|
||||
List<EvidenceViolation> violations = new ArrayList<>();
|
||||
if (draft == null) {
|
||||
violations.add(violation(EvidenceViolationCode.DRAFT_MISSING, "draft"));
|
||||
return violations;
|
||||
}
|
||||
if (draft.analysis().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_MISSING, "analysis"));
|
||||
}
|
||||
Set<String> ids = new HashSet<>();
|
||||
for (int index = 0; index < draft.analysis().size(); index++) {
|
||||
DiagnosisDraft.AnalysisItem item = draft.analysis().get(index);
|
||||
String target = "analysis[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_MISSING, target));
|
||||
continue;
|
||||
}
|
||||
if (isBlank(item.analysisId())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_ID_MISSING, target + ".analysis_id"));
|
||||
} else if (!ids.add(item.analysisId())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_ID_DUPLICATE, target + ".analysis_id"));
|
||||
}
|
||||
if (item.kind() == null) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_KIND_MISSING, target + ".kind"));
|
||||
}
|
||||
if (isBlank(item.text())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_TEXT_MISSING, target + ".text"));
|
||||
}
|
||||
if (item.toolCallIds().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_MISSING, target + ".tool_call_ids"));
|
||||
}
|
||||
}
|
||||
validateReportReferences(draft, ids, violations);
|
||||
return violations;
|
||||
}
|
||||
|
||||
private void validateReportReferences(DiagnosisDraft draft, Set<String> ids,
|
||||
List<EvidenceViolation> violations) {
|
||||
if (draft.conclusion() != null) {
|
||||
validateTextAndReferences("conclusion", draft.conclusion().text(),
|
||||
draft.conclusion().basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
for (int index = 0; index < draft.actionPlan().size(); index++) {
|
||||
DiagnosisDraft.ActionPlanItem item = draft.actionPlan().get(index);
|
||||
String target = "action_plan[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
} else {
|
||||
validateTextAndReferences(target, item.action(), item.basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
}
|
||||
for (int index = 0; index < draft.recommendations().size(); index++) {
|
||||
DiagnosisDraft.Recommendation item = draft.recommendations().get(index);
|
||||
String target = "recommendations[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
} else {
|
||||
validateTextAndReferences(target, item.text(), item.basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
}
|
||||
if (draft.limitations() == null || isBlank(draft.limitations().scope())) {
|
||||
violations.add(violation(EvidenceViolationCode.LIMITATIONS_MISSING, "limitations"));
|
||||
}
|
||||
}
|
||||
|
||||
private void validateTextAndReferences(String target, String text, List<String> references,
|
||||
Set<String> ids, List<EvidenceViolation> violations) {
|
||||
if (isBlank(text)) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
}
|
||||
if (references == null || references.isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING,
|
||||
target + ".based_on_analysis_ids"));
|
||||
return;
|
||||
}
|
||||
for (int index = 0; index < references.size(); index++) {
|
||||
String reference = references.get(index);
|
||||
if (isBlank(reference)) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING,
|
||||
target + ".based_on_analysis_ids[" + index + "]"));
|
||||
} else if (!ids.contains(reference)) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_UNKNOWN,
|
||||
target + ".based_on_analysis_ids[" + index + "]"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyInvocation(RunContext context, DiagnosisDraft.AnalysisItem analysis,
|
||||
int analysisIndex, String toolCallId,
|
||||
List<VerifiedEvidence> evidence,
|
||||
List<EvidenceViolation> violations) {
|
||||
String target = "analysis[" + analysisIndex + "].tool_call_ids";
|
||||
if (isBlank(toolCallId)) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String key;
|
||||
try {
|
||||
key = keyFactory.create(context.runId(), toolCallId);
|
||||
} catch (IllegalArgumentException exception) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_INVALID, target));
|
||||
return;
|
||||
}
|
||||
CanonicalToolInvocation invocation;
|
||||
try {
|
||||
invocation = store.find(key).orElse(null);
|
||||
} catch (RuntimeException exception) {
|
||||
violations.add(violation(EvidenceViolationCode.CANONICAL_LOOKUP_FAILED, target));
|
||||
return;
|
||||
}
|
||||
if (invocation == null) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_MISSING, target));
|
||||
return;
|
||||
}
|
||||
if (!Objects.equals(toolCallId, invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_ID_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (!invocation.isReferencableBy(context.runId()) || invocation.agentResult().isBlank()) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE, target));
|
||||
return;
|
||||
}
|
||||
if (!analysis.kind().accepts(invocation.evidenceStatus())) {
|
||||
violations.add(violation(EvidenceViolationCode.EVIDENCE_KIND_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
switch (invocation.toolName()) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE ->
|
||||
readRag(invocation, evidence, target, violations);
|
||||
case AgentToolContracts.QUERY_LOGS ->
|
||||
readLogs(invocation, evidence, target, violations);
|
||||
case AgentToolContracts.QUERY_MYSQL ->
|
||||
readMysql(invocation, evidence, target, violations);
|
||||
default -> violations.add(violation(EvidenceViolationCode.TOOL_UNSUPPORTED, target));
|
||||
}
|
||||
}
|
||||
|
||||
private void readRag(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
RagToolResult result;
|
||||
try {
|
||||
result = ragReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!Objects.equals(result.toolCallId(), invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_ID_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (result.evidenceStatus() != invocation.evidenceStatus()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_STATUS_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (result.returnedCount() != result.evidence().size()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (!result.evidence().isEmpty() || result.returnedCount() != 0) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"RAG", "KNOWLEDGE_BASE", firstText(result.query(), "knowledge query"),
|
||||
null, "No evidence matched the query scope",
|
||||
Map.of("match_count", 0, "truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.evidence().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (RagEvidence item : result.evidence()) {
|
||||
if (item == null || isBlank(item.documentId()) || isBlank(item.excerpt())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
values.put("document_id", item.documentId());
|
||||
putIfText(values, "title", item.title());
|
||||
putIfText(values, "breadcrumb", item.breadcrumb());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"RAG", firstText(item.source(), item.title(), item.documentId()),
|
||||
firstText(result.query(), "knowledge query"), null, item.excerpt(), values));
|
||||
}
|
||||
}
|
||||
|
||||
private void readLogs(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
QueryLogsToolResult result;
|
||||
try {
|
||||
result = logsReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!projectionMatches(invocation, result.toolCallId(), result.evidenceStatus(),
|
||||
target, violations)) {
|
||||
return;
|
||||
}
|
||||
if (result.sourceKind() == null || result.scope() == null
|
||||
|| result.scope().topic() == null || isBlank(result.scope().query())
|
||||
|| isBlank(result.scope().startTime()) || isBlank(result.scope().endTime())
|
||||
|| result.matchCount() < 0 || result.returnedCount() != result.events().size()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String source = result.scope().topic().name() + " (" + result.sourceKind().name() + ")";
|
||||
String scope = logScope(result.scope());
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (result.matchCount() != 0 || !result.patterns().isEmpty() || !result.events().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, null, "No log events matched the query scope",
|
||||
Map.of("match_count", 0, "truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.patterns().isEmpty() && result.events().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (LogPattern pattern : result.patterns()) {
|
||||
if (pattern == null || pattern.count() <= 0 || isBlank(pattern.example())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
values.put("count", pattern.count());
|
||||
putIfText(values, "first_seen", pattern.firstSeen());
|
||||
putIfText(values, "last_seen", pattern.lastSeen());
|
||||
putIfText(values, "level", pattern.level());
|
||||
putIfText(values, "service", pattern.service());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, pattern.lastSeen(), pattern.example(), values));
|
||||
}
|
||||
for (LogEvent event : result.events()) {
|
||||
if (event == null || isBlank(event.message())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
putIfText(values, "level", event.level());
|
||||
putIfText(values, "service", event.service());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, event.timestamp(), event.message(), values));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean projectionMatches(CanonicalToolInvocation invocation, String toolCallId,
|
||||
com.superbiz.agent.harness.contract.EvidenceStatus evidenceStatus,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
if (!Objects.equals(toolCallId, invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_ID_MISMATCH, target));
|
||||
return false;
|
||||
}
|
||||
if (evidenceStatus != invocation.evidenceStatus()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_STATUS_MISMATCH, target));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void readMysql(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
MysqlToolRequest request;
|
||||
MysqlToolResult result;
|
||||
try {
|
||||
request = mysqlRequestReader.readValue(invocation.request());
|
||||
result = mysqlResultReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!projectionMatches(invocation, result.toolCallId(), result.evidenceStatus(),
|
||||
target, violations)) {
|
||||
return;
|
||||
}
|
||||
if (isBlank(request.dataSource()) || isBlank(request.sql())
|
||||
|| result.returnedCount() != result.rows().size()
|
||||
|| result.columns().isEmpty() || hasInvalidColumns(result.columns())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String scope = "sql=" + request.sql() + " params=" + request.params();
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (!result.rows().isEmpty() || result.returnedCount() != 0) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"MYSQL", request.dataSource(), scope, null,
|
||||
"No rows matched the query scope",
|
||||
Map.of("match_count", 0, "columns", result.columns(),
|
||||
"truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.rows().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (int index = 0; index < result.rows().size(); index++) {
|
||||
Map<String, Object> row = result.rows().get(index);
|
||||
if (row == null || !row.keySet().equals(new java.util.LinkedHashSet<>(result.columns()))) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>(row);
|
||||
values.put("_row_number", index + 1);
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"MYSQL", request.dataSource(), scope, null, null, values));
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean hasInvalidColumns(List<String> columns) {
|
||||
Set<String> unique = new HashSet<>();
|
||||
for (String column : columns) {
|
||||
if (isBlank(column) || !unique.add(column)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String logScope(LogQueryScope scope) {
|
||||
return scope.topic().name() + " query=" + scope.query()
|
||||
+ " from=" + scope.startTime() + " to=" + scope.endTime();
|
||||
}
|
||||
|
||||
private static EvidenceViolation violation(EvidenceViolationCode code, String target) {
|
||||
return new EvidenceViolation(code, target);
|
||||
}
|
||||
|
||||
private static void putIfText(Map<String, Object> values, String name, String value) {
|
||||
if (!isBlank(value)) {
|
||||
values.put(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
private static String firstText(String... values) {
|
||||
for (String value : values) {
|
||||
if (!isBlank(value)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public record EvidenceGuardResult(
|
||||
List<EvidenceViolation> violations,
|
||||
VerifiedEvidenceSnapshot snapshot) {
|
||||
|
||||
public EvidenceGuardResult {
|
||||
violations = violations == null ? List.of() : List.copyOf(violations);
|
||||
if (violations.isEmpty() == (snapshot == null)) {
|
||||
throw new IllegalArgumentException(
|
||||
"valid result requires snapshot and invalid result requires violations");
|
||||
}
|
||||
}
|
||||
|
||||
public static EvidenceGuardResult valid(VerifiedEvidenceSnapshot snapshot) {
|
||||
return new EvidenceGuardResult(List.of(), snapshot);
|
||||
}
|
||||
|
||||
public static EvidenceGuardResult invalid(List<EvidenceViolation> violations) {
|
||||
return new EvidenceGuardResult(violations, null);
|
||||
}
|
||||
|
||||
public boolean valid() {
|
||||
return violations.isEmpty();
|
||||
}
|
||||
|
||||
public Optional<VerifiedEvidenceSnapshot> verifiedSnapshot() {
|
||||
return Optional.ofNullable(snapshot);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record EvidenceViolation(
|
||||
@JsonProperty("code") EvidenceViolationCode code,
|
||||
@JsonProperty("target") String target) {
|
||||
|
||||
public EvidenceViolation {
|
||||
Objects.requireNonNull(code, "code must not be null");
|
||||
if (target == null || target.isBlank()) {
|
||||
throw new IllegalArgumentException("target must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
public enum EvidenceViolationCode {
|
||||
DRAFT_MISSING,
|
||||
ANALYSIS_MISSING,
|
||||
ANALYSIS_ID_MISSING,
|
||||
ANALYSIS_ID_DUPLICATE,
|
||||
ANALYSIS_KIND_MISSING,
|
||||
ANALYSIS_TEXT_MISSING,
|
||||
TOOL_REFERENCE_MISSING,
|
||||
REPORT_TEXT_MISSING,
|
||||
ANALYSIS_REFERENCE_MISSING,
|
||||
ANALYSIS_REFERENCE_UNKNOWN,
|
||||
LIMITATIONS_MISSING,
|
||||
TOOL_REFERENCE_INVALID,
|
||||
INVOCATION_MISSING,
|
||||
INVOCATION_ID_MISMATCH,
|
||||
INVOCATION_NOT_REFERENCABLE,
|
||||
EVIDENCE_KIND_MISMATCH,
|
||||
TOOL_UNSUPPORTED,
|
||||
PROJECTION_INVALID,
|
||||
PROJECTION_ID_MISMATCH,
|
||||
PROJECTION_STATUS_MISMATCH,
|
||||
CANONICAL_LOOKUP_FAILED
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public record VerifiedAnalysisEvidence(
|
||||
@JsonProperty("analysis_id") String analysisId,
|
||||
@JsonProperty("analysis_text") String analysisText,
|
||||
@JsonProperty("analysis_kind") AnalysisKind analysisKind,
|
||||
@JsonProperty("verified_evidence") List<VerifiedEvidence> evidence) {
|
||||
|
||||
public VerifiedAnalysisEvidence {
|
||||
requireText(analysisId, "analysisId");
|
||||
requireText(analysisText, "analysisText");
|
||||
Objects.requireNonNull(analysisKind, "analysisKind must not be null");
|
||||
evidence = evidence == null ? List.of() : List.copyOf(evidence);
|
||||
if (evidence.isEmpty()) {
|
||||
throw new IllegalArgumentException("evidence must not be empty");
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public record VerifiedEvidence(
|
||||
@JsonProperty("source_type") String sourceType,
|
||||
@JsonProperty("source") String source,
|
||||
@JsonProperty("scope") String scope,
|
||||
@JsonProperty("timestamp") String timestamp,
|
||||
@JsonProperty("excerpt") String excerpt,
|
||||
@JsonProperty("values") Map<String, Object> values) {
|
||||
|
||||
public VerifiedEvidence {
|
||||
requireText(sourceType, "sourceType");
|
||||
requireText(source, "source");
|
||||
requireText(scope, "scope");
|
||||
values = immutableValues(values);
|
||||
}
|
||||
|
||||
private static Map<String, Object> immutableValues(Map<String, Object> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
return java.util.Collections.unmodifiableMap(new LinkedHashMap<>(values));
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public record VerifiedEvidenceSnapshot(
|
||||
@JsonProperty("analyses") List<VerifiedAnalysisEvidence> analyses) {
|
||||
|
||||
public VerifiedEvidenceSnapshot {
|
||||
analyses = analyses == null ? List.of() : List.copyOf(analyses);
|
||||
}
|
||||
|
||||
public static VerifiedEvidenceSnapshot empty() {
|
||||
return new VerifiedEvidenceSnapshot(List.of());
|
||||
}
|
||||
|
||||
public List<SafeFallback.VerifiedSource> verifiedSources() {
|
||||
Map<String, SafeFallback.VerifiedSource> unique = new LinkedHashMap<>();
|
||||
for (VerifiedAnalysisEvidence analysis : analyses) {
|
||||
for (VerifiedEvidence evidence : analysis.evidence()) {
|
||||
String key = evidence.sourceType() + "\u0000" + evidence.source()
|
||||
+ "\u0000" + evidence.scope();
|
||||
unique.putIfAbsent(key, new SafeFallback.VerifiedSource(
|
||||
evidence.sourceType(), evidence.source(), evidence.scope()));
|
||||
}
|
||||
}
|
||||
return List.copyOf(unique.values());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
public final class GuardModelCall {
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final ChatModel chatModel;
|
||||
private final ExecutorService executor;
|
||||
|
||||
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
|
||||
ExecutorService executor) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
|
||||
this.executor = Objects.requireNonNull(executor, "executor must not be null");
|
||||
}
|
||||
|
||||
public String call(RunContext context, Prompt prompt, Duration timeout, long maxOutputBytes) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(prompt, "prompt must not be null");
|
||||
Objects.requireNonNull(timeout, "timeout must not be null");
|
||||
if (timeout.isZero() || timeout.isNegative() || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("timeout and output limit must be positive");
|
||||
}
|
||||
core.beforeModelCall(context);
|
||||
Future<String> future = executor.submit(() -> invoke(context, prompt, maxOutputBytes));
|
||||
context.cancellation().onCancel(ignored -> future.cancel(true));
|
||||
try {
|
||||
return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
|
||||
} catch (TimeoutException exception) {
|
||||
future.cancel(true);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TIMEOUT, "Guard model attempt timed out", exception);
|
||||
} catch (CancellationException exception) {
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model attempt was cancelled", exception);
|
||||
} catch (InterruptedException exception) {
|
||||
future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Interrupted while waiting for guard model", exception);
|
||||
} catch (ExecutionException exception) {
|
||||
Throwable cause = exception.getCause();
|
||||
if (cause instanceof RunAbortedException aborted) {
|
||||
throw aborted;
|
||||
}
|
||||
if (cause instanceof BudgetExceededException exceeded) {
|
||||
throw exceeded;
|
||||
}
|
||||
if (cause instanceof GuardModelCallException guardFailure) {
|
||||
throw guardFailure;
|
||||
}
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model call failed", cause);
|
||||
}
|
||||
}
|
||||
|
||||
private String invoke(RunContext context, Prompt prompt, long maxOutputBytes) {
|
||||
ChatResponse response;
|
||||
try {
|
||||
response = chatModel.call(prompt);
|
||||
} catch (RuntimeException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model transport failed", exception);
|
||||
}
|
||||
recordUsage(context, response);
|
||||
core.checkActive(context);
|
||||
AssistantMessage output = response == null || response.getResult() == null
|
||||
? null : response.getResult().getOutput();
|
||||
if (output == null || output.getText() == null || output.getText().isBlank()
|
||||
|| (output.getToolCalls() != null && !output.getToolCalls().isEmpty())) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Guard model returned invalid message shape");
|
||||
}
|
||||
long bytes = output.getText().getBytes(StandardCharsets.UTF_8).length;
|
||||
if (bytes > maxOutputBytes) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Guard model output exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, bytes);
|
||||
return output.getText();
|
||||
}
|
||||
|
||||
private void recordUsage(RunContext context, ChatResponse response) {
|
||||
if (response == null || response.getMetadata() == null) {
|
||||
return;
|
||||
}
|
||||
Usage usage = response.getMetadata().getUsage();
|
||||
if (usage == null) {
|
||||
return;
|
||||
}
|
||||
core.recordTokens(context, nonNegative(usage.getPromptTokens()),
|
||||
nonNegative(usage.getCompletionTokens()));
|
||||
}
|
||||
|
||||
private static long nonNegative(Integer value) {
|
||||
return value == null || value < 0 ? 0L : value.longValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class GuardModelCallException extends RuntimeException {
|
||||
|
||||
private final RetryFailure failure;
|
||||
|
||||
public GuardModelCallException(RetryFailure failure, String message) {
|
||||
super(message);
|
||||
this.failure = Objects.requireNonNull(failure, "failure must not be null");
|
||||
}
|
||||
|
||||
public GuardModelCallException(RetryFailure failure, String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.failure = Objects.requireNonNull(failure, "failure must not be null");
|
||||
}
|
||||
|
||||
public RetryFailure failure() {
|
||||
return failure;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticDraftView(
|
||||
@JsonProperty("conclusion") Conclusion conclusion,
|
||||
@JsonProperty("analysis") List<Analysis> analysis,
|
||||
@JsonProperty("action_plan") List<Action> actionPlan,
|
||||
@JsonProperty("recommendations") List<Recommendation> recommendations,
|
||||
@JsonProperty("limitations") Limitations limitations) {
|
||||
|
||||
public SemanticDraftView {
|
||||
analysis = immutable(analysis);
|
||||
actionPlan = immutable(actionPlan);
|
||||
recommendations = immutable(recommendations);
|
||||
}
|
||||
|
||||
public static SemanticDraftView from(DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
Conclusion conclusion = draft.conclusion() == null ? null : new Conclusion(
|
||||
draft.conclusion().text(), draft.conclusion().basedOnAnalysisIds());
|
||||
List<Analysis> analysis = draft.analysis().stream()
|
||||
.map(item -> new Analysis(item.analysisId(), item.kind(), item.text()))
|
||||
.toList();
|
||||
List<Action> actions = draft.actionPlan().stream()
|
||||
.map(item -> new Action(item.action(), item.basedOnAnalysisIds(),
|
||||
item.requiresHumanConfirmation()))
|
||||
.toList();
|
||||
List<Recommendation> recommendations = draft.recommendations().stream()
|
||||
.map(item -> new Recommendation(item.text(), item.basedOnAnalysisIds()))
|
||||
.toList();
|
||||
Limitations limitations = draft.limitations() == null ? null : new Limitations(
|
||||
draft.limitations().scope(), draft.limitations().missingInfo());
|
||||
return new SemanticDraftView(conclusion, analysis, actions, recommendations, limitations);
|
||||
}
|
||||
|
||||
public boolean hasSameUserVisibleSemantics(SemanticDraftView other) {
|
||||
if (other == null || !Objects.equals(text(conclusion), text(other.conclusion))
|
||||
|| analysis.size() != other.analysis.size()
|
||||
|| actionPlan.size() != other.actionPlan.size()
|
||||
|| recommendations.size() != other.recommendations.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < analysis.size(); i++) {
|
||||
Analysis left = analysis.get(i);
|
||||
Analysis right = other.analysis.get(i);
|
||||
if (left.kind() != right.kind() || !Objects.equals(left.text(), right.text())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < actionPlan.size(); i++) {
|
||||
Action left = actionPlan.get(i);
|
||||
Action right = other.actionPlan.get(i);
|
||||
if (!Objects.equals(left.action(), right.action())
|
||||
|| left.requiresHumanConfirmation() != right.requiresHumanConfirmation()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < recommendations.size(); i++) {
|
||||
if (!Objects.equals(recommendations.get(i).text(), other.recommendations.get(i).text())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return limitationsEqual(limitations, other.limitations);
|
||||
}
|
||||
|
||||
private static String text(Conclusion value) {
|
||||
return value == null ? null : value.text();
|
||||
}
|
||||
|
||||
private static boolean limitationsEqual(Limitations left, Limitations right) {
|
||||
if (left == null || right == null) {
|
||||
return left == right;
|
||||
}
|
||||
return Objects.equals(left.scope(), right.scope())
|
||||
&& Objects.equals(left.missingInfo(), right.missingInfo());
|
||||
}
|
||||
|
||||
private static <T> List<T> immutable(List<T> values) {
|
||||
return values == null ? List.of() : List.copyOf(values);
|
||||
}
|
||||
|
||||
public record Conclusion(
|
||||
@JsonProperty("text") String text,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds) {
|
||||
public Conclusion {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Analysis(
|
||||
@JsonProperty("analysis_id") String analysisId,
|
||||
@JsonProperty("kind") AnalysisKind kind,
|
||||
@JsonProperty("text") String text) {
|
||||
}
|
||||
|
||||
public record Action(
|
||||
@JsonProperty("action") String action,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds,
|
||||
@JsonProperty("requires_human_confirmation") boolean requiresHumanConfirmation) {
|
||||
public Action {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Recommendation(
|
||||
@JsonProperty("text") String text,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds) {
|
||||
public Recommendation {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Limitations(
|
||||
@JsonProperty("scope") String scope,
|
||||
@JsonProperty("missing_info") List<String> missingInfo) {
|
||||
public Limitations {
|
||||
missingInfo = immutable(missingInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public final class SemanticGuard {
|
||||
|
||||
private static final Set<String> OUTPUT_FIELDS = Set.of("verdict", "reason");
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final HarnessRetryExecutor retryExecutor;
|
||||
private final GuardModelCall modelCall;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final SemanticGuardLimits limits;
|
||||
private final Consumer<RetryAttempt> attemptRecorder;
|
||||
private final String prompt;
|
||||
|
||||
public SemanticGuard(DiagnosisHarnessCore core,
|
||||
HarnessRetryExecutor retryExecutor,
|
||||
GuardModelCall modelCall,
|
||||
ObjectMapper objectMapper,
|
||||
SemanticGuardLimits limits,
|
||||
Consumer<RetryAttempt> attemptRecorder) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null");
|
||||
this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.limits = Objects.requireNonNull(limits, "limits must not be null");
|
||||
this.attemptRecorder = Objects.requireNonNull(
|
||||
attemptRecorder, "attemptRecorder must not be null");
|
||||
this.prompt = SemanticGuardPrompt.load();
|
||||
}
|
||||
|
||||
public SemanticGuardDecision review(RunContext context, SemanticGuardInput input) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(input, "input must not be null");
|
||||
String inputJson = serialize(input);
|
||||
long inputBytes = utf8Bytes(inputJson);
|
||||
if (inputBytes > limits.maxInputBytes()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard input exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, inputBytes);
|
||||
Prompt modelPrompt = new Prompt(java.util.List.of(
|
||||
new SystemMessage(prompt), new UserMessage(inputJson)));
|
||||
long startedNanos = System.nanoTime();
|
||||
return retryExecutor.execute(
|
||||
context,
|
||||
context.retryPolicies().semanticGuard(),
|
||||
() -> parse(modelCall.call(
|
||||
context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
|
||||
this::classify,
|
||||
attemptRecorder);
|
||||
}
|
||||
|
||||
private SemanticGuardDecision parse(String output) {
|
||||
JsonNode root;
|
||||
try {
|
||||
root = objectMapper.readTree(output);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.PARSE_ERROR, "SemanticGuard output is not JSON", exception);
|
||||
}
|
||||
if (root == null || !root.isObject() || !fieldNames(root).equals(OUTPUT_FIELDS)
|
||||
|| !root.path("verdict").isTextual() || !root.path("reason").isTextual()
|
||||
|| root.path("reason").asText().isBlank()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard output schema is invalid");
|
||||
}
|
||||
SemanticVerdict verdict;
|
||||
try {
|
||||
verdict = SemanticVerdict.valueOf(root.path("verdict").asText());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard verdict is invalid", exception);
|
||||
}
|
||||
return new SemanticGuardDecision(verdict, root.path("reason").asText());
|
||||
}
|
||||
|
||||
private Duration remainingTimeout(long startedNanos) {
|
||||
long elapsed = Math.max(0L, System.nanoTime() - startedNanos);
|
||||
long remaining = limits.totalTimeout().toNanos() - elapsed;
|
||||
if (remaining <= 0) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TIMEOUT, "SemanticGuard total timeout exhausted");
|
||||
}
|
||||
return Duration.ofNanos(Math.min(remaining, limits.perAttemptTimeout().toNanos()));
|
||||
}
|
||||
|
||||
private RetryFailure classify(Exception exception) {
|
||||
if (exception instanceof GuardModelCallException guardFailure) {
|
||||
return guardFailure.failure();
|
||||
}
|
||||
return RetryFailure.UNKNOWN;
|
||||
}
|
||||
|
||||
private String serialize(SemanticGuardInput input) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(input);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard input is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> fieldNames(JsonNode node) {
|
||||
Set<String> names = new HashSet<>();
|
||||
Iterator<String> fields = node.fieldNames();
|
||||
fields.forEachRemaining(names::add);
|
||||
return names;
|
||||
}
|
||||
|
||||
private static long utf8Bytes(String value) {
|
||||
return value.getBytes(StandardCharsets.UTF_8).length;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardDecision(
|
||||
@JsonProperty("verdict") SemanticVerdict verdict,
|
||||
@JsonProperty("reason") String reason) {
|
||||
|
||||
public SemanticGuardDecision {
|
||||
Objects.requireNonNull(verdict, "verdict must not be null");
|
||||
if (reason == null || reason.isBlank()) {
|
||||
throw new IllegalArgumentException("reason must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardInput(
|
||||
@JsonProperty("query") String query,
|
||||
@JsonProperty("draft") SemanticDraftView draft,
|
||||
@JsonProperty("verified_evidence") VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
|
||||
public SemanticGuardInput {
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
Objects.requireNonNull(verifiedEvidence, "verifiedEvidence must not be null");
|
||||
}
|
||||
|
||||
public static SemanticGuardInput from(String query, DiagnosisDraft draft,
|
||||
VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
return new SemanticGuardInput(query, SemanticDraftView.from(draft), verifiedEvidence);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardLimits(
|
||||
long maxInputBytes,
|
||||
long maxOutputBytes,
|
||||
Duration perAttemptTimeout,
|
||||
Duration totalTimeout) {
|
||||
|
||||
public SemanticGuardLimits {
|
||||
if (maxInputBytes <= 0 || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("byte limits must be positive");
|
||||
}
|
||||
requirePositive(perAttemptTimeout, "perAttemptTimeout");
|
||||
requirePositive(totalTimeout, "totalTimeout");
|
||||
if (totalTimeout.compareTo(perAttemptTimeout) < 0) {
|
||||
throw new IllegalArgumentException("totalTimeout must not be shorter than perAttemptTimeout");
|
||||
}
|
||||
}
|
||||
|
||||
private static void requirePositive(Duration value, String name) {
|
||||
Objects.requireNonNull(value, name + " must not be null");
|
||||
if (value.isZero() || value.isNegative()) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
value.toNanos();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
final class SemanticGuardPrompt {
|
||||
|
||||
private static final String RESOURCE_PATH = "prompts/semantic-guard-prompt.md";
|
||||
|
||||
private SemanticGuardPrompt() {
|
||||
}
|
||||
|
||||
static String load() {
|
||||
ClassPathResource resource = new ClassPathResource(RESOURCE_PATH);
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
return new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("Failed to load SemanticGuard prompt", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record DiagnosisReleaseResult(
|
||||
ReleaseOutcome outcome,
|
||||
DiagnosisDraft draft,
|
||||
SafeFallback fallback,
|
||||
VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
|
||||
public DiagnosisReleaseResult {
|
||||
Objects.requireNonNull(outcome, "outcome must not be null");
|
||||
Objects.requireNonNull(verifiedEvidence, "verifiedEvidence must not be null");
|
||||
if (outcome == ReleaseOutcome.SUCCESS) {
|
||||
Objects.requireNonNull(draft, "successful release requires draft");
|
||||
if (fallback != null) {
|
||||
throw new IllegalArgumentException("successful release must not contain fallback");
|
||||
}
|
||||
} else if (outcome == ReleaseOutcome.FALLBACK) {
|
||||
Objects.requireNonNull(fallback, "fallback release requires fallback");
|
||||
if (draft != null) {
|
||||
throw new IllegalArgumentException("fallback release must not contain draft");
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("release use case supports SUCCESS or FALLBACK only");
|
||||
}
|
||||
}
|
||||
|
||||
public static DiagnosisReleaseResult success(
|
||||
DiagnosisDraft draft, VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
return new DiagnosisReleaseResult(
|
||||
ReleaseOutcome.SUCCESS, draft, null, verifiedEvidence);
|
||||
}
|
||||
|
||||
public static DiagnosisReleaseResult fallback(SafeFallback fallback) {
|
||||
return new DiagnosisReleaseResult(
|
||||
ReleaseOutcome.FALLBACK, null, fallback, VerifiedEvidenceSnapshot.empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardDecision;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardInput;
|
||||
import com.superbiz.agent.harness.retry.RetryExecutionException;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class DiagnosisReleaseUseCase {
|
||||
|
||||
private final EvidenceGuard evidenceGuard;
|
||||
private final EvidenceRepair evidenceRepair;
|
||||
private final SemanticGuard semanticGuard;
|
||||
private final SafeFallbackFactory fallbackFactory;
|
||||
|
||||
public DiagnosisReleaseUseCase(EvidenceGuard evidenceGuard,
|
||||
EvidenceRepair evidenceRepair,
|
||||
SemanticGuard semanticGuard,
|
||||
SafeFallbackFactory fallbackFactory) {
|
||||
this.evidenceGuard = Objects.requireNonNull(evidenceGuard, "evidenceGuard must not be null");
|
||||
this.evidenceRepair = Objects.requireNonNull(evidenceRepair, "evidenceRepair must not be null");
|
||||
this.semanticGuard = Objects.requireNonNull(semanticGuard, "semanticGuard must not be null");
|
||||
this.fallbackFactory = Objects.requireNonNull(
|
||||
fallbackFactory, "fallbackFactory must not be null");
|
||||
}
|
||||
|
||||
public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
|
||||
DiagnosisDraft candidate = draft;
|
||||
EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate);
|
||||
if (!evidence.valid()) {
|
||||
try {
|
||||
candidate = evidenceRepair.repair(context, query, draft, evidence.violations());
|
||||
evidence = evidenceGuard.validate(context, candidate);
|
||||
} catch (RuntimeException exception) {
|
||||
propagateTerminal(exception);
|
||||
return evidenceFailure();
|
||||
}
|
||||
if (!evidence.valid()) {
|
||||
return evidenceFailure();
|
||||
}
|
||||
}
|
||||
|
||||
VerifiedEvidenceSnapshot snapshot = evidence.verifiedSnapshot().orElseThrow();
|
||||
SemanticGuardDecision decision;
|
||||
try {
|
||||
decision = semanticGuard.review(
|
||||
context, SemanticGuardInput.from(query, candidate, snapshot));
|
||||
} catch (RuntimeException exception) {
|
||||
propagateTerminal(exception);
|
||||
return DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.semanticUnavailable(snapshot));
|
||||
}
|
||||
return decision.verdict() == SemanticVerdict.SUPPORTED
|
||||
? DiagnosisReleaseResult.success(candidate, snapshot)
|
||||
: DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.semanticUnsupported(snapshot));
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult evidenceFailure() {
|
||||
return DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.evidenceValidationFailed());
|
||||
}
|
||||
|
||||
private void propagateTerminal(RuntimeException exception) {
|
||||
if (exception instanceof RunAbortedException
|
||||
|| exception instanceof BudgetExceededException) {
|
||||
throw exception;
|
||||
}
|
||||
if (exception instanceof RetryExecutionException retry
|
||||
&& (retry.failure() == RetryFailure.CANCELLED
|
||||
|| retry.failure() == RetryFailure.BUDGET_EXHAUSTED)) {
|
||||
throw retry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCallException;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public final class EvidenceRepair {
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final HarnessRetryExecutor retryExecutor;
|
||||
private final GuardModelCall modelCall;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ObjectReader draftReader;
|
||||
private final EvidenceRepairLimits limits;
|
||||
private final Consumer<RetryAttempt> attemptRecorder;
|
||||
private final String prompt;
|
||||
|
||||
public EvidenceRepair(DiagnosisHarnessCore core,
|
||||
HarnessRetryExecutor retryExecutor,
|
||||
GuardModelCall modelCall,
|
||||
ObjectMapper objectMapper,
|
||||
EvidenceRepairLimits limits,
|
||||
Consumer<RetryAttempt> attemptRecorder) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null");
|
||||
this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.draftReader = objectMapper.readerFor(DiagnosisDraft.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.limits = Objects.requireNonNull(limits, "limits must not be null");
|
||||
this.attemptRecorder = Objects.requireNonNull(
|
||||
attemptRecorder, "attemptRecorder must not be null");
|
||||
this.prompt = EvidenceRepairPrompt.load();
|
||||
}
|
||||
|
||||
public DiagnosisDraft repair(RunContext context, String query, DiagnosisDraft original,
|
||||
List<EvidenceViolation> violations) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(original, "original must not be null");
|
||||
List<EvidenceViolation> safeViolations = violations == null
|
||||
? List.of() : List.copyOf(violations);
|
||||
String inputJson = serialize(new RepairInput(query, original, safeViolations));
|
||||
long inputBytes = utf8Bytes(inputJson);
|
||||
if (inputBytes > limits.maxInputBytes()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Evidence repair input exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, inputBytes);
|
||||
Prompt modelPrompt = new Prompt(List.of(
|
||||
new SystemMessage(prompt), new UserMessage(inputJson)));
|
||||
SemanticDraftView originalSemantics = SemanticDraftView.from(original);
|
||||
return retryExecutor.execute(
|
||||
context,
|
||||
context.retryPolicies().evidenceRepair(),
|
||||
() -> {
|
||||
DiagnosisDraft repaired = parse(modelCall.call(
|
||||
context, modelPrompt, limits.timeout(), limits.maxOutputBytes()));
|
||||
if (!originalSemantics.hasSameUserVisibleSemantics(
|
||||
SemanticDraftView.from(repaired))) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID,
|
||||
"Evidence repair changed user-visible semantics");
|
||||
}
|
||||
return repaired;
|
||||
},
|
||||
this::classify,
|
||||
attemptRecorder);
|
||||
}
|
||||
|
||||
private DiagnosisDraft parse(String output) {
|
||||
try {
|
||||
return draftReader.readValue(output);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.PARSE_ERROR, "Evidence repair output is invalid", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private RetryFailure classify(Exception exception) {
|
||||
return exception instanceof GuardModelCallException failure
|
||||
? failure.failure() : RetryFailure.UNKNOWN;
|
||||
}
|
||||
|
||||
private String serialize(RepairInput input) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(input);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Evidence repair input is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static long utf8Bytes(String value) {
|
||||
return value.getBytes(StandardCharsets.UTF_8).length;
|
||||
}
|
||||
|
||||
private record RepairInput(
|
||||
String query,
|
||||
DiagnosisDraft draft,
|
||||
List<EvidenceViolation> violations) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
public record EvidenceRepairLimits(
|
||||
long maxInputBytes,
|
||||
long maxOutputBytes,
|
||||
Duration timeout) {
|
||||
|
||||
public EvidenceRepairLimits {
|
||||
if (maxInputBytes <= 0 || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("byte limits must be positive");
|
||||
}
|
||||
Objects.requireNonNull(timeout, "timeout must not be null");
|
||||
if (timeout.isZero() || timeout.isNegative()) {
|
||||
throw new IllegalArgumentException("timeout must be positive");
|
||||
}
|
||||
timeout.toNanos();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
final class EvidenceRepairPrompt {
|
||||
|
||||
private static final String RESOURCE_PATH = "prompts/evidence-repair-prompt.md";
|
||||
|
||||
private EvidenceRepairPrompt() {
|
||||
}
|
||||
|
||||
static String load() {
|
||||
ClassPathResource resource = new ClassPathResource(RESOURCE_PATH);
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
return new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("Failed to load evidence repair prompt", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class SafeFallbackFactory {
|
||||
|
||||
public SafeFallback evidenceValidationFailed() {
|
||||
return fallback(
|
||||
FallbackType.EVIDENCE_VALIDATION_FAILED,
|
||||
List.of(),
|
||||
"当前证据无法完成真实性校验，无法确认根因",
|
||||
"证据引用校验未通过",
|
||||
"重新收集当前诊断范围内的证据后再发起诊断");
|
||||
}
|
||||
|
||||
public SafeFallback semanticUnsupported(VerifiedEvidenceSnapshot snapshot) {
|
||||
return fallback(
|
||||
FallbackType.SEMANTIC_UNSUPPORTED,
|
||||
sources(snapshot),
|
||||
"当前证据不足，无法确认根因",
|
||||
"语义校验未通过",
|
||||
"补充当前缺失的数据后重新发起诊断");
|
||||
}
|
||||
|
||||
public SafeFallback semanticUnavailable(VerifiedEvidenceSnapshot snapshot) {
|
||||
return fallback(
|
||||
FallbackType.SEMANTIC_UNAVAILABLE,
|
||||
sources(snapshot),
|
||||
"当前证据暂时无法完成语义校验，无法确认根因",
|
||||
"语义校验暂不可用",
|
||||
"稍后重试或补充当前缺失的数据");
|
||||
}
|
||||
|
||||
private SafeFallback fallback(FallbackType type,
|
||||
List<SafeFallback.VerifiedSource> sources,
|
||||
String message,
|
||||
String limitation,
|
||||
String nextStep) {
|
||||
return new SafeFallback(
|
||||
type, null, message, sources, List.of(limitation), List.of(nextStep));
|
||||
}
|
||||
|
||||
private List<SafeFallback.VerifiedSource> sources(VerifiedEvidenceSnapshot snapshot) {
|
||||
return Objects.requireNonNull(snapshot, "snapshot must not be null").verifiedSources();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user