feat(harness): add evidence and semantic guards
This commit is contained in:
zhuyongxin
@@ -0,0 +1,435 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.LogEvent;
|
||||
import com.superbiz.agent.harness.tool.contract.LogPattern;
|
||||
import com.superbiz.agent.harness.tool.contract.LogQueryScope;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolResult;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsToolResult;
|
||||
import com.superbiz.agent.harness.tool.contract.RagEvidence;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolResult;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public final class EvidenceGuard {
|
||||
|
||||
private final CanonicalInvocationStore store;
|
||||
private final ToolCallKeyFactory keyFactory;
|
||||
private final ObjectReader ragReader;
|
||||
private final ObjectReader logsReader;
|
||||
private final ObjectReader mysqlRequestReader;
|
||||
private final ObjectReader mysqlResultReader;
|
||||
|
||||
public EvidenceGuard(CanonicalInvocationStore store,
|
||||
ToolCallKeyFactory keyFactory,
|
||||
ObjectMapper objectMapper) {
|
||||
this.store = Objects.requireNonNull(store, "store must not be null");
|
||||
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
|
||||
ObjectMapper mapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.ragReader = mapper.readerFor(RagToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.logsReader = mapper.readerFor(QueryLogsToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.mysqlRequestReader = mapper.readerFor(MysqlToolRequest.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.mysqlResultReader = mapper.readerFor(MysqlToolResult.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
}
|
||||
|
||||
public EvidenceGuardResult validate(RunContext context, DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
List<EvidenceViolation> violations = validateDraft(draft);
|
||||
if (!violations.isEmpty()) {
|
||||
return EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
List<VerifiedAnalysisEvidence> verifiedAnalyses = new ArrayList<>();
|
||||
for (int index = 0; index < draft.analysis().size(); index++) {
|
||||
DiagnosisDraft.AnalysisItem analysis = draft.analysis().get(index);
|
||||
List<VerifiedEvidence> evidence = new ArrayList<>();
|
||||
for (String toolCallId : analysis.toolCallIds()) {
|
||||
verifyInvocation(context, analysis, index, toolCallId, evidence, violations);
|
||||
}
|
||||
if (!evidence.isEmpty()) {
|
||||
verifiedAnalyses.add(new VerifiedAnalysisEvidence(
|
||||
analysis.analysisId(), analysis.text(), analysis.kind(), evidence));
|
||||
}
|
||||
}
|
||||
return violations.isEmpty()
|
||||
? EvidenceGuardResult.valid(new VerifiedEvidenceSnapshot(verifiedAnalyses))
|
||||
: EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
private List<EvidenceViolation> validateDraft(DiagnosisDraft draft) {
|
||||
List<EvidenceViolation> violations = new ArrayList<>();
|
||||
if (draft == null) {
|
||||
violations.add(violation(EvidenceViolationCode.DRAFT_MISSING, "draft"));
|
||||
return violations;
|
||||
}
|
||||
if (draft.analysis().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_MISSING, "analysis"));
|
||||
}
|
||||
Set<String> ids = new HashSet<>();
|
||||
for (int index = 0; index < draft.analysis().size(); index++) {
|
||||
DiagnosisDraft.AnalysisItem item = draft.analysis().get(index);
|
||||
String target = "analysis[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_MISSING, target));
|
||||
continue;
|
||||
}
|
||||
if (isBlank(item.analysisId())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_ID_MISSING, target + ".analysis_id"));
|
||||
} else if (!ids.add(item.analysisId())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_ID_DUPLICATE, target + ".analysis_id"));
|
||||
}
|
||||
if (item.kind() == null) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_KIND_MISSING, target + ".kind"));
|
||||
}
|
||||
if (isBlank(item.text())) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_TEXT_MISSING, target + ".text"));
|
||||
}
|
||||
if (item.toolCallIds().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_MISSING, target + ".tool_call_ids"));
|
||||
}
|
||||
}
|
||||
validateReportReferences(draft, ids, violations);
|
||||
return violations;
|
||||
}
|
||||
|
||||
private void validateReportReferences(DiagnosisDraft draft, Set<String> ids,
|
||||
List<EvidenceViolation> violations) {
|
||||
if (draft.conclusion() != null) {
|
||||
validateTextAndReferences("conclusion", draft.conclusion().text(),
|
||||
draft.conclusion().basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
for (int index = 0; index < draft.actionPlan().size(); index++) {
|
||||
DiagnosisDraft.ActionPlanItem item = draft.actionPlan().get(index);
|
||||
String target = "action_plan[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
} else {
|
||||
validateTextAndReferences(target, item.action(), item.basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
}
|
||||
for (int index = 0; index < draft.recommendations().size(); index++) {
|
||||
DiagnosisDraft.Recommendation item = draft.recommendations().get(index);
|
||||
String target = "recommendations[" + index + "]";
|
||||
if (item == null) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
} else {
|
||||
validateTextAndReferences(target, item.text(), item.basedOnAnalysisIds(), ids, violations);
|
||||
}
|
||||
}
|
||||
if (draft.limitations() == null || isBlank(draft.limitations().scope())) {
|
||||
violations.add(violation(EvidenceViolationCode.LIMITATIONS_MISSING, "limitations"));
|
||||
}
|
||||
}
|
||||
|
||||
private void validateTextAndReferences(String target, String text, List<String> references,
|
||||
Set<String> ids, List<EvidenceViolation> violations) {
|
||||
if (isBlank(text)) {
|
||||
violations.add(violation(EvidenceViolationCode.REPORT_TEXT_MISSING, target));
|
||||
}
|
||||
if (references == null || references.isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING,
|
||||
target + ".based_on_analysis_ids"));
|
||||
return;
|
||||
}
|
||||
for (int index = 0; index < references.size(); index++) {
|
||||
String reference = references.get(index);
|
||||
if (isBlank(reference)) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING,
|
||||
target + ".based_on_analysis_ids[" + index + "]"));
|
||||
} else if (!ids.contains(reference)) {
|
||||
violations.add(violation(EvidenceViolationCode.ANALYSIS_REFERENCE_UNKNOWN,
|
||||
target + ".based_on_analysis_ids[" + index + "]"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyInvocation(RunContext context, DiagnosisDraft.AnalysisItem analysis,
|
||||
int analysisIndex, String toolCallId,
|
||||
List<VerifiedEvidence> evidence,
|
||||
List<EvidenceViolation> violations) {
|
||||
String target = "analysis[" + analysisIndex + "].tool_call_ids";
|
||||
if (isBlank(toolCallId)) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String key;
|
||||
try {
|
||||
key = keyFactory.create(context.runId(), toolCallId);
|
||||
} catch (IllegalArgumentException exception) {
|
||||
violations.add(violation(EvidenceViolationCode.TOOL_REFERENCE_INVALID, target));
|
||||
return;
|
||||
}
|
||||
CanonicalToolInvocation invocation;
|
||||
try {
|
||||
invocation = store.find(key).orElse(null);
|
||||
} catch (RuntimeException exception) {
|
||||
violations.add(violation(EvidenceViolationCode.CANONICAL_LOOKUP_FAILED, target));
|
||||
return;
|
||||
}
|
||||
if (invocation == null) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_MISSING, target));
|
||||
return;
|
||||
}
|
||||
if (!Objects.equals(toolCallId, invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_ID_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (!invocation.isReferencableBy(context.runId()) || invocation.agentResult().isBlank()) {
|
||||
violations.add(violation(EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE, target));
|
||||
return;
|
||||
}
|
||||
if (!analysis.kind().accepts(invocation.evidenceStatus())) {
|
||||
violations.add(violation(EvidenceViolationCode.EVIDENCE_KIND_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
switch (invocation.toolName()) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE ->
|
||||
readRag(invocation, evidence, target, violations);
|
||||
case AgentToolContracts.QUERY_LOGS ->
|
||||
readLogs(invocation, evidence, target, violations);
|
||||
case AgentToolContracts.QUERY_MYSQL ->
|
||||
readMysql(invocation, evidence, target, violations);
|
||||
default -> violations.add(violation(EvidenceViolationCode.TOOL_UNSUPPORTED, target));
|
||||
}
|
||||
}
|
||||
|
||||
private void readRag(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
RagToolResult result;
|
||||
try {
|
||||
result = ragReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!Objects.equals(result.toolCallId(), invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_ID_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (result.evidenceStatus() != invocation.evidenceStatus()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_STATUS_MISMATCH, target));
|
||||
return;
|
||||
}
|
||||
if (result.returnedCount() != result.evidence().size()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (!result.evidence().isEmpty() || result.returnedCount() != 0) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"RAG", "KNOWLEDGE_BASE", firstText(result.query(), "knowledge query"),
|
||||
null, "No evidence matched the query scope",
|
||||
Map.of("match_count", 0, "truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.evidence().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (RagEvidence item : result.evidence()) {
|
||||
if (item == null || isBlank(item.documentId()) || isBlank(item.excerpt())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
values.put("document_id", item.documentId());
|
||||
putIfText(values, "title", item.title());
|
||||
putIfText(values, "breadcrumb", item.breadcrumb());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"RAG", firstText(item.source(), item.title(), item.documentId()),
|
||||
firstText(result.query(), "knowledge query"), null, item.excerpt(), values));
|
||||
}
|
||||
}
|
||||
|
||||
private void readLogs(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
QueryLogsToolResult result;
|
||||
try {
|
||||
result = logsReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!projectionMatches(invocation, result.toolCallId(), result.evidenceStatus(),
|
||||
target, violations)) {
|
||||
return;
|
||||
}
|
||||
if (result.sourceKind() == null || result.scope() == null
|
||||
|| result.scope().topic() == null || isBlank(result.scope().query())
|
||||
|| isBlank(result.scope().startTime()) || isBlank(result.scope().endTime())
|
||||
|| result.matchCount() < 0 || result.returnedCount() != result.events().size()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String source = result.scope().topic().name() + " (" + result.sourceKind().name() + ")";
|
||||
String scope = logScope(result.scope());
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (result.matchCount() != 0 || !result.patterns().isEmpty() || !result.events().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, null, "No log events matched the query scope",
|
||||
Map.of("match_count", 0, "truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.patterns().isEmpty() && result.events().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (LogPattern pattern : result.patterns()) {
|
||||
if (pattern == null || pattern.count() <= 0 || isBlank(pattern.example())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
values.put("count", pattern.count());
|
||||
putIfText(values, "first_seen", pattern.firstSeen());
|
||||
putIfText(values, "last_seen", pattern.lastSeen());
|
||||
putIfText(values, "level", pattern.level());
|
||||
putIfText(values, "service", pattern.service());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, pattern.lastSeen(), pattern.example(), values));
|
||||
}
|
||||
for (LogEvent event : result.events()) {
|
||||
if (event == null || isBlank(event.message())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>();
|
||||
putIfText(values, "level", event.level());
|
||||
putIfText(values, "service", event.service());
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"LOG", source, scope, event.timestamp(), event.message(), values));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean projectionMatches(CanonicalToolInvocation invocation, String toolCallId,
|
||||
com.superbiz.agent.harness.contract.EvidenceStatus evidenceStatus,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
if (!Objects.equals(toolCallId, invocation.toolCallId())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_ID_MISMATCH, target));
|
||||
return false;
|
||||
}
|
||||
if (evidenceStatus != invocation.evidenceStatus()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_STATUS_MISMATCH, target));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void readMysql(CanonicalToolInvocation invocation, List<VerifiedEvidence> evidence,
|
||||
String target, List<EvidenceViolation> violations) {
|
||||
MysqlToolRequest request;
|
||||
MysqlToolResult result;
|
||||
try {
|
||||
request = mysqlRequestReader.readValue(invocation.request());
|
||||
result = mysqlResultReader.readValue(invocation.agentResult());
|
||||
} catch (Exception exception) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
if (!projectionMatches(invocation, result.toolCallId(), result.evidenceStatus(),
|
||||
target, violations)) {
|
||||
return;
|
||||
}
|
||||
if (isBlank(request.dataSource()) || isBlank(request.sql())
|
||||
|| result.returnedCount() != result.rows().size()
|
||||
|| result.columns().isEmpty() || hasInvalidColumns(result.columns())) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
String scope = "sql=" + request.sql() + " params=" + request.params();
|
||||
if (result.evidenceStatus() == com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
if (!result.rows().isEmpty() || result.returnedCount() != 0) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"MYSQL", request.dataSource(), scope, null,
|
||||
"No rows matched the query scope",
|
||||
Map.of("match_count", 0, "columns", result.columns(),
|
||||
"truncated", result.truncated())));
|
||||
return;
|
||||
}
|
||||
if (result.rows().isEmpty()) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
for (int index = 0; index < result.rows().size(); index++) {
|
||||
Map<String, Object> row = result.rows().get(index);
|
||||
if (row == null || !row.keySet().equals(new java.util.LinkedHashSet<>(result.columns()))) {
|
||||
violations.add(violation(EvidenceViolationCode.PROJECTION_INVALID, target));
|
||||
return;
|
||||
}
|
||||
Map<String, Object> values = new LinkedHashMap<>(row);
|
||||
values.put("_row_number", index + 1);
|
||||
evidence.add(new VerifiedEvidence(
|
||||
"MYSQL", request.dataSource(), scope, null, null, values));
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean hasInvalidColumns(List<String> columns) {
|
||||
Set<String> unique = new HashSet<>();
|
||||
for (String column : columns) {
|
||||
if (isBlank(column) || !unique.add(column)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String logScope(LogQueryScope scope) {
|
||||
return scope.topic().name() + " query=" + scope.query()
|
||||
+ " from=" + scope.startTime() + " to=" + scope.endTime();
|
||||
}
|
||||
|
||||
private static EvidenceViolation violation(EvidenceViolationCode code, String target) {
|
||||
return new EvidenceViolation(code, target);
|
||||
}
|
||||
|
||||
private static void putIfText(Map<String, Object> values, String name, String value) {
|
||||
if (!isBlank(value)) {
|
||||
values.put(name, value);
|
||||
}
|
||||
}
|
||||
|
||||
private static String firstText(String... values) {
|
||||
for (String value : values) {
|
||||
if (!isBlank(value)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static boolean isBlank(String value) {
|
||||
return value == null || value.isBlank();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
public record EvidenceGuardResult(
|
||||
List<EvidenceViolation> violations,
|
||||
VerifiedEvidenceSnapshot snapshot) {
|
||||
|
||||
public EvidenceGuardResult {
|
||||
violations = violations == null ? List.of() : List.copyOf(violations);
|
||||
if (violations.isEmpty() == (snapshot == null)) {
|
||||
throw new IllegalArgumentException(
|
||||
"valid result requires snapshot and invalid result requires violations");
|
||||
}
|
||||
}
|
||||
|
||||
public static EvidenceGuardResult valid(VerifiedEvidenceSnapshot snapshot) {
|
||||
return new EvidenceGuardResult(List.of(), snapshot);
|
||||
}
|
||||
|
||||
public static EvidenceGuardResult invalid(List<EvidenceViolation> violations) {
|
||||
return new EvidenceGuardResult(violations, null);
|
||||
}
|
||||
|
||||
public boolean valid() {
|
||||
return violations.isEmpty();
|
||||
}
|
||||
|
||||
public Optional<VerifiedEvidenceSnapshot> verifiedSnapshot() {
|
||||
return Optional.ofNullable(snapshot);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record EvidenceViolation(
|
||||
@JsonProperty("code") EvidenceViolationCode code,
|
||||
@JsonProperty("target") String target) {
|
||||
|
||||
public EvidenceViolation {
|
||||
Objects.requireNonNull(code, "code must not be null");
|
||||
if (target == null || target.isBlank()) {
|
||||
throw new IllegalArgumentException("target must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
public enum EvidenceViolationCode {
|
||||
DRAFT_MISSING,
|
||||
ANALYSIS_MISSING,
|
||||
ANALYSIS_ID_MISSING,
|
||||
ANALYSIS_ID_DUPLICATE,
|
||||
ANALYSIS_KIND_MISSING,
|
||||
ANALYSIS_TEXT_MISSING,
|
||||
TOOL_REFERENCE_MISSING,
|
||||
REPORT_TEXT_MISSING,
|
||||
ANALYSIS_REFERENCE_MISSING,
|
||||
ANALYSIS_REFERENCE_UNKNOWN,
|
||||
LIMITATIONS_MISSING,
|
||||
TOOL_REFERENCE_INVALID,
|
||||
INVOCATION_MISSING,
|
||||
INVOCATION_ID_MISMATCH,
|
||||
INVOCATION_NOT_REFERENCABLE,
|
||||
EVIDENCE_KIND_MISMATCH,
|
||||
TOOL_UNSUPPORTED,
|
||||
PROJECTION_INVALID,
|
||||
PROJECTION_ID_MISMATCH,
|
||||
PROJECTION_STATUS_MISMATCH,
|
||||
CANONICAL_LOOKUP_FAILED
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public record VerifiedAnalysisEvidence(
|
||||
@JsonProperty("analysis_id") String analysisId,
|
||||
@JsonProperty("analysis_text") String analysisText,
|
||||
@JsonProperty("analysis_kind") AnalysisKind analysisKind,
|
||||
@JsonProperty("verified_evidence") List<VerifiedEvidence> evidence) {
|
||||
|
||||
public VerifiedAnalysisEvidence {
|
||||
requireText(analysisId, "analysisId");
|
||||
requireText(analysisText, "analysisText");
|
||||
Objects.requireNonNull(analysisKind, "analysisKind must not be null");
|
||||
evidence = evidence == null ? List.of() : List.copyOf(evidence);
|
||||
if (evidence.isEmpty()) {
|
||||
throw new IllegalArgumentException("evidence must not be empty");
|
||||
}
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public record VerifiedEvidence(
|
||||
@JsonProperty("source_type") String sourceType,
|
||||
@JsonProperty("source") String source,
|
||||
@JsonProperty("scope") String scope,
|
||||
@JsonProperty("timestamp") String timestamp,
|
||||
@JsonProperty("excerpt") String excerpt,
|
||||
@JsonProperty("values") Map<String, Object> values) {
|
||||
|
||||
public VerifiedEvidence {
|
||||
requireText(sourceType, "sourceType");
|
||||
requireText(source, "source");
|
||||
requireText(scope, "scope");
|
||||
values = immutableValues(values);
|
||||
}
|
||||
|
||||
private static Map<String, Object> immutableValues(Map<String, Object> values) {
|
||||
if (values == null || values.isEmpty()) {
|
||||
return Map.of();
|
||||
}
|
||||
return java.util.Collections.unmodifiableMap(new LinkedHashMap<>(values));
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public record VerifiedEvidenceSnapshot(
|
||||
@JsonProperty("analyses") List<VerifiedAnalysisEvidence> analyses) {
|
||||
|
||||
public VerifiedEvidenceSnapshot {
|
||||
analyses = analyses == null ? List.of() : List.copyOf(analyses);
|
||||
}
|
||||
|
||||
public static VerifiedEvidenceSnapshot empty() {
|
||||
return new VerifiedEvidenceSnapshot(List.of());
|
||||
}
|
||||
|
||||
public List<SafeFallback.VerifiedSource> verifiedSources() {
|
||||
Map<String, SafeFallback.VerifiedSource> unique = new LinkedHashMap<>();
|
||||
for (VerifiedAnalysisEvidence analysis : analyses) {
|
||||
for (VerifiedEvidence evidence : analysis.evidence()) {
|
||||
String key = evidence.sourceType() + "\u0000" + evidence.source()
|
||||
+ "\u0000" + evidence.scope();
|
||||
unique.putIfAbsent(key, new SafeFallback.VerifiedSource(
|
||||
evidence.sourceType(), evidence.source(), evidence.scope()));
|
||||
}
|
||||
}
|
||||
return List.copyOf(unique.values());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.CancellationException;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
|
||||
public final class GuardModelCall {
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final ChatModel chatModel;
|
||||
private final ExecutorService executor;
|
||||
|
||||
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
|
||||
ExecutorService executor) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
|
||||
this.executor = Objects.requireNonNull(executor, "executor must not be null");
|
||||
}
|
||||
|
||||
public String call(RunContext context, Prompt prompt, Duration timeout, long maxOutputBytes) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(prompt, "prompt must not be null");
|
||||
Objects.requireNonNull(timeout, "timeout must not be null");
|
||||
if (timeout.isZero() || timeout.isNegative() || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("timeout and output limit must be positive");
|
||||
}
|
||||
core.beforeModelCall(context);
|
||||
Future<String> future = executor.submit(() -> invoke(context, prompt, maxOutputBytes));
|
||||
context.cancellation().onCancel(ignored -> future.cancel(true));
|
||||
try {
|
||||
return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
|
||||
} catch (TimeoutException exception) {
|
||||
future.cancel(true);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TIMEOUT, "Guard model attempt timed out", exception);
|
||||
} catch (CancellationException exception) {
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model attempt was cancelled", exception);
|
||||
} catch (InterruptedException exception) {
|
||||
future.cancel(true);
|
||||
Thread.currentThread().interrupt();
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Interrupted while waiting for guard model", exception);
|
||||
} catch (ExecutionException exception) {
|
||||
Throwable cause = exception.getCause();
|
||||
if (cause instanceof RunAbortedException aborted) {
|
||||
throw aborted;
|
||||
}
|
||||
if (cause instanceof BudgetExceededException exceeded) {
|
||||
throw exceeded;
|
||||
}
|
||||
if (cause instanceof GuardModelCallException guardFailure) {
|
||||
throw guardFailure;
|
||||
}
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model call failed", cause);
|
||||
}
|
||||
}
|
||||
|
||||
private String invoke(RunContext context, Prompt prompt, long maxOutputBytes) {
|
||||
ChatResponse response;
|
||||
try {
|
||||
response = chatModel.call(prompt);
|
||||
} catch (RuntimeException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model transport failed", exception);
|
||||
}
|
||||
recordUsage(context, response);
|
||||
core.checkActive(context);
|
||||
AssistantMessage output = response == null || response.getResult() == null
|
||||
? null : response.getResult().getOutput();
|
||||
if (output == null || output.getText() == null || output.getText().isBlank()
|
||||
|| (output.getToolCalls() != null && !output.getToolCalls().isEmpty())) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Guard model returned invalid message shape");
|
||||
}
|
||||
long bytes = output.getText().getBytes(StandardCharsets.UTF_8).length;
|
||||
if (bytes > maxOutputBytes) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Guard model output exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, bytes);
|
||||
return output.getText();
|
||||
}
|
||||
|
||||
private void recordUsage(RunContext context, ChatResponse response) {
|
||||
if (response == null || response.getMetadata() == null) {
|
||||
return;
|
||||
}
|
||||
Usage usage = response.getMetadata().getUsage();
|
||||
if (usage == null) {
|
||||
return;
|
||||
}
|
||||
core.recordTokens(context, nonNegative(usage.getPromptTokens()),
|
||||
nonNegative(usage.getCompletionTokens()));
|
||||
}
|
||||
|
||||
private static long nonNegative(Integer value) {
|
||||
return value == null || value < 0 ? 0L : value.longValue();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class GuardModelCallException extends RuntimeException {
|
||||
|
||||
private final RetryFailure failure;
|
||||
|
||||
public GuardModelCallException(RetryFailure failure, String message) {
|
||||
super(message);
|
||||
this.failure = Objects.requireNonNull(failure, "failure must not be null");
|
||||
}
|
||||
|
||||
public GuardModelCallException(RetryFailure failure, String message, Throwable cause) {
|
||||
super(message, cause);
|
||||
this.failure = Objects.requireNonNull(failure, "failure must not be null");
|
||||
}
|
||||
|
||||
public RetryFailure failure() {
|
||||
return failure;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticDraftView(
|
||||
@JsonProperty("conclusion") Conclusion conclusion,
|
||||
@JsonProperty("analysis") List<Analysis> analysis,
|
||||
@JsonProperty("action_plan") List<Action> actionPlan,
|
||||
@JsonProperty("recommendations") List<Recommendation> recommendations,
|
||||
@JsonProperty("limitations") Limitations limitations) {
|
||||
|
||||
public SemanticDraftView {
|
||||
analysis = immutable(analysis);
|
||||
actionPlan = immutable(actionPlan);
|
||||
recommendations = immutable(recommendations);
|
||||
}
|
||||
|
||||
public static SemanticDraftView from(DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
Conclusion conclusion = draft.conclusion() == null ? null : new Conclusion(
|
||||
draft.conclusion().text(), draft.conclusion().basedOnAnalysisIds());
|
||||
List<Analysis> analysis = draft.analysis().stream()
|
||||
.map(item -> new Analysis(item.analysisId(), item.kind(), item.text()))
|
||||
.toList();
|
||||
List<Action> actions = draft.actionPlan().stream()
|
||||
.map(item -> new Action(item.action(), item.basedOnAnalysisIds(),
|
||||
item.requiresHumanConfirmation()))
|
||||
.toList();
|
||||
List<Recommendation> recommendations = draft.recommendations().stream()
|
||||
.map(item -> new Recommendation(item.text(), item.basedOnAnalysisIds()))
|
||||
.toList();
|
||||
Limitations limitations = draft.limitations() == null ? null : new Limitations(
|
||||
draft.limitations().scope(), draft.limitations().missingInfo());
|
||||
return new SemanticDraftView(conclusion, analysis, actions, recommendations, limitations);
|
||||
}
|
||||
|
||||
public boolean hasSameUserVisibleSemantics(SemanticDraftView other) {
|
||||
if (other == null || !Objects.equals(text(conclusion), text(other.conclusion))
|
||||
|| analysis.size() != other.analysis.size()
|
||||
|| actionPlan.size() != other.actionPlan.size()
|
||||
|| recommendations.size() != other.recommendations.size()) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < analysis.size(); i++) {
|
||||
Analysis left = analysis.get(i);
|
||||
Analysis right = other.analysis.get(i);
|
||||
if (left.kind() != right.kind() || !Objects.equals(left.text(), right.text())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < actionPlan.size(); i++) {
|
||||
Action left = actionPlan.get(i);
|
||||
Action right = other.actionPlan.get(i);
|
||||
if (!Objects.equals(left.action(), right.action())
|
||||
|| left.requiresHumanConfirmation() != right.requiresHumanConfirmation()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < recommendations.size(); i++) {
|
||||
if (!Objects.equals(recommendations.get(i).text(), other.recommendations.get(i).text())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return limitationsEqual(limitations, other.limitations);
|
||||
}
|
||||
|
||||
private static String text(Conclusion value) {
|
||||
return value == null ? null : value.text();
|
||||
}
|
||||
|
||||
private static boolean limitationsEqual(Limitations left, Limitations right) {
|
||||
if (left == null || right == null) {
|
||||
return left == right;
|
||||
}
|
||||
return Objects.equals(left.scope(), right.scope())
|
||||
&& Objects.equals(left.missingInfo(), right.missingInfo());
|
||||
}
|
||||
|
||||
private static <T> List<T> immutable(List<T> values) {
|
||||
return values == null ? List.of() : List.copyOf(values);
|
||||
}
|
||||
|
||||
public record Conclusion(
|
||||
@JsonProperty("text") String text,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds) {
|
||||
public Conclusion {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Analysis(
|
||||
@JsonProperty("analysis_id") String analysisId,
|
||||
@JsonProperty("kind") AnalysisKind kind,
|
||||
@JsonProperty("text") String text) {
|
||||
}
|
||||
|
||||
public record Action(
|
||||
@JsonProperty("action") String action,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds,
|
||||
@JsonProperty("requires_human_confirmation") boolean requiresHumanConfirmation) {
|
||||
public Action {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Recommendation(
|
||||
@JsonProperty("text") String text,
|
||||
@JsonProperty("based_on_analysis_ids") List<String> basedOnAnalysisIds) {
|
||||
public Recommendation {
|
||||
basedOnAnalysisIds = immutable(basedOnAnalysisIds);
|
||||
}
|
||||
}
|
||||
|
||||
public record Limitations(
|
||||
@JsonProperty("scope") String scope,
|
||||
@JsonProperty("missing_info") List<String> missingInfo) {
|
||||
public Limitations {
|
||||
missingInfo = immutable(missingInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public final class SemanticGuard {
|
||||
|
||||
private static final Set<String> OUTPUT_FIELDS = Set.of("verdict", "reason");
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final HarnessRetryExecutor retryExecutor;
|
||||
private final GuardModelCall modelCall;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final SemanticGuardLimits limits;
|
||||
private final Consumer<RetryAttempt> attemptRecorder;
|
||||
private final String prompt;
|
||||
|
||||
public SemanticGuard(DiagnosisHarnessCore core,
|
||||
HarnessRetryExecutor retryExecutor,
|
||||
GuardModelCall modelCall,
|
||||
ObjectMapper objectMapper,
|
||||
SemanticGuardLimits limits,
|
||||
Consumer<RetryAttempt> attemptRecorder) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null");
|
||||
this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.limits = Objects.requireNonNull(limits, "limits must not be null");
|
||||
this.attemptRecorder = Objects.requireNonNull(
|
||||
attemptRecorder, "attemptRecorder must not be null");
|
||||
this.prompt = SemanticGuardPrompt.load();
|
||||
}
|
||||
|
||||
public SemanticGuardDecision review(RunContext context, SemanticGuardInput input) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(input, "input must not be null");
|
||||
String inputJson = serialize(input);
|
||||
long inputBytes = utf8Bytes(inputJson);
|
||||
if (inputBytes > limits.maxInputBytes()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard input exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, inputBytes);
|
||||
Prompt modelPrompt = new Prompt(java.util.List.of(
|
||||
new SystemMessage(prompt), new UserMessage(inputJson)));
|
||||
long startedNanos = System.nanoTime();
|
||||
return retryExecutor.execute(
|
||||
context,
|
||||
context.retryPolicies().semanticGuard(),
|
||||
() -> parse(modelCall.call(
|
||||
context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
|
||||
this::classify,
|
||||
attemptRecorder);
|
||||
}
|
||||
|
||||
private SemanticGuardDecision parse(String output) {
|
||||
JsonNode root;
|
||||
try {
|
||||
root = objectMapper.readTree(output);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.PARSE_ERROR, "SemanticGuard output is not JSON", exception);
|
||||
}
|
||||
if (root == null || !root.isObject() || !fieldNames(root).equals(OUTPUT_FIELDS)
|
||||
|| !root.path("verdict").isTextual() || !root.path("reason").isTextual()
|
||||
|| root.path("reason").asText().isBlank()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard output schema is invalid");
|
||||
}
|
||||
SemanticVerdict verdict;
|
||||
try {
|
||||
verdict = SemanticVerdict.valueOf(root.path("verdict").asText());
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard verdict is invalid", exception);
|
||||
}
|
||||
return new SemanticGuardDecision(verdict, root.path("reason").asText());
|
||||
}
|
||||
|
||||
private Duration remainingTimeout(long startedNanos) {
|
||||
long elapsed = Math.max(0L, System.nanoTime() - startedNanos);
|
||||
long remaining = limits.totalTimeout().toNanos() - elapsed;
|
||||
if (remaining <= 0) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TIMEOUT, "SemanticGuard total timeout exhausted");
|
||||
}
|
||||
return Duration.ofNanos(Math.min(remaining, limits.perAttemptTimeout().toNanos()));
|
||||
}
|
||||
|
||||
private RetryFailure classify(Exception exception) {
|
||||
if (exception instanceof GuardModelCallException guardFailure) {
|
||||
return guardFailure.failure();
|
||||
}
|
||||
return RetryFailure.UNKNOWN;
|
||||
}
|
||||
|
||||
private String serialize(SemanticGuardInput input) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(input);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "SemanticGuard input is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> fieldNames(JsonNode node) {
|
||||
Set<String> names = new HashSet<>();
|
||||
Iterator<String> fields = node.fieldNames();
|
||||
fields.forEachRemaining(names::add);
|
||||
return names;
|
||||
}
|
||||
|
||||
private static long utf8Bytes(String value) {
|
||||
return value.getBytes(StandardCharsets.UTF_8).length;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardDecision(
|
||||
@JsonProperty("verdict") SemanticVerdict verdict,
|
||||
@JsonProperty("reason") String reason) {
|
||||
|
||||
public SemanticGuardDecision {
|
||||
Objects.requireNonNull(verdict, "verdict must not be null");
|
||||
if (reason == null || reason.isBlank()) {
|
||||
throw new IllegalArgumentException("reason must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardInput(
|
||||
@JsonProperty("query") String query,
|
||||
@JsonProperty("draft") SemanticDraftView draft,
|
||||
@JsonProperty("verified_evidence") VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
|
||||
public SemanticGuardInput {
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
Objects.requireNonNull(verifiedEvidence, "verifiedEvidence must not be null");
|
||||
}
|
||||
|
||||
public static SemanticGuardInput from(String query, DiagnosisDraft draft,
|
||||
VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
return new SemanticGuardInput(query, SemanticDraftView.from(draft), verifiedEvidence);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
public record SemanticGuardLimits(
|
||||
long maxInputBytes,
|
||||
long maxOutputBytes,
|
||||
Duration perAttemptTimeout,
|
||||
Duration totalTimeout) {
|
||||
|
||||
public SemanticGuardLimits {
|
||||
if (maxInputBytes <= 0 || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("byte limits must be positive");
|
||||
}
|
||||
requirePositive(perAttemptTimeout, "perAttemptTimeout");
|
||||
requirePositive(totalTimeout, "totalTimeout");
|
||||
if (totalTimeout.compareTo(perAttemptTimeout) < 0) {
|
||||
throw new IllegalArgumentException("totalTimeout must not be shorter than perAttemptTimeout");
|
||||
}
|
||||
}
|
||||
|
||||
private static void requirePositive(Duration value, String name) {
|
||||
Objects.requireNonNull(value, name + " must not be null");
|
||||
if (value.isZero() || value.isNegative()) {
|
||||
throw new IllegalArgumentException(name + " must be positive");
|
||||
}
|
||||
value.toNanos();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
final class SemanticGuardPrompt {
|
||||
|
||||
private static final String RESOURCE_PATH = "prompts/semantic-guard-prompt.md";
|
||||
|
||||
private SemanticGuardPrompt() {
|
||||
}
|
||||
|
||||
static String load() {
|
||||
ClassPathResource resource = new ClassPathResource(RESOURCE_PATH);
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
return new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("Failed to load SemanticGuard prompt", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record DiagnosisReleaseResult(
|
||||
ReleaseOutcome outcome,
|
||||
DiagnosisDraft draft,
|
||||
SafeFallback fallback,
|
||||
VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
|
||||
public DiagnosisReleaseResult {
|
||||
Objects.requireNonNull(outcome, "outcome must not be null");
|
||||
Objects.requireNonNull(verifiedEvidence, "verifiedEvidence must not be null");
|
||||
if (outcome == ReleaseOutcome.SUCCESS) {
|
||||
Objects.requireNonNull(draft, "successful release requires draft");
|
||||
if (fallback != null) {
|
||||
throw new IllegalArgumentException("successful release must not contain fallback");
|
||||
}
|
||||
} else if (outcome == ReleaseOutcome.FALLBACK) {
|
||||
Objects.requireNonNull(fallback, "fallback release requires fallback");
|
||||
if (draft != null) {
|
||||
throw new IllegalArgumentException("fallback release must not contain draft");
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("release use case supports SUCCESS or FALLBACK only");
|
||||
}
|
||||
}
|
||||
|
||||
public static DiagnosisReleaseResult success(
|
||||
DiagnosisDraft draft, VerifiedEvidenceSnapshot verifiedEvidence) {
|
||||
return new DiagnosisReleaseResult(
|
||||
ReleaseOutcome.SUCCESS, draft, null, verifiedEvidence);
|
||||
}
|
||||
|
||||
public static DiagnosisReleaseResult fallback(SafeFallback fallback) {
|
||||
return new DiagnosisReleaseResult(
|
||||
ReleaseOutcome.FALLBACK, null, fallback, VerifiedEvidenceSnapshot.empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardDecision;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardInput;
|
||||
import com.superbiz.agent.harness.retry.RetryExecutionException;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class DiagnosisReleaseUseCase {
|
||||
|
||||
private final EvidenceGuard evidenceGuard;
|
||||
private final EvidenceRepair evidenceRepair;
|
||||
private final SemanticGuard semanticGuard;
|
||||
private final SafeFallbackFactory fallbackFactory;
|
||||
|
||||
public DiagnosisReleaseUseCase(EvidenceGuard evidenceGuard,
|
||||
EvidenceRepair evidenceRepair,
|
||||
SemanticGuard semanticGuard,
|
||||
SafeFallbackFactory fallbackFactory) {
|
||||
this.evidenceGuard = Objects.requireNonNull(evidenceGuard, "evidenceGuard must not be null");
|
||||
this.evidenceRepair = Objects.requireNonNull(evidenceRepair, "evidenceRepair must not be null");
|
||||
this.semanticGuard = Objects.requireNonNull(semanticGuard, "semanticGuard must not be null");
|
||||
this.fallbackFactory = Objects.requireNonNull(
|
||||
fallbackFactory, "fallbackFactory must not be null");
|
||||
}
|
||||
|
||||
public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
|
||||
DiagnosisDraft candidate = draft;
|
||||
EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate);
|
||||
if (!evidence.valid()) {
|
||||
try {
|
||||
candidate = evidenceRepair.repair(context, query, draft, evidence.violations());
|
||||
evidence = evidenceGuard.validate(context, candidate);
|
||||
} catch (RuntimeException exception) {
|
||||
propagateTerminal(exception);
|
||||
return evidenceFailure();
|
||||
}
|
||||
if (!evidence.valid()) {
|
||||
return evidenceFailure();
|
||||
}
|
||||
}
|
||||
|
||||
VerifiedEvidenceSnapshot snapshot = evidence.verifiedSnapshot().orElseThrow();
|
||||
SemanticGuardDecision decision;
|
||||
try {
|
||||
decision = semanticGuard.review(
|
||||
context, SemanticGuardInput.from(query, candidate, snapshot));
|
||||
} catch (RuntimeException exception) {
|
||||
propagateTerminal(exception);
|
||||
return DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.semanticUnavailable(snapshot));
|
||||
}
|
||||
return decision.verdict() == SemanticVerdict.SUPPORTED
|
||||
? DiagnosisReleaseResult.success(candidate, snapshot)
|
||||
: DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.semanticUnsupported(snapshot));
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult evidenceFailure() {
|
||||
return DiagnosisReleaseResult.fallback(
|
||||
fallbackFactory.evidenceValidationFailed());
|
||||
}
|
||||
|
||||
private void propagateTerminal(RuntimeException exception) {
|
||||
if (exception instanceof RunAbortedException
|
||||
|| exception instanceof BudgetExceededException) {
|
||||
throw exception;
|
||||
}
|
||||
if (exception instanceof RetryExecutionException retry
|
||||
&& (retry.failure() == RetryFailure.CANCELLED
|
||||
|| retry.failure() == RetryFailure.BUDGET_EXHAUSTED)) {
|
||||
throw retry;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCallException;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
public final class EvidenceRepair {
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final HarnessRetryExecutor retryExecutor;
|
||||
private final GuardModelCall modelCall;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ObjectReader draftReader;
|
||||
private final EvidenceRepairLimits limits;
|
||||
private final Consumer<RetryAttempt> attemptRecorder;
|
||||
private final String prompt;
|
||||
|
||||
public EvidenceRepair(DiagnosisHarnessCore core,
|
||||
HarnessRetryExecutor retryExecutor,
|
||||
GuardModelCall modelCall,
|
||||
ObjectMapper objectMapper,
|
||||
EvidenceRepairLimits limits,
|
||||
Consumer<RetryAttempt> attemptRecorder) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null");
|
||||
this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.draftReader = objectMapper.readerFor(DiagnosisDraft.class)
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.limits = Objects.requireNonNull(limits, "limits must not be null");
|
||||
this.attemptRecorder = Objects.requireNonNull(
|
||||
attemptRecorder, "attemptRecorder must not be null");
|
||||
this.prompt = EvidenceRepairPrompt.load();
|
||||
}
|
||||
|
||||
public DiagnosisDraft repair(RunContext context, String query, DiagnosisDraft original,
|
||||
List<EvidenceViolation> violations) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(original, "original must not be null");
|
||||
List<EvidenceViolation> safeViolations = violations == null
|
||||
? List.of() : List.copyOf(violations);
|
||||
String inputJson = serialize(new RepairInput(query, original, safeViolations));
|
||||
long inputBytes = utf8Bytes(inputJson);
|
||||
if (inputBytes > limits.maxInputBytes()) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Evidence repair input exceeded limit");
|
||||
}
|
||||
core.reserveRunBytes(context, inputBytes);
|
||||
Prompt modelPrompt = new Prompt(List.of(
|
||||
new SystemMessage(prompt), new UserMessage(inputJson)));
|
||||
SemanticDraftView originalSemantics = SemanticDraftView.from(original);
|
||||
return retryExecutor.execute(
|
||||
context,
|
||||
context.retryPolicies().evidenceRepair(),
|
||||
() -> {
|
||||
DiagnosisDraft repaired = parse(modelCall.call(
|
||||
context, modelPrompt, limits.timeout(), limits.maxOutputBytes()));
|
||||
if (!originalSemantics.hasSameUserVisibleSemantics(
|
||||
SemanticDraftView.from(repaired))) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID,
|
||||
"Evidence repair changed user-visible semantics");
|
||||
}
|
||||
return repaired;
|
||||
},
|
||||
this::classify,
|
||||
attemptRecorder);
|
||||
}
|
||||
|
||||
private DiagnosisDraft parse(String output) {
|
||||
try {
|
||||
return draftReader.readValue(output);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.PARSE_ERROR, "Evidence repair output is invalid", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private RetryFailure classify(Exception exception) {
|
||||
return exception instanceof GuardModelCallException failure
|
||||
? failure.failure() : RetryFailure.UNKNOWN;
|
||||
}
|
||||
|
||||
private String serialize(RepairInput input) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(input);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.SCHEMA_INVALID, "Evidence repair input is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static long utf8Bytes(String value) {
|
||||
return value.getBytes(StandardCharsets.UTF_8).length;
|
||||
}
|
||||
|
||||
private record RepairInput(
|
||||
String query,
|
||||
DiagnosisDraft draft,
|
||||
List<EvidenceViolation> violations) {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Objects;
|
||||
|
||||
public record EvidenceRepairLimits(
|
||||
long maxInputBytes,
|
||||
long maxOutputBytes,
|
||||
Duration timeout) {
|
||||
|
||||
public EvidenceRepairLimits {
|
||||
if (maxInputBytes <= 0 || maxOutputBytes <= 0) {
|
||||
throw new IllegalArgumentException("byte limits must be positive");
|
||||
}
|
||||
Objects.requireNonNull(timeout, "timeout must not be null");
|
||||
if (timeout.isZero() || timeout.isNegative()) {
|
||||
throw new IllegalArgumentException("timeout must be positive");
|
||||
}
|
||||
timeout.toNanos();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
final class EvidenceRepairPrompt {
|
||||
|
||||
private static final String RESOURCE_PATH = "prompts/evidence-repair-prompt.md";
|
||||
|
||||
private EvidenceRepairPrompt() {
|
||||
}
|
||||
|
||||
static String load() {
|
||||
ClassPathResource resource = new ClassPathResource(RESOURCE_PATH);
|
||||
try (InputStream input = resource.getInputStream()) {
|
||||
return new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (IOException exception) {
|
||||
throw new IllegalStateException("Failed to load evidence repair prompt", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class SafeFallbackFactory {
|
||||
|
||||
public SafeFallback evidenceValidationFailed() {
|
||||
return fallback(
|
||||
FallbackType.EVIDENCE_VALIDATION_FAILED,
|
||||
List.of(),
|
||||
"当前证据无法完成真实性校验，无法确认根因",
|
||||
"证据引用校验未通过",
|
||||
"重新收集当前诊断范围内的证据后再发起诊断");
|
||||
}
|
||||
|
||||
public SafeFallback semanticUnsupported(VerifiedEvidenceSnapshot snapshot) {
|
||||
return fallback(
|
||||
FallbackType.SEMANTIC_UNSUPPORTED,
|
||||
sources(snapshot),
|
||||
"当前证据不足，无法确认根因",
|
||||
"语义校验未通过",
|
||||
"补充当前缺失的数据后重新发起诊断");
|
||||
}
|
||||
|
||||
public SafeFallback semanticUnavailable(VerifiedEvidenceSnapshot snapshot) {
|
||||
return fallback(
|
||||
FallbackType.SEMANTIC_UNAVAILABLE,
|
||||
sources(snapshot),
|
||||
"当前证据暂时无法完成语义校验，无法确认根因",
|
||||
"语义校验暂不可用",
|
||||
"稍后重试或补充当前缺失的数据");
|
||||
}
|
||||
|
||||
private SafeFallback fallback(FallbackType type,
|
||||
List<SafeFallback.VerifiedSource> sources,
|
||||
String message,
|
||||
String limitation,
|
||||
String nextStep) {
|
||||
return new SafeFallback(
|
||||
type, null, message, sources, List.of(limitation), List.of(nextStep));
|
||||
}
|
||||
|
||||
private List<SafeFallback.VerifiedSource> sources(VerifiedEvidenceSnapshot snapshot) {
|
||||
return Objects.requireNonNull(snapshot, "snapshot must not be null").verifiedSources();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
You repair only the identifier and reference structure of one DiagnosisDraft.
|
||||
|
||||
Use the supplied violation codes and targets. You may change only analysis_id, tool_call_ids, and based_on_analysis_ids.
|
||||
Do not add, remove, reorder, summarize, or rewrite any conclusion, analysis, action, recommendation, limitation, kind, or human-confirmation flag.
|
||||
You have no tools, memory, evidence store, or permission to call another agent.
|
||||
|
||||
Return exactly the complete repaired DiagnosisDraft JSON with no markdown or explanation.
|
||||
@@ -0,0 +1,10 @@
|
||||
You are the isolated SemanticGuard for one diagnosis report.
|
||||
|
||||
Review only the supplied original query, complete user-visible draft, and verified evidence snapshot.
|
||||
Check whether evidence supports each analysis, analyses support the conclusion, actions and recommendations stay within supported findings, and limitations accurately state the observed scope.
|
||||
|
||||
You have no tools, memory, Redis access, diagnosis history, or permission to call another agent.
|
||||
Do not rewrite, correct, summarize, extend, or partially approve the report.
|
||||
|
||||
Return exactly one JSON object with no markdown and no extra fields:
|
||||
{"verdict":"SUPPORTED|UNSUPPORTED","reason":"non-empty audit reason"}
|
||||
@@ -0,0 +1,296 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class EvidenceGuardTest {
|
||||
|
||||
private static final String PREFIX = "superbiz:harness:tool-call";
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final InMemoryStore store = new InMemoryStore();
|
||||
private final EvidenceGuard guard = new EvidenceGuard(
|
||||
store, new ToolCallKeyFactory(PREFIX), objectMapper);
|
||||
private final RunContext context = context("run-guard");
|
||||
|
||||
@Test
|
||||
void validCurrentRunRagReferenceProducesInternalIdFreeSnapshot() throws Exception {
|
||||
String callId = "call-rag-1";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"pool timeout\"}", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-rag-1",
|
||||
"query":"pool timeout","evidence":[{"document_id":"doc-1",
|
||||
"source":"runbook.md","title":"Pool guide","breadcrumb":"DB > Pool",
|
||||
"excerpt":"active=50 max=50"}],"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidenceSnapshot snapshot = result.verifiedSnapshot().orElseThrow();
|
||||
assertEquals("a-1", snapshot.analyses().get(0).analysisId());
|
||||
assertEquals("runbook.md", snapshot.analyses().get(0).evidence().get(0).source());
|
||||
assertEquals("active=50 max=50", snapshot.analyses().get(0).evidence().get(0).excerpt());
|
||||
String json = objectMapper.writeValueAsString(snapshot);
|
||||
assertFalse(json.contains(callId));
|
||||
assertFalse(json.contains("raw_response"));
|
||||
assertEquals(1, snapshot.verifiedSources().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noEvidenceSupportsOnlyScopedNegativeObservation() {
|
||||
String callId = "call-rag-empty";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"unknown failure\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-rag-empty",
|
||||
"query":"unknown failure","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(
|
||||
context, draft(callId, AnalysisKind.NEGATIVE_OBSERVATION));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("unknown failure", evidence.scope());
|
||||
assertEquals(0, evidence.values().get("match_count"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validLogProjectionPreservesSourceScopeTimelineAndExactMessage() {
|
||||
String callId = "call-log-1";
|
||||
ready(callId, "query_logs", """
|
||||
{"topic":"APPLICATION","query":"pool timeout","lookback_minutes":30}
|
||||
""", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-log-1",
|
||||
"source_kind":"MOCK","scope":{"topic":"APPLICATION","query":"pool timeout",
|
||||
"start_time":"2026-07-21T10:00:00Z","end_time":"2026-07-21T10:30:00Z"},
|
||||
"match_count":1,"returned_count":1,"patterns":[],
|
||||
"events":[{"timestamp":"2026-07-21T10:29:00Z","level":"ERROR",
|
||||
"service":"order-service","message":"active=50 max=50"}],"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("LOG", evidence.sourceType());
|
||||
assertEquals("APPLICATION (MOCK)", evidence.source());
|
||||
assertEquals("2026-07-21T10:29:00Z", evidence.timestamp());
|
||||
assertEquals("active=50 max=50", evidence.excerpt());
|
||||
assertTrue(evidence.scope().contains("pool timeout"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validMysqlProjectionCombinesBoundedRequestScopeAndProjectedRows() {
|
||||
String callId = "call-mysql-1";
|
||||
ready(callId, "query_mysql", """
|
||||
{"data_source":"order_readonly","sql":"SELECT status FROM biz_order WHERE id = ?",
|
||||
"params":["order-1"]}
|
||||
""", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-mysql-1",
|
||||
"columns":["status"],"rows":[{"status":"FAILED"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("MYSQL", evidence.sourceType());
|
||||
assertEquals("order_readonly", evidence.source());
|
||||
assertTrue(evidence.scope().contains("SELECT status"));
|
||||
assertTrue(evidence.scope().contains("order-1"));
|
||||
assertEquals("FAILED", evidence.values().get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateAnalysisAndBrokenReportReferenceFailBeforeStoreLookup() {
|
||||
DiagnosisDraft invalid = new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("missing")),
|
||||
List.of(
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "first", List.of("call-1")),
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "second", List.of("call-2"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem("inspect", List.of(), false)),
|
||||
List.of(), new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, invalid);
|
||||
|
||||
assertFalse(result.valid());
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_ID_DUPLICATE));
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_REFERENCE_UNKNOWN));
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING));
|
||||
assertTrue(result.verifiedSnapshot().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fabricatedAndCrossRunReferencesAreRejected() {
|
||||
EvidenceGuardResult missing = guard.validate(
|
||||
context, draft("fabricated-call", AnalysisKind.NORMAL));
|
||||
String callId = "call-cross-run";
|
||||
String currentKey = PREFIX + ":" + context.runId() + ":" + callId;
|
||||
store.records.put(currentKey, CanonicalToolInvocation.projecting(
|
||||
callId, "another-run", "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-cross-run",
|
||||
"query":"pool","evidence":[{"document_id":"doc-1","source":"guide",
|
||||
"title":"guide","breadcrumb":"pool","excerpt":"active=50"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z")));
|
||||
|
||||
EvidenceGuardResult crossRun = guard.validate(
|
||||
context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(missing, EvidenceViolationCode.INVOCATION_MISSING));
|
||||
assertTrue(hasViolation(crossRun, EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectingInvocationAndEvidenceKindMismatchAreRejected() {
|
||||
String projectingId = "call-projecting";
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + projectingId,
|
||||
CanonicalToolInvocation.projecting(
|
||||
projectingId, context.runId(), "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z")));
|
||||
EvidenceGuardResult projecting = guard.validate(
|
||||
context, draft(projectingId, AnalysisKind.NORMAL));
|
||||
|
||||
String emptyId = "call-empty-normal";
|
||||
ready(emptyId, "lookup_knowledge", "{\"query\":\"pool\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-normal",
|
||||
"query":"pool","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE);
|
||||
EvidenceGuardResult mismatch = guard.validate(
|
||||
context, draft(emptyId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(projecting, EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE));
|
||||
assertTrue(hasViolation(mismatch, EvidenceViolationCode.EVIDENCE_KIND_MISMATCH));
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectionIdOrStatusMismatchFailsClosed() {
|
||||
String callId = "call-mismatch";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"pool\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"another-call",
|
||||
"query":"pool","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.PROJECTION_ID_MISMATCH));
|
||||
}
|
||||
|
||||
@Test
|
||||
void canonicalRecordIdMustMatchTheDraftReferenceEvenUnderCorruptedKey() {
|
||||
String referencedId = "call-requested";
|
||||
CanonicalToolInvocation wrongRecord = CanonicalToolInvocation.projecting(
|
||||
"call-other", context.runId(), "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-other",
|
||||
"query":"pool","evidence":[{"document_id":"doc-1","source":"guide",
|
||||
"title":"guide","breadcrumb":"pool","excerpt":"active=50"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + referencedId, wrongRecord);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(
|
||||
context, draft(referencedId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.INVOCATION_ID_MISMATCH));
|
||||
}
|
||||
|
||||
private boolean hasViolation(EvidenceGuardResult result, EvidenceViolationCode code) {
|
||||
return result.violations().stream().anyMatch(violation -> violation.code() == code);
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft(String callId, AnalysisKind kind) {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", kind, "Pool reached its limit", List.of(callId))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("a-1"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("a-1"))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private void ready(String callId, String toolName, String request,
|
||||
String agentResult, EvidenceStatus evidenceStatus) {
|
||||
String key = PREFIX + ":" + context.runId() + ":" + callId;
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), toolName, request, Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", agentResult, evidenceStatus,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(key, invocation);
|
||||
}
|
||||
|
||||
private RunContext context(String runId) {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> runId, Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict()).startRun("session-guard", runId);
|
||||
}
|
||||
|
||||
private static final class InMemoryStore implements CanonicalInvocationStore {
|
||||
private final Map<String, CanonicalToolInvocation> records = new HashMap<>();
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 1_000_000, 64_000);
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
records.put(key, invocation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse,
|
||||
String agentResult, EvidenceStatus evidenceStatus,
|
||||
Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SemanticGuardTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutor() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedIsAValidBusinessDecisionAndIsNotRetried() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-semantic", "run-semantic");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"{\"verdict\":\"UNSUPPORTED\",\"reason\":\"evidence does not prove the root cause\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core,
|
||||
new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor),
|
||||
objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)),
|
||||
attempts::add);
|
||||
|
||||
SemanticGuardDecision decision = guard.review(
|
||||
context, SemanticGuardInput.from("Why did payment fail?", draft(), snapshot()));
|
||||
|
||||
assertEquals(SemanticVerdict.UNSUPPORTED, decision.verdict());
|
||||
assertEquals(1, model.calls.get());
|
||||
assertEquals(1, attempts.size());
|
||||
assertTrue(attempts.get(0).success());
|
||||
String prompt = model.prompts.get(0);
|
||||
assertTrue(prompt.contains("Why did payment fail?"));
|
||||
assertTrue(prompt.contains("active=50 max=50"));
|
||||
assertFalse(prompt.contains("call-rag-1"));
|
||||
assertFalse(prompt.contains("tool_call_id"));
|
||||
assertFalse(prompt.contains("raw_response"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseFailureRetriesOnceWithTheExactSameInput() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-retry", "run-retry");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"not-json", "{\"verdict\":\"SUPPORTED\",\"reason\":\"all claims are grounded\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), attempts::add);
|
||||
|
||||
SemanticGuardDecision decision = guard.review(
|
||||
context, SemanticGuardInput.from("Why did payment fail?", draft(), snapshot()));
|
||||
|
||||
assertEquals(SemanticVerdict.SUPPORTED, decision.verdict());
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(model.prompts.get(0), model.prompts.get(1));
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.PARSE_ERROR,
|
||||
attempts.get(0).failure());
|
||||
assertTrue(attempts.get(1).success());
|
||||
assertEquals(2, context.budget().snapshot().modelCalls());
|
||||
assertEquals(16, context.budget().snapshot().totalTokens());
|
||||
}
|
||||
|
||||
@Test
|
||||
void attemptTimeoutCancelsBothPermittedModelCalls() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-timeout", "run-timeout");
|
||||
BlockingChatModel model = new BlockingChatModel(2);
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofMillis(50), Duration.ofMillis(500)), attempts::add);
|
||||
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException failure = assertThrows(
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException.class,
|
||||
() -> guard.review(context,
|
||||
SemanticGuardInput.from("Why?", draft(), snapshot())));
|
||||
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.TIMEOUT, failure.failure());
|
||||
assertEquals(2, failure.attempts());
|
||||
assertTrue(model.interrupted.await(2, TimeUnit.SECONDS));
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(2, attempts.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runCancellationInterruptsPendingModelAndDoesNotRetry() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-cancel", "run-cancel");
|
||||
BlockingChatModel model = new BlockingChatModel(1);
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { });
|
||||
|
||||
CompletableFuture<SemanticGuardDecision> execution = CompletableFuture.supplyAsync(
|
||||
() -> guard.review(context,
|
||||
SemanticGuardInput.from("Why?", draft(), snapshot())));
|
||||
assertTrue(model.started.await(2, TimeUnit.SECONDS));
|
||||
core.cancel(context,
|
||||
com.superbiz.agent.harness.core.RunCancellationReason.USER_REQUESTED);
|
||||
|
||||
CompletionException thrown = assertThrows(CompletionException.class, execution::join);
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException failure =
|
||||
(com.superbiz.agent.harness.retry.RetryExecutionException) thrown.getCause();
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.CANCELLED, failure.failure());
|
||||
assertTrue(model.interrupted.await(2, TimeUnit.SECONDS));
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft() {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "Pool reached its limit", List.of("call-rag-1"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("a-1"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("a-1"))),
|
||||
new DiagnosisDraft.Limitations("order-service, last 30 minutes", List.of("No slow SQL")));
|
||||
}
|
||||
|
||||
private VerifiedEvidenceSnapshot snapshot() {
|
||||
return new VerifiedEvidenceSnapshot(List.of(new VerifiedAnalysisEvidence(
|
||||
"a-1", "Pool reached its limit", AnalysisKind.NORMAL,
|
||||
List.of(new VerifiedEvidence(
|
||||
"LOG", "APPLICATION (MOCK)", "order-service, last 30 minutes",
|
||||
"2026-07-21T10:29:00Z", "active=50 max=50", Map.of("count", 1))))));
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
private final List<String> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedChatModel(String... responses) {
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
int index = calls.getAndIncrement();
|
||||
ChatResponseMetadata metadata = ChatResponseMetadata.builder()
|
||||
.usage(new DefaultUsage(5, 3)).build();
|
||||
return new ChatResponse(
|
||||
List.of(new Generation(new AssistantMessage(responses.get(index)))), metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class BlockingChatModel implements ChatModel {
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
private final CountDownLatch started = new CountDownLatch(1);
|
||||
private final CountDownLatch interrupted;
|
||||
|
||||
private BlockingChatModel(int expectedInterruptions) {
|
||||
this.interrupted = new CountDownLatch(expectedInterruptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
calls.incrementAndGet();
|
||||
started.countDown();
|
||||
try {
|
||||
Thread.sleep(10_000);
|
||||
throw new AssertionError("blocking model should be interrupted");
|
||||
} catch (InterruptedException exception) {
|
||||
interrupted.countDown();
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
class DiagnosisReleaseUseCaseTest {
|
||||
|
||||
private static final String PREFIX = "superbiz:harness:tool-call";
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutor() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void supportedVerifiedDraftIsReleasedUnchangedWithoutRepair() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-release", "run-release");
|
||||
InMemoryStore store = new InMemoryStore();
|
||||
ready(store, context, "call-rag-1");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"{\"verdict\":\"SUPPORTED\",\"reason\":\"all claims are grounded\"}");
|
||||
HarnessRetryExecutor retries = new HarnessRetryExecutor(core);
|
||||
GuardModelCall modelCall = new GuardModelCall(core, model, executor);
|
||||
DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase(
|
||||
new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper),
|
||||
new EvidenceRepair(core, retries, modelCall, objectMapper,
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }),
|
||||
new SemanticGuard(core, retries, modelCall, objectMapper,
|
||||
new SemanticGuardLimits(100_000, 20_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }),
|
||||
new SafeFallbackFactory());
|
||||
DiagnosisDraft draft = draft("a-1", "call-rag-1", "Pool exhausted");
|
||||
|
||||
DiagnosisReleaseResult result = useCase.execute(context, "Why did payment fail?", draft);
|
||||
|
||||
assertEquals(ReleaseOutcome.SUCCESS, result.outcome());
|
||||
assertSame(draft, result.draft());
|
||||
assertNull(result.fallback());
|
||||
assertEquals(1, result.verifiedEvidence().analyses().size());
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
|
||||
Fixture fixture = fixture(
|
||||
repairedDuplicateDraft("Pool exhausted", "a-2"),
|
||||
"{\"verdict\":\"SUPPORTED\",\"reason\":\"grounded\"}");
|
||||
DiagnosisDraft original = duplicateDraft();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", original);
|
||||
|
||||
assertEquals(ReleaseOutcome.SUCCESS, result.outcome());
|
||||
assertEquals(List.of("a-1", "a-2"), result.draft().analysis().stream()
|
||||
.map(DiagnosisDraft.AnalysisItem::analysisId).toList());
|
||||
assertEquals("Pool exhausted", result.draft().conclusion().text());
|
||||
assertEquals(2, fixture.model.calls.get());
|
||||
assertFalse(fixture.model.prompts.get(1).contains("call-rag-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void repairThatChangesVisibleSemanticsReturnsEvidenceFallback() throws Exception {
|
||||
Fixture fixture = fixture(repairedDuplicateDraft("Different conclusion", "a-2"));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", duplicateDraft());
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("Pool exhausted"));
|
||||
assertFalse(json.contains("Different conclusion"));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondEvidenceValidationFailureHasNoVerifiedSources() {
|
||||
Fixture fixture = fixture(repairedDuplicateDraft("Pool exhausted", "a-1"));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", duplicateDraft());
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedNeverReleasesDraftOrAuditReason() throws Exception {
|
||||
Fixture fixture = fixture(
|
||||
"{\"verdict\":\"UNSUPPORTED\",\"reason\":\"secret-audit-reason\"}");
|
||||
DiagnosisDraft draft = draft("a-1", "call-rag-1", "Pool exhausted");
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", draft);
|
||||
|
||||
assertFallback(result, FallbackType.SEMANTIC_UNSUPPORTED, 1);
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("Pool exhausted"));
|
||||
assertFalse(json.contains("secret-audit-reason"));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoInvalidSemanticOutputsReturnUnavailableWithoutDraft() throws Exception {
|
||||
Fixture fixture = fixture("not-json", "still-not-json");
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?",
|
||||
draft("a-1", "call-rag-1", "Pool exhausted"));
|
||||
|
||||
assertFallback(result, FallbackType.SEMANTIC_UNAVAILABLE, 1);
|
||||
assertFalse(objectMapper.writeValueAsString(result).contains("Pool exhausted"));
|
||||
assertEquals(2, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
private Fixture fixture(String... responses) {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-fixture", "run-fixture");
|
||||
InMemoryStore store = new InMemoryStore();
|
||||
ready(store, context, "call-rag-1");
|
||||
ScriptedChatModel model = new ScriptedChatModel(responses);
|
||||
HarnessRetryExecutor retries = new HarnessRetryExecutor(core);
|
||||
GuardModelCall modelCall = new GuardModelCall(core, model, executor);
|
||||
DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase(
|
||||
new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper),
|
||||
new EvidenceRepair(core, retries, modelCall, objectMapper,
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }),
|
||||
new SemanticGuard(core, retries, modelCall, objectMapper,
|
||||
new SemanticGuardLimits(100_000, 20_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }),
|
||||
new SafeFallbackFactory());
|
||||
return new Fixture(context, model, useCase);
|
||||
}
|
||||
|
||||
private DiagnosisDraft duplicateDraft() {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("dup")),
|
||||
List.of(
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"dup", AnalysisKind.NORMAL, "Pool reached its limit", List.of("call-rag-1")),
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"dup", AnalysisKind.NORMAL, "Requests are waiting", List.of("call-rag-1"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("dup"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("dup"))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private String repairedDuplicateDraft(String conclusion, String secondAnalysisId) {
|
||||
return """
|
||||
{
|
||||
"conclusion":{"text":"%s","based_on_analysis_ids":["a-1"]},
|
||||
"analysis":[
|
||||
{"analysis_id":"a-1","kind":"NORMAL","text":"Pool reached its limit","tool_call_ids":["call-rag-1"]},
|
||||
{"analysis_id":"%s","kind":"NORMAL","text":"Requests are waiting","tool_call_ids":["call-rag-1"]}
|
||||
],
|
||||
"action_plan":[{"action":"Inspect long transactions","based_on_analysis_ids":["a-1"],"requires_human_confirmation":false}],
|
||||
"recommendations":[{"text":"Add saturation alert","based_on_analysis_ids":["a-1"]}],
|
||||
"limitations":{"scope":"order-service","missing_info":[]}
|
||||
}
|
||||
""".formatted(conclusion, secondAnalysisId);
|
||||
}
|
||||
|
||||
private void assertFallback(DiagnosisReleaseResult result,
|
||||
FallbackType type, int expectedSources) {
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertNull(result.draft());
|
||||
assertNotNull(result.fallback());
|
||||
assertEquals(type, result.fallback().type());
|
||||
assertNull(result.fallback().conclusion());
|
||||
assertEquals(expectedSources, result.fallback().verifiedSources().size());
|
||||
assertEquals(0, result.verifiedEvidence().analyses().size());
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft(String analysisId, String callId, String conclusion) {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion(conclusion, List.of(analysisId)),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
analysisId, AnalysisKind.NORMAL, "Pool reached its limit", List.of(callId))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of(analysisId), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of(analysisId))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private void ready(InMemoryStore store, RunContext context, String callId) {
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), "lookup_knowledge", "{\"query\":\"pool timeout\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-rag-1",
|
||||
"query":"pool timeout","evidence":[{"document_id":"doc-1",
|
||||
"source":"runbook.md","title":"Pool guide","breadcrumb":"DB > Pool",
|
||||
"excerpt":"active=50 max=50"}],"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
private final List<String> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedChatModel(String... responses) {
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
int index = calls.getAndIncrement();
|
||||
ChatResponseMetadata metadata = ChatResponseMetadata.builder()
|
||||
.usage(new DefaultUsage(5, 3)).build();
|
||||
return new ChatResponse(
|
||||
List.of(new Generation(new AssistantMessage(responses.get(index)))), metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class InMemoryStore implements CanonicalInvocationStore {
|
||||
private final Map<String, CanonicalToolInvocation> records = new HashMap<>();
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 1_000_000, 64_000);
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
records.put(key, invocation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse,
|
||||
String agentResult, EvidenceStatus evidenceStatus,
|
||||
Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
RunContext context,
|
||||
ScriptedChatModel model,
|
||||
DiagnosisReleaseUseCase useCase) {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user