feat(harness): add evidence and semantic guards
This commit is contained in:
zhuyongxin
@@ -0,0 +1,296 @@
|
||||
package com.superbiz.agent.harness.guard.evidence;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class EvidenceGuardTest {
|
||||
|
||||
private static final String PREFIX = "superbiz:harness:tool-call";
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final InMemoryStore store = new InMemoryStore();
|
||||
private final EvidenceGuard guard = new EvidenceGuard(
|
||||
store, new ToolCallKeyFactory(PREFIX), objectMapper);
|
||||
private final RunContext context = context("run-guard");
|
||||
|
||||
@Test
|
||||
void validCurrentRunRagReferenceProducesInternalIdFreeSnapshot() throws Exception {
|
||||
String callId = "call-rag-1";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"pool timeout\"}", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-rag-1",
|
||||
"query":"pool timeout","evidence":[{"document_id":"doc-1",
|
||||
"source":"runbook.md","title":"Pool guide","breadcrumb":"DB > Pool",
|
||||
"excerpt":"active=50 max=50"}],"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidenceSnapshot snapshot = result.verifiedSnapshot().orElseThrow();
|
||||
assertEquals("a-1", snapshot.analyses().get(0).analysisId());
|
||||
assertEquals("runbook.md", snapshot.analyses().get(0).evidence().get(0).source());
|
||||
assertEquals("active=50 max=50", snapshot.analyses().get(0).evidence().get(0).excerpt());
|
||||
String json = objectMapper.writeValueAsString(snapshot);
|
||||
assertFalse(json.contains(callId));
|
||||
assertFalse(json.contains("raw_response"));
|
||||
assertEquals(1, snapshot.verifiedSources().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noEvidenceSupportsOnlyScopedNegativeObservation() {
|
||||
String callId = "call-rag-empty";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"unknown failure\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-rag-empty",
|
||||
"query":"unknown failure","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(
|
||||
context, draft(callId, AnalysisKind.NEGATIVE_OBSERVATION));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("unknown failure", evidence.scope());
|
||||
assertEquals(0, evidence.values().get("match_count"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validLogProjectionPreservesSourceScopeTimelineAndExactMessage() {
|
||||
String callId = "call-log-1";
|
||||
ready(callId, "query_logs", """
|
||||
{"topic":"APPLICATION","query":"pool timeout","lookback_minutes":30}
|
||||
""", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-log-1",
|
||||
"source_kind":"MOCK","scope":{"topic":"APPLICATION","query":"pool timeout",
|
||||
"start_time":"2026-07-21T10:00:00Z","end_time":"2026-07-21T10:30:00Z"},
|
||||
"match_count":1,"returned_count":1,"patterns":[],
|
||||
"events":[{"timestamp":"2026-07-21T10:29:00Z","level":"ERROR",
|
||||
"service":"order-service","message":"active=50 max=50"}],"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("LOG", evidence.sourceType());
|
||||
assertEquals("APPLICATION (MOCK)", evidence.source());
|
||||
assertEquals("2026-07-21T10:29:00Z", evidence.timestamp());
|
||||
assertEquals("active=50 max=50", evidence.excerpt());
|
||||
assertTrue(evidence.scope().contains("pool timeout"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validMysqlProjectionCombinesBoundedRequestScopeAndProjectedRows() {
|
||||
String callId = "call-mysql-1";
|
||||
ready(callId, "query_mysql", """
|
||||
{"data_source":"order_readonly","sql":"SELECT status FROM biz_order WHERE id = ?",
|
||||
"params":["order-1"]}
|
||||
""", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-mysql-1",
|
||||
"columns":["status"],"rows":[{"status":"FAILED"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(result.valid());
|
||||
VerifiedEvidence evidence = result.verifiedSnapshot().orElseThrow()
|
||||
.analyses().get(0).evidence().get(0);
|
||||
assertEquals("MYSQL", evidence.sourceType());
|
||||
assertEquals("order_readonly", evidence.source());
|
||||
assertTrue(evidence.scope().contains("SELECT status"));
|
||||
assertTrue(evidence.scope().contains("order-1"));
|
||||
assertEquals("FAILED", evidence.values().get("status"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateAnalysisAndBrokenReportReferenceFailBeforeStoreLookup() {
|
||||
DiagnosisDraft invalid = new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("missing")),
|
||||
List.of(
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "first", List.of("call-1")),
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "second", List.of("call-2"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem("inspect", List.of(), false)),
|
||||
List.of(), new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, invalid);
|
||||
|
||||
assertFalse(result.valid());
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_ID_DUPLICATE));
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_REFERENCE_UNKNOWN));
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.ANALYSIS_REFERENCE_MISSING));
|
||||
assertTrue(result.verifiedSnapshot().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fabricatedAndCrossRunReferencesAreRejected() {
|
||||
EvidenceGuardResult missing = guard.validate(
|
||||
context, draft("fabricated-call", AnalysisKind.NORMAL));
|
||||
String callId = "call-cross-run";
|
||||
String currentKey = PREFIX + ":" + context.runId() + ":" + callId;
|
||||
store.records.put(currentKey, CanonicalToolInvocation.projecting(
|
||||
callId, "another-run", "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-cross-run",
|
||||
"query":"pool","evidence":[{"document_id":"doc-1","source":"guide",
|
||||
"title":"guide","breadcrumb":"pool","excerpt":"active=50"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z")));
|
||||
|
||||
EvidenceGuardResult crossRun = guard.validate(
|
||||
context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(missing, EvidenceViolationCode.INVOCATION_MISSING));
|
||||
assertTrue(hasViolation(crossRun, EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE));
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectingInvocationAndEvidenceKindMismatchAreRejected() {
|
||||
String projectingId = "call-projecting";
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + projectingId,
|
||||
CanonicalToolInvocation.projecting(
|
||||
projectingId, context.runId(), "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z")));
|
||||
EvidenceGuardResult projecting = guard.validate(
|
||||
context, draft(projectingId, AnalysisKind.NORMAL));
|
||||
|
||||
String emptyId = "call-empty-normal";
|
||||
ready(emptyId, "lookup_knowledge", "{\"query\":\"pool\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-normal",
|
||||
"query":"pool","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE);
|
||||
EvidenceGuardResult mismatch = guard.validate(
|
||||
context, draft(emptyId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(projecting, EvidenceViolationCode.INVOCATION_NOT_REFERENCABLE));
|
||||
assertTrue(hasViolation(mismatch, EvidenceViolationCode.EVIDENCE_KIND_MISMATCH));
|
||||
}
|
||||
|
||||
@Test
|
||||
void projectionIdOrStatusMismatchFailsClosed() {
|
||||
String callId = "call-mismatch";
|
||||
ready(callId, "lookup_knowledge", "{\"query\":\"pool\"}", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"another-call",
|
||||
"query":"pool","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(context, draft(callId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.PROJECTION_ID_MISMATCH));
|
||||
}
|
||||
|
||||
@Test
|
||||
void canonicalRecordIdMustMatchTheDraftReferenceEvenUnderCorruptedKey() {
|
||||
String referencedId = "call-requested";
|
||||
CanonicalToolInvocation wrongRecord = CanonicalToolInvocation.projecting(
|
||||
"call-other", context.runId(), "lookup_knowledge", "{\"query\":\"pool\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-other",
|
||||
"query":"pool","evidence":[{"document_id":"doc-1","source":"guide",
|
||||
"title":"guide","breadcrumb":"pool","excerpt":"active=50"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + referencedId, wrongRecord);
|
||||
|
||||
EvidenceGuardResult result = guard.validate(
|
||||
context, draft(referencedId, AnalysisKind.NORMAL));
|
||||
|
||||
assertTrue(hasViolation(result, EvidenceViolationCode.INVOCATION_ID_MISMATCH));
|
||||
}
|
||||
|
||||
private boolean hasViolation(EvidenceGuardResult result, EvidenceViolationCode code) {
|
||||
return result.violations().stream().anyMatch(violation -> violation.code() == code);
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft(String callId, AnalysisKind kind) {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", kind, "Pool reached its limit", List.of(callId))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("a-1"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("a-1"))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private void ready(String callId, String toolName, String request,
|
||||
String agentResult, EvidenceStatus evidenceStatus) {
|
||||
String key = PREFIX + ":" + context.runId() + ":" + callId;
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), toolName, request, Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", agentResult, evidenceStatus,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(key, invocation);
|
||||
}
|
||||
|
||||
private RunContext context(String runId) {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> runId, Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict()).startRun("session-guard", runId);
|
||||
}
|
||||
|
||||
private static final class InMemoryStore implements CanonicalInvocationStore {
|
||||
private final Map<String, CanonicalToolInvocation> records = new HashMap<>();
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 1_000_000, 64_000);
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
records.put(key, invocation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse,
|
||||
String agentResult, EvidenceStatus evidenceStatus,
|
||||
Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package com.superbiz.agent.harness.guard.semantic;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class SemanticGuardTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutor() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedIsAValidBusinessDecisionAndIsNotRetried() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-semantic", "run-semantic");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"{\"verdict\":\"UNSUPPORTED\",\"reason\":\"evidence does not prove the root cause\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core,
|
||||
new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor),
|
||||
objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)),
|
||||
attempts::add);
|
||||
|
||||
SemanticGuardDecision decision = guard.review(
|
||||
context, SemanticGuardInput.from("Why did payment fail?", draft(), snapshot()));
|
||||
|
||||
assertEquals(SemanticVerdict.UNSUPPORTED, decision.verdict());
|
||||
assertEquals(1, model.calls.get());
|
||||
assertEquals(1, attempts.size());
|
||||
assertTrue(attempts.get(0).success());
|
||||
String prompt = model.prompts.get(0);
|
||||
assertTrue(prompt.contains("Why did payment fail?"));
|
||||
assertTrue(prompt.contains("active=50 max=50"));
|
||||
assertFalse(prompt.contains("call-rag-1"));
|
||||
assertFalse(prompt.contains("tool_call_id"));
|
||||
assertFalse(prompt.contains("raw_response"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void parseFailureRetriesOnceWithTheExactSameInput() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-retry", "run-retry");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"not-json", "{\"verdict\":\"SUPPORTED\",\"reason\":\"all claims are grounded\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), attempts::add);
|
||||
|
||||
SemanticGuardDecision decision = guard.review(
|
||||
context, SemanticGuardInput.from("Why did payment fail?", draft(), snapshot()));
|
||||
|
||||
assertEquals(SemanticVerdict.SUPPORTED, decision.verdict());
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(model.prompts.get(0), model.prompts.get(1));
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.PARSE_ERROR,
|
||||
attempts.get(0).failure());
|
||||
assertTrue(attempts.get(1).success());
|
||||
assertEquals(2, context.budget().snapshot().modelCalls());
|
||||
assertEquals(16, context.budget().snapshot().totalTokens());
|
||||
}
|
||||
|
||||
@Test
|
||||
void attemptTimeoutCancelsBothPermittedModelCalls() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-timeout", "run-timeout");
|
||||
BlockingChatModel model = new BlockingChatModel(2);
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofMillis(50), Duration.ofMillis(500)), attempts::add);
|
||||
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException failure = assertThrows(
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException.class,
|
||||
() -> guard.review(context,
|
||||
SemanticGuardInput.from("Why?", draft(), snapshot())));
|
||||
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.TIMEOUT, failure.failure());
|
||||
assertEquals(2, failure.attempts());
|
||||
assertTrue(model.interrupted.await(2, TimeUnit.SECONDS));
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(2, attempts.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void runCancellationInterruptsPendingModelAndDoesNotRetry() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-cancel", "run-cancel");
|
||||
BlockingChatModel model = new BlockingChatModel(1);
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new SemanticGuardLimits(100_000, 10_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { });
|
||||
|
||||
CompletableFuture<SemanticGuardDecision> execution = CompletableFuture.supplyAsync(
|
||||
() -> guard.review(context,
|
||||
SemanticGuardInput.from("Why?", draft(), snapshot())));
|
||||
assertTrue(model.started.await(2, TimeUnit.SECONDS));
|
||||
core.cancel(context,
|
||||
com.superbiz.agent.harness.core.RunCancellationReason.USER_REQUESTED);
|
||||
|
||||
CompletionException thrown = assertThrows(CompletionException.class, execution::join);
|
||||
com.superbiz.agent.harness.retry.RetryExecutionException failure =
|
||||
(com.superbiz.agent.harness.retry.RetryExecutionException) thrown.getCause();
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.CANCELLED, failure.failure());
|
||||
assertTrue(model.interrupted.await(2, TimeUnit.SECONDS));
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft() {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "Pool reached its limit", List.of("call-rag-1"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("a-1"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("a-1"))),
|
||||
new DiagnosisDraft.Limitations("order-service, last 30 minutes", List.of("No slow SQL")));
|
||||
}
|
||||
|
||||
private VerifiedEvidenceSnapshot snapshot() {
|
||||
return new VerifiedEvidenceSnapshot(List.of(new VerifiedAnalysisEvidence(
|
||||
"a-1", "Pool reached its limit", AnalysisKind.NORMAL,
|
||||
List.of(new VerifiedEvidence(
|
||||
"LOG", "APPLICATION (MOCK)", "order-service, last 30 minutes",
|
||||
"2026-07-21T10:29:00Z", "active=50 max=50", Map.of("count", 1))))));
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
private final List<String> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedChatModel(String... responses) {
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
int index = calls.getAndIncrement();
|
||||
ChatResponseMetadata metadata = ChatResponseMetadata.builder()
|
||||
.usage(new DefaultUsage(5, 3)).build();
|
||||
return new ChatResponse(
|
||||
List.of(new Generation(new AssistantMessage(responses.get(index)))), metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class BlockingChatModel implements ChatModel {
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
private final CountDownLatch started = new CountDownLatch(1);
|
||||
private final CountDownLatch interrupted;
|
||||
|
||||
private BlockingChatModel(int expectedInterruptions) {
|
||||
this.interrupted = new CountDownLatch(expectedInterruptions);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
calls.incrementAndGet();
|
||||
started.countDown();
|
||||
try {
|
||||
Thread.sleep(10_000);
|
||||
throw new AssertionError("blocking model should be interrupted");
|
||||
} catch (InterruptedException exception) {
|
||||
interrupted.countDown();
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException("interrupted", exception);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
class DiagnosisReleaseUseCaseTest {
|
||||
|
||||
private static final String PREFIX = "superbiz:harness:tool-call";
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@AfterEach
|
||||
void shutdownExecutor() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void supportedVerifiedDraftIsReleasedUnchangedWithoutRepair() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-release", "run-release");
|
||||
InMemoryStore store = new InMemoryStore();
|
||||
ready(store, context, "call-rag-1");
|
||||
ScriptedChatModel model = new ScriptedChatModel(
|
||||
"{\"verdict\":\"SUPPORTED\",\"reason\":\"all claims are grounded\"}");
|
||||
HarnessRetryExecutor retries = new HarnessRetryExecutor(core);
|
||||
GuardModelCall modelCall = new GuardModelCall(core, model, executor);
|
||||
DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase(
|
||||
new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper),
|
||||
new EvidenceRepair(core, retries, modelCall, objectMapper,
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }),
|
||||
new SemanticGuard(core, retries, modelCall, objectMapper,
|
||||
new SemanticGuardLimits(100_000, 20_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }),
|
||||
new SafeFallbackFactory());
|
||||
DiagnosisDraft draft = draft("a-1", "call-rag-1", "Pool exhausted");
|
||||
|
||||
DiagnosisReleaseResult result = useCase.execute(context, "Why did payment fail?", draft);
|
||||
|
||||
assertEquals(ReleaseOutcome.SUCCESS, result.outcome());
|
||||
assertSame(draft, result.draft());
|
||||
assertNull(result.fallback());
|
||||
assertEquals(1, result.verifiedEvidence().analyses().size());
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
|
||||
Fixture fixture = fixture(
|
||||
repairedDuplicateDraft("Pool exhausted", "a-2"),
|
||||
"{\"verdict\":\"SUPPORTED\",\"reason\":\"grounded\"}");
|
||||
DiagnosisDraft original = duplicateDraft();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", original);
|
||||
|
||||
assertEquals(ReleaseOutcome.SUCCESS, result.outcome());
|
||||
assertEquals(List.of("a-1", "a-2"), result.draft().analysis().stream()
|
||||
.map(DiagnosisDraft.AnalysisItem::analysisId).toList());
|
||||
assertEquals("Pool exhausted", result.draft().conclusion().text());
|
||||
assertEquals(2, fixture.model.calls.get());
|
||||
assertFalse(fixture.model.prompts.get(1).contains("call-rag-1"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void repairThatChangesVisibleSemanticsReturnsEvidenceFallback() throws Exception {
|
||||
Fixture fixture = fixture(repairedDuplicateDraft("Different conclusion", "a-2"));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", duplicateDraft());
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("Pool exhausted"));
|
||||
assertFalse(json.contains("Different conclusion"));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void secondEvidenceValidationFailureHasNoVerifiedSources() {
|
||||
Fixture fixture = fixture(repairedDuplicateDraft("Pool exhausted", "a-1"));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", duplicateDraft());
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unsupportedNeverReleasesDraftOrAuditReason() throws Exception {
|
||||
Fixture fixture = fixture(
|
||||
"{\"verdict\":\"UNSUPPORTED\",\"reason\":\"secret-audit-reason\"}");
|
||||
DiagnosisDraft draft = draft("a-1", "call-rag-1", "Pool exhausted");
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?", draft);
|
||||
|
||||
assertFallback(result, FallbackType.SEMANTIC_UNSUPPORTED, 1);
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("Pool exhausted"));
|
||||
assertFalse(json.contains("secret-audit-reason"));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void twoInvalidSemanticOutputsReturnUnavailableWithoutDraft() throws Exception {
|
||||
Fixture fixture = fixture("not-json", "still-not-json");
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context, "Why did payment fail?",
|
||||
draft("a-1", "call-rag-1", "Pool exhausted"));
|
||||
|
||||
assertFallback(result, FallbackType.SEMANTIC_UNAVAILABLE, 1);
|
||||
assertFalse(objectMapper.writeValueAsString(result).contains("Pool exhausted"));
|
||||
assertEquals(2, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
private Fixture fixture(String... responses) {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-fixture", "run-fixture");
|
||||
InMemoryStore store = new InMemoryStore();
|
||||
ready(store, context, "call-rag-1");
|
||||
ScriptedChatModel model = new ScriptedChatModel(responses);
|
||||
HarnessRetryExecutor retries = new HarnessRetryExecutor(core);
|
||||
GuardModelCall modelCall = new GuardModelCall(core, model, executor);
|
||||
DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase(
|
||||
new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper),
|
||||
new EvidenceRepair(core, retries, modelCall, objectMapper,
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }),
|
||||
new SemanticGuard(core, retries, modelCall, objectMapper,
|
||||
new SemanticGuardLimits(100_000, 20_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }),
|
||||
new SafeFallbackFactory());
|
||||
return new Fixture(context, model, useCase);
|
||||
}
|
||||
|
||||
private DiagnosisDraft duplicateDraft() {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("Pool exhausted", List.of("dup")),
|
||||
List.of(
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"dup", AnalysisKind.NORMAL, "Pool reached its limit", List.of("call-rag-1")),
|
||||
new DiagnosisDraft.AnalysisItem(
|
||||
"dup", AnalysisKind.NORMAL, "Requests are waiting", List.of("call-rag-1"))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of("dup"), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of("dup"))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private String repairedDuplicateDraft(String conclusion, String secondAnalysisId) {
|
||||
return """
|
||||
{
|
||||
"conclusion":{"text":"%s","based_on_analysis_ids":["a-1"]},
|
||||
"analysis":[
|
||||
{"analysis_id":"a-1","kind":"NORMAL","text":"Pool reached its limit","tool_call_ids":["call-rag-1"]},
|
||||
{"analysis_id":"%s","kind":"NORMAL","text":"Requests are waiting","tool_call_ids":["call-rag-1"]}
|
||||
],
|
||||
"action_plan":[{"action":"Inspect long transactions","based_on_analysis_ids":["a-1"],"requires_human_confirmation":false}],
|
||||
"recommendations":[{"text":"Add saturation alert","based_on_analysis_ids":["a-1"]}],
|
||||
"limitations":{"scope":"order-service","missing_info":[]}
|
||||
}
|
||||
""".formatted(conclusion, secondAnalysisId);
|
||||
}
|
||||
|
||||
private void assertFallback(DiagnosisReleaseResult result,
|
||||
FallbackType type, int expectedSources) {
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertNull(result.draft());
|
||||
assertNotNull(result.fallback());
|
||||
assertEquals(type, result.fallback().type());
|
||||
assertNull(result.fallback().conclusion());
|
||||
assertEquals(expectedSources, result.fallback().verifiedSources().size());
|
||||
assertEquals(0, result.verifiedEvidence().analyses().size());
|
||||
}
|
||||
|
||||
private DiagnosisDraft draft(String analysisId, String callId, String conclusion) {
|
||||
return new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion(conclusion, List.of(analysisId)),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
analysisId, AnalysisKind.NORMAL, "Pool reached its limit", List.of(callId))),
|
||||
List.of(new DiagnosisDraft.ActionPlanItem(
|
||||
"Inspect long transactions", List.of(analysisId), false)),
|
||||
List.of(new DiagnosisDraft.Recommendation(
|
||||
"Add saturation alert", List.of(analysisId))),
|
||||
new DiagnosisDraft.Limitations("order-service", List.of()));
|
||||
}
|
||||
|
||||
private void ready(InMemoryStore store, RunContext context, String callId) {
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), "lookup_knowledge", "{\"query\":\"pool timeout\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-rag-1",
|
||||
"query":"pool timeout","evidence":[{"document_id":"doc-1",
|
||||
"source":"runbook.md","title":"Pool guide","breadcrumb":"DB > Pool",
|
||||
"excerpt":"active=50 max=50"}],"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
private final List<String> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedChatModel(String... responses) {
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
int index = calls.getAndIncrement();
|
||||
ChatResponseMetadata metadata = ChatResponseMetadata.builder()
|
||||
.usage(new DefaultUsage(5, 3)).build();
|
||||
return new ChatResponse(
|
||||
List.of(new Generation(new AssistantMessage(responses.get(index)))), metadata);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class InMemoryStore implements CanonicalInvocationStore {
|
||||
private final Map<String, CanonicalToolInvocation> records = new HashMap<>();
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(2), 1_000_000, 64_000);
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
records.put(key, invocation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse,
|
||||
String agentResult, EvidenceStatus evidenceStatus,
|
||||
Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
private record Fixture(
|
||||
RunContext context,
|
||||
ScriptedChatModel model,
|
||||
DiagnosisReleaseUseCase useCase) {
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user