feat(harness): add information gain stop and audit

This commit is contained in:
aruo
2026-07-27 01:03:34 +08:00
parent de5a5b09d9
commit d0452184ee
92 changed files with 5019 additions and 122 deletions
@@ -31,6 +31,15 @@ import static org.mockito.Mockito.mock;
class HarnessChatConfigurationTest {
@Test
void diagnosisNoGainThresholdDefaultsAndValidates() {
ChatHarnessProperties properties = new ChatHarnessProperties();
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
properties.setStopAfterConsecutiveNoGain(0);
assertThrows(IllegalArgumentException.class, properties::validate);
}
@Test
void defaultsArePositiveAndExecutorsAreBounded() {
ChatHarnessProperties properties = new ChatHarnessProperties();
@@ -0,0 +1,29 @@
package com.superbiz.agent.harness.agent;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisAgentPromptTest {
@Test
void promptLicensesAbandonmentWithoutDuplicatingToolSchema() {
String prompt = DiagnosisAgentPrompt.load();
assertTrue(prompt.contains("你不必须给出根因"));
assertTrue(prompt.contains("conclusion=null"));
assertTrue(prompt.contains("可以不调用工具"));
assertTrue(prompt.contains("GAINED"));
assertTrue(prompt.contains("NO_GAIN"));
assertTrue(prompt.contains("正确,但只是通用说明、重复内容"));
assertTrue(prompt.contains("STOP_REQUIRED"));
assertFalse(prompt.contains("lookup_knowledge"));
assertFalse(prompt.contains("query_logs"));
assertFalse(prompt.contains("query_mysql"));
assertFalse(prompt.contains("previous_observation"));
assertFalse(prompt.contains("next_action"));
assertFalse(prompt.contains("阈值"));
assertFalse(prompt.contains("计数器"));
}
}
@@ -16,6 +16,8 @@ import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
@@ -71,8 +73,9 @@ class DiagnosisAgentUseCaseTest {
"上一轮支付服务为什么超时?", "支付服务连接池已耗尽", "payment-service",
List.of("未覆盖退款服务"), List.of(new SourceDocument("doc-1", "连接池手册")));
DiagnosisDraft draft = useCase.execute(
DiagnosisAgentExecution execution = useCase.execute(
context, new DiagnosisAgentInput("那退款服务呢?", previous));
DiagnosisDraft draft = execution.draft();
assertEquals("退款服务出现连接池等待", draft.conclusion().text());
assertEquals(AnalysisKind.NORMAL, draft.analysis().get(0).kind());
@@ -109,8 +112,9 @@ class DiagnosisAgentUseCaseTest {
toolCall("call-empty-1", "{\"query\":\"refund timeout\"}"),
new AssistantMessage(noEvidenceDraft("call-empty-1")));
DiagnosisDraft draft = useCase(core, model, tools, LARGE_LIMITS)
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("退款为什么超时?", null));
DiagnosisDraft draft = execution.draft();
assertNull(draft.conclusion());
assertEquals(AnalysisKind.NEGATIVE_OBSERVATION, draft.analysis().get(0).kind());
@@ -119,6 +123,32 @@ class DiagnosisAgentUseCaseTest {
assertEquals(1, toolCalls.get());
}
@Test
void ignoredStopInstructionCannotExecuteAnotherTool() {
DiagnosisHarnessCore core = core(new RunBudgetLimits(6, 6, 6,
100, 100, 200, 100_000));
RunContext context = core.startRun("session-stop", "run-stop");
AtomicInteger toolCalls = new AtomicInteger();
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
toolCalls.incrementAndGet();
return ToolBoundaryResult.ready(id, noEvidence(id), EvidenceStatus.NO_EVIDENCE);
});
ScriptedChatModel model = new ScriptedChatModel(1, 1,
toolCall("call-empty-1", "{\"query\":\"first\"}"),
toolCall("call-empty-2", "{\"query\":\"second\"}"),
toolCall("call-ignored-stop", "{\"query\":\"third\"}"));
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, execution.stopReason());
assertEquals(3, model.calls());
assertEquals(2, toolCalls.get());
assertEquals(2, context.budget().snapshot().toolCalls());
}
@Test
void fencedOutputFailsClosedWithoutAgentRetry() {
DiagnosisHarnessCore core = core(defaultBudget());
@@ -126,14 +156,45 @@ class DiagnosisAgentUseCaseTest {
new AssistantMessage("```json\n" + supportedDraft("call-1") + "\n```"));
RunContext context = core.startRun("session-fenced", "run-fenced");
assertThrows(DiagnosisAgentOutputException.class, () ->
DiagnosisAgentOutputException failure = assertThrows(
DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
assertFalse(failure.progress().hasObservedFacts());
assertEquals(1, model.calls());
assertEquals(1, context.budget().snapshot().modelCalls());
}
@Test
void invalidDraftCarriesOnlyProjectedProgressForOuterReleaseBoundary() {
DiagnosisHarnessCore core = core(defaultBudget());
ScriptedChatModel model = new ScriptedChatModel(1, 1,
new AssistantMessage("not-json"));
RunContext context = core.startRun("session-invalid-progress", "run-invalid-progress");
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
List.of(new com.superbiz.agent.harness.contract.SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new com.superbiz.agent.harness.contract.SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown", "已完成知识范围检查")),
List.of(), null);
DiagnosisAgentFactory factory = new DiagnosisAgentFactory(
model, core, tools(errorInvoker()), objectMapper, List.of());
DiagnosisAgentUseCase useCase = new DiagnosisAgentUseCase(
core, factory, objectMapper, LARGE_LIMITS, ignored -> progress);
DiagnosisAgentOutputException failure = assertThrows(
DiagnosisAgentOutputException.class,
() -> useCase.execute(
context, new DiagnosisAgentInput("诊断未知故障", null)));
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
assertEquals("已完成知识范围检查",
failure.progress().observedFacts().get(0).summary());
assertFalse(failure.getMessage().contains("not-json"));
}
@Test
void modelCallBudgetStopsNextReactRoundBeforeChatModel() {
DiagnosisHarnessCore core = core(new RunBudgetLimits(1, 2, 2,
@@ -147,10 +208,11 @@ class DiagnosisAgentUseCaseTest {
toolCall("call-budget-1", "{\"query\":\"timeout\"}"),
new AssistantMessage(supportedDraft("call-budget-1")));
assertThrows(DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
assertEquals(1, model.calls());
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
}
@@ -163,10 +225,11 @@ class DiagnosisAgentUseCaseTest {
ScriptedChatModel model = new ScriptedChatModel(6, 6,
new AssistantMessage(supportedDraft("call-unused")));
assertThrows(DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
DiagnosisAgentExecution execution = useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
assertEquals(1, model.calls());
assertEquals(12, context.budget().snapshot().totalTokens());
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
@@ -261,7 +324,8 @@ class DiagnosisAgentUseCaseTest {
return AssistantMessage.builder()
.content("")
.toolCalls(List.of(new AssistantMessage.ToolCall(
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE, arguments)))
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"input\":" + arguments + "}")))
.build();
}
@@ -0,0 +1,58 @@
package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
import org.springframework.ai.chat.metadata.DefaultUsage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
class HarnessModelInterceptorTest {
@Test
void extractsProviderUsageWithoutPersistingModelContent() {
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 100_000),
HarnessRetryPolicies.strict());
RunContext context = core.startRun("session-agent-token", "run-agent-token");
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessModelInterceptor interceptor = new HarnessModelInterceptor(
core, context,
new ModelCallAuditor(core, trace::add, mock(AgentStepRepository.class)));
AssistantMessage assistant = new AssistantMessage("secret-model-content");
ChatResponse response = new ChatResponse(
List.of(new Generation(assistant)),
ChatResponseMetadata.builder().usage(new DefaultUsage(21, 4)).build());
ModelResponse actual = interceptor.interceptModel(
ModelRequest.builder().messages(List.of()).build(),
ignored -> ModelResponse.of(assistant, response));
assertEquals(response, actual.getChatResponse());
assertEquals(25, context.budget().snapshot().totalTokens());
assertEquals(25, context.modelCalls().snapshot().totalTokens());
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, trace.get(0).eventType());
assertEquals("DIAGNOSIS_AGENT", trace.get(0).details().get("component"));
assertFalse(trace.get(0).details().toString().contains("secret-model-content"));
}
}
@@ -4,6 +4,8 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
@@ -23,6 +25,7 @@ import org.springframework.ai.tool.execution.ToolExecutionException;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@@ -52,7 +55,9 @@ class HarnessToolInterceptorTest {
AgentToolContracts.QUERY_MYSQL),
callbacks.stream().map(callback -> callback.getToolDefinition().name()).toList());
callbacks.forEach(callback -> {
assertFalse(callback.getToolDefinition().inputSchema().contains("tool_call_id"));
assertTrue(callback.getToolDefinition().inputSchema().contains("previous_observation"));
assertTrue(callback.getToolDefinition().inputSchema().contains("information_gain"));
assertTrue(callback.getToolDefinition().inputSchema().contains("\"input\""));
assertFalse(callback.getToolDefinition().description().isBlank());
assertThrows(ToolExecutionException.class, () -> callback.call("{}"));
});
@@ -72,7 +77,8 @@ class HarnessToolInterceptorTest {
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7", "{\"query\":\"timeout\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7",
"{\"input\":{\"query\":\"timeout\"}}"),
ignored -> {
throw new AssertionError("registered Tool must not bypass Harness interceptor");
});
@@ -82,6 +88,7 @@ class HarnessToolInterceptorTest {
assertEquals("framework-call-7", envelope.getValue().toolCallId());
assertEquals(context.runId(), envelope.getValue().runId());
assertEquals(AgentToolContracts.LOOKUP_KNOWLEDGE, envelope.getValue().toolName());
assertEquals("{\"query\":\"timeout\"}", envelope.getValue().requestJson());
assertEquals("framework-call-7", response.getToolCallId());
assertTrue(response.getResult().contains("framework-call-7"));
}
@@ -94,7 +101,8 @@ class HarnessToolInterceptorTest {
context(3, 3, 3), tools, objectMapper);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error", "{\"query\":\"secret\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error",
"{\"input\":{\"query\":\"secret\"}}"),
ignored -> null);
JsonNode observation = objectMapper.readTree(response.getResult());
@@ -142,13 +150,164 @@ class HarnessToolInterceptorTest {
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1", "{\"query\":\"one\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"),
ignored -> null);
assertEquals(1, invocations.get());
assertEquals(1, context.budget().snapshot().toolCalls());
}
@Test
void requiresExactPreviousEvaluationBeforeAnotherToolExecutes() throws Exception {
AtomicInteger invocations = new AtomicInteger();
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
ToolCallResponse missing = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
ToolCallResponse accepted = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
"input":{"query":"three"}}
"""), ignored -> null);
assertTrue(missing.isError());
assertEquals("INVALID_PROGRESS_PROTOCOL",
objectMapper.readTree(missing.getResult()).path("error_code").asText());
assertFalse(accepted.isError());
assertEquals(2, invocations.get());
DiagnosisTraceAuditEvent rejected = trace.stream()
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
.findFirst().orElseThrow();
assertEquals("call-2", rejected.details().get("tool_call_id"));
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
String details = rejected.details().toString();
assertFalse(details.contains("two"));
assertFalse(details.contains("previous_observation"));
}
@Test
void duplicateSuccessfulScopeIsNotExecutedAndCanSaturateRun() throws Exception {
AtomicInteger invocations = new AtomicInteger();
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"same\"}}"), ignored -> null);
ToolCallResponse duplicate = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"NO_GAIN"},
"input":{"query":"same"}}
"""), ignored -> null);
JsonNode observation = objectMapper.readTree(duplicate.getResult());
assertEquals(1, invocations.get());
assertTrue(observation.path("stop_required").asBoolean());
assertEquals("INFORMATION_SATURATED", observation.path("reason").asText());
assertTrue(trace.stream().anyMatch(event ->
event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED
&& "DUPLICATE_SCOPE".equals(event.details().get("error_code"))));
}
@Test
void observationContractRejectionIsTracedWithoutPayload() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) ->
ToolBoundaryResult.ready(id, """
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-contract",
"query":"secret-query","evidence":[],"returned_count":0,"truncated":false}
""", EvidenceStatus.EVIDENCE_FOUND));
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper, trace::add);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-contract",
"{\"input\":{\"query\":\"secret-query\"}}"), ignored -> null);
assertTrue(response.isError());
DiagnosisTraceAuditEvent rejected = trace.stream()
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
.findFirst().orElseThrow();
assertEquals("OBSERVATION_CONTRACT_MISMATCH", rejected.details().get("error_code"));
assertFalse(rejected.details().toString().contains("secret-query"));
}
@Test
void modelInformationGainIsRecordedWithoutRawScope() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> ready(id));
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"secret-query-value\"}}"), ignored -> null);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
"input":{"query":"different-secret-query"}}
"""), ignored -> null);
assertEquals(1, trace.size());
DiagnosisTraceAuditEvent event = trace.get(0);
assertEquals(TraceEventType.TOOL_PROGRESS, event.eventType());
assertEquals("MODEL", event.details().get("producer"));
assertEquals("GAINED", event.details().get("information_gain"));
assertTrue(event.details().get("scope_summary").toString()
.startsWith(AgentToolContracts.LOOKUP_KNOWLEDGE + "#"));
assertFalse(event.details().toString().contains("secret-query-value"));
}
@Test
void harnessProgressTraceDoesNotContainSqlParamsPayloadOrBudget() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
EvidenceToolInvoker unused = (context, id, arguments) ->
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
EvidenceToolInvoker mysql = (context, id, arguments) -> ToolBoundaryResult.ready(
id,
"{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
+ "\",\"columns\":[],\"rows\":[],\"returned_count\":0,"
+ "\"truncated\":false}",
EvidenceStatus.NO_EVIDENCE);
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, mysql);
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.QUERY_MYSQL, "call-sql", """
{"input":{"data_source":"orders","sql":"select * from secret_table where tenant_id = ?",
"params":["tenant-secret"]}}
"""), ignored -> null);
String recorded = trace.toString();
assertEquals(List.of(TraceEventType.TOOL_PROGRESS),
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
assertTrue(recorded.contains("HARNESS"));
assertTrue(recorded.contains("NO_GAIN"));
assertFalse(recorded.contains("secret_table"));
assertFalse(recorded.contains("tenant-secret"));
assertFalse(recorded.contains("raw_response"));
assertFalse(recorded.contains("remaining"));
assertFalse(recorded.contains("prompt"));
assertFalse(recorded.contains("thought"));
}
private HarnessEvidenceTools fakeTools(EvidenceToolInvoker ragInvoker) {
EvidenceToolInvoker unused = (context, id, arguments) ->
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
@@ -162,7 +321,8 @@ class HarnessToolInterceptorTest {
private ToolBoundaryResult ready(String toolCallId) {
return ToolBoundaryResult.ready(toolCallId,
"{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\""
+ toolCallId + "\",\"evidence\":[]}",
+ toolCallId + "\",\"query\":\"scope\",\"evidence\":[],"
+ "\"returned_count\":1,\"truncated\":false}",
EvidenceStatus.EVIDENCE_FOUND);
}
@@ -0,0 +1,57 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ToolResultViewProjectorTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolResultViewProjector projector = new ToolResultViewProjector(objectMapper);
@Test
void separatesRagControlDataFromModelObservation() throws Exception {
String canonical = """
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
"evidence":[{"document_id":"doc-1","excerpt":"generic"}],"returned_count":1,
"relevance_level":"REFERENCE","truncated":false,"raw_score":0.42}
""";
ToolControlView control = projector.controlView(canonical);
JsonNode model = objectMapper.readTree(projector.modelObservation(
AgentToolContracts.LOOKUP_KNOWLEDGE, canonical, "{\"query\":\"unknown\"}",
true, DiagnosisStopReason.INFORMATION_SATURATED));
assertEquals(1, control.returnedCount());
assertEquals(RagRelevanceLevel.REFERENCE, control.relevanceLevel());
assertEquals("REFERENCE", model.path("relevance_level").asText());
assertEquals("unknown", model.path("scope").path("query").asText());
assertTrue(model.path("stop_required").asBoolean());
assertFalse(model.has("returned_count"));
assertFalse(model.has("raw_score"));
}
@Test
void mysqlObservationUsesNormalizedScopeWithoutInternalFields() throws Exception {
String canonical = """
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-2","columns":["id"],
"rows":[{"id":1}],"returned_count":1,"truncated":false}
""";
JsonNode model = objectMapper.readTree(projector.modelObservation(
AgentToolContracts.QUERY_MYSQL, canonical,
"{\"data_source\":\"orders\",\"sql\":\"SELECT id FROM orders\",\"params\":[]}",
false, null));
assertEquals("orders", model.path("scope").path("data_source").asText());
assertEquals(1, model.path("rows").size());
assertFalse(model.has("returned_count"));
}
}
@@ -9,14 +9,17 @@ import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
import com.superbiz.agent.harness.contract.AnalysisKind;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.contract.PreviousTurn;
import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import org.junit.jupiter.api.Test;
@@ -157,6 +160,88 @@ class ChatApplicationUseCaseTest {
store.startedContext.lifecycle().state());
}
@Test
void handledDiagnosisBudgetFallbackIsPersistedWithoutReclassification() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
SafeFallback fallback = new SafeFallback(
FallbackType.INSUFFICIENT_EVIDENCE,
null,
"已完成有限范围的检查,但现有证据不足以确认根因",
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of("当前已检查范围不足以支持根因结论"),
List.of("补充故障上下文后重试"),
"DIAGNOSIS_COLLECTION",
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"该知识检索范围内未发现可用文档证据")),
List.of());
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK, new FallbackContent(fallback), null, true);
});
ChatApplicationResult result = useCase.execute(
new ChatApplicationRequest("diagnose unknown issue", "session-budget-fallback"));
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
assertEquals(ChatContentType.SAFE_FALLBACK, result.contentType());
FallbackContent content = (FallbackContent) result.content();
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, content.fallback().type());
assertEquals(1, content.fallback().observedFacts().size());
assertEquals(ReleaseOutcome.FALLBACK, store.finishedOutcome);
assertTrue(store.finishedSafeContentJson.contains("knowledge_base"));
}
@Test
void unhandledDiagnosisBudgetFailureDoesNotBuildFallbackInApplication() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.budget().reserveToolCall("lookup_knowledge");
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
throw new IllegalStateException("unhandled diagnosis failure");
});
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
useCase.execute(new ChatApplicationRequest(
"diagnose unknown issue", "session-budget-unhandled")));
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
}
@Test
void diagnosisBudgetExhaustionWithoutToolCallsRemainsFailure() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "model budget exhausted");
throw new IllegalStateException("diagnosis failed before evidence collection");
});
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
useCase.execute(new ChatApplicationRequest(
"diagnose unavailable model", "session-budget-no-tools")));
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
}
@Test
void clientDisconnectControlCancelsSameRunAndPersistsCancelled() {
FakeStore store = new FakeStore();
@@ -252,6 +337,7 @@ class ChatApplicationUseCaseTest {
private IntentType finishedIntent;
private ReleaseOutcome finishedOutcome;
private PublishedResult finishedPublishedResult;
private String finishedSafeContentJson;
@Override
public Optional<RoutingHistory> findLatestRoutingHistory(String sessionId) {
@@ -283,6 +369,7 @@ class ChatApplicationUseCaseTest {
finishedIntent = intent;
finishedOutcome = outcome;
finishedPublishedResult = publishedResult;
finishedSafeContentJson = safeContentJson;
}
}
}
@@ -14,6 +14,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
@@ -101,6 +102,8 @@ class PublishedResultPersistenceTest {
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
RunContext context = core().startRun("session-1", "run-fallback");
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
@@ -111,6 +114,7 @@ class PublishedResultPersistenceTest {
assertEquals(ReleaseOutcome.FALLBACK, entity.getReleaseOutcome());
assertNull(entity.getPublishedResult());
assertEquals(12, entity.getTotalDurationMs());
assertEquals(2, entity.getStepCount());
}
@Test
@@ -0,0 +1,128 @@
package com.superbiz.agent.harness.application.executor;
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
import com.superbiz.agent.harness.application.ChatApplicationStatus;
import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
import com.superbiz.agent.harness.application.persistence.PreviousTurnLimits;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.release.DiagnosisReleaseResult;
import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
import com.superbiz.agent.harness.release.SafeFallbackFactory;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class DiagnosisChatExecutorTest {
@Test
void invalidDraftWithVerifiedProgressUsesDeterministicReleaseFallback() {
RunContext context = context("run-invalid-progress");
DiagnosisProgressSnapshot progress = progress();
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
new IllegalArgumentException("redacted"),
DiagnosisAgentOutputException.Kind.SCHEMA_INVALID,
321L,
progress);
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(progress, List.of());
when(release.releaseInvalidDraft(context, progress))
.thenReturn(DiagnosisReleaseResult.fallback(fallback));
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
agent,
release,
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
trace::add);
List<ChatApplicationStatus> statuses = new ArrayList<>();
DiagnosisExecutionResult result = executor.execute(
context, "诊断未知故障", null, statuses::add);
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE,
((com.superbiz.agent.harness.application.FallbackContent) result.content())
.fallback().type());
assertEquals(List.of(
ChatApplicationStatus.DIAGNOSIS_RUNNING,
ChatApplicationStatus.SAFETY_VALIDATING), statuses);
assertEquals(List.of(TraceEventType.AGENT_DRAFT_INVALID),
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
assertEquals("SCHEMA_INVALID", trace.get(0).details().get("failure_kind"));
assertEquals(321L, trace.get(0).details().get("output_bytes"));
assertEquals(true, trace.get(0).details().get("has_publishable_progress"));
assertEquals(3, trace.get(0).details().size());
}
@Test
void invalidDraftWithoutVerifiedProgressStillFailsClosed() {
RunContext context = context("run-invalid-empty");
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
null,
DiagnosisAgentOutputException.Kind.INVALID_JSON,
12L,
DiagnosisProgressSnapshot.empty());
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
agent,
release,
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
trace::add);
DiagnosisAgentOutputException actual = assertThrows(
DiagnosisAgentOutputException.class,
() -> executor.execute(context, "诊断未知故障", null, ignored -> { }));
assertSame(failure, actual);
verifyNoInteractions(release);
assertEquals(false, trace.get(0).details().get("has_publishable_progress"));
}
private RunContext context(String runId) {
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused-run", Duration.ofMinutes(5),
new RunBudgetLimits(4, 3, 2, 100, 100, 200, 100_000),
HarnessRetryPolicies.strict());
return core.startRun("session-executor", runId);
}
private DiagnosisProgressSnapshot progress() {
return new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"已完成知识范围检查")),
List.of(), null);
}
}
@@ -0,0 +1,114 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ModelCallAuditorTest {
@Test
void recordsComponentUsageIntoRunBudgetTraceAndAgentStep() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-token", "run-token");
AgentStepRepository steps = mock(AgentStepRepository.class);
AgentStep step = AgentStep.builder().id(7L).runId(context.runId()).stepIndex(0).build();
when(steps.findByRunIdAndStepIndex(context.runId(), 0)).thenReturn(Optional.of(step));
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
ModelCallAuditor auditor = new ModelCallAuditor(core, trace::add, steps);
core.beforeModelCall(context);
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
auditor.recordUsage(context, call, 13, 5, true);
assertEquals(18, context.budget().snapshot().totalTokens());
assertEquals(18, context.modelCalls().snapshot().totalTokens());
assertEquals(18, step.getTokenCount());
verify(steps).save(step);
DiagnosisTraceAuditEvent event = trace.get(0);
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, event.eventType());
assertEquals("DIAGNOSIS_AGENT", event.details().get("component"));
assertEquals(1, event.details().get("component_round"));
assertEquals(13L, event.details().get("input_tokens"));
assertEquals(5L, event.details().get("output_tokens"));
assertEquals(18L, event.details().get("total_tokens"));
assertEquals(true, event.details().get("usage_available"));
String details = event.details().toString().toLowerCase();
assertFalse(details.contains("prompt"));
assertFalse(details.contains("content"));
assertFalse(details.contains("response"));
assertFalse(details.contains("argument"));
}
@Test
void unavailableUsageIsVisibleWithoutFabricatingTokens() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-missing", "run-missing");
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
ModelCallAuditor auditor = new ModelCallAuditor(
core, trace::add, mock(AgentStepRepository.class));
core.beforeModelCall(context);
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.INTENT_ROUTER);
auditor.recordUsage(context, call, 0, 0, false);
ModelCallLedger.Snapshot snapshot = context.modelCalls().snapshot();
assertEquals(1, snapshot.auditedCallCount());
assertEquals(1, snapshot.usageUnavailableCount());
assertEquals(0, snapshot.totalTokens());
assertEquals(0, context.budget().snapshot().totalTokens());
assertEquals(false, trace.get(0).details().get("usage_available"));
assertFalse(trace.get(0).details().containsKey("input_tokens"));
assertFalse(trace.get(0).details().containsKey("output_tokens"));
assertFalse(trace.get(0).details().containsKey("total_tokens"));
DiagnosisTraceAuditEvent finished = TraceAuditEvents.runFinished(
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
com.superbiz.agent.harness.contract.ReleaseOutcome.FAILED, 1);
assertEquals(false, finished.details().get("tokens_reconciled"));
}
@Test
void runFinishedExposesTokenReconciliationWithoutContent() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-finish", "run-finish");
ModelCallAuditor auditor = new ModelCallAuditor(
core, DiagnosisTraceRecorder.noop(), mock(AgentStepRepository.class));
core.beforeModelCall(context);
auditor.recordUsage(context,
auditor.begin(context, ModelCallComponent.INTENT_ROUTER), 8, 2, true);
DiagnosisTraceAuditEvent event = TraceAuditEvents.runFinished(
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, 50);
assertEquals(10L, event.details().get("run_total_tokens"));
assertEquals(10L, event.details().get("audited_total_tokens"));
assertEquals(true, event.details().get("tokens_reconciled"));
assertEquals(0, event.details().get("usage_unavailable_count"));
assertTrue(event.details().containsKey("audited_model_call_count"));
assertFalse(event.details().toString().contains("secret"));
}
private DiagnosisHarnessCore core() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
new RunBudgetLimits(20, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
HarnessRetryPolicies.strict());
}
}
@@ -0,0 +1,185 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisProgressProjectorTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolCallKeyFactory keys = new ToolCallKeyFactory("test:tool");
@Test
void projectsReadyCurrentRunRecordsInExecutionOrder() {
RunContext context = context();
context.progress().recordCompleted(
new CompletedToolCall("call-1", AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}"),
EvidenceStatus.EVIDENCE_FOUND);
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
records.put(keys.create(context.runId(), "call-1"), ready(
"call-1", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"""
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
"evidence":[{"document_id":"doc-1","source":"guide.md","excerpt":"generic guidance"}],
"returned_count":1,"truncated":false}
""", EvidenceStatus.EVIDENCE_FOUND));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertEquals(1, snapshot.verifiedSources().size());
assertEquals("guide.md", snapshot.verifiedSources().get(0).source());
assertEquals(1, snapshot.observedFacts().size());
assertEquals("generic guidance", snapshot.observedFacts().get(0).summary());
assertTrue(snapshot.limitations().isEmpty());
}
@Test
void excludesMissingExpiredProjectingCrossRunAndErrorRecords() {
RunContext context = context();
context.progress().recordCompleted(call("missing"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("expired"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("projecting"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("cross"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("error"), EvidenceStatus.NO_EVIDENCE);
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
records.put(keys.create(context.runId(), "projecting"),
CanonicalToolInvocation.projecting(
"projecting", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}", Instant.now()));
records.put(keys.create(context.runId(), "cross"), ready(
"cross", "other-run", AgentToolContracts.LOOKUP_KNOWLEDGE,
noEvidence("cross"), EvidenceStatus.NO_EVIDENCE));
records.put(keys.create(context.runId(), "error"),
CanonicalToolInvocation.projecting(
"error", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}", Instant.now())
.markError(null, "TOOL_EXECUTION_ERROR", Instant.now()));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertTrue(snapshot.observedFacts().isEmpty());
assertFalse(snapshot.limitations().isEmpty());
}
@Test
void boundsProjectedFactsAndReportsTruncation() {
RunContext context = context();
context.progress().recordCompleted(call("many"), EvidenceStatus.EVIDENCE_FOUND);
StringBuilder evidence = new StringBuilder();
for (int index = 0; index < 13; index++) {
if (index > 0) {
evidence.append(',');
}
evidence.append("{\"document_id\":\"doc-")
.append(index)
.append("\",\"source\":\"guide-")
.append(index)
.append("\",\"excerpt\":\"")
.append("x".repeat(400))
.append("\"}");
}
String result = "{\"evidence_status\":\"EVIDENCE_FOUND\","
+ "\"tool_call_id\":\"many\",\"query\":\"unknown\",\"evidence\":["
+ evidence + "],\"returned_count\":13,\"truncated\":false}";
Map<String, CanonicalToolInvocation> records = Map.of(
keys.create(context.runId(), "many"),
ready("many", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
result, EvidenceStatus.EVIDENCE_FOUND));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertEquals(12, snapshot.observedFacts().size());
assertTrue(snapshot.observedFacts().stream()
.allMatch(fact -> fact.summary().length() <= 320));
assertTrue(snapshot.limitations().contains("已检查事实较多,展示内容已截断"));
}
private DiagnosisProgressProjector projector(Map<String, CanonicalToolInvocation> records) {
return new DiagnosisProgressProjector(new MapStore(records), keys, objectMapper);
}
private CompletedToolCall call(String id) {
return new CompletedToolCall(
id, AgentToolContracts.LOOKUP_KNOWLEDGE, "{\"query\":\"" + id + "\"}");
}
private CanonicalToolInvocation ready(String id, String runId, String toolName,
String agentResult, EvidenceStatus status) {
return CanonicalToolInvocation.projecting(
id, runId, toolName, "{\"query\":\"unknown\"}", Instant.now())
.markReady("{}", agentResult, status, Instant.now());
}
private String noEvidence(String id) {
return "{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
+ "\",\"query\":\"unknown\",\"evidence\":[],\"returned_count\":0,"
+ "\"truncated\":false}";
}
private RunContext context() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "run-1", Duration.ofMinutes(5),
new RunBudgetLimits(10, 10, 10, 1000, 1000, 2000, 100_000),
HarnessRetryPolicies.strict(), 10)
.startRun("session-1", "run-1");
}
private static final class MapStore implements CanonicalInvocationStore {
private final Map<String, CanonicalToolInvocation> records;
private final CanonicalInvocationLimits limits =
new CanonicalInvocationLimits(Duration.ofHours(1), 1_000_000, 100_000);
private MapStore(Map<String, CanonicalToolInvocation> records) {
this.records = records;
}
@Override
public CanonicalInvocationLimits limits() {
return limits;
}
@Override
public void begin(String key, CanonicalToolInvocation invocation) {
throw new UnsupportedOperationException();
}
@Override
public Optional<CanonicalToolInvocation> find(String key) {
return Optional.ofNullable(records.get(key));
}
@Override
public CanonicalToolInvocation markReady(String key, String rawResponse, String agentResult,
EvidenceStatus evidenceStatus, Instant completedAt) {
throw new UnsupportedOperationException();
}
@Override
public CanonicalToolInvocation markError(String key, String rawResponse,
String errorCode, Instant completedAt) {
throw new UnsupportedOperationException();
}
}
}
@@ -0,0 +1,67 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisProgressTrackerTest {
@Test
void gainedResetsConsecutiveNoGainAndConsumesPendingCallOnce() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordDuplicateScope();
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
DiagnosisProgressSnapshotState state = tracker.snapshot();
assertEquals(0, state.consecutiveNoGain());
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
new PreviousObservation("call-1", InformationGain.NO_GAIN)));
}
@Test
void rejectsMissingOrOutOfOrderEvaluation() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(null));
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
new PreviousObservation("call-other", InformationGain.GAINED)));
assertEquals("call-1", tracker.snapshot().pendingToolCallId());
}
@Test
void deterministicNoEvidenceSaturatesAndStopInstructionIsClaimedOnce() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.NO_EVIDENCE);
tracker.recordCompleted(call("call-2", "scope-2"), EvidenceStatus.NO_EVIDENCE);
DiagnosisProgressSnapshotState state = tracker.snapshot();
assertEquals(DiagnosisCollectionState.SATURATED, state.collectionState());
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, state.stopReason());
assertTrue(tracker.claimStopInstruction());
assertFalse(tracker.claimStopInstruction());
}
@Test
void onlySuccessfulCompletedScopesParticipateInDuplicateDetection() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3);
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-1"));
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
assertTrue(tracker.isDuplicate("lookup_knowledge", "scope-1"));
assertFalse(tracker.isDuplicate("query_logs", "scope-1"));
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
}
private CompletedToolCall call(String id, String scope) {
return new CompletedToolCall(id, "lookup_knowledge", scope);
}
}
@@ -0,0 +1,52 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.LogTopic;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
class ToolScopeNormalizerTest {
private final ToolScopeNormalizer normalizer = new ToolScopeNormalizer(new ObjectMapper());
@Test
void trimsRagParametersWithoutSemanticRewriting() {
String first = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest(" refund timeout "));
String same = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest("refund timeout"));
String rewritten = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest("refund request is slow"));
assertEquals(first, same);
assertNotEquals(first, rewritten);
}
@Test
void normalizesMissingLogLookbackToAdapterDefault() {
String implicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", null));
String explicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", 30));
assertEquals(implicit, explicit);
}
@Test
void mysqlScopeIncludesLogicalSourceSqlAndParameters() {
String first = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(1)));
String otherParam = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(2)));
assertNotEquals(first, otherParam);
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.release;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.contract.AnalysisKind;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
@@ -9,6 +10,7 @@ import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
@@ -16,6 +18,8 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
@@ -91,6 +95,101 @@ class DiagnosisReleaseUseCaseTest {
assertEquals(1, model.calls.get());
}
@Test
void noConclusionWithVerifiedNegativeReferenceSkipsRepairAndSemanticGuard() {
Fixture fixture = fixture();
readyNoEvidence(fixture.store, fixture.context, "call-empty-1");
DiagnosisDraft draft = new DiagnosisDraft(
null,
List.of(new DiagnosisDraft.AnalysisItem(
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
"知识库范围内未找到匹配证据", List.of("call-empty-1"))),
List.of(),
List.of(),
new DiagnosisDraft.Limitations(
"query=unknown", List.of("故障服务运行日志")));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.completed(draft, progress(null)));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals(1, result.fallback().observedFacts().size());
assertEquals(0, fixture.model.calls.get());
assertEquals(List.of(
TraceEventType.EVIDENCE_GUARD_INITIAL,
TraceEventType.RELEASE_DECISION),
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
}
@Test
void noConclusionWithoutToolCallsPublishesMissingContext() {
Fixture fixture = fixture();
DiagnosisDraft draft = new DiagnosisDraft(
null, List.of(), List.of(), List.of(),
new DiagnosisDraft.Limitations(
"未指定故障范围", List.of("企业标识", "故障发生时间")));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断切换企业失败的问题",
DiagnosisAgentExecution.completed(draft, DiagnosisProgressSnapshot.empty()));
assertFallback(result, FallbackType.MISSING_REQUIRED_CONTEXT, 0);
assertEquals(0, fixture.model.calls.get());
}
@Test
void fabricatedNoConclusionReferenceFailsWithoutModelRepair() {
Fixture fixture = fixture();
DiagnosisDraft draft = new DiagnosisDraft(
null,
List.of(new DiagnosisDraft.AnalysisItem(
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
"声称未找到证据", List.of("missing-call"))),
List.of(), List.of(),
new DiagnosisDraft.Limitations("query=unknown", List.of()));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.completed(draft, progress(null)));
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
assertEquals(0, fixture.model.calls.get());
}
@Test
void controlledSaturationPublishesProgressWithoutAnotherModelCall() {
Fixture fixture = fixture();
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.stopped(
progress(DiagnosisStopReason.INFORMATION_SATURATED),
DiagnosisStopReason.INFORMATION_SATURATED));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals(0, fixture.model.calls.get());
}
@Test
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
Fixture fixture = fixture();
DiagnosisReleaseResult result = fixture.useCase.releaseInvalidDraft(
fixture.context, progress(null));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals("该知识检索范围内未发现可用文档证据",
result.fallback().observedFacts().get(0).summary());
assertEquals(0, fixture.model.calls.get());
assertEquals(List.of(TraceEventType.RELEASE_DECISION),
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
}
@Test
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
Fixture fixture = fixture(
@@ -204,7 +303,7 @@ class DiagnosisReleaseUseCaseTest {
Duration.ofSeconds(2), Duration.ofSeconds(3)),
ignored -> { }, trace),
new SafeFallbackFactory(), trace);
return new Fixture(context, model, useCase, trace);
return new Fixture(context, store, model, useCase, trace);
}
private DiagnosisDraft duplicateDraft() {
@@ -274,6 +373,29 @@ class DiagnosisReleaseUseCaseTest {
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
}
private void readyNoEvidence(InMemoryStore store, RunContext context, String callId) {
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
callId, context.runId(), "lookup_knowledge", "{\"query\":\"unknown\"}",
Instant.parse("2026-07-21T10:00:00Z"))
.markReady("raw-must-not-be-read", """
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-1",
"query":"unknown","evidence":[],"returned_count":0,"truncated":false}
""", EvidenceStatus.NO_EVIDENCE,
Instant.parse("2026-07-21T10:00:01Z"));
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
}
private DiagnosisProgressSnapshot progress(DiagnosisStopReason stopReason) {
return new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"该知识检索范围内未发现可用文档证据")),
List.of(),
stopReason);
}
private DiagnosisHarnessCore core() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
@@ -337,6 +459,7 @@ class DiagnosisReleaseUseCaseTest {
private record Fixture(
RunContext context,
InMemoryStore store,
ScriptedChatModel model,
DiagnosisReleaseUseCase useCase,
RecordingTraceRecorder trace) {
@@ -0,0 +1,47 @@
package com.superbiz.agent.harness.release;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SafeFallbackFactoryTest {
@Test
void insufficientEvidenceReusesVerifiedProgress() {
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "guide.md", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "guide.md", "query=unknown", "未找到匹配的故障证据")),
List.of("部分记录已过期"),
DiagnosisStopReason.BUDGET_LIMIT_REACHED);
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(
progress, List.of("企业标识"));
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, fallback.type());
assertEquals("DIAGNOSIS_COLLECTION", fallback.failureStage());
assertEquals(progress.verifiedSources(), fallback.verifiedSources());
assertEquals(progress.observedFacts(), fallback.observedFacts());
assertTrue(fallback.limitations().contains("仍缺少:企业标识"));
}
@Test
void missingRequiredContextContainsNoFabricatedProgress() {
SafeFallback fallback = new SafeFallbackFactory().missingRequiredContext(
List.of("企业标识", "故障发生时间"));
assertEquals(FallbackType.MISSING_REQUIRED_CONTEXT, fallback.type());
assertEquals("DIAGNOSIS_INPUT", fallback.failureStage());
assertTrue(fallback.verifiedSources().isEmpty());
assertTrue(fallback.observedFacts().isEmpty());
assertEquals(2, fallback.limitations().size());
}
}
@@ -0,0 +1,51 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.PreviousObservation;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class ToolCallEnvelopeContractTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void serializesSharedControlFieldsInSnakeCase() throws Exception {
RagToolCall call = new RagToolCall(
new PreviousObservation("call-1", InformationGain.NO_GAIN),
new RagToolRequest("refund timeout"));
JsonNode json = objectMapper.readTree(objectMapper.writeValueAsString(call));
assertEquals("call-1", json.path("previous_observation").path("tool_call_id").asText());
assertEquals("NO_GAIN", json.path("previous_observation").path("information_gain").asText());
assertEquals("refund timeout", json.path("input").path("query").asText());
}
@Test
void deserializesAllTypedBusinessInputs() throws Exception {
QueryLogsToolCall logs = objectMapper.readValue("""
{"input":{"topic":"APPLICATION","query":"timeout","lookback_minutes":30}}
""", QueryLogsToolCall.class);
MysqlToolCall mysql = objectMapper.readValue("""
{"input":{"data_source":"orders","sql":"SELECT id FROM orders WHERE id = ?","params":[1]}}
""", MysqlToolCall.class);
assertEquals(LogTopic.APPLICATION, logs.input().topic());
assertEquals(List.of(1), mysql.input().params());
}
@Test
void rejectsUnknownInformationGain() {
assertThrows(Exception.class, () -> objectMapper.readValue("""
{"previous_observation":{"tool_call_id":"call-1","information_gain":"UNKNOWN"},
"input":{"query":"timeout"}}
""", RagToolCall.class));
}
}
@@ -40,6 +40,25 @@ class RagResultProjectorTest {
assertFalse(projected.agentResult().contains("score"));
}
@Test
void preservesCamelAndSnakeCaseReferenceLevel() throws Exception {
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
String camel = """
{"found":true,"relevanceLevel":"REFERENCE",
"evidenceBlocks":[{"source":"doc","content":"generic guidance"}]}
""";
String snake = camel.replace("relevanceLevel", "relevance_level");
JsonNode camelResult = objectMapper.readTree(
projector.project(new RagToolRequest("unknown"), "call-1", camel).agentResult());
JsonNode snakeResult = objectMapper.readTree(
projector.project(new RagToolRequest("unknown"), "call-2", snake).agentResult());
assertEquals("REFERENCE", camelResult.path("relevance_level").asText());
assertEquals("REFERENCE", snakeResult.path("relevance_level").asText());
assertEquals("EVIDENCE_FOUND", camelResult.path("evidence_status").asText());
}
@Test
void returnsNoEvidenceForEmptyUsableBlocks() throws Exception {
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
@@ -48,6 +67,8 @@ class RagResultProjectorTest {
"{\"found\":false,\"evidenceBlocks\":[{\"source\":\"doc\",\"content\":\"\"}]}");
assertEquals(EvidenceStatus.NO_EVIDENCE, projected.evidenceStatus());
assertEquals(0, objectMapper.readTree(projected.agentResult()).path("evidence").size());
JsonNode json = objectMapper.readTree(projected.agentResult());
assertEquals(0, json.path("evidence").size());
assertFalse(json.has("relevance_level"));
}
}