feat(harness): add information gain stop and audit
This commit is contained in:
@@ -31,6 +31,15 @@ import static org.mockito.Mockito.mock;
|
||||
|
||||
class HarnessChatConfigurationTest {
|
||||
|
||||
@Test
|
||||
void diagnosisNoGainThresholdDefaultsAndValidates() {
|
||||
ChatHarnessProperties properties = new ChatHarnessProperties();
|
||||
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
|
||||
|
||||
properties.setStopAfterConsecutiveNoGain(0);
|
||||
assertThrows(IllegalArgumentException.class, properties::validate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsArePositiveAndExecutorsAreBounded() {
|
||||
ChatHarnessProperties properties = new ChatHarnessProperties();
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisAgentPromptTest {
|
||||
|
||||
@Test
|
||||
void promptLicensesAbandonmentWithoutDuplicatingToolSchema() {
|
||||
String prompt = DiagnosisAgentPrompt.load();
|
||||
|
||||
assertTrue(prompt.contains("你不必须给出根因"));
|
||||
assertTrue(prompt.contains("conclusion=null"));
|
||||
assertTrue(prompt.contains("可以不调用工具"));
|
||||
assertTrue(prompt.contains("GAINED"));
|
||||
assertTrue(prompt.contains("NO_GAIN"));
|
||||
assertTrue(prompt.contains("正确,但只是通用说明、重复内容"));
|
||||
assertTrue(prompt.contains("STOP_REQUIRED"));
|
||||
assertFalse(prompt.contains("lookup_knowledge"));
|
||||
assertFalse(prompt.contains("query_logs"));
|
||||
assertFalse(prompt.contains("query_mysql"));
|
||||
assertFalse(prompt.contains("previous_observation"));
|
||||
assertFalse(prompt.contains("next_action"));
|
||||
assertFalse(prompt.contains("阈值"));
|
||||
assertFalse(prompt.contains("计数器"));
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,8 @@ import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
@@ -71,8 +73,9 @@ class DiagnosisAgentUseCaseTest {
|
||||
"上一轮支付服务为什么超时?", "支付服务连接池已耗尽", "payment-service",
|
||||
List.of("未覆盖退款服务"), List.of(new SourceDocument("doc-1", "连接池手册")));
|
||||
|
||||
DiagnosisDraft draft = useCase.execute(
|
||||
DiagnosisAgentExecution execution = useCase.execute(
|
||||
context, new DiagnosisAgentInput("那退款服务呢?", previous));
|
||||
DiagnosisDraft draft = execution.draft();
|
||||
|
||||
assertEquals("退款服务出现连接池等待", draft.conclusion().text());
|
||||
assertEquals(AnalysisKind.NORMAL, draft.analysis().get(0).kind());
|
||||
@@ -109,8 +112,9 @@ class DiagnosisAgentUseCaseTest {
|
||||
toolCall("call-empty-1", "{\"query\":\"refund timeout\"}"),
|
||||
new AssistantMessage(noEvidenceDraft("call-empty-1")));
|
||||
|
||||
DiagnosisDraft draft = useCase(core, model, tools, LARGE_LIMITS)
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("退款为什么超时?", null));
|
||||
DiagnosisDraft draft = execution.draft();
|
||||
|
||||
assertNull(draft.conclusion());
|
||||
assertEquals(AnalysisKind.NEGATIVE_OBSERVATION, draft.analysis().get(0).kind());
|
||||
@@ -119,6 +123,32 @@ class DiagnosisAgentUseCaseTest {
|
||||
assertEquals(1, toolCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoredStopInstructionCannotExecuteAnotherTool() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(6, 6, 6,
|
||||
100, 100, 200, 100_000));
|
||||
RunContext context = core.startRun("session-stop", "run-stop");
|
||||
AtomicInteger toolCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
toolCalls.incrementAndGet();
|
||||
return ToolBoundaryResult.ready(id, noEvidence(id), EvidenceStatus.NO_EVIDENCE);
|
||||
});
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
toolCall("call-empty-1", "{\"query\":\"first\"}"),
|
||||
toolCall("call-empty-2", "{\"query\":\"second\"}"),
|
||||
toolCall("call-ignored-stop", "{\"query\":\"third\"}"));
|
||||
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, execution.stopReason());
|
||||
assertEquals(3, model.calls());
|
||||
assertEquals(2, toolCalls.get());
|
||||
assertEquals(2, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fencedOutputFailsClosedWithoutAgentRetry() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
@@ -126,14 +156,45 @@ class DiagnosisAgentUseCaseTest {
|
||||
new AssistantMessage("```json\n" + supportedDraft("call-1") + "\n```"));
|
||||
RunContext context = core.startRun("session-fenced", "run-fenced");
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
DiagnosisAgentOutputException failure = assertThrows(
|
||||
DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
|
||||
assertFalse(failure.progress().hasObservedFacts());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(1, context.budget().snapshot().modelCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftCarriesOnlyProjectedProgressForOuterReleaseBoundary() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
new AssistantMessage("not-json"));
|
||||
RunContext context = core.startRun("session-invalid-progress", "run-invalid-progress");
|
||||
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
|
||||
List.of(new com.superbiz.agent.harness.contract.SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new com.superbiz.agent.harness.contract.SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown", "已完成知识范围检查")),
|
||||
List.of(), null);
|
||||
DiagnosisAgentFactory factory = new DiagnosisAgentFactory(
|
||||
model, core, tools(errorInvoker()), objectMapper, List.of());
|
||||
DiagnosisAgentUseCase useCase = new DiagnosisAgentUseCase(
|
||||
core, factory, objectMapper, LARGE_LIMITS, ignored -> progress);
|
||||
|
||||
DiagnosisAgentOutputException failure = assertThrows(
|
||||
DiagnosisAgentOutputException.class,
|
||||
() -> useCase.execute(
|
||||
context, new DiagnosisAgentInput("诊断未知故障", null)));
|
||||
|
||||
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
|
||||
assertEquals("已完成知识范围检查",
|
||||
failure.progress().observedFacts().get(0).summary());
|
||||
assertFalse(failure.getMessage().contains("not-json"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void modelCallBudgetStopsNextReactRoundBeforeChatModel() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(1, 2, 2,
|
||||
@@ -147,10 +208,11 @@ class DiagnosisAgentUseCaseTest {
|
||||
toolCall("call-budget-1", "{\"query\":\"timeout\"}"),
|
||||
new AssistantMessage(supportedDraft("call-budget-1")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
}
|
||||
@@ -163,10 +225,11 @@ class DiagnosisAgentUseCaseTest {
|
||||
ScriptedChatModel model = new ScriptedChatModel(6, 6,
|
||||
new AssistantMessage(supportedDraft("call-unused")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(12, context.budget().snapshot().totalTokens());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
@@ -261,7 +324,8 @@ class DiagnosisAgentUseCaseTest {
|
||||
return AssistantMessage.builder()
|
||||
.content("")
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE, arguments)))
|
||||
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"input\":" + arguments + "}")))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class HarnessModelInterceptorTest {
|
||||
|
||||
@Test
|
||||
void extractsProviderUsageWithoutPersistingModelContent() {
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
RunContext context = core.startRun("session-agent-token", "run-agent-token");
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessModelInterceptor interceptor = new HarnessModelInterceptor(
|
||||
core, context,
|
||||
new ModelCallAuditor(core, trace::add, mock(AgentStepRepository.class)));
|
||||
AssistantMessage assistant = new AssistantMessage("secret-model-content");
|
||||
ChatResponse response = new ChatResponse(
|
||||
List.of(new Generation(assistant)),
|
||||
ChatResponseMetadata.builder().usage(new DefaultUsage(21, 4)).build());
|
||||
|
||||
ModelResponse actual = interceptor.interceptModel(
|
||||
ModelRequest.builder().messages(List.of()).build(),
|
||||
ignored -> ModelResponse.of(assistant, response));
|
||||
|
||||
assertEquals(response, actual.getChatResponse());
|
||||
assertEquals(25, context.budget().snapshot().totalTokens());
|
||||
assertEquals(25, context.modelCalls().snapshot().totalTokens());
|
||||
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, trace.get(0).eventType());
|
||||
assertEquals("DIAGNOSIS_AGENT", trace.get(0).details().get("component"));
|
||||
assertFalse(trace.get(0).details().toString().contains("secret-model-content"));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
@@ -23,6 +25,7 @@ import org.springframework.ai.tool.execution.ToolExecutionException;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@@ -52,7 +55,9 @@ class HarnessToolInterceptorTest {
|
||||
AgentToolContracts.QUERY_MYSQL),
|
||||
callbacks.stream().map(callback -> callback.getToolDefinition().name()).toList());
|
||||
callbacks.forEach(callback -> {
|
||||
assertFalse(callback.getToolDefinition().inputSchema().contains("tool_call_id"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("previous_observation"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("information_gain"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("\"input\""));
|
||||
assertFalse(callback.getToolDefinition().description().isBlank());
|
||||
assertThrows(ToolExecutionException.class, () -> callback.call("{}"));
|
||||
});
|
||||
@@ -72,7 +77,8 @@ class HarnessToolInterceptorTest {
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7", "{\"query\":\"timeout\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7",
|
||||
"{\"input\":{\"query\":\"timeout\"}}"),
|
||||
ignored -> {
|
||||
throw new AssertionError("registered Tool must not bypass Harness interceptor");
|
||||
});
|
||||
@@ -82,6 +88,7 @@ class HarnessToolInterceptorTest {
|
||||
assertEquals("framework-call-7", envelope.getValue().toolCallId());
|
||||
assertEquals(context.runId(), envelope.getValue().runId());
|
||||
assertEquals(AgentToolContracts.LOOKUP_KNOWLEDGE, envelope.getValue().toolName());
|
||||
assertEquals("{\"query\":\"timeout\"}", envelope.getValue().requestJson());
|
||||
assertEquals("framework-call-7", response.getToolCallId());
|
||||
assertTrue(response.getResult().contains("framework-call-7"));
|
||||
}
|
||||
@@ -94,7 +101,8 @@ class HarnessToolInterceptorTest {
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error", "{\"query\":\"secret\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error",
|
||||
"{\"input\":{\"query\":\"secret\"}}"),
|
||||
ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(response.getResult());
|
||||
@@ -142,13 +150,164 @@ class HarnessToolInterceptorTest {
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1", "{\"query\":\"one\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"),
|
||||
ignored -> null);
|
||||
|
||||
assertEquals(1, invocations.get());
|
||||
assertEquals(1, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiresExactPreviousEvaluationBeforeAnotherToolExecutes() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
|
||||
ToolCallResponse missing = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
|
||||
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
|
||||
ToolCallResponse accepted = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
|
||||
"input":{"query":"three"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
assertTrue(missing.isError());
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL",
|
||||
objectMapper.readTree(missing.getResult()).path("error_code").asText());
|
||||
assertFalse(accepted.isError());
|
||||
assertEquals(2, invocations.get());
|
||||
DiagnosisTraceAuditEvent rejected = trace.stream()
|
||||
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("call-2", rejected.details().get("tool_call_id"));
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
|
||||
String details = rejected.details().toString();
|
||||
assertFalse(details.contains("two"));
|
||||
assertFalse(details.contains("previous_observation"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateSuccessfulScopeIsNotExecutedAndCanSaturateRun() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"same\"}}"), ignored -> null);
|
||||
ToolCallResponse duplicate = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"NO_GAIN"},
|
||||
"input":{"query":"same"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(duplicate.getResult());
|
||||
assertEquals(1, invocations.get());
|
||||
assertTrue(observation.path("stop_required").asBoolean());
|
||||
assertEquals("INFORMATION_SATURATED", observation.path("reason").asText());
|
||||
assertTrue(trace.stream().anyMatch(event ->
|
||||
event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED
|
||||
&& "DUPLICATE_SCOPE".equals(event.details().get("error_code"))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void observationContractRejectionIsTracedWithoutPayload() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) ->
|
||||
ToolBoundaryResult.ready(id, """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-contract",
|
||||
"query":"secret-query","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND));
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper, trace::add);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-contract",
|
||||
"{\"input\":{\"query\":\"secret-query\"}}"), ignored -> null);
|
||||
|
||||
assertTrue(response.isError());
|
||||
DiagnosisTraceAuditEvent rejected = trace.stream()
|
||||
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("OBSERVATION_CONTRACT_MISMATCH", rejected.details().get("error_code"));
|
||||
assertFalse(rejected.details().toString().contains("secret-query"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void modelInformationGainIsRecordedWithoutRawScope() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> ready(id));
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"secret-query-value\"}}"), ignored -> null);
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
|
||||
"input":{"query":"different-secret-query"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
assertEquals(1, trace.size());
|
||||
DiagnosisTraceAuditEvent event = trace.get(0);
|
||||
assertEquals(TraceEventType.TOOL_PROGRESS, event.eventType());
|
||||
assertEquals("MODEL", event.details().get("producer"));
|
||||
assertEquals("GAINED", event.details().get("information_gain"));
|
||||
assertTrue(event.details().get("scope_summary").toString()
|
||||
.startsWith(AgentToolContracts.LOOKUP_KNOWLEDGE + "#"));
|
||||
assertFalse(event.details().toString().contains("secret-query-value"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void harnessProgressTraceDoesNotContainSqlParamsPayloadOrBudget() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
EvidenceToolInvoker unused = (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
EvidenceToolInvoker mysql = (context, id, arguments) -> ToolBoundaryResult.ready(
|
||||
id,
|
||||
"{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"columns\":[],\"rows\":[],\"returned_count\":0,"
|
||||
+ "\"truncated\":false}",
|
||||
EvidenceStatus.NO_EVIDENCE);
|
||||
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, mysql);
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.QUERY_MYSQL, "call-sql", """
|
||||
{"input":{"data_source":"orders","sql":"select * from secret_table where tenant_id = ?",
|
||||
"params":["tenant-secret"]}}
|
||||
"""), ignored -> null);
|
||||
|
||||
String recorded = trace.toString();
|
||||
assertEquals(List.of(TraceEventType.TOOL_PROGRESS),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertTrue(recorded.contains("HARNESS"));
|
||||
assertTrue(recorded.contains("NO_GAIN"));
|
||||
assertFalse(recorded.contains("secret_table"));
|
||||
assertFalse(recorded.contains("tenant-secret"));
|
||||
assertFalse(recorded.contains("raw_response"));
|
||||
assertFalse(recorded.contains("remaining"));
|
||||
assertFalse(recorded.contains("prompt"));
|
||||
assertFalse(recorded.contains("thought"));
|
||||
}
|
||||
|
||||
private HarnessEvidenceTools fakeTools(EvidenceToolInvoker ragInvoker) {
|
||||
EvidenceToolInvoker unused = (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
@@ -162,7 +321,8 @@ class HarnessToolInterceptorTest {
|
||||
private ToolBoundaryResult ready(String toolCallId) {
|
||||
return ToolBoundaryResult.ready(toolCallId,
|
||||
"{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\""
|
||||
+ toolCallId + "\",\"evidence\":[]}",
|
||||
+ toolCallId + "\",\"query\":\"scope\",\"evidence\":[],"
|
||||
+ "\"returned_count\":1,\"truncated\":false}",
|
||||
EvidenceStatus.EVIDENCE_FOUND);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ToolResultViewProjectorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolResultViewProjector projector = new ToolResultViewProjector(objectMapper);
|
||||
|
||||
@Test
|
||||
void separatesRagControlDataFromModelObservation() throws Exception {
|
||||
String canonical = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
|
||||
"evidence":[{"document_id":"doc-1","excerpt":"generic"}],"returned_count":1,
|
||||
"relevance_level":"REFERENCE","truncated":false,"raw_score":0.42}
|
||||
""";
|
||||
|
||||
ToolControlView control = projector.controlView(canonical);
|
||||
JsonNode model = objectMapper.readTree(projector.modelObservation(
|
||||
AgentToolContracts.LOOKUP_KNOWLEDGE, canonical, "{\"query\":\"unknown\"}",
|
||||
true, DiagnosisStopReason.INFORMATION_SATURATED));
|
||||
|
||||
assertEquals(1, control.returnedCount());
|
||||
assertEquals(RagRelevanceLevel.REFERENCE, control.relevanceLevel());
|
||||
assertEquals("REFERENCE", model.path("relevance_level").asText());
|
||||
assertEquals("unknown", model.path("scope").path("query").asText());
|
||||
assertTrue(model.path("stop_required").asBoolean());
|
||||
assertFalse(model.has("returned_count"));
|
||||
assertFalse(model.has("raw_score"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mysqlObservationUsesNormalizedScopeWithoutInternalFields() throws Exception {
|
||||
String canonical = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-2","columns":["id"],
|
||||
"rows":[{"id":1}],"returned_count":1,"truncated":false}
|
||||
""";
|
||||
|
||||
JsonNode model = objectMapper.readTree(projector.modelObservation(
|
||||
AgentToolContracts.QUERY_MYSQL, canonical,
|
||||
"{\"data_source\":\"orders\",\"sql\":\"SELECT id FROM orders\",\"params\":[]}",
|
||||
false, null));
|
||||
|
||||
assertEquals("orders", model.path("scope").path("data_source").asText());
|
||||
assertEquals(1, model.path("rows").size());
|
||||
assertFalse(model.has("returned_count"));
|
||||
}
|
||||
}
|
||||
@@ -9,14 +9,17 @@ import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.PreviousTurn;
|
||||
import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -157,6 +160,88 @@ class ChatApplicationUseCaseTest {
|
||||
store.startedContext.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void handledDiagnosisBudgetFallbackIsPersistedWithoutReclassification() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
|
||||
SafeFallback fallback = new SafeFallback(
|
||||
FallbackType.INSUFFICIENT_EVIDENCE,
|
||||
null,
|
||||
"已完成有限范围的检查,但现有证据不足以确认根因",
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of("当前已检查范围不足以支持根因结论"),
|
||||
List.of("补充故障上下文后重试"),
|
||||
"DIAGNOSIS_COLLECTION",
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"该知识检索范围内未发现可用文档证据")),
|
||||
List.of());
|
||||
return new DiagnosisExecutionResult(
|
||||
ReleaseOutcome.FALLBACK, new FallbackContent(fallback), null, true);
|
||||
});
|
||||
|
||||
ChatApplicationResult result = useCase.execute(
|
||||
new ChatApplicationRequest("diagnose unknown issue", "session-budget-fallback"));
|
||||
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertEquals(ChatContentType.SAFE_FALLBACK, result.contentType());
|
||||
FallbackContent content = (FallbackContent) result.content();
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, content.fallback().type());
|
||||
assertEquals(1, content.fallback().observedFacts().size());
|
||||
assertEquals(ReleaseOutcome.FALLBACK, store.finishedOutcome);
|
||||
assertTrue(store.finishedSafeContentJson.contains("knowledge_base"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unhandledDiagnosisBudgetFailureDoesNotBuildFallbackInApplication() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.budget().reserveToolCall("lookup_knowledge");
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
|
||||
throw new IllegalStateException("unhandled diagnosis failure");
|
||||
});
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest(
|
||||
"diagnose unknown issue", "session-budget-unhandled")));
|
||||
|
||||
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
|
||||
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
|
||||
}
|
||||
|
||||
@Test
|
||||
void diagnosisBudgetExhaustionWithoutToolCallsRemainsFailure() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "model budget exhausted");
|
||||
throw new IllegalStateException("diagnosis failed before evidence collection");
|
||||
});
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest(
|
||||
"diagnose unavailable model", "session-budget-no-tools")));
|
||||
|
||||
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
|
||||
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientDisconnectControlCancelsSameRunAndPersistsCancelled() {
|
||||
FakeStore store = new FakeStore();
|
||||
@@ -252,6 +337,7 @@ class ChatApplicationUseCaseTest {
|
||||
private IntentType finishedIntent;
|
||||
private ReleaseOutcome finishedOutcome;
|
||||
private PublishedResult finishedPublishedResult;
|
||||
private String finishedSafeContentJson;
|
||||
|
||||
@Override
|
||||
public Optional<RoutingHistory> findLatestRoutingHistory(String sessionId) {
|
||||
@@ -283,6 +369,7 @@ class ChatApplicationUseCaseTest {
|
||||
finishedIntent = intent;
|
||||
finishedOutcome = outcome;
|
||||
finishedPublishedResult = publishedResult;
|
||||
finishedSafeContentJson = safeContentJson;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
@@ -14,6 +14,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
@@ -101,6 +102,8 @@ class PublishedResultPersistenceTest {
|
||||
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
|
||||
RunContext context = core().startRun("session-1", "run-fallback");
|
||||
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
|
||||
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
|
||||
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
|
||||
@@ -111,6 +114,7 @@ class PublishedResultPersistenceTest {
|
||||
assertEquals(ReleaseOutcome.FALLBACK, entity.getReleaseOutcome());
|
||||
assertNull(entity.getPublishedResult());
|
||||
assertEquals(12, entity.getTotalDurationMs());
|
||||
assertEquals(2, entity.getStepCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package com.superbiz.agent.harness.application.executor;
|
||||
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
|
||||
import com.superbiz.agent.harness.application.ChatApplicationStatus;
|
||||
import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
|
||||
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
|
||||
import com.superbiz.agent.harness.application.persistence.PreviousTurnLimits;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.release.DiagnosisReleaseResult;
|
||||
import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
|
||||
import com.superbiz.agent.harness.release.SafeFallbackFactory;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DiagnosisChatExecutorTest {
|
||||
|
||||
@Test
|
||||
void invalidDraftWithVerifiedProgressUsesDeterministicReleaseFallback() {
|
||||
RunContext context = context("run-invalid-progress");
|
||||
DiagnosisProgressSnapshot progress = progress();
|
||||
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an invalid Draft",
|
||||
new IllegalArgumentException("redacted"),
|
||||
DiagnosisAgentOutputException.Kind.SCHEMA_INVALID,
|
||||
321L,
|
||||
progress);
|
||||
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
|
||||
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
|
||||
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
|
||||
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(progress, List.of());
|
||||
when(release.releaseInvalidDraft(context, progress))
|
||||
.thenReturn(DiagnosisReleaseResult.fallback(fallback));
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
|
||||
agent,
|
||||
release,
|
||||
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
|
||||
trace::add);
|
||||
List<ChatApplicationStatus> statuses = new ArrayList<>();
|
||||
|
||||
DiagnosisExecutionResult result = executor.execute(
|
||||
context, "诊断未知故障", null, statuses::add);
|
||||
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE,
|
||||
((com.superbiz.agent.harness.application.FallbackContent) result.content())
|
||||
.fallback().type());
|
||||
assertEquals(List.of(
|
||||
ChatApplicationStatus.DIAGNOSIS_RUNNING,
|
||||
ChatApplicationStatus.SAFETY_VALIDATING), statuses);
|
||||
assertEquals(List.of(TraceEventType.AGENT_DRAFT_INVALID),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertEquals("SCHEMA_INVALID", trace.get(0).details().get("failure_kind"));
|
||||
assertEquals(321L, trace.get(0).details().get("output_bytes"));
|
||||
assertEquals(true, trace.get(0).details().get("has_publishable_progress"));
|
||||
assertEquals(3, trace.get(0).details().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftWithoutVerifiedProgressStillFailsClosed() {
|
||||
RunContext context = context("run-invalid-empty");
|
||||
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an invalid Draft",
|
||||
null,
|
||||
DiagnosisAgentOutputException.Kind.INVALID_JSON,
|
||||
12L,
|
||||
DiagnosisProgressSnapshot.empty());
|
||||
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
|
||||
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
|
||||
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
|
||||
agent,
|
||||
release,
|
||||
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
|
||||
trace::add);
|
||||
|
||||
DiagnosisAgentOutputException actual = assertThrows(
|
||||
DiagnosisAgentOutputException.class,
|
||||
() -> executor.execute(context, "诊断未知故障", null, ignored -> { }));
|
||||
|
||||
assertSame(failure, actual);
|
||||
verifyNoInteractions(release);
|
||||
assertEquals(false, trace.get(0).details().get("has_publishable_progress"));
|
||||
}
|
||||
|
||||
private RunContext context(String runId) {
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused-run", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(4, 3, 2, 100, 100, 200, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
return core.startRun("session-executor", runId);
|
||||
}
|
||||
|
||||
private DiagnosisProgressSnapshot progress() {
|
||||
return new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"已完成知识范围检查")),
|
||||
List.of(), null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ModelCallAuditorTest {
|
||||
|
||||
@Test
|
||||
void recordsComponentUsageIntoRunBudgetTraceAndAgentStep() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-token", "run-token");
|
||||
AgentStepRepository steps = mock(AgentStepRepository.class);
|
||||
AgentStep step = AgentStep.builder().id(7L).runId(context.runId()).stepIndex(0).build();
|
||||
when(steps.findByRunIdAndStepIndex(context.runId(), 0)).thenReturn(Optional.of(step));
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(core, trace::add, steps);
|
||||
|
||||
core.beforeModelCall(context);
|
||||
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
auditor.recordUsage(context, call, 13, 5, true);
|
||||
|
||||
assertEquals(18, context.budget().snapshot().totalTokens());
|
||||
assertEquals(18, context.modelCalls().snapshot().totalTokens());
|
||||
assertEquals(18, step.getTokenCount());
|
||||
verify(steps).save(step);
|
||||
DiagnosisTraceAuditEvent event = trace.get(0);
|
||||
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, event.eventType());
|
||||
assertEquals("DIAGNOSIS_AGENT", event.details().get("component"));
|
||||
assertEquals(1, event.details().get("component_round"));
|
||||
assertEquals(13L, event.details().get("input_tokens"));
|
||||
assertEquals(5L, event.details().get("output_tokens"));
|
||||
assertEquals(18L, event.details().get("total_tokens"));
|
||||
assertEquals(true, event.details().get("usage_available"));
|
||||
String details = event.details().toString().toLowerCase();
|
||||
assertFalse(details.contains("prompt"));
|
||||
assertFalse(details.contains("content"));
|
||||
assertFalse(details.contains("response"));
|
||||
assertFalse(details.contains("argument"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unavailableUsageIsVisibleWithoutFabricatingTokens() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-missing", "run-missing");
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(
|
||||
core, trace::add, mock(AgentStepRepository.class));
|
||||
|
||||
core.beforeModelCall(context);
|
||||
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.INTENT_ROUTER);
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
|
||||
ModelCallLedger.Snapshot snapshot = context.modelCalls().snapshot();
|
||||
assertEquals(1, snapshot.auditedCallCount());
|
||||
assertEquals(1, snapshot.usageUnavailableCount());
|
||||
assertEquals(0, snapshot.totalTokens());
|
||||
assertEquals(0, context.budget().snapshot().totalTokens());
|
||||
assertEquals(false, trace.get(0).details().get("usage_available"));
|
||||
assertFalse(trace.get(0).details().containsKey("input_tokens"));
|
||||
assertFalse(trace.get(0).details().containsKey("output_tokens"));
|
||||
assertFalse(trace.get(0).details().containsKey("total_tokens"));
|
||||
DiagnosisTraceAuditEvent finished = TraceAuditEvents.runFinished(
|
||||
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
|
||||
com.superbiz.agent.harness.contract.ReleaseOutcome.FAILED, 1);
|
||||
assertEquals(false, finished.details().get("tokens_reconciled"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runFinishedExposesTokenReconciliationWithoutContent() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-finish", "run-finish");
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(
|
||||
core, DiagnosisTraceRecorder.noop(), mock(AgentStepRepository.class));
|
||||
core.beforeModelCall(context);
|
||||
auditor.recordUsage(context,
|
||||
auditor.begin(context, ModelCallComponent.INTENT_ROUTER), 8, 2, true);
|
||||
|
||||
DiagnosisTraceAuditEvent event = TraceAuditEvents.runFinished(
|
||||
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
|
||||
com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, 50);
|
||||
|
||||
assertEquals(10L, event.details().get("run_total_tokens"));
|
||||
assertEquals(10L, event.details().get("audited_total_tokens"));
|
||||
assertEquals(true, event.details().get("tokens_reconciled"));
|
||||
assertEquals(0, event.details().get("usage_unavailable_count"));
|
||||
assertTrue(event.details().containsKey("audited_model_call_count"));
|
||||
assertFalse(event.details().toString().contains("secret"));
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(20, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisProgressProjectorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolCallKeyFactory keys = new ToolCallKeyFactory("test:tool");
|
||||
|
||||
@Test
|
||||
void projectsReadyCurrentRunRecordsInExecutionOrder() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(
|
||||
new CompletedToolCall("call-1", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}"),
|
||||
EvidenceStatus.EVIDENCE_FOUND);
|
||||
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
|
||||
records.put(keys.create(context.runId(), "call-1"), ready(
|
||||
"call-1", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"""
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
|
||||
"evidence":[{"document_id":"doc-1","source":"guide.md","excerpt":"generic guidance"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertEquals(1, snapshot.verifiedSources().size());
|
||||
assertEquals("guide.md", snapshot.verifiedSources().get(0).source());
|
||||
assertEquals(1, snapshot.observedFacts().size());
|
||||
assertEquals("generic guidance", snapshot.observedFacts().get(0).summary());
|
||||
assertTrue(snapshot.limitations().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void excludesMissingExpiredProjectingCrossRunAndErrorRecords() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(call("missing"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("expired"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("projecting"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("cross"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("error"), EvidenceStatus.NO_EVIDENCE);
|
||||
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
|
||||
records.put(keys.create(context.runId(), "projecting"),
|
||||
CanonicalToolInvocation.projecting(
|
||||
"projecting", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}", Instant.now()));
|
||||
records.put(keys.create(context.runId(), "cross"), ready(
|
||||
"cross", "other-run", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
noEvidence("cross"), EvidenceStatus.NO_EVIDENCE));
|
||||
records.put(keys.create(context.runId(), "error"),
|
||||
CanonicalToolInvocation.projecting(
|
||||
"error", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}", Instant.now())
|
||||
.markError(null, "TOOL_EXECUTION_ERROR", Instant.now()));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertTrue(snapshot.observedFacts().isEmpty());
|
||||
assertFalse(snapshot.limitations().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void boundsProjectedFactsAndReportsTruncation() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(call("many"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
StringBuilder evidence = new StringBuilder();
|
||||
for (int index = 0; index < 13; index++) {
|
||||
if (index > 0) {
|
||||
evidence.append(',');
|
||||
}
|
||||
evidence.append("{\"document_id\":\"doc-")
|
||||
.append(index)
|
||||
.append("\",\"source\":\"guide-")
|
||||
.append(index)
|
||||
.append("\",\"excerpt\":\"")
|
||||
.append("x".repeat(400))
|
||||
.append("\"}");
|
||||
}
|
||||
String result = "{\"evidence_status\":\"EVIDENCE_FOUND\","
|
||||
+ "\"tool_call_id\":\"many\",\"query\":\"unknown\",\"evidence\":["
|
||||
+ evidence + "],\"returned_count\":13,\"truncated\":false}";
|
||||
Map<String, CanonicalToolInvocation> records = Map.of(
|
||||
keys.create(context.runId(), "many"),
|
||||
ready("many", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
result, EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertEquals(12, snapshot.observedFacts().size());
|
||||
assertTrue(snapshot.observedFacts().stream()
|
||||
.allMatch(fact -> fact.summary().length() <= 320));
|
||||
assertTrue(snapshot.limitations().contains("已检查事实较多,展示内容已截断"));
|
||||
}
|
||||
|
||||
private DiagnosisProgressProjector projector(Map<String, CanonicalToolInvocation> records) {
|
||||
return new DiagnosisProgressProjector(new MapStore(records), keys, objectMapper);
|
||||
}
|
||||
|
||||
private CompletedToolCall call(String id) {
|
||||
return new CompletedToolCall(
|
||||
id, AgentToolContracts.LOOKUP_KNOWLEDGE, "{\"query\":\"" + id + "\"}");
|
||||
}
|
||||
|
||||
private CanonicalToolInvocation ready(String id, String runId, String toolName,
|
||||
String agentResult, EvidenceStatus status) {
|
||||
return CanonicalToolInvocation.projecting(
|
||||
id, runId, toolName, "{\"query\":\"unknown\"}", Instant.now())
|
||||
.markReady("{}", agentResult, status, Instant.now());
|
||||
}
|
||||
|
||||
private String noEvidence(String id) {
|
||||
return "{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"query\":\"unknown\",\"evidence\":[],\"returned_count\":0,"
|
||||
+ "\"truncated\":false}";
|
||||
}
|
||||
|
||||
private RunContext context() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "run-1", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 1000, 1000, 2000, 100_000),
|
||||
HarnessRetryPolicies.strict(), 10)
|
||||
.startRun("session-1", "run-1");
|
||||
}
|
||||
|
||||
private static final class MapStore implements CanonicalInvocationStore {
|
||||
|
||||
private final Map<String, CanonicalToolInvocation> records;
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(1), 1_000_000, 100_000);
|
||||
|
||||
private MapStore(Map<String, CanonicalToolInvocation> records) {
|
||||
this.records = records;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse, String agentResult,
|
||||
EvidenceStatus evidenceStatus, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisProgressTrackerTest {
|
||||
|
||||
@Test
|
||||
void gainedResetsConsecutiveNoGainAndConsumesPendingCallOnce() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordDuplicateScope();
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
|
||||
|
||||
DiagnosisProgressSnapshotState state = tracker.snapshot();
|
||||
assertEquals(0, state.consecutiveNoGain());
|
||||
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
|
||||
new PreviousObservation("call-1", InformationGain.NO_GAIN)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsMissingOrOutOfOrderEvaluation() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(null));
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
|
||||
new PreviousObservation("call-other", InformationGain.GAINED)));
|
||||
assertEquals("call-1", tracker.snapshot().pendingToolCallId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deterministicNoEvidenceSaturatesAndStopInstructionIsClaimedOnce() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.NO_EVIDENCE);
|
||||
tracker.recordCompleted(call("call-2", "scope-2"), EvidenceStatus.NO_EVIDENCE);
|
||||
|
||||
DiagnosisProgressSnapshotState state = tracker.snapshot();
|
||||
assertEquals(DiagnosisCollectionState.SATURATED, state.collectionState());
|
||||
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, state.stopReason());
|
||||
assertTrue(tracker.claimStopInstruction());
|
||||
assertFalse(tracker.claimStopInstruction());
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlySuccessfulCompletedScopesParticipateInDuplicateDetection() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3);
|
||||
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-1"));
|
||||
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
assertTrue(tracker.isDuplicate("lookup_knowledge", "scope-1"));
|
||||
assertFalse(tracker.isDuplicate("query_logs", "scope-1"));
|
||||
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
|
||||
}
|
||||
|
||||
private CompletedToolCall call(String id, String scope) {
|
||||
return new CompletedToolCall(id, "lookup_knowledge", scope);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.LogTopic;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
|
||||
class ToolScopeNormalizerTest {
|
||||
|
||||
private final ToolScopeNormalizer normalizer = new ToolScopeNormalizer(new ObjectMapper());
|
||||
|
||||
@Test
|
||||
void trimsRagParametersWithoutSemanticRewriting() {
|
||||
String first = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest(" refund timeout "));
|
||||
String same = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest("refund timeout"));
|
||||
String rewritten = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest("refund request is slow"));
|
||||
|
||||
assertEquals(first, same);
|
||||
assertNotEquals(first, rewritten);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesMissingLogLookbackToAdapterDefault() {
|
||||
String implicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
|
||||
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", null));
|
||||
String explicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
|
||||
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", 30));
|
||||
|
||||
assertEquals(implicit, explicit);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mysqlScopeIncludesLogicalSourceSqlAndParameters() {
|
||||
String first = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
|
||||
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(1)));
|
||||
String otherParam = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
|
||||
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(2)));
|
||||
|
||||
assertNotEquals(first, otherParam);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
@@ -9,6 +10,7 @@ import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
@@ -16,6 +18,8 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
@@ -91,6 +95,101 @@ class DiagnosisReleaseUseCaseTest {
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noConclusionWithVerifiedNegativeReferenceSkipsRepairAndSemanticGuard() {
|
||||
Fixture fixture = fixture();
|
||||
readyNoEvidence(fixture.store, fixture.context, "call-empty-1");
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null,
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
|
||||
"知识库范围内未找到匹配证据", List.of("call-empty-1"))),
|
||||
List.of(),
|
||||
List.of(),
|
||||
new DiagnosisDraft.Limitations(
|
||||
"query=unknown", List.of("故障服务运行日志")));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.completed(draft, progress(null)));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals(1, result.fallback().observedFacts().size());
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
assertEquals(List.of(
|
||||
TraceEventType.EVIDENCE_GUARD_INITIAL,
|
||||
TraceEventType.RELEASE_DECISION),
|
||||
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noConclusionWithoutToolCallsPublishesMissingContext() {
|
||||
Fixture fixture = fixture();
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null, List.of(), List.of(), List.of(),
|
||||
new DiagnosisDraft.Limitations(
|
||||
"未指定故障范围", List.of("企业标识", "故障发生时间")));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断切换企业失败的问题",
|
||||
DiagnosisAgentExecution.completed(draft, DiagnosisProgressSnapshot.empty()));
|
||||
|
||||
assertFallback(result, FallbackType.MISSING_REQUIRED_CONTEXT, 0);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fabricatedNoConclusionReferenceFailsWithoutModelRepair() {
|
||||
Fixture fixture = fixture();
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null,
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
|
||||
"声称未找到证据", List.of("missing-call"))),
|
||||
List.of(), List.of(),
|
||||
new DiagnosisDraft.Limitations("query=unknown", List.of()));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.completed(draft, progress(null)));
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void controlledSaturationPublishesProgressWithoutAnotherModelCall() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.stopped(
|
||||
progress(DiagnosisStopReason.INFORMATION_SATURATED),
|
||||
DiagnosisStopReason.INFORMATION_SATURATED));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.releaseInvalidDraft(
|
||||
fixture.context, progress(null));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals("该知识检索范围内未发现可用文档证据",
|
||||
result.fallback().observedFacts().get(0).summary());
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
assertEquals(List.of(TraceEventType.RELEASE_DECISION),
|
||||
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
|
||||
Fixture fixture = fixture(
|
||||
@@ -204,7 +303,7 @@ class DiagnosisReleaseUseCaseTest {
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)),
|
||||
ignored -> { }, trace),
|
||||
new SafeFallbackFactory(), trace);
|
||||
return new Fixture(context, model, useCase, trace);
|
||||
return new Fixture(context, store, model, useCase, trace);
|
||||
}
|
||||
|
||||
private DiagnosisDraft duplicateDraft() {
|
||||
@@ -274,6 +373,29 @@ class DiagnosisReleaseUseCaseTest {
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private void readyNoEvidence(InMemoryStore store, RunContext context, String callId) {
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), "lookup_knowledge", "{\"query\":\"unknown\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-1",
|
||||
"query":"unknown","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private DiagnosisProgressSnapshot progress(DiagnosisStopReason stopReason) {
|
||||
return new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"该知识检索范围内未发现可用文档证据")),
|
||||
List.of(),
|
||||
stopReason);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
@@ -337,6 +459,7 @@ class DiagnosisReleaseUseCaseTest {
|
||||
|
||||
private record Fixture(
|
||||
RunContext context,
|
||||
InMemoryStore store,
|
||||
ScriptedChatModel model,
|
||||
DiagnosisReleaseUseCase useCase,
|
||||
RecordingTraceRecorder trace) {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SafeFallbackFactoryTest {
|
||||
|
||||
@Test
|
||||
void insufficientEvidenceReusesVerifiedProgress() {
|
||||
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "guide.md", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "guide.md", "query=unknown", "未找到匹配的故障证据")),
|
||||
List.of("部分记录已过期"),
|
||||
DiagnosisStopReason.BUDGET_LIMIT_REACHED);
|
||||
|
||||
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(
|
||||
progress, List.of("企业标识"));
|
||||
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, fallback.type());
|
||||
assertEquals("DIAGNOSIS_COLLECTION", fallback.failureStage());
|
||||
assertEquals(progress.verifiedSources(), fallback.verifiedSources());
|
||||
assertEquals(progress.observedFacts(), fallback.observedFacts());
|
||||
assertTrue(fallback.limitations().contains("仍缺少:企业标识"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingRequiredContextContainsNoFabricatedProgress() {
|
||||
SafeFallback fallback = new SafeFallbackFactory().missingRequiredContext(
|
||||
List.of("企业标识", "故障发生时间"));
|
||||
|
||||
assertEquals(FallbackType.MISSING_REQUIRED_CONTEXT, fallback.type());
|
||||
assertEquals("DIAGNOSIS_INPUT", fallback.failureStage());
|
||||
assertTrue(fallback.verifiedSources().isEmpty());
|
||||
assertTrue(fallback.observedFacts().isEmpty());
|
||||
assertEquals(2, fallback.limitations().size());
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class ToolCallEnvelopeContractTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void serializesSharedControlFieldsInSnakeCase() throws Exception {
|
||||
RagToolCall call = new RagToolCall(
|
||||
new PreviousObservation("call-1", InformationGain.NO_GAIN),
|
||||
new RagToolRequest("refund timeout"));
|
||||
|
||||
JsonNode json = objectMapper.readTree(objectMapper.writeValueAsString(call));
|
||||
|
||||
assertEquals("call-1", json.path("previous_observation").path("tool_call_id").asText());
|
||||
assertEquals("NO_GAIN", json.path("previous_observation").path("information_gain").asText());
|
||||
assertEquals("refund timeout", json.path("input").path("query").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deserializesAllTypedBusinessInputs() throws Exception {
|
||||
QueryLogsToolCall logs = objectMapper.readValue("""
|
||||
{"input":{"topic":"APPLICATION","query":"timeout","lookback_minutes":30}}
|
||||
""", QueryLogsToolCall.class);
|
||||
MysqlToolCall mysql = objectMapper.readValue("""
|
||||
{"input":{"data_source":"orders","sql":"SELECT id FROM orders WHERE id = ?","params":[1]}}
|
||||
""", MysqlToolCall.class);
|
||||
|
||||
assertEquals(LogTopic.APPLICATION, logs.input().topic());
|
||||
assertEquals(List.of(1), mysql.input().params());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnknownInformationGain() {
|
||||
assertThrows(Exception.class, () -> objectMapper.readValue("""
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"UNKNOWN"},
|
||||
"input":{"query":"timeout"}}
|
||||
""", RagToolCall.class));
|
||||
}
|
||||
}
|
||||
+22
-1
@@ -40,6 +40,25 @@ class RagResultProjectorTest {
|
||||
assertFalse(projected.agentResult().contains("score"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesCamelAndSnakeCaseReferenceLevel() throws Exception {
|
||||
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
|
||||
String camel = """
|
||||
{"found":true,"relevanceLevel":"REFERENCE",
|
||||
"evidenceBlocks":[{"source":"doc","content":"generic guidance"}]}
|
||||
""";
|
||||
String snake = camel.replace("relevanceLevel", "relevance_level");
|
||||
|
||||
JsonNode camelResult = objectMapper.readTree(
|
||||
projector.project(new RagToolRequest("unknown"), "call-1", camel).agentResult());
|
||||
JsonNode snakeResult = objectMapper.readTree(
|
||||
projector.project(new RagToolRequest("unknown"), "call-2", snake).agentResult());
|
||||
|
||||
assertEquals("REFERENCE", camelResult.path("relevance_level").asText());
|
||||
assertEquals("REFERENCE", snakeResult.path("relevance_level").asText());
|
||||
assertEquals("EVIDENCE_FOUND", camelResult.path("evidence_status").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsNoEvidenceForEmptyUsableBlocks() throws Exception {
|
||||
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
|
||||
@@ -48,6 +67,8 @@ class RagResultProjectorTest {
|
||||
"{\"found\":false,\"evidenceBlocks\":[{\"source\":\"doc\",\"content\":\"\"}]}");
|
||||
|
||||
assertEquals(EvidenceStatus.NO_EVIDENCE, projected.evidenceStatus());
|
||||
assertEquals(0, objectMapper.readTree(projected.agentResult()).path("evidence").size());
|
||||
JsonNode json = objectMapper.readTree(projected.agentResult());
|
||||
assertEquals(0, json.path("evidence").size());
|
||||
assertFalse(json.has("relevance_level"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user