feat(harness): add single diagnosis react agent
This commit is contained in:
zhuyongxin
@@ -0,0 +1,366 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.OverAllState;
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPosition;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.HookPositions;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.ModelHook;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.PreviousTurn;
|
||||
import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.ToolResponseMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisAgentUseCaseTest {
|
||||
|
||||
private static final DiagnosisAgentLimits LARGE_LIMITS =
|
||||
new DiagnosisAgentLimits(20_000, 20_000, 40_000, 40_000);
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void singleAgentRunsFrameworkToolLoopAndReturnsTypedDraft() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(4, 3, 2,
|
||||
100, 100, 200, 100_000));
|
||||
RunContext context = core.startRun("session-agent", "run-agent");
|
||||
AtomicInteger toolCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
toolCalls.incrementAndGet();
|
||||
return ToolBoundaryResult.ready(id, evidence(id), EvidenceStatus.EVIDENCE_FOUND);
|
||||
});
|
||||
ScriptedChatModel model = new ScriptedChatModel(3, 2,
|
||||
toolCall("call-rag-1", "{\"query\":\"refund timeout\"}"),
|
||||
new AssistantMessage(supportedDraft("call-rag-1")));
|
||||
CapturingAuditHook audit = new CapturingAuditHook();
|
||||
DiagnosisAgentUseCase useCase = useCase(core, model, tools, LARGE_LIMITS, audit);
|
||||
PreviousTurn previous = new PreviousTurn(
|
||||
"上一轮支付服务为什么超时？", "支付服务连接池已耗尽", "payment-service",
|
||||
List.of("未覆盖退款服务"), List.of(new SourceDocument("doc-1", "连接池手册")));
|
||||
|
||||
DiagnosisDraft draft = useCase.execute(
|
||||
context, new DiagnosisAgentInput("那退款服务呢？", previous));
|
||||
|
||||
assertEquals("退款服务出现连接池等待", draft.conclusion().text());
|
||||
assertEquals(AnalysisKind.NORMAL, draft.analysis().get(0).kind());
|
||||
assertEquals(List.of("call-rag-1"), draft.analysis().get(0).toolCallIds());
|
||||
assertEquals(2, model.calls());
|
||||
assertEquals(1, toolCalls.get());
|
||||
assertEquals(2, context.budget().snapshot().modelCalls());
|
||||
assertEquals(1, context.budget().snapshot().toolCalls());
|
||||
assertEquals(10, context.budget().snapshot().totalTokens());
|
||||
assertTrue(model.prompts().get(0).contains("那退款服务呢？"));
|
||||
assertTrue(model.prompts().get(0).contains("支付服务连接池已耗尽"));
|
||||
assertTrue(model.instructions().get(1).stream()
|
||||
.filter(ToolResponseMessage.class::isInstance)
|
||||
.map(ToolResponseMessage.class::cast)
|
||||
.flatMap(message -> message.getResponses().stream())
|
||||
.anyMatch(response -> response.id().equals("call-rag-1")
|
||||
&& response.responseData().contains("call-rag-1")));
|
||||
assertEquals(List.of("session-agent", "session-agent"), audit.sessionIds);
|
||||
assertEquals(List.of("run-agent", "run-agent"), audit.runIds);
|
||||
assertEquals(RunState.RUNNING, context.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noEvidenceStopsWithNullConclusionAndNoRetry() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
RunContext context = core.startRun("session-no-evidence", "run-no-evidence");
|
||||
AtomicInteger toolCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
toolCalls.incrementAndGet();
|
||||
return ToolBoundaryResult.ready(id, noEvidence(id), EvidenceStatus.NO_EVIDENCE);
|
||||
});
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
toolCall("call-empty-1", "{\"query\":\"refund timeout\"}"),
|
||||
new AssistantMessage(noEvidenceDraft("call-empty-1")));
|
||||
|
||||
DiagnosisDraft draft = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("退款为什么超时？", null));
|
||||
|
||||
assertNull(draft.conclusion());
|
||||
assertEquals(AnalysisKind.NEGATIVE_OBSERVATION, draft.analysis().get(0).kind());
|
||||
assertTrue(draft.limitations().missingInfo().contains("退款服务运行日志"));
|
||||
assertEquals(2, model.calls());
|
||||
assertEquals(1, toolCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fencedOutputFailsClosedWithoutAgentRetry() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
new AssistantMessage("```json\n" + supportedDraft("call-1") + "\n```"));
|
||||
RunContext context = core.startRun("session-fenced", "run-fenced");
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(1, context.budget().snapshot().modelCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void modelCallBudgetStopsNextReactRoundBeforeChatModel() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(1, 2, 2,
|
||||
100, 100, 200, 100_000));
|
||||
RunContext context = core.startRun("session-model-budget", "run-model-budget");
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
return ToolBoundaryResult.ready(id, evidence(id), EvidenceStatus.EVIDENCE_FOUND);
|
||||
});
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
toolCall("call-budget-1", "{\"query\":\"timeout\"}"),
|
||||
new AssistantMessage(supportedDraft("call-budget-1")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void actualTokenUsageExhaustionStopsDraftWithoutRetry() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(2, 2, 2,
|
||||
10, 10, 10, 100_000));
|
||||
RunContext context = core.startRun("session-token-budget", "run-token-budget");
|
||||
ScriptedChatModel model = new ScriptedChatModel(6, 6,
|
||||
new AssistantMessage(supportedDraft("call-unused")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(12, context.budget().snapshot().totalTokens());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void contextLimitRejectsOriginalQueryBeforeModelCall() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
RunContext context = core.startRun("session-context-budget", "run-context-budget");
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
new AssistantMessage(supportedDraft("call-unused")));
|
||||
DiagnosisAgentLimits limits = new DiagnosisAgentLimits(4, 100, 200, 1_000);
|
||||
|
||||
DiagnosisAgentLimitException failure = assertThrows(DiagnosisAgentLimitException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), limits)
|
||||
.execute(context, new DiagnosisAgentInput("超过四个字节", null)));
|
||||
|
||||
assertEquals("query", failure.boundary());
|
||||
assertEquals(0, model.calls());
|
||||
assertEquals(0, context.budget().snapshot().modelCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void draftLimitRejectsOversizedOutputWithoutSecondCall() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
RunContext context = core.startRun("session-draft-budget", "run-draft-budget");
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
new AssistantMessage(supportedDraft("call-unused")));
|
||||
DiagnosisAgentLimits limits = new DiagnosisAgentLimits(1_000, 1_000, 2_000, 20);
|
||||
|
||||
DiagnosisAgentLimitException failure = assertThrows(DiagnosisAgentLimitException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), limits)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals("draft", failure.boundary());
|
||||
assertEquals(1, model.calls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsInvalidLimitConfiguration() {
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> new DiagnosisAgentLimits(0, 1, 1, 1));
|
||||
assertThrows(IllegalArgumentException.class,
|
||||
() -> new DiagnosisAgentInput(" ", null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void generatedDraftSchemaAllowsExplicitNullConclusion() throws Exception {
|
||||
JsonNode schema = objectMapper.readTree(new DiagnosisDraftOutputSchema(objectMapper).getJsonSchema());
|
||||
|
||||
JsonNode conclusionTypes = schema.path("properties").path("conclusion").path("type");
|
||||
assertTrue(conclusionTypes.isArray());
|
||||
assertTrue(java.util.stream.StreamSupport.stream(conclusionTypes.spliterator(), false)
|
||||
.map(JsonNode::asText)
|
||||
.anyMatch("object"::equals));
|
||||
assertTrue(java.util.stream.StreamSupport.stream(conclusionTypes.spliterator(), false)
|
||||
.map(JsonNode::asText)
|
||||
.anyMatch("null"::equals));
|
||||
}
|
||||
|
||||
private DiagnosisAgentUseCase useCase(DiagnosisHarnessCore core,
|
||||
ChatModel model,
|
||||
HarnessEvidenceTools tools,
|
||||
DiagnosisAgentLimits limits,
|
||||
ModelHook... hooks) {
|
||||
DiagnosisAgentFactory factory = new DiagnosisAgentFactory(
|
||||
model, core, tools, objectMapper, List.of(hooks));
|
||||
return new DiagnosisAgentUseCase(core, factory, objectMapper, limits);
|
||||
}
|
||||
|
||||
private HarnessEvidenceTools tools(EvidenceToolInvoker ragInvoker) {
|
||||
EvidenceToolInvoker unused = errorInvoker();
|
||||
return new HarnessEvidenceTools(ragInvoker, unused, unused);
|
||||
}
|
||||
|
||||
private EvidenceToolInvoker errorInvoker() {
|
||||
return (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core(RunBudgetLimits budget) {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused-run", Duration.ofMinutes(5),
|
||||
budget, HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private RunBudgetLimits defaultBudget() {
|
||||
return new RunBudgetLimits(4, 3, 2, 100, 100, 200, 100_000);
|
||||
}
|
||||
|
||||
private AssistantMessage toolCall(String id, String arguments) {
|
||||
return AssistantMessage.builder()
|
||||
.content("")
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE, arguments)))
|
||||
.build();
|
||||
}
|
||||
|
||||
private String evidence(String id) {
|
||||
return "{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"query\":\"refund timeout\",\"evidence\":[{\"document_id\":\"doc-1\","
|
||||
+ "\"source\":\"runbook\",\"title\":\"退款连接池\",\"breadcrumb\":[\"运行\"],"
|
||||
+ "\"excerpt\":\"active=50 max=50\"}],\"returned_count\":1,\"truncated\":false}";
|
||||
}
|
||||
|
||||
private String noEvidence(String id) {
|
||||
return "{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"query\":\"refund timeout\",\"evidence\":[],\"returned_count\":0,\"truncated\":false}";
|
||||
}
|
||||
|
||||
private String supportedDraft(String toolCallId) {
|
||||
return """
|
||||
{
|
||||
"conclusion":{"text":"退款服务出现连接池等待","based_on_analysis_ids":["a-1"]},
|
||||
"analysis":[{"analysis_id":"a-1","kind":"NORMAL","text":"连接池 active 达到上限","tool_call_ids":["%s"]}],
|
||||
"action_plan":[{"action":"检查连接归还路径","based_on_analysis_ids":["a-1"],"requires_human_confirmation":false}],
|
||||
"recommendations":[{"text":"增加连接池饱和告警","based_on_analysis_ids":["a-1"]}],
|
||||
"limitations":{"scope":"refund-service","missing_info":[]}
|
||||
}
|
||||
""".formatted(toolCallId);
|
||||
}
|
||||
|
||||
private String noEvidenceDraft(String toolCallId) {
|
||||
return """
|
||||
{
|
||||
"conclusion":null,
|
||||
"analysis":[{"analysis_id":"a-1","kind":"NEGATIVE_OBSERVATION","text":"知识库查询范围内未找到退款超时证据","tool_call_ids":["%s"]}],
|
||||
"action_plan":[],
|
||||
"recommendations":[],
|
||||
"limitations":{"scope":"lookup query=refund timeout","missing_info":["退款服务运行日志"]}
|
||||
}
|
||||
""".formatted(toolCallId);
|
||||
}
|
||||
|
||||
private static final class ScriptedChatModel implements ChatModel {
|
||||
|
||||
private final int promptTokens;
|
||||
private final int completionTokens;
|
||||
private final List<AssistantMessage> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final List<List<Message>> instructions = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedChatModel(int promptTokens, int completionTokens,
|
||||
AssistantMessage... responses) {
|
||||
this.promptTokens = promptTokens;
|
||||
this.completionTokens = completionTokens;
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
instructions.add(List.copyOf(prompt.getInstructions()));
|
||||
int index = calls.getAndIncrement();
|
||||
if (index >= responses.size()) {
|
||||
throw new AssertionError("unexpected model retry");
|
||||
}
|
||||
ChatResponseMetadata metadata = ChatResponseMetadata.builder()
|
||||
.usage(new DefaultUsage(promptTokens, completionTokens))
|
||||
.build();
|
||||
return new ChatResponse(List.of(new Generation(responses.get(index))), metadata);
|
||||
}
|
||||
|
||||
int calls() {
|
||||
return calls.get();
|
||||
}
|
||||
|
||||
List<String> prompts() {
|
||||
return prompts;
|
||||
}
|
||||
|
||||
List<List<Message>> instructions() {
|
||||
return instructions;
|
||||
}
|
||||
}
|
||||
|
||||
@HookPositions({HookPosition.BEFORE_MODEL})
|
||||
private static final class CapturingAuditHook extends ModelHook {
|
||||
|
||||
private final List<String> sessionIds = new ArrayList<>();
|
||||
private final List<String> runIds = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "capturing_agent_step_audit";
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<java.util.Map<String, Object>> beforeModel(
|
||||
OverAllState state, RunnableConfig config) {
|
||||
sessionIds.add(config.metadata("sessionId").map(Object::toString).orElseThrow());
|
||||
runIds.add(config.metadata("runId").map(Object::toString).orElseThrow());
|
||||
return CompletableFuture.completedFuture(java.util.Map.of());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.RagToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolCallRequestEnvelope;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.execution.ToolExecutionException;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class HarnessToolInterceptorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void exposesOnlyFrozenToolDefinitionsWithoutAgentSuppliedCallId() {
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> ready(id));
|
||||
|
||||
List<ToolCallback> callbacks = tools.callbacks();
|
||||
|
||||
assertEquals(List.of(
|
||||
AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
AgentToolContracts.QUERY_LOGS,
|
||||
AgentToolContracts.QUERY_MYSQL),
|
||||
callbacks.stream().map(callback -> callback.getToolDefinition().name()).toList());
|
||||
callbacks.forEach(callback -> {
|
||||
assertFalse(callback.getToolDefinition().inputSchema().contains("tool_call_id"));
|
||||
assertFalse(callback.getToolDefinition().description().isBlank());
|
||||
assertThrows(ToolExecutionException.class, () -> callback.call("{}"));
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void adapterBridgeUsesExactFrameworkIdAndRunIdentity() {
|
||||
RagToolAdapter ragAdapter = mock(RagToolAdapter.class);
|
||||
QueryLogsToolAdapter logsAdapter = mock(QueryLogsToolAdapter.class);
|
||||
MysqlToolAdapter mysqlAdapter = mock(MysqlToolAdapter.class);
|
||||
RunContext context = context(3, 3, 3);
|
||||
when(ragAdapter.execute(eq(context), any())).thenAnswer(invocation -> {
|
||||
ToolCallRequestEnvelope envelope = invocation.getArgument(1);
|
||||
return ready(envelope.toolCallId());
|
||||
});
|
||||
HarnessEvidenceTools tools = HarnessEvidenceTools.fromAdapters(ragAdapter, logsAdapter, mysqlAdapter);
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7", "{\"query\":\"timeout\"}"),
|
||||
ignored -> {
|
||||
throw new AssertionError("registered Tool must not bypass Harness interceptor");
|
||||
});
|
||||
|
||||
ArgumentCaptor<ToolCallRequestEnvelope> envelope = ArgumentCaptor.forClass(ToolCallRequestEnvelope.class);
|
||||
verify(ragAdapter).execute(eq(context), envelope.capture());
|
||||
assertEquals("framework-call-7", envelope.getValue().toolCallId());
|
||||
assertEquals(context.runId(), envelope.getValue().runId());
|
||||
assertEquals(AgentToolContracts.LOOKUP_KNOWLEDGE, envelope.getValue().toolName());
|
||||
assertEquals("framework-call-7", response.getToolCallId());
|
||||
assertTrue(response.getResult().contains("framework-call-7"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsStableErrorObservationWithoutRawFailureDetail() throws Exception {
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.TOOL_EXECUTION_ERROR));
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error", "{\"query\":\"secret\"}"),
|
||||
ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(response.getResult());
|
||||
assertTrue(response.isError());
|
||||
assertEquals("ERROR", observation.path("evidence_status").asText());
|
||||
assertEquals("call-error", observation.path("tool_call_id").asText());
|
||||
assertEquals("TOOL_EXECUTION_ERROR", observation.path("error_code").asText());
|
||||
assertFalse(response.getResult().contains("secret"));
|
||||
assertFalse(response.getResult().contains("raw_response"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void delegatesUnknownToolWithoutCreatingEvidenceInvocation() {
|
||||
AtomicInteger evidenceCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
evidenceCalls.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
AtomicInteger handlerCalls = new AtomicInteger();
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request("unknown_tool", "call-unknown", "{}"),
|
||||
request -> {
|
||||
handlerCalls.incrementAndGet();
|
||||
return ToolCallResponse.error(request.getToolCallId(), request.getToolName(), "not registered");
|
||||
});
|
||||
|
||||
assertTrue(response.isError());
|
||||
assertEquals(1, handlerCalls.get());
|
||||
assertEquals(0, evidenceCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneFrameworkActionConsumesOneToolBudgetWithoutRetry() {
|
||||
DiagnosisHarnessCore core = core(3, 1, 1, 10_000);
|
||||
RunContext context = core.startRun("session-tool-budget", "run-tool-budget");
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = fakeTools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1", "{\"query\":\"one\"}"),
|
||||
ignored -> null);
|
||||
|
||||
assertEquals(1, invocations.get());
|
||||
assertEquals(1, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
private HarnessEvidenceTools fakeTools(EvidenceToolInvoker ragInvoker) {
|
||||
EvidenceToolInvoker unused = (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
return new HarnessEvidenceTools(ragInvoker, unused, unused);
|
||||
}
|
||||
|
||||
private ToolCallRequest request(String toolName, String toolCallId, String arguments) {
|
||||
return new ToolCallRequest(toolName, arguments, toolCallId, java.util.Map.of());
|
||||
}
|
||||
|
||||
private ToolBoundaryResult ready(String toolCallId) {
|
||||
return ToolBoundaryResult.ready(toolCallId,
|
||||
"{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\""
|
||||
+ toolCallId + "\",\"evidence\":[]}",
|
||||
EvidenceStatus.EVIDENCE_FOUND);
|
||||
}
|
||||
|
||||
private RunContext context(int maxModelCalls, int maxToolCalls, int maxCallsPerTool) {
|
||||
return core(maxModelCalls, maxToolCalls, maxCallsPerTool, 10_000)
|
||||
.startRun("session-tool", "run-tool");
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core(int maxModelCalls, int maxToolCalls,
|
||||
int maxCallsPerTool, long maxTokens) {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(),
|
||||
() -> "unused-run",
|
||||
Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(maxModelCalls, maxToolCalls, maxCallsPerTool,
|
||||
maxTokens, maxTokens, maxTokens, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user