feat(harness): add information gain stop and audit

This commit is contained in:
aruo
2026-07-27 01:03:34 +08:00
parent de5a5b09d9
commit d0452184ee
92 changed files with 5019 additions and 122 deletions
@@ -27,6 +27,7 @@ public class ChatHarnessProperties {
private int maxModelCalls = 24;
private int maxToolCalls = 24;
private int maxCallsPerTool = 8;
private int stopAfterConsecutiveNoGain = 2;
private long maxInputTokens = 100_000;
private long maxOutputTokens = 100_000;
private long maxTotalTokens = 200_000;
@@ -71,6 +72,7 @@ public class ChatHarnessProperties {
positive(maxModelCalls, "maxModelCalls");
positive(maxToolCalls, "maxToolCalls");
positive(maxCallsPerTool, "maxCallsPerTool");
positive(stopAfterConsecutiveNoGain, "stopAfterConsecutiveNoGain");
positive(maxInputTokens, "maxInputTokens");
positive(maxOutputTokens, "maxOutputTokens");
positive(maxTotalTokens, "maxTotalTokens");
@@ -27,6 +27,7 @@ import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
import com.superbiz.agent.harness.release.EvidenceRepair;
import com.superbiz.agent.harness.release.EvidenceRepairLimits;
import com.superbiz.agent.harness.release.SafeFallbackFactory;
import com.superbiz.agent.harness.progress.DiagnosisProgressProjector;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
@@ -50,6 +51,7 @@ import com.superbiz.agent.harness.application.KnowledgeQueryOperation;
import com.superbiz.agent.harness.application.SystemChatOperation;
import com.superbiz.agent.harness.application.IntentRouting;
import com.superbiz.agent.harness.audit.HarnessAgentAuditHook;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
import com.superbiz.agent.repository.AgentStepRepository;
@@ -106,7 +108,8 @@ public class HarnessChatConfiguration {
new RunBudgetLimits(properties.getMaxModelCalls(), properties.getMaxToolCalls(),
properties.getMaxCallsPerTool(), properties.getMaxInputTokens(),
properties.getMaxOutputTokens(), properties.getMaxTotalTokens(), properties.getMaxRunBytes()),
HarnessRetryPolicies.strict());
HarnessRetryPolicies.strict(),
properties.getStopAfterConsecutiveNoGain());
}
@Bean
@@ -116,8 +119,16 @@ public class HarnessChatConfiguration {
@Bean
public GuardModelCall guardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor) {
return new GuardModelCall(core, chatModel, executor);
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor,
ModelCallAuditor modelCallAuditor) {
return new GuardModelCall(core, chatModel, executor, modelCallAuditor);
}
@Bean
public ModelCallAuditor modelCallAuditor(DiagnosisHarnessCore core,
DiagnosisTraceRecorder traceRecorder,
AgentStepRepository steps) {
return new ModelCallAuditor(core, traceRecorder, steps);
}
@Bean
@@ -237,20 +248,30 @@ public class HarnessChatConfiguration {
HarnessEvidenceTools tools, ObjectMapper mapper,
AgentStepRepository steps,
AgentReasoningAuditRepository reasoningAudits,
DiagnosisTraceRecorder traceRecorder) {
DiagnosisTraceRecorder traceRecorder,
ModelCallAuditor modelCallAuditor) {
return new DiagnosisAgentFactory(chatModel, core, tools, mapper,
List.of(new HarnessAgentAuditHook(
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)));
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)),
traceRecorder, modelCallAuditor);
}
@Bean
public DiagnosisAgentUseCase diagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory factory,
ObjectMapper mapper,
ChatHarnessProperties properties) {
ChatHarnessProperties properties,
DiagnosisProgressProjector progressProjector) {
return new DiagnosisAgentUseCase(core, factory, mapper, new DiagnosisAgentLimits(
properties.getDiagnosisMaxQueryBytes(), properties.getDiagnosisMaxPreviousTurnBytes(),
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()));
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()),
progressProjector);
}
@Bean
public DiagnosisProgressProjector diagnosisProgressProjector(
CanonicalInvocationStore store, ToolCallKeyFactory keyFactory, ObjectMapper mapper) {
return new DiagnosisProgressProjector(store, keyFactory, mapper);
}
@Bean
@@ -281,13 +302,19 @@ public class HarnessChatConfiguration {
attempt -> { }, traceRecorder);
}
@Bean
public SafeFallbackFactory safeFallbackFactory() {
return new SafeFallbackFactory();
}
@Bean
public DiagnosisReleaseUseCase diagnosisReleaseUseCase(EvidenceGuard evidenceGuard,
EvidenceRepair repair,
SemanticGuard semanticGuard,
DiagnosisTraceRecorder traceRecorder) {
DiagnosisTraceRecorder traceRecorder,
SafeFallbackFactory fallbackFactory) {
return new DiagnosisReleaseUseCase(
evidenceGuard, repair, semanticGuard, new SafeFallbackFactory(), traceRecorder);
evidenceGuard, repair, semanticGuard, fallbackFactory, traceRecorder);
}
@Bean
@@ -327,9 +354,10 @@ public class HarnessChatConfiguration {
@Bean
public DiagnosisOperation diagnosisOperation(DiagnosisAgentUseCase agent,
DiagnosisReleaseUseCase release,
PublishedResultPolicy policy) {
return new DiagnosisChatExecutor(agent, release, policy);
DiagnosisReleaseUseCase release,
PublishedResultPolicy policy,
DiagnosisTraceRecorder traceRecorder) {
return new DiagnosisChatExecutor(agent, release, policy, traceRecorder);
}
@Bean
@@ -0,0 +1,32 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import java.util.Objects;
public record DiagnosisAgentExecution(
DiagnosisDraft draft,
DiagnosisProgressSnapshot progress,
DiagnosisStopReason stopReason) {
public DiagnosisAgentExecution {
Objects.requireNonNull(progress, "progress must not be null");
if (draft == null && stopReason == null) {
throw new IllegalArgumentException("Execution without Draft requires a stop reason");
}
}
public static DiagnosisAgentExecution completed(
DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
return new DiagnosisAgentExecution(
Objects.requireNonNull(draft, "draft must not be null"), progress, null);
}
public static DiagnosisAgentExecution stopped(
DiagnosisProgressSnapshot progress, DiagnosisStopReason stopReason) {
return new DiagnosisAgentExecution(
null, progress, Objects.requireNonNull(stopReason, "stopReason must not be null"));
}
}
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.agent.hook.Hook;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import org.springframework.ai.chat.model.ChatModel;
@@ -19,6 +21,8 @@ public final class DiagnosisAgentFactory {
private final HarnessEvidenceTools evidenceTools;
private final ObjectMapper objectMapper;
private final List<Hook> auditHooks;
private final DiagnosisTraceRecorder traceRecorder;
private final ModelCallAuditor modelCallAuditor;
private final String prompt;
public DiagnosisAgentFactory(ChatModel chatModel,
@@ -26,11 +30,35 @@ public final class DiagnosisAgentFactory {
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks) {
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
DiagnosisTraceRecorder.noop(), new ModelCallAuditor(core));
}
public DiagnosisAgentFactory(ChatModel chatModel,
DiagnosisHarnessCore core,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks,
DiagnosisTraceRecorder traceRecorder) {
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
traceRecorder, new ModelCallAuditor(core));
}
public DiagnosisAgentFactory(ChatModel chatModel,
DiagnosisHarnessCore core,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks,
DiagnosisTraceRecorder traceRecorder,
ModelCallAuditor modelCallAuditor) {
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
this.core = Objects.requireNonNull(core, "core must not be null");
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
this.auditHooks = List.copyOf(Objects.requireNonNull(auditHooks, "auditHooks must not be null"));
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
this.modelCallAuditor = Objects.requireNonNull(
modelCallAuditor, "modelCallAuditor must not be null");
this.prompt = DiagnosisAgentPrompt.load();
}
@@ -43,8 +71,9 @@ public final class DiagnosisAgentFactory {
.systemPrompt(prompt)
.tools(evidenceTools.callbacks())
.interceptors(
new HarnessModelInterceptor(core, context),
new HarnessToolInterceptor(context, evidenceTools, objectMapper))
new HarnessModelInterceptor(core, context, modelCallAuditor),
new HarnessToolInterceptor(
context, evidenceTools, objectMapper, traceRecorder))
.hooks(auditHooks)
.outputSchema(new DiagnosisDraftOutputSchema(objectMapper).getFormat())
.returnReasoningContents(true)
@@ -1,12 +1,59 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import java.util.Objects;
public final class DiagnosisAgentOutputException extends RuntimeException {
public enum Kind {
EXECUTION_FAILED,
EMPTY_DRAFT,
INVALID_JSON,
SCHEMA_INVALID
}
private final Kind kind;
private final long outputBytes;
private final DiagnosisProgressSnapshot progress;
public DiagnosisAgentOutputException(String message) {
super(message);
this(message, null);
}
public DiagnosisAgentOutputException(String message, Throwable cause) {
this(message, cause, Kind.EXECUTION_FAILED, 0L, DiagnosisProgressSnapshot.empty());
}
public DiagnosisAgentOutputException(String message,
Throwable cause,
Kind kind,
long outputBytes,
DiagnosisProgressSnapshot progress) {
super(message, cause);
if (outputBytes < 0) {
throw new IllegalArgumentException("outputBytes must not be negative");
}
this.kind = Objects.requireNonNull(kind, "kind must not be null");
this.outputBytes = outputBytes;
this.progress = Objects.requireNonNull(progress, "progress must not be null");
}
public Kind kind() {
return kind;
}
public long outputBytes() {
return outputBytes;
}
public DiagnosisProgressSnapshot progress() {
return progress;
}
public boolean isDraftContractFailure() {
return kind == Kind.EMPTY_DRAFT
|| kind == Kind.INVALID_JSON
|| kind == Kind.SCHEMA_INVALID;
}
}
@@ -2,13 +2,20 @@ package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.BudgetExceededException;
import com.superbiz.agent.harness.core.RunAbortedException;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.progress.DiagnosisProgressProjection;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import org.springframework.ai.chat.messages.AssistantMessage;
import java.nio.charset.StandardCharsets;
@@ -23,11 +30,20 @@ public final class DiagnosisAgentUseCase {
private final ObjectMapper objectMapper;
private final ObjectReader draftReader;
private final DiagnosisAgentLimits limits;
private final DiagnosisProgressProjection progressProjection;
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory agentFactory,
ObjectMapper objectMapper,
DiagnosisAgentLimits limits) {
this(core, agentFactory, objectMapper, limits, DiagnosisProgressProjection.empty());
}
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory agentFactory,
ObjectMapper objectMapper,
DiagnosisAgentLimits limits,
DiagnosisProgressProjection progressProjection) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.agentFactory = Objects.requireNonNull(agentFactory, "agentFactory must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
@@ -35,9 +51,11 @@ public final class DiagnosisAgentUseCase {
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
this.limits = Objects.requireNonNull(limits, "limits must not be null");
this.progressProjection = Objects.requireNonNull(
progressProjection, "progressProjection must not be null");
}
public DiagnosisDraft execute(RunContext context, DiagnosisAgentInput input) {
public DiagnosisAgentExecution execute(RunContext context, DiagnosisAgentInput input) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(input, "input must not be null");
core.checkActive(context);
@@ -64,23 +82,85 @@ public final class DiagnosisAgentUseCase {
AssistantMessage response;
try {
response = agent.call(inputJson, config);
core.checkActive(context);
} catch (Exception e) {
DiagnosisAgentExecution controlled = controlledExecution(context, e);
if (controlled != null) {
return controlled;
}
throw new DiagnosisAgentOutputException("Diagnosis Agent execution failed", e);
}
core.checkActive(context);
String output = response == null ? null : response.getText();
if (output == null || output.isBlank()) {
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an empty Draft");
throw new DiagnosisAgentOutputException(
"Diagnosis Agent returned an empty Draft",
null,
DiagnosisAgentOutputException.Kind.EMPTY_DRAFT,
0L,
progressProjection.project(context));
}
long draftBytes = utf8Bytes(output);
checkLimit("draft", draftBytes, limits.maxDraftBytes());
core.reserveRunBytes(context, draftBytes);
try {
return draftReader.readValue(output);
} catch (JsonProcessingException e) {
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an invalid Draft", e);
core.reserveRunBytes(context, draftBytes);
} catch (RuntimeException exception) {
DiagnosisAgentExecution controlled = controlledExecution(context, exception);
if (controlled != null) {
return controlled;
}
throw exception;
}
try {
DiagnosisDraft draft = draftReader.readValue(output);
return DiagnosisAgentExecution.completed(draft, progressProjection.project(context));
} catch (JsonProcessingException e) {
DiagnosisAgentOutputException.Kind kind = e instanceof JsonParseException
? DiagnosisAgentOutputException.Kind.INVALID_JSON
: DiagnosisAgentOutputException.Kind.SCHEMA_INVALID;
throw new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
e,
kind,
draftBytes,
progressProjection.project(context));
}
}
private DiagnosisAgentExecution controlledExecution(RunContext context, Throwable failure) {
DiagnosisCollectionStoppedException stopped = findCause(
failure, DiagnosisCollectionStoppedException.class);
if (stopped != null) {
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), stopped.stopReason());
}
RunAbortedException aborted = findCause(failure, RunAbortedException.class);
if (aborted != null) {
if (aborted.termination().state() == RunState.BUDGET_EXHAUSTED) {
context.progress().markBudgetLimitReached();
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
throw aborted;
}
BudgetExceededException budget = findCause(failure, BudgetExceededException.class);
if (budget != null || context.lifecycle().state() == RunState.BUDGET_EXHAUSTED) {
context.progress().markBudgetLimitReached();
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
return null;
}
private static <T extends Throwable> T findCause(Throwable failure, Class<T> type) {
Throwable current = failure;
while (current != null) {
if (type.isInstance(current)) {
return type.cast(current);
}
current = current.getCause();
}
return null;
}
private String writeJson(Object value) {
@@ -0,0 +1,17 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
public final class DiagnosisCollectionStoppedException extends RuntimeException {
private final DiagnosisStopReason stopReason;
public DiagnosisCollectionStoppedException(DiagnosisStopReason stopReason) {
super("Diagnosis collection stopped: " + stopReason);
this.stopReason = stopReason;
}
public DiagnosisStopReason stopReason() {
return stopReason;
}
}
@@ -1,5 +1,8 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
@@ -8,8 +11,11 @@ import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import com.superbiz.agent.harness.tool.boundary.ToolCallRequestEnvelope;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.MysqlToolCall;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsToolCall;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import com.superbiz.agent.harness.tool.contract.RagToolCall;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.function.FunctionToolCallback;
@@ -36,11 +42,11 @@ public final class HarnessEvidenceTools {
this.invokers = Map.copyOf(registered);
this.callbacks = List.of(
definition(AgentToolContracts.LOOKUP_KNOWLEDGE,
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolRequest.class),
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolCall.class),
definition(AgentToolContracts.QUERY_LOGS,
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsRequest.class),
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsToolCall.class),
definition(AgentToolContracts.QUERY_MYSQL,
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolRequest.class));
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolCall.class));
}
public static HarnessEvidenceTools fromAdapters(RagToolAdapter ragAdapter,
@@ -72,6 +78,51 @@ public final class HarnessEvidenceTools {
return invoker.invoke(context, toolCallId, arguments);
}
public ParsedAgentToolCall parse(String toolName, String arguments, ObjectMapper objectMapper) {
Objects.requireNonNull(objectMapper, "objectMapper must not be null");
try {
ObjectReader reader = objectMapper.reader()
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
return switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
RagToolCall call = reader.forType(RagToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
case AgentToolContracts.QUERY_LOGS -> {
QueryLogsToolCall call = reader.forType(QueryLogsToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
case AgentToolContracts.QUERY_MYSQL -> {
MysqlToolCall call = reader.forType(MysqlToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
};
} catch (IllegalArgumentException exception) {
throw exception;
} catch (Exception exception) {
throw new IllegalArgumentException("Tool Call Envelope is invalid", exception);
}
}
private static ParsedAgentToolCall parsed(
com.superbiz.agent.harness.progress.PreviousObservation previousObservation,
Object input,
ObjectMapper objectMapper) throws Exception {
return new ParsedAgentToolCall(
previousObservation, input, objectMapper.writeValueAsString(input));
}
private static void requireInput(Object input) {
if (input == null) {
throw new IllegalArgumentException("Tool Call Envelope input is required");
}
}
private static EvidenceToolInvoker bridge(String toolName, AdapterCall adapter) {
return (context, toolCallId, arguments) -> adapter.execute(
context,
@@ -6,6 +6,9 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.model.ChatResponse;
@@ -15,10 +18,17 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
private final DiagnosisHarnessCore core;
private final RunContext context;
private final ModelCallAuditor auditor;
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context) {
this(core, context, new ModelCallAuditor(core));
}
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context,
ModelCallAuditor auditor) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.context = Objects.requireNonNull(context, "context must not be null");
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
}
@Override
@@ -31,23 +41,32 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
Objects.requireNonNull(request, "request must not be null");
Objects.requireNonNull(handler, "handler must not be null");
core.beforeModelCall(context);
ModelResponse response = handler.call(request);
recordUsage(response == null ? null : response.getChatResponse());
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
ModelResponse response;
try {
response = handler.call(request);
} catch (RuntimeException exception) {
auditor.recordUsage(context, call, 0, 0, false);
throw exception;
}
recordUsage(call, response == null ? null : response.getChatResponse());
core.checkActive(context);
return response;
}
private void recordUsage(ChatResponse response) {
private void recordUsage(ModelCallLedger.Call call, ChatResponse response) {
if (response == null || response.getMetadata() == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
Usage usage = response.getMetadata().getUsage();
if (usage == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
long inputTokens = nonNegative(usage.getPromptTokens());
long outputTokens = nonNegative(usage.getCompletionTokens());
core.recordTokens(context, inputTokens, outputTokens);
auditor.recordUsage(context, call, inputTokens, outputTokens, true);
}
private static long nonNegative(Integer value) {
@@ -6,8 +6,16 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolInterceptor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.contract.InvocationStatus;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.CompletedToolCall;
import com.superbiz.agent.harness.progress.DiagnosisCollectionState;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.ToolScopeNormalizer;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import java.util.LinkedHashMap;
@@ -19,13 +27,26 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
private final RunContext context;
private final HarnessEvidenceTools evidenceTools;
private final ObjectMapper objectMapper;
private final ToolScopeNormalizer scopeNormalizer;
private final ToolResultViewProjector viewProjector;
private final DiagnosisTraceRecorder traceRecorder;
public HarnessToolInterceptor(RunContext context,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper) {
this(context, evidenceTools, objectMapper, DiagnosisTraceRecorder.noop());
}
public HarnessToolInterceptor(RunContext context,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
DiagnosisTraceRecorder traceRecorder) {
this.context = Objects.requireNonNull(context, "context must not be null");
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
this.scopeNormalizer = new ToolScopeNormalizer(objectMapper);
this.viewProjector = new ToolResultViewProjector(objectMapper);
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
}
@Override
@@ -41,10 +62,76 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
return handler.call(request);
}
DiagnosisProgressSnapshotState before = context.progress().snapshot();
if (before.collectionState() == DiagnosisCollectionState.SATURATED
&& before.stopInstructionDelivered()) {
recordRejection(request, "COLLECTION_STOPPED");
throw new DiagnosisCollectionStoppedException(before.stopReason());
}
ParsedAgentToolCall call;
String normalizedScope;
try {
call = evidenceTools.parse(request.getToolName(), request.getArguments(), objectMapper);
context.progress().applyPreviousObservation(call.previousObservation());
DiagnosisProgressSnapshotState evaluated = context.progress().snapshot();
recordModelProgress(before, call, evaluated);
if (evaluated.collectionState() == DiagnosisCollectionState.SATURATED) {
recordRejection(request, "INFORMATION_SATURATED");
return stopRequired(request, evaluated.stopReason());
}
normalizedScope = scopeNormalizer.normalize(request.getToolName(), call.businessInput());
} catch (IllegalArgumentException | IllegalStateException exception) {
recordRejection(request, "INVALID_PROGRESS_PROTOCOL");
return safeError(request, "INVALID_PROGRESS_PROTOCOL");
}
if (context.progress().isDuplicate(request.getToolName(), normalizedScope)) {
recordRejection(request, "DUPLICATE_SCOPE");
context.progress().recordDuplicateScope();
DiagnosisProgressSnapshotState duplicate = context.progress().snapshot();
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
InformationGain.NO_GAIN, "HARNESS", duplicate);
if (duplicate.collectionState() == DiagnosisCollectionState.SATURATED) {
return stopRequired(request, duplicate.stopReason());
}
return duplicateScope(request);
}
ToolBoundaryResult result = evidenceTools.invoke(
context, request.getToolName(), request.getToolCallId(), request.getArguments());
context, request.getToolName(), request.getToolCallId(), call.businessArguments());
if (result.status() == InvocationStatus.READY) {
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), result.agentResult());
ToolControlView control = viewProjector.controlView(result.agentResult());
if (control.evidenceStatus() != result.evidenceStatus()) {
recordRejection(request, "OBSERVATION_CONTRACT_MISMATCH");
return safeError(request, "OBSERVATION_CONTRACT_MISMATCH");
}
context.progress().recordCompleted(
new CompletedToolCall(
request.getToolCallId(), request.getToolName(), normalizedScope),
result.evidenceStatus());
DiagnosisProgressSnapshotState completed = context.progress().snapshot();
if (result.evidenceStatus()
== com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
InformationGain.NO_GAIN, "HARNESS", completed);
}
boolean stopRequired = completed.collectionState() == DiagnosisCollectionState.SATURATED
&& context.progress().claimStopInstruction();
if (stopRequired) {
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(), completed));
}
String observation = viewProjector.modelObservation(
request.getToolName(), result.agentResult(), normalizedScope,
stopRequired, completed.stopReason());
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), observation);
}
if ("BUDGET_EXHAUSTED".equals(result.errorCode())) {
context.progress().markBudgetLimitReached();
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(),
context.progress().snapshot()));
}
return ToolCallResponse.builder()
.toolCallId(request.getToolCallId())
@@ -55,6 +142,90 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
.build();
}
private ToolCallResponse stopRequired(ToolCallRequest request, DiagnosisStopReason reason) {
if (!context.progress().claimStopInstruction()) {
throw new DiagnosisCollectionStoppedException(reason);
}
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(),
context.progress().snapshot()));
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("tool_call_id", request.getToolCallId());
observation.put("stop_required", true);
observation.put("reason", reason.name());
return ToolCallResponse.of(
request.getToolCallId(), request.getToolName(), writeObservation(observation));
}
private ToolCallResponse duplicateScope(ToolCallRequest request) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("tool_call_id", request.getToolCallId());
observation.put("observation_status", "DUPLICATE_SCOPE");
observation.put("information_gain", "NO_GAIN");
observation.put("stop_required", false);
return ToolCallResponse.of(
request.getToolCallId(), request.getToolName(), writeObservation(observation));
}
private ToolCallResponse safeError(ToolCallRequest request, String errorCode) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", "ERROR");
observation.put("tool_call_id", request.getToolCallId());
observation.put("error_code", errorCode);
return ToolCallResponse.builder()
.toolCallId(request.getToolCallId())
.toolName(request.getToolName())
.content(writeObservation(observation))
.status("error")
.metadata(Map.of("error", true))
.build();
}
private void recordRejection(ToolCallRequest request, String errorCode) {
traceRecorder.record(TraceAuditEvents.toolRequestRejected(
context, request.getToolCallId(), request.getToolName(), errorCode));
}
private String writeObservation(Map<String, Object> observation) {
try {
return objectMapper.writeValueAsString(observation);
} catch (JsonProcessingException exception) {
return "{\"evidence_status\":\"ERROR\",\"error_code\":\"SERIALIZATION_ERROR\"}";
}
}
private void recordModelProgress(
DiagnosisProgressSnapshotState before,
ParsedAgentToolCall call,
DiagnosisProgressSnapshotState after) {
if (call.previousObservation() == null) {
return;
}
before.completedToolCalls().stream()
.filter(completed -> completed.toolCallId()
.equals(call.previousObservation().toolCallId()))
.findFirst()
.ifPresent(completed -> recordProgress(
completed.toolCallId(), completed.toolName(), completed.normalizedScope(),
call.previousObservation().informationGain(), "MODEL", after));
}
private void recordProgress(
String toolCallId,
String toolName,
String normalizedScope,
InformationGain informationGain,
String producer,
DiagnosisProgressSnapshotState state) {
traceRecorder.record(TraceAuditEvents.toolProgress(
context, toolCallId, toolName, scopeSummary(toolName, normalizedScope),
informationGain, producer, state));
}
private String scopeSummary(String toolName, String normalizedScope) {
return toolName + "#" + String.format("%08x", normalizedScope.hashCode());
}
private String errorObservation(ToolBoundaryResult result) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", result.evidenceStatus());
@@ -0,0 +1,18 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record ParsedAgentToolCall(
PreviousObservation previousObservation,
Object businessInput,
String businessArguments) {
public ParsedAgentToolCall {
if (businessInput == null) {
throw new IllegalArgumentException("businessInput must not be null");
}
if (businessArguments == null || businessArguments.isBlank()) {
throw new IllegalArgumentException("businessArguments must not be blank");
}
}
}
@@ -0,0 +1,11 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
public record ToolControlView(
EvidenceStatus evidenceStatus,
int returnedCount,
RagRelevanceLevel relevanceLevel,
boolean truncated) {
}
@@ -0,0 +1,120 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import java.util.List;
import java.util.Objects;
public final class ToolResultViewProjector {
private static final List<String> COMMON_FIELDS = List.of(
"evidence_status", "tool_call_id", "truncated");
private final ObjectMapper objectMapper;
public ToolResultViewProjector(ObjectMapper objectMapper) {
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
public ToolControlView controlView(String agentResult) {
JsonNode root = readObject(agentResult);
EvidenceStatus status;
try {
status = EvidenceStatus.valueOf(requiredText(root, "evidence_status"));
} catch (IllegalArgumentException exception) {
throw new IllegalArgumentException("Invalid evidence status", exception);
}
RagRelevanceLevel relevance = null;
String relevanceText = text(root, "relevance_level");
if (!relevanceText.isBlank()) {
try {
relevance = RagRelevanceLevel.valueOf(relevanceText);
} catch (IllegalArgumentException exception) {
throw new IllegalArgumentException("Invalid relevance level", exception);
}
}
return new ToolControlView(
status,
Math.max(0, root.path("returned_count").asInt(0)),
relevance,
root.path("truncated").asBoolean(false));
}
public String modelObservation(String toolName,
String agentResult,
String normalizedScope,
boolean stopRequired,
DiagnosisStopReason stopReason) {
JsonNode root = readObject(agentResult);
ObjectNode observation = objectMapper.createObjectNode();
copy(root, observation, COMMON_FIELDS);
switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
ObjectNode scope = observation.putObject("scope");
scope.put("query", text(root, "query"));
copy(root, observation, List.of("evidence", "relevance_level"));
}
case AgentToolContracts.QUERY_LOGS ->
copy(root, observation, List.of("source_kind", "scope", "patterns", "events"));
case AgentToolContracts.QUERY_MYSQL -> {
try {
observation.set("scope", objectMapper.readTree(normalizedScope));
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Normalized scope is invalid", exception);
}
copy(root, observation, List.of("columns", "rows"));
}
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
}
if (stopRequired) {
observation.put("stop_required", true);
observation.put("reason", Objects.requireNonNull(stopReason, "stopReason must not be null").name());
}
try {
return objectMapper.writeValueAsString(observation);
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Model observation is not serializable", exception);
}
}
private JsonNode readObject(String value) {
try {
JsonNode root = objectMapper.readTree(value);
if (root == null || !root.isObject()) {
throw new IllegalArgumentException("Tool result must be a JSON object");
}
return root;
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Tool result must be valid JSON", exception);
}
}
private static void copy(JsonNode source, ObjectNode target, List<String> fields) {
for (String field : fields) {
JsonNode value = source.get(field);
if (value != null && !value.isNull()) {
target.set(field, value);
}
}
}
private static String requiredText(JsonNode root, String field) {
String value = text(root, field);
if (value.isBlank()) {
throw new IllegalArgumentException(field + " must not be blank");
}
return value;
}
private static String text(JsonNode root, String field) {
JsonNode value = root.get(field);
return value == null || value.isNull() ? "" : value.asText("");
}
}
@@ -107,8 +107,7 @@ public final class ChatApplicationUseCase {
PathResult path = executePath(
intent, context, request.query(), previousTurn.orElse(null), observer);
core.checkActive(context);
core.completeSuccess(context);
completePath(context, intent, path);
String safeJson = write(path.content());
persistFinish(context, intent, path.outcome(), safeJson,
path.publishedResult(), durationMillis(startedNanos));
@@ -131,6 +130,20 @@ public final class ChatApplicationUseCase {
}
}
private void completePath(RunContext context, IntentType intent, PathResult path) {
if (path.handledBudgetTermination()) {
if (intent != IntentType.DIAGNOSIS
|| path.outcome() != ReleaseOutcome.FALLBACK
|| path.publishedResult() != null
|| context.lifecycle().state() != RunState.BUDGET_EXHAUSTED) {
throw new IllegalStateException("Invalid handled Diagnosis budget fallback");
}
return;
}
core.checkActive(context);
core.completeSuccess(context);
}
private PathResult executePath(IntentType intent,
RunContext context,
String query,
@@ -140,18 +153,19 @@ public final class ChatApplicationUseCase {
case SYSTEM_CHAT -> {
observer.onStatus(ChatApplicationStatus.SYSTEM_RESPONDING);
yield new PathResult(
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null);
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null, false);
}
case KNOWLEDGE_QUERY -> {
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_SEARCHING);
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_ANSWERING);
yield new PathResult(
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null);
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null, false);
}
case DIAGNOSIS -> {
DiagnosisExecutionResult result = diagnosis.execute(
context, query, previousTurn, observer::onStatus);
yield new PathResult(result.outcome(), result.content(), result.publishedResult());
yield new PathResult(result.outcome(), result.content(), result.publishedResult(),
result.handledBudgetTermination());
}
};
}
@@ -251,7 +265,8 @@ public final class ChatApplicationUseCase {
private record PathResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
PublishedResult publishedResult,
boolean handledBudgetTermination) {
}
private static final class CoreRunControl implements ChatRunControl {
@@ -8,7 +8,15 @@ import java.util.Objects;
public record DiagnosisExecutionResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
PublishedResult publishedResult,
boolean handledBudgetTermination) {
public DiagnosisExecutionResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
this(outcome, content, publishedResult, false);
}
public DiagnosisExecutionResult {
Objects.requireNonNull(outcome, "outcome must not be null");
@@ -25,5 +33,9 @@ public record DiagnosisExecutionResult(
if (outcome != ReleaseOutcome.SUCCESS && publishedResult != null) {
throw new IllegalArgumentException("only success can contain PublishedResult");
}
if (handledBudgetTermination && outcome != ReleaseOutcome.FALLBACK) {
throw new IllegalArgumentException(
"handled budget termination requires fallback release");
}
}
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.application.executor;
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
import com.superbiz.agent.harness.application.ChatApplicationStatus;
import com.superbiz.agent.harness.application.DiagnosisContent;
@@ -8,7 +10,8 @@ import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
import com.superbiz.agent.harness.application.DiagnosisOperation;
import com.superbiz.agent.harness.application.FallbackContent;
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.contract.PreviousTurn;
import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
@@ -25,13 +28,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
private final DiagnosisAgentUseCase diagnosisAgent;
private final DiagnosisReleaseUseCase releaseUseCase;
private final PublishedResultPolicy publishedPolicy;
private final DiagnosisTraceRecorder traceRecorder;
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
DiagnosisReleaseUseCase releaseUseCase,
PublishedResultPolicy publishedPolicy) {
this(diagnosisAgent, releaseUseCase, publishedPolicy, DiagnosisTraceRecorder.noop());
}
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
DiagnosisReleaseUseCase releaseUseCase,
PublishedResultPolicy publishedPolicy,
DiagnosisTraceRecorder traceRecorder) {
this.diagnosisAgent = Objects.requireNonNull(diagnosisAgent, "diagnosisAgent must not be null");
this.releaseUseCase = Objects.requireNonNull(releaseUseCase, "releaseUseCase must not be null");
this.publishedPolicy = Objects.requireNonNull(publishedPolicy, "publishedPolicy must not be null");
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
}
@Override
@@ -41,15 +53,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
Consumer<ChatApplicationStatus> statusSink) {
Objects.requireNonNull(statusSink, "statusSink must not be null");
statusSink.accept(ChatApplicationStatus.DIAGNOSIS_RUNNING);
DiagnosisDraft draft = diagnosisAgent.execute(
context, new DiagnosisAgentInput(query, previousTurn));
DiagnosisAgentExecution execution;
try {
execution = diagnosisAgent.execute(
context, new DiagnosisAgentInput(query, previousTurn));
} catch (DiagnosisAgentOutputException exception) {
return recoverInvalidDraft(context, exception, statusSink);
}
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, draft);
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, execution);
if (released.outcome() == ReleaseOutcome.FALLBACK) {
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK,
new FallbackContent(released.fallback()),
null);
null,
execution.stopReason()
== com.superbiz.agent.harness.progress.DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
PublishedResult published = publishedPolicy.create(
query, released.draft(), released.verifiedEvidence()).orElse(null);
@@ -60,4 +79,26 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
released.verifiedEvidence().verifiedSources()),
published);
}
private DiagnosisExecutionResult recoverInvalidDraft(
RunContext context,
DiagnosisAgentOutputException exception,
Consumer<ChatApplicationStatus> statusSink) {
if (!exception.isDraftContractFailure()) {
throw exception;
}
boolean hasProgress = exception.progress().hasObservedFacts();
traceRecorder.record(TraceAuditEvents.agentDraftInvalid(
context, exception.kind(), exception.outputBytes(), hasProgress));
if (!hasProgress) {
throw exception;
}
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
DiagnosisReleaseResult released = releaseUseCase.releaseInvalidDraft(
context, exception.progress());
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK,
new FallbackContent(released.fallback()),
null);
}
}
@@ -16,6 +16,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagEvidence;
@@ -106,7 +107,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
}
core.reserveRunBytes(context, bytes);
String output = modelCall.call(
context,
context, ModelCallComponent.KNOWLEDGE_ANSWER,
new Prompt(List.of(new SystemMessage(prompt), new UserMessage(modelInput))),
limits.modelTimeout(),
limits.maxModelOutputBytes());
@@ -7,6 +7,7 @@ import com.superbiz.agent.harness.application.SystemChatOperation;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -47,7 +48,7 @@ public final class SystemChatExecutor implements SystemChatOperation {
}
core.reserveRunBytes(context, bytes);
String answer = modelCall.call(
context,
context, ModelCallComponent.SYSTEM_CHAT,
new Prompt(List.of(new SystemMessage(SYSTEM_PROMPT), new UserMessage(query))),
limits.timeout(),
limits.maxOutputBytes());
@@ -12,6 +12,7 @@ import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.core.RunBudgetUsage;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.repository.ChatSessionRepository;
import com.superbiz.agent.repository.DiagnosisRunRepository;
import org.springframework.stereotype.Component;
@@ -119,6 +120,8 @@ public class JpaChatRunStore implements ChatRunStore {
run.setTotalDurationMs(Math.max(0, durationMs));
RunBudgetUsage usage = context.budget().snapshot();
run.setTotalTokenCount(saturatingInt(usage.totalTokens()));
run.setStepCount(context.modelCalls().componentCallCount(
ModelCallComponent.DIAGNOSIS_AGENT));
run.setToolCallCount(saturatingInt(usage.toolCalls()));
runs.save(run);
if (outcome == ReleaseOutcome.SUCCESS || outcome == ReleaseOutcome.FALLBACK) {
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.application.IntentRouting;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
@@ -86,7 +87,8 @@ public final class IntentRouter implements IntentRouting {
context,
context.retryPolicies().intentRouter(),
() -> parse(modelCall.call(
context, prompt, remaining(started), limits.maxOutputBytes())),
context, ModelCallComponent.INTENT_ROUTER,
prompt, remaining(started), limits.maxOutputBytes())),
this::classify,
attempt -> {
attemptRecorder.accept(attempt);
@@ -0,0 +1,85 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.repository.AgentStepRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Objects;
public final class ModelCallAuditor {
private static final Logger log = LoggerFactory.getLogger(ModelCallAuditor.class);
private final DiagnosisHarnessCore core;
private final DiagnosisTraceRecorder traceRecorder;
private final AgentStepRepository agentSteps;
public ModelCallAuditor(DiagnosisHarnessCore core) {
this(core, DiagnosisTraceRecorder.noop(), null);
}
public ModelCallAuditor(DiagnosisHarnessCore core,
DiagnosisTraceRecorder traceRecorder,
AgentStepRepository agentSteps) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
this.agentSteps = agentSteps;
}
public ModelCallLedger.Call begin(RunContext context, ModelCallComponent component) {
Objects.requireNonNull(context, "context must not be null");
return context.modelCalls().begin(component);
}
public void recordUsage(RunContext context, ModelCallLedger.Call call,
long inputTokens, long outputTokens, boolean usageAvailable) {
Objects.requireNonNull(context, "context must not be null");
if (!context.modelCalls().record(call, inputTokens, outputTokens, usageAvailable)) {
return;
}
try {
if (usageAvailable) {
core.recordTokens(context, inputTokens, outputTokens);
}
} finally {
traceRecorder.record(TraceAuditEvents.modelTokenUsage(
context, call, inputTokens, outputTokens, usageAvailable));
updateAgentStep(context, call, inputTokens, outputTokens, usageAvailable);
}
}
private void updateAgentStep(RunContext context, ModelCallLedger.Call call,
long inputTokens, long outputTokens, boolean usageAvailable) {
if (!usageAvailable || agentSteps == null
|| call.component() != ModelCallComponent.DIAGNOSIS_AGENT) {
return;
}
try {
agentSteps.findByRunIdAndStepIndex(context.runId(), call.componentRound() - 1)
.ifPresent(step -> saveTokens(step, inputTokens, outputTokens));
} catch (RuntimeException exception) {
log.warn("Failed to persist AgentStep token audit: runId={}, round={}",
context.runId(), call.componentRound());
}
}
private void saveTokens(AgentStep step, long inputTokens, long outputTokens) {
step.setTokenCount(saturatingInt(safeAdd(inputTokens, outputTokens)));
agentSteps.save(step);
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
private static int saturatingInt(long value) {
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, value);
}
}
@@ -0,0 +1,20 @@
package com.superbiz.agent.harness.audit;
public enum ModelCallComponent {
INTENT_ROUTER(TracePhase.ROUTING),
SYSTEM_CHAT(TracePhase.AGENT),
KNOWLEDGE_ANSWER(TracePhase.AGENT),
DIAGNOSIS_AGENT(TracePhase.AGENT),
EVIDENCE_REPAIR(TracePhase.EVIDENCE),
SEMANTIC_GUARD(TracePhase.SEMANTIC);
private final TracePhase tracePhase;
ModelCallComponent(TracePhase tracePhase) {
this.tracePhase = tracePhase;
}
public TracePhase tracePhase() {
return tracePhase;
}
}
@@ -0,0 +1,86 @@
package com.superbiz.agent.harness.audit;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public final class ModelCallLedger {
private final Map<ModelCallComponent, Integer> componentRounds =
new EnumMap<>(ModelCallComponent.class);
private final Set<Call> auditedCalls = new HashSet<>();
private int startedCallCount;
private int usageUnavailableCount;
private long inputTokens;
private long outputTokens;
private long totalTokens;
public synchronized Call begin(ModelCallComponent component) {
Objects.requireNonNull(component, "component must not be null");
int round = componentRounds.merge(component, 1, Integer::sum);
startedCallCount++;
return new Call(component, round);
}
public synchronized boolean record(Call call, long input, long output, boolean usageAvailable) {
Objects.requireNonNull(call, "call must not be null");
if (input < 0 || output < 0) {
throw new IllegalArgumentException("token counts must not be negative");
}
if (!auditedCalls.add(call)) {
return false;
}
if (!usageAvailable) {
usageUnavailableCount++;
return true;
}
inputTokens = safeAdd(inputTokens, input);
outputTokens = safeAdd(outputTokens, output);
totalTokens = safeAdd(totalTokens, safeAdd(input, output));
return true;
}
public synchronized int componentCallCount(ModelCallComponent component) {
Objects.requireNonNull(component, "component must not be null");
return componentRounds.getOrDefault(component, 0);
}
public synchronized Snapshot snapshot() {
return new Snapshot(
startedCallCount,
auditedCalls.size(),
usageUnavailableCount,
inputTokens,
outputTokens,
totalTokens);
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
public record Call(ModelCallComponent component, int componentRound) {
public Call {
Objects.requireNonNull(component, "component must not be null");
if (componentRound < 1) {
throw new IllegalArgumentException("componentRound must be positive");
}
}
}
public record Snapshot(
int startedCallCount,
int auditedCallCount,
int usageUnavailableCount,
long inputTokens,
long outputTokens,
long totalTokens) {
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
@@ -8,6 +9,8 @@ import com.superbiz.agent.harness.contract.SemanticVerdict;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.retry.RetryAttempt;
import java.util.ArrayList;
@@ -33,11 +36,27 @@ public final class TraceAuditEvents {
public static DiagnosisTraceAuditEvent runFinished(
RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) {
var budget = context.budget().snapshot();
ModelCallLedger.Snapshot modelCalls = context.modelCalls().snapshot();
Map<String, Object> details = new LinkedHashMap<>();
if (intent != null) {
details.put("intent", intent.name());
}
details.put("release_outcome", outcome.name());
details.put("model_call_count", budget.modelCalls());
details.put("run_input_tokens", budget.inputTokens());
details.put("run_output_tokens", budget.outputTokens());
details.put("run_total_tokens", budget.totalTokens());
details.put("audited_model_call_count", modelCalls.auditedCallCount());
details.put("audited_input_tokens", modelCalls.inputTokens());
details.put("audited_output_tokens", modelCalls.outputTokens());
details.put("audited_total_tokens", modelCalls.totalTokens());
details.put("usage_unavailable_count", modelCalls.usageUnavailableCount());
details.put("tokens_reconciled", budget.modelCalls() == modelCalls.auditedCallCount()
&& modelCalls.usageUnavailableCount() == 0
&& budget.inputTokens() == modelCalls.inputTokens()
&& budget.outputTokens() == modelCalls.outputTokens()
&& budget.totalTokens() == modelCalls.totalTokens());
return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED,
terminalStatus(outcome), null, durationMs, details);
}
@@ -66,6 +85,41 @@ public final class TraceAuditEvents {
TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details);
}
public static DiagnosisTraceAuditEvent modelTokenUsage(
RunContext context,
ModelCallLedger.Call call,
long inputTokens,
long outputTokens,
boolean usageAvailable) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("component", call.component().name());
details.put("component_round", call.componentRound());
details.put("usage_available", usageAvailable);
if (usageAvailable) {
details.put("input_tokens", inputTokens);
details.put("output_tokens", outputTokens);
details.put("total_tokens", safeAdd(inputTokens, outputTokens));
}
return new DiagnosisTraceAuditEvent(
context.sessionId(), context.runId(), call.component().tracePhase(),
TraceEventType.MODEL_TOKEN_USAGE,
usageAvailable ? TraceEventStatus.SUCCEEDED : TraceEventStatus.UNAVAILABLE,
call.componentRound(), null, details);
}
public static DiagnosisTraceAuditEvent agentDraftInvalid(
RunContext context,
DiagnosisAgentOutputException.Kind kind,
long outputBytes,
boolean hasPublishableProgress) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("failure_kind", kind.name());
details.put("output_bytes", outputBytes);
details.put("has_publishable_progress", hasPublishableProgress);
return event(context, TracePhase.AGENT, TraceEventType.AGENT_DRAFT_INVALID,
TraceEventStatus.REJECTED, null, null, details);
}
public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", tool.toolCallId());
@@ -84,6 +138,55 @@ public final class TraceAuditEvents {
null, tool.durationMs(), details);
}
public static DiagnosisTraceAuditEvent toolRequestRejected(
RunContext context, String toolCallId, String toolName, String errorCode) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", safeIdentifier(toolCallId));
details.put("tool_name", safeIdentifier(toolName));
details.put("error_code", safeIdentifier(errorCode));
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
TraceEventStatus.REJECTED, null, null, details);
}
public static DiagnosisTraceAuditEvent toolProgress(
RunContext context,
String toolCallId,
String toolName,
String scopeSummary,
InformationGain informationGain,
String producer,
DiagnosisProgressSnapshotState state) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", toolCallId);
details.put("tool_name", toolName);
details.put("scope_summary", scopeSummary);
details.put("information_gain", informationGain.name());
details.put("producer", producer);
details.put("consecutive_no_gain", state.consecutiveNoGain());
details.put("collection_state", state.collectionState().name());
if (state.stopReason() != null) {
details.put("stop_reason", state.stopReason().name());
}
return event(context, TracePhase.TOOL, TraceEventType.TOOL_PROGRESS,
TraceEventStatus.SUCCEEDED, null, null, details);
}
public static DiagnosisTraceAuditEvent collectionStop(
RunContext context,
String toolCallId,
String toolName,
DiagnosisProgressSnapshotState state) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", toolCallId);
details.put("tool_name", toolName);
details.put("collection_state", state.collectionState().name());
if (state.stopReason() != null) {
details.put("stop_reason", state.stopReason().name());
}
return event(context, TracePhase.TOOL, TraceEventType.COLLECTION_STOP,
TraceEventStatus.SUCCEEDED, null, null, details);
}
public static DiagnosisTraceAuditEvent evidenceValidation(
RunContext context, TraceEventType type,
EvidenceGuardResult result, DiagnosisDraft draft) {
@@ -198,6 +301,18 @@ public final class TraceAuditEvents {
return new ToolReferences(List.copyOf(safeIds), invalid);
}
private static String safeIdentifier(String value) {
return value != null && SAFE_TOOL_CALL_ID.matcher(value).matches() ? value : "INVALID";
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
private record ToolReferences(List<String> safeIds, int invalidCount) {
}
}
@@ -4,8 +4,13 @@ public enum TraceEventType {
RUN_STARTED,
ROUTING_ATTEMPT,
ROUTING_DECISION,
MODEL_TOKEN_USAGE,
AGENT_MODEL_STEP,
AGENT_DRAFT_INVALID,
TOOL_INVOCATION,
TOOL_REQUEST_REJECTED,
TOOL_PROGRESS,
COLLECTION_STOP,
EVIDENCE_GUARD_INITIAL,
EVIDENCE_REPAIR_ATTEMPT,
EVIDENCE_GUARD_RECHECK,
@@ -3,5 +3,8 @@ package com.superbiz.agent.harness.contract;
public enum FallbackType {
EVIDENCE_VALIDATION_FAILED,
SEMANTIC_UNSUPPORTED,
SEMANTIC_UNAVAILABLE
SEMANTIC_UNAVAILABLE,
BUDGET_EXHAUSTED,
INSUFFICIENT_EVIDENCE,
MISSING_REQUIRED_CONTEXT
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.core;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import java.time.Clock;
import java.time.Duration;
@@ -15,17 +17,31 @@ public final class DiagnosisHarnessCore {
private final Duration maxRunDuration;
private final RunBudgetLimits budgetLimits;
private final HarnessRetryPolicies retryPolicies;
private final int stopAfterConsecutiveNoGain;
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
Duration maxRunDuration,
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies) {
this(clock, runIdSupplier, maxRunDuration, budgetLimits, retryPolicies, 2);
}
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
Duration maxRunDuration,
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies,
int stopAfterConsecutiveNoGain) {
this.clock = Objects.requireNonNull(clock, "clock must not be null");
this.runIdSupplier = Objects.requireNonNull(runIdSupplier, "runIdSupplier must not be null");
this.maxRunDuration = requirePositive(maxRunDuration, "maxRunDuration");
this.budgetLimits = Objects.requireNonNull(budgetLimits, "budgetLimits must not be null");
this.retryPolicies = Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
}
public RunContext startRun(String sessionId) {
@@ -42,8 +58,10 @@ public final class DiagnosisHarnessCore {
deadline,
cancellation,
new RunBudget(budgetLimits),
new ModelCallLedger(),
retryPolicies,
lifecycle);
lifecycle,
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain));
cancellation.onCancel(reason -> lifecycle.finish(terminalState(reason), reason.name()));
return context;
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.core;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import java.time.Instant;
import java.util.Objects;
@@ -14,8 +16,10 @@ public record RunContext(
Instant deadline,
RunCancellation cancellation,
RunBudget budget,
ModelCallLedger modelCalls,
HarnessRetryPolicies retryPolicies,
RunLifecycle lifecycle) {
RunLifecycle lifecycle,
DiagnosisProgressTracker progress) {
public RunContext {
requireText(sessionId, "sessionId");
@@ -23,8 +27,10 @@ public record RunContext(
Objects.requireNonNull(deadline, "deadline must not be null");
Objects.requireNonNull(cancellation, "cancellation must not be null");
Objects.requireNonNull(budget, "budget must not be null");
Objects.requireNonNull(modelCalls, "modelCalls must not be null");
Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
Objects.requireNonNull(lifecycle, "lifecycle must not be null");
Objects.requireNonNull(progress, "progress must not be null");
}
private static void requireText(String value, String name) {
@@ -79,6 +79,40 @@ public final class EvidenceGuard {
: EvidenceGuardResult.invalid(violations);
}
public EvidenceGuardResult validateNoConclusionReferences(
RunContext context, DiagnosisDraft draft) {
Objects.requireNonNull(context, "context must not be null");
if (draft == null) {
return EvidenceGuardResult.invalid(List.of(
violation(EvidenceViolationCode.DRAFT_MISSING, "draft")));
}
if (draft.conclusion() != null) {
throw new IllegalArgumentException("no-conclusion validation requires conclusion=null");
}
List<EvidenceViolation> violations = new ArrayList<>();
for (int index = 0; index < draft.analysis().size(); index++) {
DiagnosisDraft.AnalysisItem analysis = draft.analysis().get(index);
if (analysis == null || analysis.toolCallIds().isEmpty()) {
continue;
}
if (analysis.kind() == null) {
violations.add(violation(
EvidenceViolationCode.ANALYSIS_KIND_MISSING,
"analysis[" + index + "].kind"));
continue;
}
List<VerifiedEvidence> ignoredEvidence = new ArrayList<>();
for (String toolCallId : analysis.toolCallIds()) {
verifyInvocation(
context, analysis, index, toolCallId, ignoredEvidence, violations);
}
}
return violations.isEmpty()
? EvidenceGuardResult.valid(VerifiedEvidenceSnapshot.empty())
: EvidenceGuardResult.invalid(violations);
}
private List<EvidenceViolation> validateDraft(DiagnosisDraft draft) {
List<EvidenceViolation> violations = new ArrayList<>();
if (draft == null) {
@@ -4,6 +4,9 @@ import com.superbiz.agent.harness.core.BudgetExceededException;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunAbortedException;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import com.superbiz.agent.harness.retry.RetryFailure;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.metadata.Usage;
@@ -26,15 +29,23 @@ public final class GuardModelCall {
private final DiagnosisHarnessCore core;
private final ChatModel chatModel;
private final ExecutorService executor;
private final ModelCallAuditor auditor;
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
ExecutorService executor) {
this(core, chatModel, executor, new ModelCallAuditor(core));
}
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
ExecutorService executor, ModelCallAuditor auditor) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
this.executor = Objects.requireNonNull(executor, "executor must not be null");
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
}
public String call(RunContext context, Prompt prompt, Duration timeout, long maxOutputBytes) {
public String call(RunContext context, ModelCallComponent component,
Prompt prompt, Duration timeout, long maxOutputBytes) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(prompt, "prompt must not be null");
Objects.requireNonNull(timeout, "timeout must not be null");
@@ -42,20 +53,24 @@ public final class GuardModelCall {
throw new IllegalArgumentException("timeout and output limit must be positive");
}
core.beforeModelCall(context);
Future<String> future = executor.submit(() -> invoke(context, prompt, maxOutputBytes));
ModelCallLedger.Call call = auditor.begin(context, component);
Future<String> future = executor.submit(() -> invoke(context, call, prompt, maxOutputBytes));
context.cancellation().onCancel(ignored -> future.cancel(true));
try {
return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
} catch (TimeoutException exception) {
future.cancel(true);
auditor.recordUsage(context, call, 0, 0, false);
throw new GuardModelCallException(
RetryFailure.TIMEOUT, "Guard model attempt timed out", exception);
} catch (CancellationException exception) {
auditor.recordUsage(context, call, 0, 0, false);
core.checkActive(context);
throw new GuardModelCallException(
RetryFailure.TRANSPORT, "Guard model attempt was cancelled", exception);
} catch (InterruptedException exception) {
future.cancel(true);
auditor.recordUsage(context, call, 0, 0, false);
Thread.currentThread().interrupt();
core.checkActive(context);
throw new GuardModelCallException(
@@ -76,15 +91,17 @@ public final class GuardModelCall {
}
}
private String invoke(RunContext context, Prompt prompt, long maxOutputBytes) {
private String invoke(RunContext context, ModelCallLedger.Call call,
Prompt prompt, long maxOutputBytes) {
ChatResponse response;
try {
response = chatModel.call(prompt);
} catch (RuntimeException exception) {
auditor.recordUsage(context, call, 0, 0, false);
throw new GuardModelCallException(
RetryFailure.TRANSPORT, "Guard model transport failed", exception);
}
recordUsage(context, response);
recordUsage(context, call, response);
core.checkActive(context);
AssistantMessage output = response == null || response.getResult() == null
? null : response.getResult().getOutput();
@@ -102,16 +119,18 @@ public final class GuardModelCall {
return output.getText();
}
private void recordUsage(RunContext context, ChatResponse response) {
private void recordUsage(RunContext context, ModelCallLedger.Call call, ChatResponse response) {
if (response == null || response.getMetadata() == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
Usage usage = response.getMetadata().getUsage();
if (usage == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
core.recordTokens(context, nonNegative(usage.getPromptTokens()),
nonNegative(usage.getCompletionTokens()));
auditor.recordUsage(context, call, nonNegative(usage.getPromptTokens()),
nonNegative(usage.getCompletionTokens()), true);
}
private static long nonNegative(Integer value) {
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.SemanticVerdict;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
@@ -81,7 +82,8 @@ public final class SemanticGuard {
context,
context.retryPolicies().semanticGuard(),
() -> parse(modelCall.call(
context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
context, ModelCallComponent.SEMANTIC_GUARD,
modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
this::classify,
attempt -> {
attemptRecorder.accept(attempt);
@@ -0,0 +1,19 @@
package com.superbiz.agent.harness.progress;
public record CompletedToolCall(
String toolCallId,
String toolName,
String normalizedScope) {
public CompletedToolCall {
requireText(toolCallId, "toolCallId");
requireText(toolName, "toolName");
requireText(normalizedScope, "normalizedScope");
}
private static void requireText(String value, String name) {
if (value == null || value.isBlank()) {
throw new IllegalArgumentException(name + " must not be blank");
}
}
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum DiagnosisCollectionState {
COLLECTING,
SATURATED
}
@@ -0,0 +1,13 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.core.RunContext;
@FunctionalInterface
public interface DiagnosisProgressProjection {
DiagnosisProgressSnapshot project(RunContext context);
static DiagnosisProgressProjection empty() {
return ignored -> DiagnosisProgressSnapshot.empty();
}
}
@@ -0,0 +1,228 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public final class DiagnosisProgressProjector implements DiagnosisProgressProjection {
private static final int MAX_FACTS = 12;
private static final int MAX_SUMMARY_CHARS = 320;
private static final int MAX_SCOPE_CHARS = 320;
private final CanonicalInvocationStore store;
private final ToolCallKeyFactory keyFactory;
private final ObjectMapper objectMapper;
public DiagnosisProgressProjector(CanonicalInvocationStore store,
ToolCallKeyFactory keyFactory,
ObjectMapper objectMapper) {
this.store = Objects.requireNonNull(store, "store must not be null");
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
@Override
public DiagnosisProgressSnapshot project(RunContext context) {
Objects.requireNonNull(context, "context must not be null");
DiagnosisProgressSnapshotState state = context.progress().snapshot();
Map<String, SafeFallback.VerifiedSource> sources = new LinkedHashMap<>();
Map<String, SafeFallback.ObservedFact> facts = new LinkedHashMap<>();
List<String> limitations = new ArrayList<>();
for (CompletedToolCall completed : state.completedToolCalls()) {
CanonicalToolInvocation invocation = resolve(context, completed, limitations);
if (invocation == null) {
continue;
}
try {
projectInvocation(completed, invocation, sources, facts);
} catch (RuntimeException exception) {
addLimitation(limitations, "部分已完成的工具结果格式无法验证,未纳入已检查事实");
}
if (facts.size() >= MAX_FACTS) {
addLimitation(limitations, "已检查事实较多,展示内容已截断");
break;
}
}
return new DiagnosisProgressSnapshot(
List.copyOf(sources.values()),
List.copyOf(facts.values()),
List.copyOf(limitations),
state.stopReason());
}
private CanonicalToolInvocation resolve(RunContext context,
CompletedToolCall completed,
List<String> limitations) {
try {
String key = keyFactory.create(context.runId(), completed.toolCallId());
CanonicalToolInvocation invocation = store.find(key).orElse(null);
if (invocation == null
|| !invocation.isReferencableBy(context.runId())
|| !completed.toolCallId().equals(invocation.toolCallId())
|| !completed.toolName().equals(invocation.toolName())) {
addLimitation(limitations, "部分已完成的工具记录无法验证,未纳入已检查事实");
return null;
}
return invocation;
} catch (RuntimeException exception) {
addLimitation(limitations, "部分已完成的工具记录暂时不可读取,未纳入已检查事实");
return null;
}
}
private void projectInvocation(CompletedToolCall completed,
CanonicalToolInvocation invocation,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
JsonNode root = readObject(invocation.agentResult());
String scope = publicScope(completed.toolName(), completed.normalizedScope(), root);
switch (completed.toolName()) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> projectRag(root, scope, sources, facts);
case AgentToolContracts.QUERY_LOGS -> projectLogs(root, scope, sources, facts);
case AgentToolContracts.QUERY_MYSQL -> projectMysql(root, scope, sources, facts);
default -> throw new IllegalArgumentException("Unsupported evidence Tool");
}
}
private void projectRag(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
JsonNode evidence = root.path("evidence");
if (!evidence.isArray() || evidence.isEmpty()) {
addFact(sources, facts, "RAG", "knowledge_base", scope,
"该知识检索范围内未发现可用文档证据");
return;
}
for (JsonNode item : evidence) {
String source = firstNonBlank(text(item, "source"), text(item, "title"),
text(item, "document_id"), "knowledge_base");
addFact(sources, facts, "RAG", source, scope,
firstNonBlank(text(item, "excerpt"), "已找到候选知识片段"));
}
}
private void projectLogs(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
String source = firstNonBlank(text(root, "source_kind"), "logs");
JsonNode events = root.path("events");
if (!events.isArray() || events.isEmpty()) {
addFact(sources, facts, "LOG", source, scope,
"该日志查询范围内未发现匹配事件");
return;
}
for (JsonNode event : events) {
addFact(sources, facts, "LOG", source, scope,
firstNonBlank(text(event, "message"), "已找到匹配日志事件"));
}
}
private void projectMysql(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
String source = mysqlSource(scope);
JsonNode rows = root.path("rows");
if (!rows.isArray() || rows.isEmpty()) {
addFact(sources, facts, "MYSQL", source, scope,
"该只读数据查询范围内未发现匹配记录");
return;
}
for (JsonNode row : rows) {
addFact(sources, facts, "MYSQL", source, scope, row.toString());
}
}
private String publicScope(String toolName, String normalizedScope, JsonNode result) {
if (AgentToolContracts.LOOKUP_KNOWLEDGE.equals(toolName)) {
return bounded("query=" + text(result, "query"), MAX_SCOPE_CHARS);
}
if (AgentToolContracts.QUERY_LOGS.equals(toolName)) {
JsonNode scope = result.path("scope");
return bounded(scope.isObject() ? scope.toString() : normalizedScope, MAX_SCOPE_CHARS);
}
try {
JsonNode scope = objectMapper.readTree(normalizedScope);
return bounded("data_source=" + text(scope, "data_source"), MAX_SCOPE_CHARS);
} catch (JsonProcessingException exception) {
return "data_source=unknown";
}
}
private String mysqlSource(String scope) {
int separator = scope.indexOf('=');
return separator < 0 ? "mysql" : scope.substring(separator + 1);
}
private void addFact(Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts,
String sourceType,
String source,
String scope,
String summary) {
if (facts.size() >= MAX_FACTS) {
return;
}
String safeSource = bounded(source, 160);
String safeScope = bounded(scope, MAX_SCOPE_CHARS);
String safeSummary = bounded(summary, MAX_SUMMARY_CHARS);
String sourceKey = sourceType + '\u0000' + safeSource + '\u0000' + safeScope;
sources.putIfAbsent(sourceKey,
new SafeFallback.VerifiedSource(sourceType, safeSource, safeScope));
String factKey = sourceKey + '\u0000' + safeSummary;
facts.putIfAbsent(factKey,
new SafeFallback.ObservedFact(sourceType, safeSource, safeScope, safeSummary));
}
private JsonNode readObject(String value) {
try {
JsonNode root = objectMapper.readTree(value);
if (root == null || !root.isObject()) {
throw new IllegalArgumentException("Canonical Agent result must be an object");
}
return root;
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Canonical Agent result is invalid", exception);
}
}
private static void addLimitation(List<String> limitations, String value) {
if (!limitations.contains(value)) {
limitations.add(value);
}
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
return value;
}
}
return "unknown";
}
private static String text(JsonNode node, String field) {
JsonNode value = node == null ? null : node.get(field);
return value == null || value.isNull() ? "" : value.asText("");
}
private static String bounded(String value, int max) {
String safe = value == null ? "" : value;
return safe.length() <= max ? safe : safe.substring(0, max);
}
}
@@ -0,0 +1,26 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.SafeFallback;
import java.util.List;
public record DiagnosisProgressSnapshot(
List<SafeFallback.VerifiedSource> verifiedSources,
List<SafeFallback.ObservedFact> observedFacts,
List<String> limitations,
DiagnosisStopReason stopReason) {
public DiagnosisProgressSnapshot {
verifiedSources = verifiedSources == null ? List.of() : List.copyOf(verifiedSources);
observedFacts = observedFacts == null ? List.of() : List.copyOf(observedFacts);
limitations = limitations == null ? List.of() : List.copyOf(limitations);
}
public static DiagnosisProgressSnapshot empty() {
return new DiagnosisProgressSnapshot(List.of(), List.of(), List.of(), null);
}
public boolean hasObservedFacts() {
return !observedFacts.isEmpty();
}
}
@@ -0,0 +1,16 @@
package com.superbiz.agent.harness.progress;
import java.util.List;
public record DiagnosisProgressSnapshotState(
int consecutiveNoGain,
DiagnosisCollectionState collectionState,
DiagnosisStopReason stopReason,
String pendingToolCallId,
boolean stopInstructionDelivered,
List<CompletedToolCall> completedToolCalls) {
public DiagnosisProgressSnapshotState {
completedToolCalls = completedToolCalls == null ? List.of() : List.copyOf(completedToolCalls);
}
}
@@ -0,0 +1,118 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public final class DiagnosisProgressTracker {
private final int stopAfterConsecutiveNoGain;
private final Set<ToolScopeIdentity> completedScopes = new LinkedHashSet<>();
private final List<CompletedToolCall> completedToolCalls = new ArrayList<>();
private int consecutiveNoGain;
private DiagnosisCollectionState collectionState = DiagnosisCollectionState.COLLECTING;
private DiagnosisStopReason stopReason;
private String pendingToolCallId;
private boolean stopInstructionDelivered;
public DiagnosisProgressTracker(int stopAfterConsecutiveNoGain) {
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
}
public synchronized void applyPreviousObservation(PreviousObservation observation) {
if (pendingToolCallId == null) {
if (observation != null) {
throw new IllegalArgumentException("No Tool observation is pending evaluation");
}
return;
}
if (observation == null) {
throw new IllegalArgumentException("Previous Tool observation must be evaluated");
}
if (!pendingToolCallId.equals(observation.toolCallId())) {
throw new IllegalArgumentException("Previous Tool observation ID is out of order");
}
pendingToolCallId = null;
applyGain(observation.informationGain());
}
public synchronized boolean isDuplicate(String toolName, String normalizedScope) {
return completedScopes.contains(new ToolScopeIdentity(toolName, normalizedScope));
}
public synchronized void recordDuplicateScope() {
applyGain(InformationGain.NO_GAIN);
}
public synchronized void recordCompleted(CompletedToolCall call, EvidenceStatus evidenceStatus) {
if (collectionState == DiagnosisCollectionState.SATURATED) {
throw new IllegalStateException("Cannot record Tool completion after saturation");
}
if (evidenceStatus != EvidenceStatus.EVIDENCE_FOUND
&& evidenceStatus != EvidenceStatus.NO_EVIDENCE) {
throw new IllegalArgumentException("Completed Tool requires a successful evidence status");
}
ToolScopeIdentity scope = new ToolScopeIdentity(call.toolName(), call.normalizedScope());
if (!completedScopes.add(scope)) {
throw new IllegalStateException("Completed Tool scope was already recorded");
}
completedToolCalls.add(call);
if (evidenceStatus == EvidenceStatus.NO_EVIDENCE) {
applyGain(InformationGain.NO_GAIN);
} else {
pendingToolCallId = call.toolCallId();
}
}
public synchronized boolean claimStopInstruction() {
if (collectionState != DiagnosisCollectionState.SATURATED) {
return false;
}
if (stopInstructionDelivered) {
return false;
}
stopInstructionDelivered = true;
return true;
}
public synchronized void markBudgetLimitReached() {
if (stopReason == null) {
stopReason = DiagnosisStopReason.BUDGET_LIMIT_REACHED;
}
}
public synchronized DiagnosisProgressSnapshotState snapshot() {
return new DiagnosisProgressSnapshotState(
consecutiveNoGain,
collectionState,
stopReason,
pendingToolCallId,
stopInstructionDelivered,
completedToolCalls);
}
public int stopAfterConsecutiveNoGain() {
return stopAfterConsecutiveNoGain;
}
private void applyGain(InformationGain gain) {
if (collectionState == DiagnosisCollectionState.SATURATED) {
throw new IllegalStateException("Collection is already saturated");
}
if (gain == InformationGain.GAINED) {
consecutiveNoGain = 0;
return;
}
consecutiveNoGain++;
if (consecutiveNoGain >= stopAfterConsecutiveNoGain) {
collectionState = DiagnosisCollectionState.SATURATED;
stopReason = DiagnosisStopReason.INFORMATION_SATURATED;
}
}
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum DiagnosisStopReason {
INFORMATION_SATURATED,
BUDGET_LIMIT_REACHED
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum InformationGain {
GAINED,
NO_GAIN
}
@@ -0,0 +1,17 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.annotation.JsonProperty;
public record PreviousObservation(
@JsonProperty("tool_call_id") String toolCallId,
@JsonProperty("information_gain") InformationGain informationGain) {
public PreviousObservation {
if (toolCallId == null || toolCallId.isBlank()) {
throw new IllegalArgumentException("toolCallId must not be blank");
}
if (informationGain == null) {
throw new IllegalArgumentException("informationGain must not be null");
}
}
}
@@ -0,0 +1,13 @@
package com.superbiz.agent.harness.progress;
public record ToolScopeIdentity(String toolName, String normalizedScope) {
public ToolScopeIdentity {
if (toolName == null || toolName.isBlank()) {
throw new IllegalArgumentException("toolName must not be blank");
}
if (normalizedScope == null || normalizedScope.isBlank()) {
throw new IllegalArgumentException("normalizedScope must not be blank");
}
}
}
@@ -0,0 +1,76 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
public final class ToolScopeNormalizer {
private static final int DEFAULT_LOG_LOOKBACK_MINUTES = 30;
private final ObjectMapper objectMapper;
public ToolScopeNormalizer(ObjectMapper objectMapper) {
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
public String normalize(String toolName, Object input) {
Objects.requireNonNull(input, "input must not be null");
Map<String, Object> scope = switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> ragScope(requireType(input, RagToolRequest.class));
case AgentToolContracts.QUERY_LOGS -> logScope(requireType(input, QueryLogsRequest.class));
case AgentToolContracts.QUERY_MYSQL -> mysqlScope(requireType(input, MysqlToolRequest.class));
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
};
try {
return objectMapper.writeValueAsString(scope);
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Tool scope is not serializable", exception);
}
}
private static Map<String, Object> ragScope(RagToolRequest request) {
return ordered("query", normalizeText(request.query()));
}
private static Map<String, Object> logScope(QueryLogsRequest request) {
Map<String, Object> scope = new LinkedHashMap<>();
scope.put("topic", request.topic() == null ? null : request.topic().name());
scope.put("query", normalizeText(request.query()));
scope.put("lookback_minutes", request.lookbackMinutes() == null
? DEFAULT_LOG_LOOKBACK_MINUTES : request.lookbackMinutes());
return scope;
}
private static Map<String, Object> mysqlScope(MysqlToolRequest request) {
Map<String, Object> scope = new LinkedHashMap<>();
scope.put("data_source", normalizeText(request.dataSource()));
scope.put("sql", normalizeText(request.sql()));
scope.put("params", request.params());
return scope;
}
private static Map<String, Object> ordered(String name, Object value) {
Map<String, Object> result = new LinkedHashMap<>();
result.put(name, value);
return result;
}
private static String normalizeText(String value) {
return value == null ? "" : value.trim();
}
private static <T> T requireType(Object input, Class<T> type) {
if (!type.isInstance(input)) {
throw new IllegalArgumentException("Unexpected Tool input type: " + input.getClass().getName());
}
return type.cast(input);
}
}
@@ -1,5 +1,6 @@
package com.superbiz.agent.harness.release;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
@@ -15,10 +16,13 @@ import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardDecision;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardInput;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.retry.RetryExecutionException;
import com.superbiz.agent.harness.retry.RetryFailure;
import java.util.Objects;
import java.util.List;
public final class DiagnosisReleaseUseCase {
@@ -50,12 +54,44 @@ public final class DiagnosisReleaseUseCase {
}
public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) {
return execute(context, query, DiagnosisAgentExecution.completed(
Objects.requireNonNull(draft, "draft must not be null"),
DiagnosisProgressSnapshot.empty()));
}
public DiagnosisReleaseResult execute(
RunContext context, String query, DiagnosisAgentExecution execution) {
Objects.requireNonNull(context, "context must not be null");
if (query == null || query.isBlank()) {
throw new IllegalArgumentException("query must not be blank");
}
Objects.requireNonNull(draft, "draft must not be null");
Objects.requireNonNull(execution, "execution must not be null");
DiagnosisDraft draft = execution.draft();
if (draft == null) {
return releaseControlledStop(context, execution.progress(), execution.stopReason());
}
if (draft.conclusion() == null) {
return releaseNoConclusion(context, draft, execution.progress());
}
return releaseConclusion(context, query, draft);
}
public DiagnosisReleaseResult releaseInvalidDraft(
RunContext context, DiagnosisProgressSnapshot progress) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(progress, "progress must not be null");
if (!progress.hasObservedFacts()) {
throw new IllegalStateException(
"Invalid Diagnosis Draft has no verified publishable progress");
}
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, List.of()),
FallbackType.INSUFFICIENT_EVIDENCE);
}
private DiagnosisReleaseResult releaseConclusion(
RunContext context, String query, DiagnosisDraft draft) {
DiagnosisDraft candidate = draft;
EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate);
traceRecorder.record(TraceAuditEvents.evidenceValidation(
@@ -101,6 +137,61 @@ public final class DiagnosisReleaseUseCase {
return DiagnosisReleaseResult.fallback(fallbackFactory.semanticUnsupported(snapshot));
}
private DiagnosisReleaseResult releaseNoConclusion(
RunContext context, DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
EvidenceGuardResult evidence = evidenceGuard.validateNoConclusionReferences(context, draft);
traceRecorder.record(TraceAuditEvents.evidenceValidation(
context, TraceEventType.EVIDENCE_GUARD_INITIAL, evidence, draft));
if (!evidence.valid()) {
return evidenceFailure(context, evidence);
}
List<String> missingInfo = missingInfo(draft);
if (progress.hasObservedFacts()) {
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, missingInfo),
FallbackType.INSUFFICIENT_EVIDENCE);
}
if (!missingInfo.isEmpty()) {
return progressFallback(context,
fallbackFactory.missingRequiredContext(missingInfo),
FallbackType.MISSING_REQUIRED_CONTEXT);
}
throw new IllegalStateException(
"No-conclusion Diagnosis has neither verified progress nor missing context");
}
private DiagnosisReleaseResult releaseControlledStop(
RunContext context,
DiagnosisProgressSnapshot progress,
DiagnosisStopReason stopReason) {
if (stopReason != DiagnosisStopReason.INFORMATION_SATURATED
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED) {
throw new IllegalStateException("Unsupported Diagnosis stop reason");
}
if (!progress.hasObservedFacts()) {
throw new IllegalStateException(
"Controlled Diagnosis stop has no verified publishable progress");
}
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, List.of()),
FallbackType.INSUFFICIENT_EVIDENCE);
}
private DiagnosisReleaseResult progressFallback(
RunContext context,
com.superbiz.agent.harness.contract.SafeFallback fallback,
FallbackType type) {
traceRecorder.record(TraceAuditEvents.releaseDecision(
context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, type));
return DiagnosisReleaseResult.fallback(fallback);
}
private List<String> missingInfo(DiagnosisDraft draft) {
return draft.limitations() == null
? List.of() : draft.limitations().missingInfo();
}
private DiagnosisReleaseResult evidenceFailure(
RunContext context, EvidenceGuardResult evidence) {
traceRecorder.record(TraceAuditEvents.releaseDecision(
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
@@ -92,7 +93,8 @@ public final class EvidenceRepair {
context.retryPolicies().evidenceRepair(),
() -> {
DiagnosisDraft repaired = parse(modelCall.call(
context, modelPrompt, limits.timeout(), limits.maxOutputBytes()));
context, ModelCallComponent.EVIDENCE_REPAIR,
modelPrompt, limits.timeout(), limits.maxOutputBytes()));
if (!originalSemantics.hasSameUserVisibleSemantics(
SemanticDraftView.from(repaired))) {
throw new GuardModelCallException(
@@ -5,6 +5,7 @@ import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -53,6 +54,47 @@ public final class SafeFallbackFactory {
List.of());
}
public SafeFallback insufficientEvidence(
DiagnosisProgressSnapshot progress, List<String> missingInfo) {
Objects.requireNonNull(progress, "progress must not be null");
if (!progress.hasObservedFacts()) {
throw new IllegalArgumentException(
"insufficient-evidence fallback requires verified progress");
}
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
List<String> limitations = new ArrayList<>(progress.limitations());
limitations.add("当前已检查范围不足以支持根因结论");
safeMissingInfo.forEach(item -> limitations.add("仍缺少:" + item));
return fallback(
FallbackType.INSUFFICIENT_EVIDENCE,
progress.verifiedSources(),
"已完成有限范围的检查,但现有证据不足以确认根因",
List.copyOf(limitations),
List.of(safeMissingInfo.isEmpty()
? "补充故障对象、发生时间、错误信息或新的可查询范围后重试"
: "补充缺失信息后,在新的明确范围内继续诊断"),
"DIAGNOSIS_COLLECTION",
progress.observedFacts(),
List.of());
}
public SafeFallback missingRequiredContext(List<String> missingInfo) {
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
if (safeMissingInfo.isEmpty()) {
throw new IllegalArgumentException(
"missing-context fallback requires missing information");
}
return fallback(
FallbackType.MISSING_REQUIRED_CONTEXT,
List.of(),
"当前缺少执行定向诊断所需的信息,暂时无法开始有效查询或确认根因",
safeMissingInfo.stream().map(item -> "缺少:" + item).toList(),
List.of("补充上述故障上下文后重试"),
"DIAGNOSIS_INPUT",
List.of(),
List.of());
}
private SafeFallback fallback(FallbackType type,
List<SafeFallback.VerifiedSource> sources,
String message,
@@ -100,6 +142,20 @@ public final class SafeFallbackFactory {
return List.copyOf(result);
}
private List<String> boundedMissingInfo(List<String> missingInfo) {
List<String> result = new ArrayList<>();
for (String item : missingInfo == null ? List.<String>of() : missingInfo) {
String safe = bounded(item);
if (safe != null && !result.contains(safe)) {
result.add(safe);
}
if (result.size() >= 8) {
break;
}
}
return List.copyOf(result);
}
private String bounded(String value) {
if (value == null || value.isBlank()) {
return null;
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record MysqlToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") MysqlToolRequest input) {
}
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record QueryLogsToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") QueryLogsRequest input) {
}
@@ -0,0 +1,7 @@
package com.superbiz.agent.harness.tool.contract;
public enum RagRelevanceLevel {
PRECISE,
HIGHLY_RELEVANT,
REFERENCE
}
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record RagToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") RagToolRequest input) {
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import java.util.List;
@@ -11,9 +12,20 @@ public record RagToolResult(
@JsonProperty("query") String query,
@JsonProperty("evidence") List<RagEvidence> evidence,
@JsonProperty("returned_count") int returnedCount,
@JsonProperty("relevance_level") @JsonInclude(JsonInclude.Include.NON_NULL)
RagRelevanceLevel relevanceLevel,
@JsonProperty("truncated") boolean truncated) {
public RagToolResult {
evidence = ToolContractCollections.immutable(evidence);
}
public RagToolResult(EvidenceStatus evidenceStatus,
String toolCallId,
String query,
List<RagEvidence> evidence,
int returnedCount,
boolean truncated) {
this(evidenceStatus, toolCallId, query, evidence, returnedCount, null, truncated);
}
}
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.tool.boundary.ProjectedToolResult;
import com.superbiz.agent.harness.tool.contract.RagEvidence;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import com.superbiz.agent.harness.tool.contract.RagToolResult;
@@ -81,7 +82,10 @@ public final class RagResultProjector {
EvidenceStatus status = evidence.isEmpty()
? EvidenceStatus.NO_EVIDENCE
: EvidenceStatus.EVIDENCE_FOUND;
RagToolResult result = new RagToolResult(status, toolCallId, query, evidence, evidence.size(), truncated);
RagRelevanceLevel relevanceLevel = status == EvidenceStatus.EVIDENCE_FOUND
? relevanceLevel(root) : null;
RagToolResult result = new RagToolResult(
status, toolCallId, query, evidence, evidence.size(), relevanceLevel, truncated);
result = fitBudget(result, truncated);
return new ProjectedToolResult(objectMapper.writeValueAsString(result), result.evidenceStatus());
}
@@ -94,7 +98,8 @@ public final class RagResultProjector {
reduced.remove(reduced.size() - 1);
current = new RagToolResult(
reduced.isEmpty() ? EvidenceStatus.NO_EVIDENCE : EvidenceStatus.EVIDENCE_FOUND,
current.toolCallId(), current.query(), reduced, reduced.size(), true);
current.toolCallId(), current.query(), reduced, reduced.size(),
reduced.isEmpty() ? null : current.relevanceLevel(), true);
}
if (bytes(objectMapper.writeValueAsString(current)) > limits.maxAgentUtf8Bytes()) {
throw new IllegalArgumentException("RAG projection exceeds total budget");
@@ -120,6 +125,21 @@ public final class RagResultProjector {
return value == null || value.isBlank() ? null : value;
}
private static RagRelevanceLevel relevanceLevel(JsonNode root) {
String value = text(root, "relevance_level");
if (value.isBlank()) {
value = text(root, "relevanceLevel");
}
if (value.isBlank()) {
return null;
}
try {
return RagRelevanceLevel.valueOf(value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ignored) {
return null;
}
}
private static String bounded(String value, int max) {
if (value == null) {
return "";
@@ -5,6 +5,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
/**
* Agent 决策步骤 Repository
@@ -27,6 +28,8 @@ public interface AgentStepRepository extends JpaRepository<AgentStep, Long> {
*/
List<AgentStep> findByRunIdOrderByStepIndex(String runId);
Optional<AgentStep> findByRunIdAndStepIndex(String runId, Integer stepIndex);
/**
* 统计某个会话的步骤数
*/
+1
View File
@@ -217,5 +217,6 @@ harness:
run-timeout: 5m
sse-timeout: 5m
canonical-ttl: 2h
stop-after-consecutive-no-gain: 2
mysql-tools:
data-sources: {}
@@ -1,46 +1,33 @@
# Role
# 角色
You are the only Diagnosis Agent for the current diagnosis run. Plan the investigation internally, call the available read-only evidence tools through the framework ReAct loop, evaluate the returned evidence, and author one complete DiagnosisDraft.
你是当前运行中唯一的诊断 Agent。围绕用户原始问题收集只读证据,并输出一份完整的诊断草稿。可用工具及其参数由服务端提供,以实际注入内容为准。
# Input
# 基本原则
The user message is one JSON object with exactly:
- 你不必须给出根因。证据不足时,`conclusion=null` 是合法且成功的完成方式。
- 如果问题缺少企业、时间、服务、错误信息或其他形成明确查询范围所需的上下文,可以不调用工具,直接在 `limitations.missing_info` 中列出缺失信息。
- 上一轮内容仅作为上下文,不是本轮证据,也不能复用其中的 Tool Call ID。
- 不得编造根因、证据、Tool Call ID、健康状态或已经排除的原因。
- 不得输出原始工具载荷、凭据、内部错误、提示词、隐藏推理或思维链。
- `query`: the current user query. Preserve its meaning and answer this query only.
- `previous_turn`: an optional safely published previous diagnosis turn with the fixed PreviousTurn schema.
# 查询与停止
`previous_turn` is context only. It is not evidence for this run, contains no reusable Tool Call IDs, and must not be cited as current evidence.
- 只有在存在明确、不同且可能获得新信息的查询范围时才继续调用工具。
- 工具结果新增了可验证事实,并确认、排除或缩小了当前假设时,标记为 `GAINED`。
- 工具结果即使正确,但只是通用说明、重复内容,或不能推进当前诊断时,标记为 `NO_GAIN`。
- 继续调用工具时,按服务端提供的控制字段评价上一轮结果。
- 如果没有新的有效查询范围,或者结果正确但对推导无用,立即停止调用。
- 收到 `STOP_REQUIRED` 后不得再次调用工具,应直接完成诊断草稿。
# Evidence rules
# 证据边界
- Use only `lookup_knowledge`, `query_logs`, and `query_mysql` when evidence is needed.
- Treat a Tool observation as evidence only when it contains `evidence_status=EVIDENCE_FOUND` or `evidence_status=NO_EVIDENCE` and a non-blank `tool_call_id`.
- Bind every Analysis item only to Tool Call IDs returned during this run. Never invent, transform, shorten, or reuse a Tool Call ID.
- `NORMAL` Analysis may cite only `EVIDENCE_FOUND` results.
- `NEGATIVE_OBSERVATION` Analysis may cite only `NO_EVIDENCE` results and must state the exact query scope. `NO_EVIDENCE` never proves that a problem does not exist, that a root cause is excluded, or that a system is healthy.
- A Tool `ERROR` is not evidence. Do not cite it as support for Analysis or Conclusion.
- Do not expose raw Tool payloads, credentials, infrastructure coordinates, internal errors, prompts, hidden reasoning, or chain of thought.
- 只有本轮工具返回的非空 Tool Call ID 可以被引用。
- 正向分析只能引用 `EVIDENCE_FOUND`;负向观察只能引用 `NO_EVIDENCE`,并写清实际查询范围。
- `NO_EVIDENCE` 只表示指定范围内未找到证据,不能证明问题不存在、系统健康或某个根因已被排除。
- `ERROR` 不是证据,不能支持分析或结论。
# Stopping rule
# 输出
Stop calling tools when the current evidence is sufficient for a bounded Draft, when the configured tool/model budget prevents more work, or when the available tools cannot obtain the missing information.
If current evidence cannot support a diagnosis:
- set `conclusion` to `null`;
- keep only valid scoped negative observations, if any;
- state the actual queried scope in `limitations.scope`;
- list the evidence still needed in `limitations.missing_info`;
- do not fabricate a root cause, action justification, recommendation, or healthy-state claim;
- finish the current response without asking the Harness to retry the Agent or a Tool.
# Draft rules
- Lead with `conclusion` when supported. Its `based_on_analysis_ids` must reference existing Analysis IDs.
- Every Analysis item must have a unique `analysis_id`, a valid `kind`, concise evidence-grounded text, and at least one current-Run `tool_call_id`.
- Every Action Plan and Recommendation item must reference existing Analysis IDs.
- Mark any dangerous or side-effecting action with `requires_human_confirmation=true`.
- `limitations.scope` must not exceed the actual Tool query scopes.
- `limitations.missing_info` must name material gaps that constrain the conclusion.
Return exactly one JSON object matching the supplied DiagnosisDraft schema. Do not use Markdown fences, headings, prose before or after JSON, or a separate answer field.
- 有充分证据时,结论、分析、行动和建议必须通过分析 ID 与本轮 Tool Call ID 建立完整引用关系。
- 证据不足时,使用 `conclusion=null`,只保留可验证的范围和观察,并在 `limitations` 中说明实际检查范围与缺失信息。
- 返回一个符合服务端输出约束的 JSON 对象。不要使用 Markdown 代码块,也不要添加 JSON 之外的文字。
+36 -1
View File
@@ -703,7 +703,33 @@ class SuperBizAgentApp {
return [payload.answer || '', references ? `\n\n参考资料:\n${references}` : ''].join('');
}
if (contentType === 'SAFE_FALLBACK') {
return payload.fallback?.message || '当前证据不足,无法确认根因';
const fallback = payload.fallback || {};
const sections = [];
sections.push(`## 当前判断\n${fallback.message || '当前证据不足,无法确认根因'}`);
if (fallback.observed_facts?.length) {
const facts = fallback.observed_facts.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**(${item.scope || '范围未提供'}):${item.summary || '已完成检查'}`
).join('\n');
sections.push(`## 已完成排查\n${facts}`);
} else if (fallback.verified_sources?.length) {
const sources = fallback.verified_sources.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
).join('\n');
sections.push(`## 已检查范围\n${sources}`);
}
if (fallback.validation_issues?.length) {
const issues = fallback.validation_issues.map(item =>
`- ${item.code || 'VALIDATION_FAILED'}${item.target ? `:${item.target}` : ''}`
).join('\n');
sections.push(`## 校验问题\n${issues}`);
}
if (fallback.limitations?.length) {
sections.push(`## 限制\n${fallback.limitations.map(item => `- ${item}`).join('\n')}`);
}
if (fallback.next_steps?.length) {
sections.push(`## 下一步\n${fallback.next_steps.map(item => `- ${item}`).join('\n')}`);
}
return sections.join('\n\n');
}
if (contentType === 'DIAGNOSIS_REPORT') {
const report = payload.report || {};
@@ -713,6 +739,15 @@ class SuperBizAgentApp {
if (report.action_plan?.length) sections.push(`## 行动计划\n${report.action_plan.map(item => `- ${item.action}`).join('\n')}`);
if (report.recommendations?.length) sections.push(`## 建议\n${report.recommendations.map(item => `- ${item.text}`).join('\n')}`);
if (report.limitations?.scope) sections.push(`## 限制\n${report.limitations.scope}`);
if (report.limitations?.missing_info?.length) {
sections.push(`## 仍缺信息\n${report.limitations.missing_info.map(item => `- ${item}`).join('\n')}`);
}
if (payload.references?.length) {
const references = payload.references.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
).join('\n');
sections.push(`## 已检查范围\n${references}`);
}
return sections.join('\n\n');
}
throw new Error(`未知 content_type: ${contentType}`);
+1 -1
View File
@@ -111,6 +111,6 @@
</div>
</div>
<script src="app.js?v=20260707-session-picker"></script>
<script src="app.js?v=20260725-diagnosis-details"></script>
</body>
</html>
@@ -31,6 +31,15 @@ import static org.mockito.Mockito.mock;
class HarnessChatConfigurationTest {
@Test
void diagnosisNoGainThresholdDefaultsAndValidates() {
ChatHarnessProperties properties = new ChatHarnessProperties();
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
properties.setStopAfterConsecutiveNoGain(0);
assertThrows(IllegalArgumentException.class, properties::validate);
}
@Test
void defaultsArePositiveAndExecutorsAreBounded() {
ChatHarnessProperties properties = new ChatHarnessProperties();
@@ -0,0 +1,29 @@
package com.superbiz.agent.harness.agent;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisAgentPromptTest {
@Test
void promptLicensesAbandonmentWithoutDuplicatingToolSchema() {
String prompt = DiagnosisAgentPrompt.load();
assertTrue(prompt.contains("你不必须给出根因"));
assertTrue(prompt.contains("conclusion=null"));
assertTrue(prompt.contains("可以不调用工具"));
assertTrue(prompt.contains("GAINED"));
assertTrue(prompt.contains("NO_GAIN"));
assertTrue(prompt.contains("正确,但只是通用说明、重复内容"));
assertTrue(prompt.contains("STOP_REQUIRED"));
assertFalse(prompt.contains("lookup_knowledge"));
assertFalse(prompt.contains("query_logs"));
assertFalse(prompt.contains("query_mysql"));
assertFalse(prompt.contains("previous_observation"));
assertFalse(prompt.contains("next_action"));
assertFalse(prompt.contains("阈值"));
assertFalse(prompt.contains("计数器"));
}
}
@@ -16,6 +16,8 @@ import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
@@ -71,8 +73,9 @@ class DiagnosisAgentUseCaseTest {
"上一轮支付服务为什么超时?", "支付服务连接池已耗尽", "payment-service",
List.of("未覆盖退款服务"), List.of(new SourceDocument("doc-1", "连接池手册")));
DiagnosisDraft draft = useCase.execute(
DiagnosisAgentExecution execution = useCase.execute(
context, new DiagnosisAgentInput("那退款服务呢?", previous));
DiagnosisDraft draft = execution.draft();
assertEquals("退款服务出现连接池等待", draft.conclusion().text());
assertEquals(AnalysisKind.NORMAL, draft.analysis().get(0).kind());
@@ -109,8 +112,9 @@ class DiagnosisAgentUseCaseTest {
toolCall("call-empty-1", "{\"query\":\"refund timeout\"}"),
new AssistantMessage(noEvidenceDraft("call-empty-1")));
DiagnosisDraft draft = useCase(core, model, tools, LARGE_LIMITS)
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("退款为什么超时?", null));
DiagnosisDraft draft = execution.draft();
assertNull(draft.conclusion());
assertEquals(AnalysisKind.NEGATIVE_OBSERVATION, draft.analysis().get(0).kind());
@@ -119,6 +123,32 @@ class DiagnosisAgentUseCaseTest {
assertEquals(1, toolCalls.get());
}
@Test
void ignoredStopInstructionCannotExecuteAnotherTool() {
DiagnosisHarnessCore core = core(new RunBudgetLimits(6, 6, 6,
100, 100, 200, 100_000));
RunContext context = core.startRun("session-stop", "run-stop");
AtomicInteger toolCalls = new AtomicInteger();
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
toolCalls.incrementAndGet();
return ToolBoundaryResult.ready(id, noEvidence(id), EvidenceStatus.NO_EVIDENCE);
});
ScriptedChatModel model = new ScriptedChatModel(1, 1,
toolCall("call-empty-1", "{\"query\":\"first\"}"),
toolCall("call-empty-2", "{\"query\":\"second\"}"),
toolCall("call-ignored-stop", "{\"query\":\"third\"}"));
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, execution.stopReason());
assertEquals(3, model.calls());
assertEquals(2, toolCalls.get());
assertEquals(2, context.budget().snapshot().toolCalls());
}
@Test
void fencedOutputFailsClosedWithoutAgentRetry() {
DiagnosisHarnessCore core = core(defaultBudget());
@@ -126,14 +156,45 @@ class DiagnosisAgentUseCaseTest {
new AssistantMessage("```json\n" + supportedDraft("call-1") + "\n```"));
RunContext context = core.startRun("session-fenced", "run-fenced");
assertThrows(DiagnosisAgentOutputException.class, () ->
DiagnosisAgentOutputException failure = assertThrows(
DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
assertFalse(failure.progress().hasObservedFacts());
assertEquals(1, model.calls());
assertEquals(1, context.budget().snapshot().modelCalls());
}
@Test
void invalidDraftCarriesOnlyProjectedProgressForOuterReleaseBoundary() {
DiagnosisHarnessCore core = core(defaultBudget());
ScriptedChatModel model = new ScriptedChatModel(1, 1,
new AssistantMessage("not-json"));
RunContext context = core.startRun("session-invalid-progress", "run-invalid-progress");
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
List.of(new com.superbiz.agent.harness.contract.SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new com.superbiz.agent.harness.contract.SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown", "已完成知识范围检查")),
List.of(), null);
DiagnosisAgentFactory factory = new DiagnosisAgentFactory(
model, core, tools(errorInvoker()), objectMapper, List.of());
DiagnosisAgentUseCase useCase = new DiagnosisAgentUseCase(
core, factory, objectMapper, LARGE_LIMITS, ignored -> progress);
DiagnosisAgentOutputException failure = assertThrows(
DiagnosisAgentOutputException.class,
() -> useCase.execute(
context, new DiagnosisAgentInput("诊断未知故障", null)));
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
assertEquals("已完成知识范围检查",
failure.progress().observedFacts().get(0).summary());
assertFalse(failure.getMessage().contains("not-json"));
}
@Test
void modelCallBudgetStopsNextReactRoundBeforeChatModel() {
DiagnosisHarnessCore core = core(new RunBudgetLimits(1, 2, 2,
@@ -147,10 +208,11 @@ class DiagnosisAgentUseCaseTest {
toolCall("call-budget-1", "{\"query\":\"timeout\"}"),
new AssistantMessage(supportedDraft("call-budget-1")));
assertThrows(DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
assertEquals(1, model.calls());
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
}
@@ -163,10 +225,11 @@ class DiagnosisAgentUseCaseTest {
ScriptedChatModel model = new ScriptedChatModel(6, 6,
new AssistantMessage(supportedDraft("call-unused")));
assertThrows(DiagnosisAgentOutputException.class, () ->
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
DiagnosisAgentExecution execution = useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断超时", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
assertEquals(1, model.calls());
assertEquals(12, context.budget().snapshot().totalTokens());
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
@@ -261,7 +324,8 @@ class DiagnosisAgentUseCaseTest {
return AssistantMessage.builder()
.content("")
.toolCalls(List.of(new AssistantMessage.ToolCall(
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE, arguments)))
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"input\":" + arguments + "}")))
.build();
}
@@ -0,0 +1,58 @@
package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
import org.springframework.ai.chat.metadata.DefaultUsage;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.mock;
class HarnessModelInterceptorTest {
@Test
void extractsProviderUsageWithoutPersistingModelContent() {
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 100_000),
HarnessRetryPolicies.strict());
RunContext context = core.startRun("session-agent-token", "run-agent-token");
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessModelInterceptor interceptor = new HarnessModelInterceptor(
core, context,
new ModelCallAuditor(core, trace::add, mock(AgentStepRepository.class)));
AssistantMessage assistant = new AssistantMessage("secret-model-content");
ChatResponse response = new ChatResponse(
List.of(new Generation(assistant)),
ChatResponseMetadata.builder().usage(new DefaultUsage(21, 4)).build());
ModelResponse actual = interceptor.interceptModel(
ModelRequest.builder().messages(List.of()).build(),
ignored -> ModelResponse.of(assistant, response));
assertEquals(response, actual.getChatResponse());
assertEquals(25, context.budget().snapshot().totalTokens());
assertEquals(25, context.modelCalls().snapshot().totalTokens());
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, trace.get(0).eventType());
assertEquals("DIAGNOSIS_AGENT", trace.get(0).details().get("component"));
assertFalse(trace.get(0).details().toString().contains("secret-model-content"));
}
}
@@ -4,6 +4,8 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
@@ -23,6 +25,7 @@ import org.springframework.ai.tool.execution.ToolExecutionException;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
@@ -52,7 +55,9 @@ class HarnessToolInterceptorTest {
AgentToolContracts.QUERY_MYSQL),
callbacks.stream().map(callback -> callback.getToolDefinition().name()).toList());
callbacks.forEach(callback -> {
assertFalse(callback.getToolDefinition().inputSchema().contains("tool_call_id"));
assertTrue(callback.getToolDefinition().inputSchema().contains("previous_observation"));
assertTrue(callback.getToolDefinition().inputSchema().contains("information_gain"));
assertTrue(callback.getToolDefinition().inputSchema().contains("\"input\""));
assertFalse(callback.getToolDefinition().description().isBlank());
assertThrows(ToolExecutionException.class, () -> callback.call("{}"));
});
@@ -72,7 +77,8 @@ class HarnessToolInterceptorTest {
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7", "{\"query\":\"timeout\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7",
"{\"input\":{\"query\":\"timeout\"}}"),
ignored -> {
throw new AssertionError("registered Tool must not bypass Harness interceptor");
});
@@ -82,6 +88,7 @@ class HarnessToolInterceptorTest {
assertEquals("framework-call-7", envelope.getValue().toolCallId());
assertEquals(context.runId(), envelope.getValue().runId());
assertEquals(AgentToolContracts.LOOKUP_KNOWLEDGE, envelope.getValue().toolName());
assertEquals("{\"query\":\"timeout\"}", envelope.getValue().requestJson());
assertEquals("framework-call-7", response.getToolCallId());
assertTrue(response.getResult().contains("framework-call-7"));
}
@@ -94,7 +101,8 @@ class HarnessToolInterceptorTest {
context(3, 3, 3), tools, objectMapper);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error", "{\"query\":\"secret\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error",
"{\"input\":{\"query\":\"secret\"}}"),
ignored -> null);
JsonNode observation = objectMapper.readTree(response.getResult());
@@ -142,13 +150,164 @@ class HarnessToolInterceptorTest {
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1", "{\"query\":\"one\"}"),
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"),
ignored -> null);
assertEquals(1, invocations.get());
assertEquals(1, context.budget().snapshot().toolCalls());
}
@Test
void requiresExactPreviousEvaluationBeforeAnotherToolExecutes() throws Exception {
AtomicInteger invocations = new AtomicInteger();
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
ToolCallResponse missing = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
ToolCallResponse accepted = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
"input":{"query":"three"}}
"""), ignored -> null);
assertTrue(missing.isError());
assertEquals("INVALID_PROGRESS_PROTOCOL",
objectMapper.readTree(missing.getResult()).path("error_code").asText());
assertFalse(accepted.isError());
assertEquals(2, invocations.get());
DiagnosisTraceAuditEvent rejected = trace.stream()
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
.findFirst().orElseThrow();
assertEquals("call-2", rejected.details().get("tool_call_id"));
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
String details = rejected.details().toString();
assertFalse(details.contains("two"));
assertFalse(details.contains("previous_observation"));
}
@Test
void duplicateSuccessfulScopeIsNotExecutedAndCanSaturateRun() throws Exception {
AtomicInteger invocations = new AtomicInteger();
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"same\"}}"), ignored -> null);
ToolCallResponse duplicate = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"NO_GAIN"},
"input":{"query":"same"}}
"""), ignored -> null);
JsonNode observation = objectMapper.readTree(duplicate.getResult());
assertEquals(1, invocations.get());
assertTrue(observation.path("stop_required").asBoolean());
assertEquals("INFORMATION_SATURATED", observation.path("reason").asText());
assertTrue(trace.stream().anyMatch(event ->
event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED
&& "DUPLICATE_SCOPE".equals(event.details().get("error_code"))));
}
@Test
void observationContractRejectionIsTracedWithoutPayload() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) ->
ToolBoundaryResult.ready(id, """
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-contract",
"query":"secret-query","evidence":[],"returned_count":0,"truncated":false}
""", EvidenceStatus.EVIDENCE_FOUND));
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper, trace::add);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-contract",
"{\"input\":{\"query\":\"secret-query\"}}"), ignored -> null);
assertTrue(response.isError());
DiagnosisTraceAuditEvent rejected = trace.stream()
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
.findFirst().orElseThrow();
assertEquals("OBSERVATION_CONTRACT_MISMATCH", rejected.details().get("error_code"));
assertFalse(rejected.details().toString().contains("secret-query"));
}
@Test
void modelInformationGainIsRecordedWithoutRawScope() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> ready(id));
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"secret-query-value\"}}"), ignored -> null);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
"input":{"query":"different-secret-query"}}
"""), ignored -> null);
assertEquals(1, trace.size());
DiagnosisTraceAuditEvent event = trace.get(0);
assertEquals(TraceEventType.TOOL_PROGRESS, event.eventType());
assertEquals("MODEL", event.details().get("producer"));
assertEquals("GAINED", event.details().get("information_gain"));
assertTrue(event.details().get("scope_summary").toString()
.startsWith(AgentToolContracts.LOOKUP_KNOWLEDGE + "#"));
assertFalse(event.details().toString().contains("secret-query-value"));
}
@Test
void harnessProgressTraceDoesNotContainSqlParamsPayloadOrBudget() {
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
EvidenceToolInvoker unused = (context, id, arguments) ->
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
EvidenceToolInvoker mysql = (context, id, arguments) -> ToolBoundaryResult.ready(
id,
"{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
+ "\",\"columns\":[],\"rows\":[],\"returned_count\":0,"
+ "\"truncated\":false}",
EvidenceStatus.NO_EVIDENCE);
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, mysql);
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(4, 4, 4), tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.QUERY_MYSQL, "call-sql", """
{"input":{"data_source":"orders","sql":"select * from secret_table where tenant_id = ?",
"params":["tenant-secret"]}}
"""), ignored -> null);
String recorded = trace.toString();
assertEquals(List.of(TraceEventType.TOOL_PROGRESS),
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
assertTrue(recorded.contains("HARNESS"));
assertTrue(recorded.contains("NO_GAIN"));
assertFalse(recorded.contains("secret_table"));
assertFalse(recorded.contains("tenant-secret"));
assertFalse(recorded.contains("raw_response"));
assertFalse(recorded.contains("remaining"));
assertFalse(recorded.contains("prompt"));
assertFalse(recorded.contains("thought"));
}
private HarnessEvidenceTools fakeTools(EvidenceToolInvoker ragInvoker) {
EvidenceToolInvoker unused = (context, id, arguments) ->
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
@@ -162,7 +321,8 @@ class HarnessToolInterceptorTest {
private ToolBoundaryResult ready(String toolCallId) {
return ToolBoundaryResult.ready(toolCallId,
"{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\""
+ toolCallId + "\",\"evidence\":[]}",
+ toolCallId + "\",\"query\":\"scope\",\"evidence\":[],"
+ "\"returned_count\":1,\"truncated\":false}",
EvidenceStatus.EVIDENCE_FOUND);
}
@@ -0,0 +1,57 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class ToolResultViewProjectorTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolResultViewProjector projector = new ToolResultViewProjector(objectMapper);
@Test
void separatesRagControlDataFromModelObservation() throws Exception {
String canonical = """
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
"evidence":[{"document_id":"doc-1","excerpt":"generic"}],"returned_count":1,
"relevance_level":"REFERENCE","truncated":false,"raw_score":0.42}
""";
ToolControlView control = projector.controlView(canonical);
JsonNode model = objectMapper.readTree(projector.modelObservation(
AgentToolContracts.LOOKUP_KNOWLEDGE, canonical, "{\"query\":\"unknown\"}",
true, DiagnosisStopReason.INFORMATION_SATURATED));
assertEquals(1, control.returnedCount());
assertEquals(RagRelevanceLevel.REFERENCE, control.relevanceLevel());
assertEquals("REFERENCE", model.path("relevance_level").asText());
assertEquals("unknown", model.path("scope").path("query").asText());
assertTrue(model.path("stop_required").asBoolean());
assertFalse(model.has("returned_count"));
assertFalse(model.has("raw_score"));
}
@Test
void mysqlObservationUsesNormalizedScopeWithoutInternalFields() throws Exception {
String canonical = """
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-2","columns":["id"],
"rows":[{"id":1}],"returned_count":1,"truncated":false}
""";
JsonNode model = objectMapper.readTree(projector.modelObservation(
AgentToolContracts.QUERY_MYSQL, canonical,
"{\"data_source\":\"orders\",\"sql\":\"SELECT id FROM orders\",\"params\":[]}",
false, null));
assertEquals("orders", model.path("scope").path("data_source").asText());
assertEquals(1, model.path("rows").size());
assertFalse(model.has("returned_count"));
}
}
@@ -9,14 +9,17 @@ import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
import com.superbiz.agent.harness.contract.AnalysisKind;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.contract.PreviousTurn;
import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import org.junit.jupiter.api.Test;
@@ -157,6 +160,88 @@ class ChatApplicationUseCaseTest {
store.startedContext.lifecycle().state());
}
@Test
void handledDiagnosisBudgetFallbackIsPersistedWithoutReclassification() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
SafeFallback fallback = new SafeFallback(
FallbackType.INSUFFICIENT_EVIDENCE,
null,
"已完成有限范围的检查,但现有证据不足以确认根因",
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of("当前已检查范围不足以支持根因结论"),
List.of("补充故障上下文后重试"),
"DIAGNOSIS_COLLECTION",
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"该知识检索范围内未发现可用文档证据")),
List.of());
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK, new FallbackContent(fallback), null, true);
});
ChatApplicationResult result = useCase.execute(
new ChatApplicationRequest("diagnose unknown issue", "session-budget-fallback"));
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
assertEquals(ChatContentType.SAFE_FALLBACK, result.contentType());
FallbackContent content = (FallbackContent) result.content();
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, content.fallback().type());
assertEquals(1, content.fallback().observedFacts().size());
assertEquals(ReleaseOutcome.FALLBACK, store.finishedOutcome);
assertTrue(store.finishedSafeContentJson.contains("knowledge_base"));
}
@Test
void unhandledDiagnosisBudgetFailureDoesNotBuildFallbackInApplication() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.budget().reserveToolCall("lookup_knowledge");
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
throw new IllegalStateException("unhandled diagnosis failure");
});
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
useCase.execute(new ChatApplicationRequest(
"diagnose unknown issue", "session-budget-unhandled")));
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
}
@Test
void diagnosisBudgetExhaustionWithoutToolCallsRemainsFailure() {
FakeStore store = new FakeStore();
ChatApplicationUseCase useCase = useCase(
store,
(context, input) -> IntentType.DIAGNOSIS,
(context, query) -> new SystemChatContent("unused"),
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
(context, query, previous, status) -> {
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "model budget exhausted");
throw new IllegalStateException("diagnosis failed before evidence collection");
});
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
useCase.execute(new ChatApplicationRequest(
"diagnose unavailable model", "session-budget-no-tools")));
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
}
@Test
void clientDisconnectControlCancelsSameRunAndPersistsCancelled() {
FakeStore store = new FakeStore();
@@ -252,6 +337,7 @@ class ChatApplicationUseCaseTest {
private IntentType finishedIntent;
private ReleaseOutcome finishedOutcome;
private PublishedResult finishedPublishedResult;
private String finishedSafeContentJson;
@Override
public Optional<RoutingHistory> findLatestRoutingHistory(String sessionId) {
@@ -283,6 +369,7 @@ class ChatApplicationUseCaseTest {
finishedIntent = intent;
finishedOutcome = outcome;
finishedPublishedResult = publishedResult;
finishedSafeContentJson = safeContentJson;
}
}
}
@@ -14,6 +14,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
@@ -101,6 +102,8 @@ class PublishedResultPersistenceTest {
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
RunContext context = core().startRun("session-1", "run-fallback");
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
@@ -111,6 +114,7 @@ class PublishedResultPersistenceTest {
assertEquals(ReleaseOutcome.FALLBACK, entity.getReleaseOutcome());
assertNull(entity.getPublishedResult());
assertEquals(12, entity.getTotalDurationMs());
assertEquals(2, entity.getStepCount());
}
@Test
@@ -0,0 +1,128 @@
package com.superbiz.agent.harness.application.executor;
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
import com.superbiz.agent.harness.application.ChatApplicationStatus;
import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
import com.superbiz.agent.harness.application.persistence.PreviousTurnLimits;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.TraceEventType;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.release.DiagnosisReleaseResult;
import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
import com.superbiz.agent.harness.release.SafeFallbackFactory;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
class DiagnosisChatExecutorTest {
@Test
void invalidDraftWithVerifiedProgressUsesDeterministicReleaseFallback() {
RunContext context = context("run-invalid-progress");
DiagnosisProgressSnapshot progress = progress();
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
new IllegalArgumentException("redacted"),
DiagnosisAgentOutputException.Kind.SCHEMA_INVALID,
321L,
progress);
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(progress, List.of());
when(release.releaseInvalidDraft(context, progress))
.thenReturn(DiagnosisReleaseResult.fallback(fallback));
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
agent,
release,
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
trace::add);
List<ChatApplicationStatus> statuses = new ArrayList<>();
DiagnosisExecutionResult result = executor.execute(
context, "诊断未知故障", null, statuses::add);
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE,
((com.superbiz.agent.harness.application.FallbackContent) result.content())
.fallback().type());
assertEquals(List.of(
ChatApplicationStatus.DIAGNOSIS_RUNNING,
ChatApplicationStatus.SAFETY_VALIDATING), statuses);
assertEquals(List.of(TraceEventType.AGENT_DRAFT_INVALID),
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
assertEquals("SCHEMA_INVALID", trace.get(0).details().get("failure_kind"));
assertEquals(321L, trace.get(0).details().get("output_bytes"));
assertEquals(true, trace.get(0).details().get("has_publishable_progress"));
assertEquals(3, trace.get(0).details().size());
}
@Test
void invalidDraftWithoutVerifiedProgressStillFailsClosed() {
RunContext context = context("run-invalid-empty");
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
null,
DiagnosisAgentOutputException.Kind.INVALID_JSON,
12L,
DiagnosisProgressSnapshot.empty());
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
agent,
release,
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
trace::add);
DiagnosisAgentOutputException actual = assertThrows(
DiagnosisAgentOutputException.class,
() -> executor.execute(context, "诊断未知故障", null, ignored -> { }));
assertSame(failure, actual);
verifyNoInteractions(release);
assertEquals(false, trace.get(0).details().get("has_publishable_progress"));
}
private RunContext context(String runId) {
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused-run", Duration.ofMinutes(5),
new RunBudgetLimits(4, 3, 2, 100, 100, 200, 100_000),
HarnessRetryPolicies.strict());
return core.startRun("session-executor", runId);
}
private DiagnosisProgressSnapshot progress() {
return new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"已完成知识范围检查")),
List.of(), null);
}
}
@@ -0,0 +1,114 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.repository.AgentStepRepository;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class ModelCallAuditorTest {
@Test
void recordsComponentUsageIntoRunBudgetTraceAndAgentStep() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-token", "run-token");
AgentStepRepository steps = mock(AgentStepRepository.class);
AgentStep step = AgentStep.builder().id(7L).runId(context.runId()).stepIndex(0).build();
when(steps.findByRunIdAndStepIndex(context.runId(), 0)).thenReturn(Optional.of(step));
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
ModelCallAuditor auditor = new ModelCallAuditor(core, trace::add, steps);
core.beforeModelCall(context);
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
auditor.recordUsage(context, call, 13, 5, true);
assertEquals(18, context.budget().snapshot().totalTokens());
assertEquals(18, context.modelCalls().snapshot().totalTokens());
assertEquals(18, step.getTokenCount());
verify(steps).save(step);
DiagnosisTraceAuditEvent event = trace.get(0);
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, event.eventType());
assertEquals("DIAGNOSIS_AGENT", event.details().get("component"));
assertEquals(1, event.details().get("component_round"));
assertEquals(13L, event.details().get("input_tokens"));
assertEquals(5L, event.details().get("output_tokens"));
assertEquals(18L, event.details().get("total_tokens"));
assertEquals(true, event.details().get("usage_available"));
String details = event.details().toString().toLowerCase();
assertFalse(details.contains("prompt"));
assertFalse(details.contains("content"));
assertFalse(details.contains("response"));
assertFalse(details.contains("argument"));
}
@Test
void unavailableUsageIsVisibleWithoutFabricatingTokens() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-missing", "run-missing");
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
ModelCallAuditor auditor = new ModelCallAuditor(
core, trace::add, mock(AgentStepRepository.class));
core.beforeModelCall(context);
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.INTENT_ROUTER);
auditor.recordUsage(context, call, 0, 0, false);
ModelCallLedger.Snapshot snapshot = context.modelCalls().snapshot();
assertEquals(1, snapshot.auditedCallCount());
assertEquals(1, snapshot.usageUnavailableCount());
assertEquals(0, snapshot.totalTokens());
assertEquals(0, context.budget().snapshot().totalTokens());
assertEquals(false, trace.get(0).details().get("usage_available"));
assertFalse(trace.get(0).details().containsKey("input_tokens"));
assertFalse(trace.get(0).details().containsKey("output_tokens"));
assertFalse(trace.get(0).details().containsKey("total_tokens"));
DiagnosisTraceAuditEvent finished = TraceAuditEvents.runFinished(
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
com.superbiz.agent.harness.contract.ReleaseOutcome.FAILED, 1);
assertEquals(false, finished.details().get("tokens_reconciled"));
}
@Test
void runFinishedExposesTokenReconciliationWithoutContent() {
DiagnosisHarnessCore core = core();
RunContext context = core.startRun("session-finish", "run-finish");
ModelCallAuditor auditor = new ModelCallAuditor(
core, DiagnosisTraceRecorder.noop(), mock(AgentStepRepository.class));
core.beforeModelCall(context);
auditor.recordUsage(context,
auditor.begin(context, ModelCallComponent.INTENT_ROUTER), 8, 2, true);
DiagnosisTraceAuditEvent event = TraceAuditEvents.runFinished(
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, 50);
assertEquals(10L, event.details().get("run_total_tokens"));
assertEquals(10L, event.details().get("audited_total_tokens"));
assertEquals(true, event.details().get("tokens_reconciled"));
assertEquals(0, event.details().get("usage_unavailable_count"));
assertTrue(event.details().containsKey("audited_model_call_count"));
assertFalse(event.details().toString().contains("secret"));
}
private DiagnosisHarnessCore core() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
new RunBudgetLimits(20, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
HarnessRetryPolicies.strict());
}
}
@@ -0,0 +1,185 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
import org.junit.jupiter.api.Test;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisProgressProjectorTest {
private final ObjectMapper objectMapper = new ObjectMapper();
private final ToolCallKeyFactory keys = new ToolCallKeyFactory("test:tool");
@Test
void projectsReadyCurrentRunRecordsInExecutionOrder() {
RunContext context = context();
context.progress().recordCompleted(
new CompletedToolCall("call-1", AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}"),
EvidenceStatus.EVIDENCE_FOUND);
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
records.put(keys.create(context.runId(), "call-1"), ready(
"call-1", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"""
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
"evidence":[{"document_id":"doc-1","source":"guide.md","excerpt":"generic guidance"}],
"returned_count":1,"truncated":false}
""", EvidenceStatus.EVIDENCE_FOUND));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertEquals(1, snapshot.verifiedSources().size());
assertEquals("guide.md", snapshot.verifiedSources().get(0).source());
assertEquals(1, snapshot.observedFacts().size());
assertEquals("generic guidance", snapshot.observedFacts().get(0).summary());
assertTrue(snapshot.limitations().isEmpty());
}
@Test
void excludesMissingExpiredProjectingCrossRunAndErrorRecords() {
RunContext context = context();
context.progress().recordCompleted(call("missing"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("expired"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("projecting"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("cross"), EvidenceStatus.NO_EVIDENCE);
context.progress().recordCompleted(call("error"), EvidenceStatus.NO_EVIDENCE);
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
records.put(keys.create(context.runId(), "projecting"),
CanonicalToolInvocation.projecting(
"projecting", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}", Instant.now()));
records.put(keys.create(context.runId(), "cross"), ready(
"cross", "other-run", AgentToolContracts.LOOKUP_KNOWLEDGE,
noEvidence("cross"), EvidenceStatus.NO_EVIDENCE));
records.put(keys.create(context.runId(), "error"),
CanonicalToolInvocation.projecting(
"error", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
"{\"query\":\"unknown\"}", Instant.now())
.markError(null, "TOOL_EXECUTION_ERROR", Instant.now()));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertTrue(snapshot.observedFacts().isEmpty());
assertFalse(snapshot.limitations().isEmpty());
}
@Test
void boundsProjectedFactsAndReportsTruncation() {
RunContext context = context();
context.progress().recordCompleted(call("many"), EvidenceStatus.EVIDENCE_FOUND);
StringBuilder evidence = new StringBuilder();
for (int index = 0; index < 13; index++) {
if (index > 0) {
evidence.append(',');
}
evidence.append("{\"document_id\":\"doc-")
.append(index)
.append("\",\"source\":\"guide-")
.append(index)
.append("\",\"excerpt\":\"")
.append("x".repeat(400))
.append("\"}");
}
String result = "{\"evidence_status\":\"EVIDENCE_FOUND\","
+ "\"tool_call_id\":\"many\",\"query\":\"unknown\",\"evidence\":["
+ evidence + "],\"returned_count\":13,\"truncated\":false}";
Map<String, CanonicalToolInvocation> records = Map.of(
keys.create(context.runId(), "many"),
ready("many", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
result, EvidenceStatus.EVIDENCE_FOUND));
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
assertEquals(12, snapshot.observedFacts().size());
assertTrue(snapshot.observedFacts().stream()
.allMatch(fact -> fact.summary().length() <= 320));
assertTrue(snapshot.limitations().contains("已检查事实较多,展示内容已截断"));
}
private DiagnosisProgressProjector projector(Map<String, CanonicalToolInvocation> records) {
return new DiagnosisProgressProjector(new MapStore(records), keys, objectMapper);
}
private CompletedToolCall call(String id) {
return new CompletedToolCall(
id, AgentToolContracts.LOOKUP_KNOWLEDGE, "{\"query\":\"" + id + "\"}");
}
private CanonicalToolInvocation ready(String id, String runId, String toolName,
String agentResult, EvidenceStatus status) {
return CanonicalToolInvocation.projecting(
id, runId, toolName, "{\"query\":\"unknown\"}", Instant.now())
.markReady("{}", agentResult, status, Instant.now());
}
private String noEvidence(String id) {
return "{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
+ "\",\"query\":\"unknown\",\"evidence\":[],\"returned_count\":0,"
+ "\"truncated\":false}";
}
private RunContext context() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "run-1", Duration.ofMinutes(5),
new RunBudgetLimits(10, 10, 10, 1000, 1000, 2000, 100_000),
HarnessRetryPolicies.strict(), 10)
.startRun("session-1", "run-1");
}
private static final class MapStore implements CanonicalInvocationStore {
private final Map<String, CanonicalToolInvocation> records;
private final CanonicalInvocationLimits limits =
new CanonicalInvocationLimits(Duration.ofHours(1), 1_000_000, 100_000);
private MapStore(Map<String, CanonicalToolInvocation> records) {
this.records = records;
}
@Override
public CanonicalInvocationLimits limits() {
return limits;
}
@Override
public void begin(String key, CanonicalToolInvocation invocation) {
throw new UnsupportedOperationException();
}
@Override
public Optional<CanonicalToolInvocation> find(String key) {
return Optional.ofNullable(records.get(key));
}
@Override
public CanonicalToolInvocation markReady(String key, String rawResponse, String agentResult,
EvidenceStatus evidenceStatus, Instant completedAt) {
throw new UnsupportedOperationException();
}
@Override
public CanonicalToolInvocation markError(String key, String rawResponse,
String errorCode, Instant completedAt) {
throw new UnsupportedOperationException();
}
}
}
@@ -0,0 +1,67 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisProgressTrackerTest {
@Test
void gainedResetsConsecutiveNoGainAndConsumesPendingCallOnce() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordDuplicateScope();
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
DiagnosisProgressSnapshotState state = tracker.snapshot();
assertEquals(0, state.consecutiveNoGain());
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
new PreviousObservation("call-1", InformationGain.NO_GAIN)));
}
@Test
void rejectsMissingOrOutOfOrderEvaluation() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(null));
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
new PreviousObservation("call-other", InformationGain.GAINED)));
assertEquals("call-1", tracker.snapshot().pendingToolCallId());
}
@Test
void deterministicNoEvidenceSaturatesAndStopInstructionIsClaimedOnce() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.NO_EVIDENCE);
tracker.recordCompleted(call("call-2", "scope-2"), EvidenceStatus.NO_EVIDENCE);
DiagnosisProgressSnapshotState state = tracker.snapshot();
assertEquals(DiagnosisCollectionState.SATURATED, state.collectionState());
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, state.stopReason());
assertTrue(tracker.claimStopInstruction());
assertFalse(tracker.claimStopInstruction());
}
@Test
void onlySuccessfulCompletedScopesParticipateInDuplicateDetection() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3);
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-1"));
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
assertTrue(tracker.isDuplicate("lookup_knowledge", "scope-1"));
assertFalse(tracker.isDuplicate("query_logs", "scope-1"));
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
}
private CompletedToolCall call(String id, String scope) {
return new CompletedToolCall(id, "lookup_knowledge", scope);
}
}
@@ -0,0 +1,52 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.LogTopic;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
class ToolScopeNormalizerTest {
private final ToolScopeNormalizer normalizer = new ToolScopeNormalizer(new ObjectMapper());
@Test
void trimsRagParametersWithoutSemanticRewriting() {
String first = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest(" refund timeout "));
String same = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest("refund timeout"));
String rewritten = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
new RagToolRequest("refund request is slow"));
assertEquals(first, same);
assertNotEquals(first, rewritten);
}
@Test
void normalizesMissingLogLookbackToAdapterDefault() {
String implicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", null));
String explicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", 30));
assertEquals(implicit, explicit);
}
@Test
void mysqlScopeIncludesLogicalSourceSqlAndParameters() {
String first = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(1)));
String otherParam = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(2)));
assertNotEquals(first, otherParam);
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.release;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.contract.AnalysisKind;
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
@@ -9,6 +10,7 @@ import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
@@ -16,6 +18,8 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
@@ -91,6 +95,101 @@ class DiagnosisReleaseUseCaseTest {
assertEquals(1, model.calls.get());
}
@Test
void noConclusionWithVerifiedNegativeReferenceSkipsRepairAndSemanticGuard() {
Fixture fixture = fixture();
readyNoEvidence(fixture.store, fixture.context, "call-empty-1");
DiagnosisDraft draft = new DiagnosisDraft(
null,
List.of(new DiagnosisDraft.AnalysisItem(
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
"知识库范围内未找到匹配证据", List.of("call-empty-1"))),
List.of(),
List.of(),
new DiagnosisDraft.Limitations(
"query=unknown", List.of("故障服务运行日志")));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.completed(draft, progress(null)));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals(1, result.fallback().observedFacts().size());
assertEquals(0, fixture.model.calls.get());
assertEquals(List.of(
TraceEventType.EVIDENCE_GUARD_INITIAL,
TraceEventType.RELEASE_DECISION),
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
}
@Test
void noConclusionWithoutToolCallsPublishesMissingContext() {
Fixture fixture = fixture();
DiagnosisDraft draft = new DiagnosisDraft(
null, List.of(), List.of(), List.of(),
new DiagnosisDraft.Limitations(
"未指定故障范围", List.of("企业标识", "故障发生时间")));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断切换企业失败的问题",
DiagnosisAgentExecution.completed(draft, DiagnosisProgressSnapshot.empty()));
assertFallback(result, FallbackType.MISSING_REQUIRED_CONTEXT, 0);
assertEquals(0, fixture.model.calls.get());
}
@Test
void fabricatedNoConclusionReferenceFailsWithoutModelRepair() {
Fixture fixture = fixture();
DiagnosisDraft draft = new DiagnosisDraft(
null,
List.of(new DiagnosisDraft.AnalysisItem(
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
"声称未找到证据", List.of("missing-call"))),
List.of(), List.of(),
new DiagnosisDraft.Limitations("query=unknown", List.of()));
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.completed(draft, progress(null)));
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
assertEquals(0, fixture.model.calls.get());
}
@Test
void controlledSaturationPublishesProgressWithoutAnotherModelCall() {
Fixture fixture = fixture();
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.stopped(
progress(DiagnosisStopReason.INFORMATION_SATURATED),
DiagnosisStopReason.INFORMATION_SATURATED));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals(0, fixture.model.calls.get());
}
@Test
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
Fixture fixture = fixture();
DiagnosisReleaseResult result = fixture.useCase.releaseInvalidDraft(
fixture.context, progress(null));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals("该知识检索范围内未发现可用文档证据",
result.fallback().observedFacts().get(0).summary());
assertEquals(0, fixture.model.calls.get());
assertEquals(List.of(TraceEventType.RELEASE_DECISION),
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
}
@Test
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
Fixture fixture = fixture(
@@ -204,7 +303,7 @@ class DiagnosisReleaseUseCaseTest {
Duration.ofSeconds(2), Duration.ofSeconds(3)),
ignored -> { }, trace),
new SafeFallbackFactory(), trace);
return new Fixture(context, model, useCase, trace);
return new Fixture(context, store, model, useCase, trace);
}
private DiagnosisDraft duplicateDraft() {
@@ -274,6 +373,29 @@ class DiagnosisReleaseUseCaseTest {
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
}
private void readyNoEvidence(InMemoryStore store, RunContext context, String callId) {
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
callId, context.runId(), "lookup_knowledge", "{\"query\":\"unknown\"}",
Instant.parse("2026-07-21T10:00:00Z"))
.markReady("raw-must-not-be-read", """
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-1",
"query":"unknown","evidence":[],"returned_count":0,"truncated":false}
""", EvidenceStatus.NO_EVIDENCE,
Instant.parse("2026-07-21T10:00:01Z"));
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
}
private DiagnosisProgressSnapshot progress(DiagnosisStopReason stopReason) {
return new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "knowledge_base", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "knowledge_base", "query=unknown",
"该知识检索范围内未发现可用文档证据")),
List.of(),
stopReason);
}
private DiagnosisHarnessCore core() {
return new DiagnosisHarnessCore(
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
@@ -337,6 +459,7 @@ class DiagnosisReleaseUseCaseTest {
private record Fixture(
RunContext context,
InMemoryStore store,
ScriptedChatModel model,
DiagnosisReleaseUseCase useCase,
RecordingTraceRecorder trace) {
@@ -0,0 +1,47 @@
package com.superbiz.agent.harness.release;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SafeFallbackFactoryTest {
@Test
void insufficientEvidenceReusesVerifiedProgress() {
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
List.of(new SafeFallback.VerifiedSource(
"RAG", "guide.md", "query=unknown")),
List.of(new SafeFallback.ObservedFact(
"RAG", "guide.md", "query=unknown", "未找到匹配的故障证据")),
List.of("部分记录已过期"),
DiagnosisStopReason.BUDGET_LIMIT_REACHED);
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(
progress, List.of("企业标识"));
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, fallback.type());
assertEquals("DIAGNOSIS_COLLECTION", fallback.failureStage());
assertEquals(progress.verifiedSources(), fallback.verifiedSources());
assertEquals(progress.observedFacts(), fallback.observedFacts());
assertTrue(fallback.limitations().contains("仍缺少:企业标识"));
}
@Test
void missingRequiredContextContainsNoFabricatedProgress() {
SafeFallback fallback = new SafeFallbackFactory().missingRequiredContext(
List.of("企业标识", "故障发生时间"));
assertEquals(FallbackType.MISSING_REQUIRED_CONTEXT, fallback.type());
assertEquals("DIAGNOSIS_INPUT", fallback.failureStage());
assertTrue(fallback.verifiedSources().isEmpty());
assertTrue(fallback.observedFacts().isEmpty());
assertEquals(2, fallback.limitations().size());
}
}
@@ -0,0 +1,51 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.PreviousObservation;
import org.junit.jupiter.api.Test;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class ToolCallEnvelopeContractTest {
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void serializesSharedControlFieldsInSnakeCase() throws Exception {
RagToolCall call = new RagToolCall(
new PreviousObservation("call-1", InformationGain.NO_GAIN),
new RagToolRequest("refund timeout"));
JsonNode json = objectMapper.readTree(objectMapper.writeValueAsString(call));
assertEquals("call-1", json.path("previous_observation").path("tool_call_id").asText());
assertEquals("NO_GAIN", json.path("previous_observation").path("information_gain").asText());
assertEquals("refund timeout", json.path("input").path("query").asText());
}
@Test
void deserializesAllTypedBusinessInputs() throws Exception {
QueryLogsToolCall logs = objectMapper.readValue("""
{"input":{"topic":"APPLICATION","query":"timeout","lookback_minutes":30}}
""", QueryLogsToolCall.class);
MysqlToolCall mysql = objectMapper.readValue("""
{"input":{"data_source":"orders","sql":"SELECT id FROM orders WHERE id = ?","params":[1]}}
""", MysqlToolCall.class);
assertEquals(LogTopic.APPLICATION, logs.input().topic());
assertEquals(List.of(1), mysql.input().params());
}
@Test
void rejectsUnknownInformationGain() {
assertThrows(Exception.class, () -> objectMapper.readValue("""
{"previous_observation":{"tool_call_id":"call-1","information_gain":"UNKNOWN"},
"input":{"query":"timeout"}}
""", RagToolCall.class));
}
}
@@ -40,6 +40,25 @@ class RagResultProjectorTest {
assertFalse(projected.agentResult().contains("score"));
}
@Test
void preservesCamelAndSnakeCaseReferenceLevel() throws Exception {
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
String camel = """
{"found":true,"relevanceLevel":"REFERENCE",
"evidenceBlocks":[{"source":"doc","content":"generic guidance"}]}
""";
String snake = camel.replace("relevanceLevel", "relevance_level");
JsonNode camelResult = objectMapper.readTree(
projector.project(new RagToolRequest("unknown"), "call-1", camel).agentResult());
JsonNode snakeResult = objectMapper.readTree(
projector.project(new RagToolRequest("unknown"), "call-2", snake).agentResult());
assertEquals("REFERENCE", camelResult.path("relevance_level").asText());
assertEquals("REFERENCE", snakeResult.path("relevance_level").asText());
assertEquals("EVIDENCE_FOUND", camelResult.path("evidence_status").asText());
}
@Test
void returnsNoEvidenceForEmptyUsableBlocks() throws Exception {
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
@@ -48,6 +67,8 @@ class RagResultProjectorTest {
"{\"found\":false,\"evidenceBlocks\":[{\"source\":\"doc\",\"content\":\"\"}]}");
assertEquals(EvidenceStatus.NO_EVIDENCE, projected.evidenceStatus());
assertEquals(0, objectMapper.readTree(projected.agentResult()).path("evidence").size());
JsonNode json = objectMapper.readTree(projected.agentResult());
assertEquals(0, json.path("evidence").size());
assertFalse(json.has("relevance_level"));
}
}