feat(harness): add information gain stop and audit

This commit is contained in:
aruo
2026-07-27 01:03:34 +08:00
parent de5a5b09d9
commit d0452184ee
92 changed files with 5019 additions and 122 deletions
@@ -27,6 +27,7 @@ public class ChatHarnessProperties {
private int maxModelCalls = 24;
private int maxToolCalls = 24;
private int maxCallsPerTool = 8;
private int stopAfterConsecutiveNoGain = 2;
private long maxInputTokens = 100_000;
private long maxOutputTokens = 100_000;
private long maxTotalTokens = 200_000;
@@ -71,6 +72,7 @@ public class ChatHarnessProperties {
positive(maxModelCalls, "maxModelCalls");
positive(maxToolCalls, "maxToolCalls");
positive(maxCallsPerTool, "maxCallsPerTool");
positive(stopAfterConsecutiveNoGain, "stopAfterConsecutiveNoGain");
positive(maxInputTokens, "maxInputTokens");
positive(maxOutputTokens, "maxOutputTokens");
positive(maxTotalTokens, "maxTotalTokens");
@@ -27,6 +27,7 @@ import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
import com.superbiz.agent.harness.release.EvidenceRepair;
import com.superbiz.agent.harness.release.EvidenceRepairLimits;
import com.superbiz.agent.harness.release.SafeFallbackFactory;
import com.superbiz.agent.harness.progress.DiagnosisProgressProjector;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
@@ -50,6 +51,7 @@ import com.superbiz.agent.harness.application.KnowledgeQueryOperation;
import com.superbiz.agent.harness.application.SystemChatOperation;
import com.superbiz.agent.harness.application.IntentRouting;
import com.superbiz.agent.harness.audit.HarnessAgentAuditHook;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
import com.superbiz.agent.repository.AgentStepRepository;
@@ -106,7 +108,8 @@ public class HarnessChatConfiguration {
new RunBudgetLimits(properties.getMaxModelCalls(), properties.getMaxToolCalls(),
properties.getMaxCallsPerTool(), properties.getMaxInputTokens(),
properties.getMaxOutputTokens(), properties.getMaxTotalTokens(), properties.getMaxRunBytes()),
HarnessRetryPolicies.strict());
HarnessRetryPolicies.strict(),
properties.getStopAfterConsecutiveNoGain());
}
@Bean
@@ -116,8 +119,16 @@ public class HarnessChatConfiguration {
@Bean
public GuardModelCall guardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor) {
return new GuardModelCall(core, chatModel, executor);
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor,
ModelCallAuditor modelCallAuditor) {
return new GuardModelCall(core, chatModel, executor, modelCallAuditor);
}
@Bean
public ModelCallAuditor modelCallAuditor(DiagnosisHarnessCore core,
DiagnosisTraceRecorder traceRecorder,
AgentStepRepository steps) {
return new ModelCallAuditor(core, traceRecorder, steps);
}
@Bean
@@ -237,20 +248,30 @@ public class HarnessChatConfiguration {
HarnessEvidenceTools tools, ObjectMapper mapper,
AgentStepRepository steps,
AgentReasoningAuditRepository reasoningAudits,
DiagnosisTraceRecorder traceRecorder) {
DiagnosisTraceRecorder traceRecorder,
ModelCallAuditor modelCallAuditor) {
return new DiagnosisAgentFactory(chatModel, core, tools, mapper,
List.of(new HarnessAgentAuditHook(
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)));
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)),
traceRecorder, modelCallAuditor);
}
@Bean
public DiagnosisAgentUseCase diagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory factory,
ObjectMapper mapper,
ChatHarnessProperties properties) {
ChatHarnessProperties properties,
DiagnosisProgressProjector progressProjector) {
return new DiagnosisAgentUseCase(core, factory, mapper, new DiagnosisAgentLimits(
properties.getDiagnosisMaxQueryBytes(), properties.getDiagnosisMaxPreviousTurnBytes(),
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()));
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()),
progressProjector);
}
@Bean
public DiagnosisProgressProjector diagnosisProgressProjector(
CanonicalInvocationStore store, ToolCallKeyFactory keyFactory, ObjectMapper mapper) {
return new DiagnosisProgressProjector(store, keyFactory, mapper);
}
@Bean
@@ -281,13 +302,19 @@ public class HarnessChatConfiguration {
attempt -> { }, traceRecorder);
}
@Bean
public SafeFallbackFactory safeFallbackFactory() {
return new SafeFallbackFactory();
}
@Bean
public DiagnosisReleaseUseCase diagnosisReleaseUseCase(EvidenceGuard evidenceGuard,
EvidenceRepair repair,
SemanticGuard semanticGuard,
DiagnosisTraceRecorder traceRecorder) {
DiagnosisTraceRecorder traceRecorder,
SafeFallbackFactory fallbackFactory) {
return new DiagnosisReleaseUseCase(
evidenceGuard, repair, semanticGuard, new SafeFallbackFactory(), traceRecorder);
evidenceGuard, repair, semanticGuard, fallbackFactory, traceRecorder);
}
@Bean
@@ -327,9 +354,10 @@ public class HarnessChatConfiguration {
@Bean
public DiagnosisOperation diagnosisOperation(DiagnosisAgentUseCase agent,
DiagnosisReleaseUseCase release,
PublishedResultPolicy policy) {
return new DiagnosisChatExecutor(agent, release, policy);
DiagnosisReleaseUseCase release,
PublishedResultPolicy policy,
DiagnosisTraceRecorder traceRecorder) {
return new DiagnosisChatExecutor(agent, release, policy, traceRecorder);
}
@Bean
@@ -0,0 +1,32 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import java.util.Objects;
public record DiagnosisAgentExecution(
DiagnosisDraft draft,
DiagnosisProgressSnapshot progress,
DiagnosisStopReason stopReason) {
public DiagnosisAgentExecution {
Objects.requireNonNull(progress, "progress must not be null");
if (draft == null && stopReason == null) {
throw new IllegalArgumentException("Execution without Draft requires a stop reason");
}
}
public static DiagnosisAgentExecution completed(
DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
return new DiagnosisAgentExecution(
Objects.requireNonNull(draft, "draft must not be null"), progress, null);
}
public static DiagnosisAgentExecution stopped(
DiagnosisProgressSnapshot progress, DiagnosisStopReason stopReason) {
return new DiagnosisAgentExecution(
null, progress, Objects.requireNonNull(stopReason, "stopReason must not be null"));
}
}
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.alibaba.cloud.ai.graph.agent.hook.Hook;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import org.springframework.ai.chat.model.ChatModel;
@@ -19,6 +21,8 @@ public final class DiagnosisAgentFactory {
private final HarnessEvidenceTools evidenceTools;
private final ObjectMapper objectMapper;
private final List<Hook> auditHooks;
private final DiagnosisTraceRecorder traceRecorder;
private final ModelCallAuditor modelCallAuditor;
private final String prompt;
public DiagnosisAgentFactory(ChatModel chatModel,
@@ -26,11 +30,35 @@ public final class DiagnosisAgentFactory {
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks) {
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
DiagnosisTraceRecorder.noop(), new ModelCallAuditor(core));
}
public DiagnosisAgentFactory(ChatModel chatModel,
DiagnosisHarnessCore core,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks,
DiagnosisTraceRecorder traceRecorder) {
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
traceRecorder, new ModelCallAuditor(core));
}
public DiagnosisAgentFactory(ChatModel chatModel,
DiagnosisHarnessCore core,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
List<? extends Hook> auditHooks,
DiagnosisTraceRecorder traceRecorder,
ModelCallAuditor modelCallAuditor) {
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
this.core = Objects.requireNonNull(core, "core must not be null");
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
this.auditHooks = List.copyOf(Objects.requireNonNull(auditHooks, "auditHooks must not be null"));
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
this.modelCallAuditor = Objects.requireNonNull(
modelCallAuditor, "modelCallAuditor must not be null");
this.prompt = DiagnosisAgentPrompt.load();
}
@@ -43,8 +71,9 @@ public final class DiagnosisAgentFactory {
.systemPrompt(prompt)
.tools(evidenceTools.callbacks())
.interceptors(
new HarnessModelInterceptor(core, context),
new HarnessToolInterceptor(context, evidenceTools, objectMapper))
new HarnessModelInterceptor(core, context, modelCallAuditor),
new HarnessToolInterceptor(
context, evidenceTools, objectMapper, traceRecorder))
.hooks(auditHooks)
.outputSchema(new DiagnosisDraftOutputSchema(objectMapper).getFormat())
.returnReasoningContents(true)
@@ -1,12 +1,59 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import java.util.Objects;
public final class DiagnosisAgentOutputException extends RuntimeException {
public enum Kind {
EXECUTION_FAILED,
EMPTY_DRAFT,
INVALID_JSON,
SCHEMA_INVALID
}
private final Kind kind;
private final long outputBytes;
private final DiagnosisProgressSnapshot progress;
public DiagnosisAgentOutputException(String message) {
super(message);
this(message, null);
}
public DiagnosisAgentOutputException(String message, Throwable cause) {
this(message, cause, Kind.EXECUTION_FAILED, 0L, DiagnosisProgressSnapshot.empty());
}
public DiagnosisAgentOutputException(String message,
Throwable cause,
Kind kind,
long outputBytes,
DiagnosisProgressSnapshot progress) {
super(message, cause);
if (outputBytes < 0) {
throw new IllegalArgumentException("outputBytes must not be negative");
}
this.kind = Objects.requireNonNull(kind, "kind must not be null");
this.outputBytes = outputBytes;
this.progress = Objects.requireNonNull(progress, "progress must not be null");
}
public Kind kind() {
return kind;
}
public long outputBytes() {
return outputBytes;
}
public DiagnosisProgressSnapshot progress() {
return progress;
}
public boolean isDraftContractFailure() {
return kind == Kind.EMPTY_DRAFT
|| kind == Kind.INVALID_JSON
|| kind == Kind.SCHEMA_INVALID;
}
}
@@ -2,13 +2,20 @@ package com.superbiz.agent.harness.agent;
import com.alibaba.cloud.ai.graph.RunnableConfig;
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.BudgetExceededException;
import com.superbiz.agent.harness.core.RunAbortedException;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.core.RunState;
import com.superbiz.agent.harness.progress.DiagnosisProgressProjection;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import org.springframework.ai.chat.messages.AssistantMessage;
import java.nio.charset.StandardCharsets;
@@ -23,11 +30,20 @@ public final class DiagnosisAgentUseCase {
private final ObjectMapper objectMapper;
private final ObjectReader draftReader;
private final DiagnosisAgentLimits limits;
private final DiagnosisProgressProjection progressProjection;
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory agentFactory,
ObjectMapper objectMapper,
DiagnosisAgentLimits limits) {
this(core, agentFactory, objectMapper, limits, DiagnosisProgressProjection.empty());
}
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
DiagnosisAgentFactory agentFactory,
ObjectMapper objectMapper,
DiagnosisAgentLimits limits,
DiagnosisProgressProjection progressProjection) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.agentFactory = Objects.requireNonNull(agentFactory, "agentFactory must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
@@ -35,9 +51,11 @@ public final class DiagnosisAgentUseCase {
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
this.limits = Objects.requireNonNull(limits, "limits must not be null");
this.progressProjection = Objects.requireNonNull(
progressProjection, "progressProjection must not be null");
}
public DiagnosisDraft execute(RunContext context, DiagnosisAgentInput input) {
public DiagnosisAgentExecution execute(RunContext context, DiagnosisAgentInput input) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(input, "input must not be null");
core.checkActive(context);
@@ -64,23 +82,85 @@ public final class DiagnosisAgentUseCase {
AssistantMessage response;
try {
response = agent.call(inputJson, config);
core.checkActive(context);
} catch (Exception e) {
DiagnosisAgentExecution controlled = controlledExecution(context, e);
if (controlled != null) {
return controlled;
}
throw new DiagnosisAgentOutputException("Diagnosis Agent execution failed", e);
}
core.checkActive(context);
String output = response == null ? null : response.getText();
if (output == null || output.isBlank()) {
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an empty Draft");
throw new DiagnosisAgentOutputException(
"Diagnosis Agent returned an empty Draft",
null,
DiagnosisAgentOutputException.Kind.EMPTY_DRAFT,
0L,
progressProjection.project(context));
}
long draftBytes = utf8Bytes(output);
checkLimit("draft", draftBytes, limits.maxDraftBytes());
core.reserveRunBytes(context, draftBytes);
try {
return draftReader.readValue(output);
} catch (JsonProcessingException e) {
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an invalid Draft", e);
core.reserveRunBytes(context, draftBytes);
} catch (RuntimeException exception) {
DiagnosisAgentExecution controlled = controlledExecution(context, exception);
if (controlled != null) {
return controlled;
}
throw exception;
}
try {
DiagnosisDraft draft = draftReader.readValue(output);
return DiagnosisAgentExecution.completed(draft, progressProjection.project(context));
} catch (JsonProcessingException e) {
DiagnosisAgentOutputException.Kind kind = e instanceof JsonParseException
? DiagnosisAgentOutputException.Kind.INVALID_JSON
: DiagnosisAgentOutputException.Kind.SCHEMA_INVALID;
throw new DiagnosisAgentOutputException(
"Diagnosis Agent returned an invalid Draft",
e,
kind,
draftBytes,
progressProjection.project(context));
}
}
private DiagnosisAgentExecution controlledExecution(RunContext context, Throwable failure) {
DiagnosisCollectionStoppedException stopped = findCause(
failure, DiagnosisCollectionStoppedException.class);
if (stopped != null) {
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), stopped.stopReason());
}
RunAbortedException aborted = findCause(failure, RunAbortedException.class);
if (aborted != null) {
if (aborted.termination().state() == RunState.BUDGET_EXHAUSTED) {
context.progress().markBudgetLimitReached();
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
throw aborted;
}
BudgetExceededException budget = findCause(failure, BudgetExceededException.class);
if (budget != null || context.lifecycle().state() == RunState.BUDGET_EXHAUSTED) {
context.progress().markBudgetLimitReached();
return DiagnosisAgentExecution.stopped(
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
return null;
}
private static <T extends Throwable> T findCause(Throwable failure, Class<T> type) {
Throwable current = failure;
while (current != null) {
if (type.isInstance(current)) {
return type.cast(current);
}
current = current.getCause();
}
return null;
}
private String writeJson(Object value) {
@@ -0,0 +1,17 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
public final class DiagnosisCollectionStoppedException extends RuntimeException {
private final DiagnosisStopReason stopReason;
public DiagnosisCollectionStoppedException(DiagnosisStopReason stopReason) {
super("Diagnosis collection stopped: " + stopReason);
this.stopReason = stopReason;
}
public DiagnosisStopReason stopReason() {
return stopReason;
}
}
@@ -1,5 +1,8 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
@@ -8,8 +11,11 @@ import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import com.superbiz.agent.harness.tool.boundary.ToolCallRequestEnvelope;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.MysqlToolCall;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsToolCall;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import com.superbiz.agent.harness.tool.contract.RagToolCall;
import org.springframework.ai.tool.ToolCallback;
import org.springframework.ai.tool.function.FunctionToolCallback;
@@ -36,11 +42,11 @@ public final class HarnessEvidenceTools {
this.invokers = Map.copyOf(registered);
this.callbacks = List.of(
definition(AgentToolContracts.LOOKUP_KNOWLEDGE,
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolRequest.class),
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolCall.class),
definition(AgentToolContracts.QUERY_LOGS,
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsRequest.class),
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsToolCall.class),
definition(AgentToolContracts.QUERY_MYSQL,
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolRequest.class));
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolCall.class));
}
public static HarnessEvidenceTools fromAdapters(RagToolAdapter ragAdapter,
@@ -72,6 +78,51 @@ public final class HarnessEvidenceTools {
return invoker.invoke(context, toolCallId, arguments);
}
public ParsedAgentToolCall parse(String toolName, String arguments, ObjectMapper objectMapper) {
Objects.requireNonNull(objectMapper, "objectMapper must not be null");
try {
ObjectReader reader = objectMapper.reader()
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
return switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
RagToolCall call = reader.forType(RagToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
case AgentToolContracts.QUERY_LOGS -> {
QueryLogsToolCall call = reader.forType(QueryLogsToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
case AgentToolContracts.QUERY_MYSQL -> {
MysqlToolCall call = reader.forType(MysqlToolCall.class).readValue(arguments);
requireInput(call == null ? null : call.input());
yield parsed(call.previousObservation(), call.input(), objectMapper);
}
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
};
} catch (IllegalArgumentException exception) {
throw exception;
} catch (Exception exception) {
throw new IllegalArgumentException("Tool Call Envelope is invalid", exception);
}
}
private static ParsedAgentToolCall parsed(
com.superbiz.agent.harness.progress.PreviousObservation previousObservation,
Object input,
ObjectMapper objectMapper) throws Exception {
return new ParsedAgentToolCall(
previousObservation, input, objectMapper.writeValueAsString(input));
}
private static void requireInput(Object input) {
if (input == null) {
throw new IllegalArgumentException("Tool Call Envelope input is required");
}
}
private static EvidenceToolInvoker bridge(String toolName, AdapterCall adapter) {
return (context, toolCallId, arguments) -> adapter.execute(
context,
@@ -6,6 +6,9 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import org.springframework.ai.chat.metadata.Usage;
import org.springframework.ai.chat.model.ChatResponse;
@@ -15,10 +18,17 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
private final DiagnosisHarnessCore core;
private final RunContext context;
private final ModelCallAuditor auditor;
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context) {
this(core, context, new ModelCallAuditor(core));
}
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context,
ModelCallAuditor auditor) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.context = Objects.requireNonNull(context, "context must not be null");
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
}
@Override
@@ -31,23 +41,32 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
Objects.requireNonNull(request, "request must not be null");
Objects.requireNonNull(handler, "handler must not be null");
core.beforeModelCall(context);
ModelResponse response = handler.call(request);
recordUsage(response == null ? null : response.getChatResponse());
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
ModelResponse response;
try {
response = handler.call(request);
} catch (RuntimeException exception) {
auditor.recordUsage(context, call, 0, 0, false);
throw exception;
}
recordUsage(call, response == null ? null : response.getChatResponse());
core.checkActive(context);
return response;
}
private void recordUsage(ChatResponse response) {
private void recordUsage(ModelCallLedger.Call call, ChatResponse response) {
if (response == null || response.getMetadata() == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
Usage usage = response.getMetadata().getUsage();
if (usage == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
long inputTokens = nonNegative(usage.getPromptTokens());
long outputTokens = nonNegative(usage.getCompletionTokens());
core.recordTokens(context, inputTokens, outputTokens);
auditor.recordUsage(context, call, inputTokens, outputTokens, true);
}
private static long nonNegative(Integer value) {
@@ -6,8 +6,16 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolInterceptor;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.contract.InvocationStatus;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.CompletedToolCall;
import com.superbiz.agent.harness.progress.DiagnosisCollectionState;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.ToolScopeNormalizer;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import java.util.LinkedHashMap;
@@ -19,13 +27,26 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
private final RunContext context;
private final HarnessEvidenceTools evidenceTools;
private final ObjectMapper objectMapper;
private final ToolScopeNormalizer scopeNormalizer;
private final ToolResultViewProjector viewProjector;
private final DiagnosisTraceRecorder traceRecorder;
public HarnessToolInterceptor(RunContext context,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper) {
this(context, evidenceTools, objectMapper, DiagnosisTraceRecorder.noop());
}
public HarnessToolInterceptor(RunContext context,
HarnessEvidenceTools evidenceTools,
ObjectMapper objectMapper,
DiagnosisTraceRecorder traceRecorder) {
this.context = Objects.requireNonNull(context, "context must not be null");
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
this.scopeNormalizer = new ToolScopeNormalizer(objectMapper);
this.viewProjector = new ToolResultViewProjector(objectMapper);
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
}
@Override
@@ -41,10 +62,76 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
return handler.call(request);
}
DiagnosisProgressSnapshotState before = context.progress().snapshot();
if (before.collectionState() == DiagnosisCollectionState.SATURATED
&& before.stopInstructionDelivered()) {
recordRejection(request, "COLLECTION_STOPPED");
throw new DiagnosisCollectionStoppedException(before.stopReason());
}
ParsedAgentToolCall call;
String normalizedScope;
try {
call = evidenceTools.parse(request.getToolName(), request.getArguments(), objectMapper);
context.progress().applyPreviousObservation(call.previousObservation());
DiagnosisProgressSnapshotState evaluated = context.progress().snapshot();
recordModelProgress(before, call, evaluated);
if (evaluated.collectionState() == DiagnosisCollectionState.SATURATED) {
recordRejection(request, "INFORMATION_SATURATED");
return stopRequired(request, evaluated.stopReason());
}
normalizedScope = scopeNormalizer.normalize(request.getToolName(), call.businessInput());
} catch (IllegalArgumentException | IllegalStateException exception) {
recordRejection(request, "INVALID_PROGRESS_PROTOCOL");
return safeError(request, "INVALID_PROGRESS_PROTOCOL");
}
if (context.progress().isDuplicate(request.getToolName(), normalizedScope)) {
recordRejection(request, "DUPLICATE_SCOPE");
context.progress().recordDuplicateScope();
DiagnosisProgressSnapshotState duplicate = context.progress().snapshot();
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
InformationGain.NO_GAIN, "HARNESS", duplicate);
if (duplicate.collectionState() == DiagnosisCollectionState.SATURATED) {
return stopRequired(request, duplicate.stopReason());
}
return duplicateScope(request);
}
ToolBoundaryResult result = evidenceTools.invoke(
context, request.getToolName(), request.getToolCallId(), request.getArguments());
context, request.getToolName(), request.getToolCallId(), call.businessArguments());
if (result.status() == InvocationStatus.READY) {
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), result.agentResult());
ToolControlView control = viewProjector.controlView(result.agentResult());
if (control.evidenceStatus() != result.evidenceStatus()) {
recordRejection(request, "OBSERVATION_CONTRACT_MISMATCH");
return safeError(request, "OBSERVATION_CONTRACT_MISMATCH");
}
context.progress().recordCompleted(
new CompletedToolCall(
request.getToolCallId(), request.getToolName(), normalizedScope),
result.evidenceStatus());
DiagnosisProgressSnapshotState completed = context.progress().snapshot();
if (result.evidenceStatus()
== com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
InformationGain.NO_GAIN, "HARNESS", completed);
}
boolean stopRequired = completed.collectionState() == DiagnosisCollectionState.SATURATED
&& context.progress().claimStopInstruction();
if (stopRequired) {
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(), completed));
}
String observation = viewProjector.modelObservation(
request.getToolName(), result.agentResult(), normalizedScope,
stopRequired, completed.stopReason());
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), observation);
}
if ("BUDGET_EXHAUSTED".equals(result.errorCode())) {
context.progress().markBudgetLimitReached();
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(),
context.progress().snapshot()));
}
return ToolCallResponse.builder()
.toolCallId(request.getToolCallId())
@@ -55,6 +142,90 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
.build();
}
private ToolCallResponse stopRequired(ToolCallRequest request, DiagnosisStopReason reason) {
if (!context.progress().claimStopInstruction()) {
throw new DiagnosisCollectionStoppedException(reason);
}
traceRecorder.record(TraceAuditEvents.collectionStop(
context, request.getToolCallId(), request.getToolName(),
context.progress().snapshot()));
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("tool_call_id", request.getToolCallId());
observation.put("stop_required", true);
observation.put("reason", reason.name());
return ToolCallResponse.of(
request.getToolCallId(), request.getToolName(), writeObservation(observation));
}
private ToolCallResponse duplicateScope(ToolCallRequest request) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("tool_call_id", request.getToolCallId());
observation.put("observation_status", "DUPLICATE_SCOPE");
observation.put("information_gain", "NO_GAIN");
observation.put("stop_required", false);
return ToolCallResponse.of(
request.getToolCallId(), request.getToolName(), writeObservation(observation));
}
private ToolCallResponse safeError(ToolCallRequest request, String errorCode) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", "ERROR");
observation.put("tool_call_id", request.getToolCallId());
observation.put("error_code", errorCode);
return ToolCallResponse.builder()
.toolCallId(request.getToolCallId())
.toolName(request.getToolName())
.content(writeObservation(observation))
.status("error")
.metadata(Map.of("error", true))
.build();
}
private void recordRejection(ToolCallRequest request, String errorCode) {
traceRecorder.record(TraceAuditEvents.toolRequestRejected(
context, request.getToolCallId(), request.getToolName(), errorCode));
}
private String writeObservation(Map<String, Object> observation) {
try {
return objectMapper.writeValueAsString(observation);
} catch (JsonProcessingException exception) {
return "{\"evidence_status\":\"ERROR\",\"error_code\":\"SERIALIZATION_ERROR\"}";
}
}
private void recordModelProgress(
DiagnosisProgressSnapshotState before,
ParsedAgentToolCall call,
DiagnosisProgressSnapshotState after) {
if (call.previousObservation() == null) {
return;
}
before.completedToolCalls().stream()
.filter(completed -> completed.toolCallId()
.equals(call.previousObservation().toolCallId()))
.findFirst()
.ifPresent(completed -> recordProgress(
completed.toolCallId(), completed.toolName(), completed.normalizedScope(),
call.previousObservation().informationGain(), "MODEL", after));
}
private void recordProgress(
String toolCallId,
String toolName,
String normalizedScope,
InformationGain informationGain,
String producer,
DiagnosisProgressSnapshotState state) {
traceRecorder.record(TraceAuditEvents.toolProgress(
context, toolCallId, toolName, scopeSummary(toolName, normalizedScope),
informationGain, producer, state));
}
private String scopeSummary(String toolName, String normalizedScope) {
return toolName + "#" + String.format("%08x", normalizedScope.hashCode());
}
private String errorObservation(ToolBoundaryResult result) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", result.evidenceStatus());
@@ -0,0 +1,18 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record ParsedAgentToolCall(
PreviousObservation previousObservation,
Object businessInput,
String businessArguments) {
public ParsedAgentToolCall {
if (businessInput == null) {
throw new IllegalArgumentException("businessInput must not be null");
}
if (businessArguments == null || businessArguments.isBlank()) {
throw new IllegalArgumentException("businessArguments must not be blank");
}
}
}
@@ -0,0 +1,11 @@
package com.superbiz.agent.harness.agent;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
public record ToolControlView(
EvidenceStatus evidenceStatus,
int returnedCount,
RagRelevanceLevel relevanceLevel,
boolean truncated) {
}
@@ -0,0 +1,120 @@
package com.superbiz.agent.harness.agent;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import java.util.List;
import java.util.Objects;
public final class ToolResultViewProjector {
private static final List<String> COMMON_FIELDS = List.of(
"evidence_status", "tool_call_id", "truncated");
private final ObjectMapper objectMapper;
public ToolResultViewProjector(ObjectMapper objectMapper) {
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
public ToolControlView controlView(String agentResult) {
JsonNode root = readObject(agentResult);
EvidenceStatus status;
try {
status = EvidenceStatus.valueOf(requiredText(root, "evidence_status"));
} catch (IllegalArgumentException exception) {
throw new IllegalArgumentException("Invalid evidence status", exception);
}
RagRelevanceLevel relevance = null;
String relevanceText = text(root, "relevance_level");
if (!relevanceText.isBlank()) {
try {
relevance = RagRelevanceLevel.valueOf(relevanceText);
} catch (IllegalArgumentException exception) {
throw new IllegalArgumentException("Invalid relevance level", exception);
}
}
return new ToolControlView(
status,
Math.max(0, root.path("returned_count").asInt(0)),
relevance,
root.path("truncated").asBoolean(false));
}
public String modelObservation(String toolName,
String agentResult,
String normalizedScope,
boolean stopRequired,
DiagnosisStopReason stopReason) {
JsonNode root = readObject(agentResult);
ObjectNode observation = objectMapper.createObjectNode();
copy(root, observation, COMMON_FIELDS);
switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
ObjectNode scope = observation.putObject("scope");
scope.put("query", text(root, "query"));
copy(root, observation, List.of("evidence", "relevance_level"));
}
case AgentToolContracts.QUERY_LOGS ->
copy(root, observation, List.of("source_kind", "scope", "patterns", "events"));
case AgentToolContracts.QUERY_MYSQL -> {
try {
observation.set("scope", objectMapper.readTree(normalizedScope));
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Normalized scope is invalid", exception);
}
copy(root, observation, List.of("columns", "rows"));
}
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
}
if (stopRequired) {
observation.put("stop_required", true);
observation.put("reason", Objects.requireNonNull(stopReason, "stopReason must not be null").name());
}
try {
return objectMapper.writeValueAsString(observation);
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Model observation is not serializable", exception);
}
}
private JsonNode readObject(String value) {
try {
JsonNode root = objectMapper.readTree(value);
if (root == null || !root.isObject()) {
throw new IllegalArgumentException("Tool result must be a JSON object");
}
return root;
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Tool result must be valid JSON", exception);
}
}
private static void copy(JsonNode source, ObjectNode target, List<String> fields) {
for (String field : fields) {
JsonNode value = source.get(field);
if (value != null && !value.isNull()) {
target.set(field, value);
}
}
}
private static String requiredText(JsonNode root, String field) {
String value = text(root, field);
if (value.isBlank()) {
throw new IllegalArgumentException(field + " must not be blank");
}
return value;
}
private static String text(JsonNode root, String field) {
JsonNode value = root.get(field);
return value == null || value.isNull() ? "" : value.asText("");
}
}
@@ -107,8 +107,7 @@ public final class ChatApplicationUseCase {
PathResult path = executePath(
intent, context, request.query(), previousTurn.orElse(null), observer);
core.checkActive(context);
core.completeSuccess(context);
completePath(context, intent, path);
String safeJson = write(path.content());
persistFinish(context, intent, path.outcome(), safeJson,
path.publishedResult(), durationMillis(startedNanos));
@@ -131,6 +130,20 @@ public final class ChatApplicationUseCase {
}
}
private void completePath(RunContext context, IntentType intent, PathResult path) {
if (path.handledBudgetTermination()) {
if (intent != IntentType.DIAGNOSIS
|| path.outcome() != ReleaseOutcome.FALLBACK
|| path.publishedResult() != null
|| context.lifecycle().state() != RunState.BUDGET_EXHAUSTED) {
throw new IllegalStateException("Invalid handled Diagnosis budget fallback");
}
return;
}
core.checkActive(context);
core.completeSuccess(context);
}
private PathResult executePath(IntentType intent,
RunContext context,
String query,
@@ -140,18 +153,19 @@ public final class ChatApplicationUseCase {
case SYSTEM_CHAT -> {
observer.onStatus(ChatApplicationStatus.SYSTEM_RESPONDING);
yield new PathResult(
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null);
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null, false);
}
case KNOWLEDGE_QUERY -> {
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_SEARCHING);
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_ANSWERING);
yield new PathResult(
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null);
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null, false);
}
case DIAGNOSIS -> {
DiagnosisExecutionResult result = diagnosis.execute(
context, query, previousTurn, observer::onStatus);
yield new PathResult(result.outcome(), result.content(), result.publishedResult());
yield new PathResult(result.outcome(), result.content(), result.publishedResult(),
result.handledBudgetTermination());
}
};
}
@@ -251,7 +265,8 @@ public final class ChatApplicationUseCase {
private record PathResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
PublishedResult publishedResult,
boolean handledBudgetTermination) {
}
private static final class CoreRunControl implements ChatRunControl {
@@ -8,7 +8,15 @@ import java.util.Objects;
public record DiagnosisExecutionResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
PublishedResult publishedResult,
boolean handledBudgetTermination) {
public DiagnosisExecutionResult(
ReleaseOutcome outcome,
ChatApplicationContent content,
PublishedResult publishedResult) {
this(outcome, content, publishedResult, false);
}
public DiagnosisExecutionResult {
Objects.requireNonNull(outcome, "outcome must not be null");
@@ -25,5 +33,9 @@ public record DiagnosisExecutionResult(
if (outcome != ReleaseOutcome.SUCCESS && publishedResult != null) {
throw new IllegalArgumentException("only success can contain PublishedResult");
}
if (handledBudgetTermination && outcome != ReleaseOutcome.FALLBACK) {
throw new IllegalArgumentException(
"handled budget termination requires fallback release");
}
}
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.application.executor;
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
import com.superbiz.agent.harness.application.ChatApplicationStatus;
import com.superbiz.agent.harness.application.DiagnosisContent;
@@ -8,7 +10,8 @@ import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
import com.superbiz.agent.harness.application.DiagnosisOperation;
import com.superbiz.agent.harness.application.FallbackContent;
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.contract.PreviousTurn;
import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
@@ -25,13 +28,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
private final DiagnosisAgentUseCase diagnosisAgent;
private final DiagnosisReleaseUseCase releaseUseCase;
private final PublishedResultPolicy publishedPolicy;
private final DiagnosisTraceRecorder traceRecorder;
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
DiagnosisReleaseUseCase releaseUseCase,
PublishedResultPolicy publishedPolicy) {
this(diagnosisAgent, releaseUseCase, publishedPolicy, DiagnosisTraceRecorder.noop());
}
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
DiagnosisReleaseUseCase releaseUseCase,
PublishedResultPolicy publishedPolicy,
DiagnosisTraceRecorder traceRecorder) {
this.diagnosisAgent = Objects.requireNonNull(diagnosisAgent, "diagnosisAgent must not be null");
this.releaseUseCase = Objects.requireNonNull(releaseUseCase, "releaseUseCase must not be null");
this.publishedPolicy = Objects.requireNonNull(publishedPolicy, "publishedPolicy must not be null");
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
}
@Override
@@ -41,15 +53,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
Consumer<ChatApplicationStatus> statusSink) {
Objects.requireNonNull(statusSink, "statusSink must not be null");
statusSink.accept(ChatApplicationStatus.DIAGNOSIS_RUNNING);
DiagnosisDraft draft = diagnosisAgent.execute(
context, new DiagnosisAgentInput(query, previousTurn));
DiagnosisAgentExecution execution;
try {
execution = diagnosisAgent.execute(
context, new DiagnosisAgentInput(query, previousTurn));
} catch (DiagnosisAgentOutputException exception) {
return recoverInvalidDraft(context, exception, statusSink);
}
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, draft);
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, execution);
if (released.outcome() == ReleaseOutcome.FALLBACK) {
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK,
new FallbackContent(released.fallback()),
null);
null,
execution.stopReason()
== com.superbiz.agent.harness.progress.DiagnosisStopReason.BUDGET_LIMIT_REACHED);
}
PublishedResult published = publishedPolicy.create(
query, released.draft(), released.verifiedEvidence()).orElse(null);
@@ -60,4 +79,26 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
released.verifiedEvidence().verifiedSources()),
published);
}
private DiagnosisExecutionResult recoverInvalidDraft(
RunContext context,
DiagnosisAgentOutputException exception,
Consumer<ChatApplicationStatus> statusSink) {
if (!exception.isDraftContractFailure()) {
throw exception;
}
boolean hasProgress = exception.progress().hasObservedFacts();
traceRecorder.record(TraceAuditEvents.agentDraftInvalid(
context, exception.kind(), exception.outputBytes(), hasProgress));
if (!hasProgress) {
throw exception;
}
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
DiagnosisReleaseResult released = releaseUseCase.releaseInvalidDraft(
context, exception.progress());
return new DiagnosisExecutionResult(
ReleaseOutcome.FALLBACK,
new FallbackContent(released.fallback()),
null);
}
}
@@ -16,6 +16,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.RagEvidence;
@@ -106,7 +107,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
}
core.reserveRunBytes(context, bytes);
String output = modelCall.call(
context,
context, ModelCallComponent.KNOWLEDGE_ANSWER,
new Prompt(List.of(new SystemMessage(prompt), new UserMessage(modelInput))),
limits.modelTimeout(),
limits.maxModelOutputBytes());
@@ -7,6 +7,7 @@ import com.superbiz.agent.harness.application.SystemChatOperation;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.prompt.Prompt;
@@ -47,7 +48,7 @@ public final class SystemChatExecutor implements SystemChatOperation {
}
core.reserveRunBytes(context, bytes);
String answer = modelCall.call(
context,
context, ModelCallComponent.SYSTEM_CHAT,
new Prompt(List.of(new SystemMessage(SYSTEM_PROMPT), new UserMessage(query))),
limits.timeout(),
limits.maxOutputBytes());
@@ -12,6 +12,7 @@ import com.superbiz.agent.harness.contract.PublishedResult;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
import com.superbiz.agent.harness.core.RunBudgetUsage;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.repository.ChatSessionRepository;
import com.superbiz.agent.repository.DiagnosisRunRepository;
import org.springframework.stereotype.Component;
@@ -119,6 +120,8 @@ public class JpaChatRunStore implements ChatRunStore {
run.setTotalDurationMs(Math.max(0, durationMs));
RunBudgetUsage usage = context.budget().snapshot();
run.setTotalTokenCount(saturatingInt(usage.totalTokens()));
run.setStepCount(context.modelCalls().componentCallCount(
ModelCallComponent.DIAGNOSIS_AGENT));
run.setToolCallCount(saturatingInt(usage.toolCalls()));
runs.save(run);
if (outcome == ReleaseOutcome.SUCCESS || outcome == ReleaseOutcome.FALLBACK) {
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.application.IntentRouting;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
@@ -86,7 +87,8 @@ public final class IntentRouter implements IntentRouting {
context,
context.retryPolicies().intentRouter(),
() -> parse(modelCall.call(
context, prompt, remaining(started), limits.maxOutputBytes())),
context, ModelCallComponent.INTENT_ROUTER,
prompt, remaining(started), limits.maxOutputBytes())),
this::classify,
attempt -> {
attemptRecorder.accept(attempt);
@@ -0,0 +1,85 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.domain.entity.AgentStep;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.repository.AgentStepRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Objects;
public final class ModelCallAuditor {
private static final Logger log = LoggerFactory.getLogger(ModelCallAuditor.class);
private final DiagnosisHarnessCore core;
private final DiagnosisTraceRecorder traceRecorder;
private final AgentStepRepository agentSteps;
public ModelCallAuditor(DiagnosisHarnessCore core) {
this(core, DiagnosisTraceRecorder.noop(), null);
}
public ModelCallAuditor(DiagnosisHarnessCore core,
DiagnosisTraceRecorder traceRecorder,
AgentStepRepository agentSteps) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
this.agentSteps = agentSteps;
}
public ModelCallLedger.Call begin(RunContext context, ModelCallComponent component) {
Objects.requireNonNull(context, "context must not be null");
return context.modelCalls().begin(component);
}
public void recordUsage(RunContext context, ModelCallLedger.Call call,
long inputTokens, long outputTokens, boolean usageAvailable) {
Objects.requireNonNull(context, "context must not be null");
if (!context.modelCalls().record(call, inputTokens, outputTokens, usageAvailable)) {
return;
}
try {
if (usageAvailable) {
core.recordTokens(context, inputTokens, outputTokens);
}
} finally {
traceRecorder.record(TraceAuditEvents.modelTokenUsage(
context, call, inputTokens, outputTokens, usageAvailable));
updateAgentStep(context, call, inputTokens, outputTokens, usageAvailable);
}
}
private void updateAgentStep(RunContext context, ModelCallLedger.Call call,
long inputTokens, long outputTokens, boolean usageAvailable) {
if (!usageAvailable || agentSteps == null
|| call.component() != ModelCallComponent.DIAGNOSIS_AGENT) {
return;
}
try {
agentSteps.findByRunIdAndStepIndex(context.runId(), call.componentRound() - 1)
.ifPresent(step -> saveTokens(step, inputTokens, outputTokens));
} catch (RuntimeException exception) {
log.warn("Failed to persist AgentStep token audit: runId={}, round={}",
context.runId(), call.componentRound());
}
}
private void saveTokens(AgentStep step, long inputTokens, long outputTokens) {
step.setTokenCount(saturatingInt(safeAdd(inputTokens, outputTokens)));
agentSteps.save(step);
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
private static int saturatingInt(long value) {
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, value);
}
}
@@ -0,0 +1,20 @@
package com.superbiz.agent.harness.audit;
public enum ModelCallComponent {
INTENT_ROUTER(TracePhase.ROUTING),
SYSTEM_CHAT(TracePhase.AGENT),
KNOWLEDGE_ANSWER(TracePhase.AGENT),
DIAGNOSIS_AGENT(TracePhase.AGENT),
EVIDENCE_REPAIR(TracePhase.EVIDENCE),
SEMANTIC_GUARD(TracePhase.SEMANTIC);
private final TracePhase tracePhase;
ModelCallComponent(TracePhase tracePhase) {
this.tracePhase = tracePhase;
}
public TracePhase tracePhase() {
return tracePhase;
}
}
@@ -0,0 +1,86 @@
package com.superbiz.agent.harness.audit;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
public final class ModelCallLedger {
private final Map<ModelCallComponent, Integer> componentRounds =
new EnumMap<>(ModelCallComponent.class);
private final Set<Call> auditedCalls = new HashSet<>();
private int startedCallCount;
private int usageUnavailableCount;
private long inputTokens;
private long outputTokens;
private long totalTokens;
public synchronized Call begin(ModelCallComponent component) {
Objects.requireNonNull(component, "component must not be null");
int round = componentRounds.merge(component, 1, Integer::sum);
startedCallCount++;
return new Call(component, round);
}
public synchronized boolean record(Call call, long input, long output, boolean usageAvailable) {
Objects.requireNonNull(call, "call must not be null");
if (input < 0 || output < 0) {
throw new IllegalArgumentException("token counts must not be negative");
}
if (!auditedCalls.add(call)) {
return false;
}
if (!usageAvailable) {
usageUnavailableCount++;
return true;
}
inputTokens = safeAdd(inputTokens, input);
outputTokens = safeAdd(outputTokens, output);
totalTokens = safeAdd(totalTokens, safeAdd(input, output));
return true;
}
public synchronized int componentCallCount(ModelCallComponent component) {
Objects.requireNonNull(component, "component must not be null");
return componentRounds.getOrDefault(component, 0);
}
public synchronized Snapshot snapshot() {
return new Snapshot(
startedCallCount,
auditedCalls.size(),
usageUnavailableCount,
inputTokens,
outputTokens,
totalTokens);
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
public record Call(ModelCallComponent component, int componentRound) {
public Call {
Objects.requireNonNull(component, "component must not be null");
if (componentRound < 1) {
throw new IllegalArgumentException("componentRound must be positive");
}
}
}
public record Snapshot(
int startedCallCount,
int auditedCallCount,
int usageUnavailableCount,
long inputTokens,
long outputTokens,
long totalTokens) {
}
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.audit;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
import com.superbiz.agent.harness.contract.FallbackType;
import com.superbiz.agent.harness.contract.IntentType;
import com.superbiz.agent.harness.contract.ReleaseOutcome;
@@ -8,6 +9,8 @@ import com.superbiz.agent.harness.contract.SemanticVerdict;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.retry.RetryAttempt;
import java.util.ArrayList;
@@ -33,11 +36,27 @@ public final class TraceAuditEvents {
public static DiagnosisTraceAuditEvent runFinished(
RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) {
var budget = context.budget().snapshot();
ModelCallLedger.Snapshot modelCalls = context.modelCalls().snapshot();
Map<String, Object> details = new LinkedHashMap<>();
if (intent != null) {
details.put("intent", intent.name());
}
details.put("release_outcome", outcome.name());
details.put("model_call_count", budget.modelCalls());
details.put("run_input_tokens", budget.inputTokens());
details.put("run_output_tokens", budget.outputTokens());
details.put("run_total_tokens", budget.totalTokens());
details.put("audited_model_call_count", modelCalls.auditedCallCount());
details.put("audited_input_tokens", modelCalls.inputTokens());
details.put("audited_output_tokens", modelCalls.outputTokens());
details.put("audited_total_tokens", modelCalls.totalTokens());
details.put("usage_unavailable_count", modelCalls.usageUnavailableCount());
details.put("tokens_reconciled", budget.modelCalls() == modelCalls.auditedCallCount()
&& modelCalls.usageUnavailableCount() == 0
&& budget.inputTokens() == modelCalls.inputTokens()
&& budget.outputTokens() == modelCalls.outputTokens()
&& budget.totalTokens() == modelCalls.totalTokens());
return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED,
terminalStatus(outcome), null, durationMs, details);
}
@@ -66,6 +85,41 @@ public final class TraceAuditEvents {
TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details);
}
public static DiagnosisTraceAuditEvent modelTokenUsage(
RunContext context,
ModelCallLedger.Call call,
long inputTokens,
long outputTokens,
boolean usageAvailable) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("component", call.component().name());
details.put("component_round", call.componentRound());
details.put("usage_available", usageAvailable);
if (usageAvailable) {
details.put("input_tokens", inputTokens);
details.put("output_tokens", outputTokens);
details.put("total_tokens", safeAdd(inputTokens, outputTokens));
}
return new DiagnosisTraceAuditEvent(
context.sessionId(), context.runId(), call.component().tracePhase(),
TraceEventType.MODEL_TOKEN_USAGE,
usageAvailable ? TraceEventStatus.SUCCEEDED : TraceEventStatus.UNAVAILABLE,
call.componentRound(), null, details);
}
public static DiagnosisTraceAuditEvent agentDraftInvalid(
RunContext context,
DiagnosisAgentOutputException.Kind kind,
long outputBytes,
boolean hasPublishableProgress) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("failure_kind", kind.name());
details.put("output_bytes", outputBytes);
details.put("has_publishable_progress", hasPublishableProgress);
return event(context, TracePhase.AGENT, TraceEventType.AGENT_DRAFT_INVALID,
TraceEventStatus.REJECTED, null, null, details);
}
public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", tool.toolCallId());
@@ -84,6 +138,55 @@ public final class TraceAuditEvents {
null, tool.durationMs(), details);
}
public static DiagnosisTraceAuditEvent toolRequestRejected(
RunContext context, String toolCallId, String toolName, String errorCode) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", safeIdentifier(toolCallId));
details.put("tool_name", safeIdentifier(toolName));
details.put("error_code", safeIdentifier(errorCode));
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
TraceEventStatus.REJECTED, null, null, details);
}
public static DiagnosisTraceAuditEvent toolProgress(
RunContext context,
String toolCallId,
String toolName,
String scopeSummary,
InformationGain informationGain,
String producer,
DiagnosisProgressSnapshotState state) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", toolCallId);
details.put("tool_name", toolName);
details.put("scope_summary", scopeSummary);
details.put("information_gain", informationGain.name());
details.put("producer", producer);
details.put("consecutive_no_gain", state.consecutiveNoGain());
details.put("collection_state", state.collectionState().name());
if (state.stopReason() != null) {
details.put("stop_reason", state.stopReason().name());
}
return event(context, TracePhase.TOOL, TraceEventType.TOOL_PROGRESS,
TraceEventStatus.SUCCEEDED, null, null, details);
}
public static DiagnosisTraceAuditEvent collectionStop(
RunContext context,
String toolCallId,
String toolName,
DiagnosisProgressSnapshotState state) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", toolCallId);
details.put("tool_name", toolName);
details.put("collection_state", state.collectionState().name());
if (state.stopReason() != null) {
details.put("stop_reason", state.stopReason().name());
}
return event(context, TracePhase.TOOL, TraceEventType.COLLECTION_STOP,
TraceEventStatus.SUCCEEDED, null, null, details);
}
public static DiagnosisTraceAuditEvent evidenceValidation(
RunContext context, TraceEventType type,
EvidenceGuardResult result, DiagnosisDraft draft) {
@@ -198,6 +301,18 @@ public final class TraceAuditEvents {
return new ToolReferences(List.copyOf(safeIds), invalid);
}
private static String safeIdentifier(String value) {
return value != null && SAFE_TOOL_CALL_ID.matcher(value).matches() ? value : "INVALID";
}
private static long safeAdd(long left, long right) {
try {
return Math.addExact(left, right);
} catch (ArithmeticException exception) {
return Long.MAX_VALUE;
}
}
private record ToolReferences(List<String> safeIds, int invalidCount) {
}
}
@@ -4,8 +4,13 @@ public enum TraceEventType {
RUN_STARTED,
ROUTING_ATTEMPT,
ROUTING_DECISION,
MODEL_TOKEN_USAGE,
AGENT_MODEL_STEP,
AGENT_DRAFT_INVALID,
TOOL_INVOCATION,
TOOL_REQUEST_REJECTED,
TOOL_PROGRESS,
COLLECTION_STOP,
EVIDENCE_GUARD_INITIAL,
EVIDENCE_REPAIR_ATTEMPT,
EVIDENCE_GUARD_RECHECK,
@@ -3,5 +3,8 @@ package com.superbiz.agent.harness.contract;
public enum FallbackType {
EVIDENCE_VALIDATION_FAILED,
SEMANTIC_UNSUPPORTED,
SEMANTIC_UNAVAILABLE
SEMANTIC_UNAVAILABLE,
BUDGET_EXHAUSTED,
INSUFFICIENT_EVIDENCE,
MISSING_REQUIRED_CONTEXT
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.core;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import java.time.Clock;
import java.time.Duration;
@@ -15,17 +17,31 @@ public final class DiagnosisHarnessCore {
private final Duration maxRunDuration;
private final RunBudgetLimits budgetLimits;
private final HarnessRetryPolicies retryPolicies;
private final int stopAfterConsecutiveNoGain;
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
Duration maxRunDuration,
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies) {
this(clock, runIdSupplier, maxRunDuration, budgetLimits, retryPolicies, 2);
}
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
Duration maxRunDuration,
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies,
int stopAfterConsecutiveNoGain) {
this.clock = Objects.requireNonNull(clock, "clock must not be null");
this.runIdSupplier = Objects.requireNonNull(runIdSupplier, "runIdSupplier must not be null");
this.maxRunDuration = requirePositive(maxRunDuration, "maxRunDuration");
this.budgetLimits = Objects.requireNonNull(budgetLimits, "budgetLimits must not be null");
this.retryPolicies = Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
}
public RunContext startRun(String sessionId) {
@@ -42,8 +58,10 @@ public final class DiagnosisHarnessCore {
deadline,
cancellation,
new RunBudget(budgetLimits),
new ModelCallLedger(),
retryPolicies,
lifecycle);
lifecycle,
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain));
cancellation.onCancel(reason -> lifecycle.finish(terminalState(reason), reason.name()));
return context;
}
@@ -1,6 +1,8 @@
package com.superbiz.agent.harness.core;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import java.time.Instant;
import java.util.Objects;
@@ -14,8 +16,10 @@ public record RunContext(
Instant deadline,
RunCancellation cancellation,
RunBudget budget,
ModelCallLedger modelCalls,
HarnessRetryPolicies retryPolicies,
RunLifecycle lifecycle) {
RunLifecycle lifecycle,
DiagnosisProgressTracker progress) {
public RunContext {
requireText(sessionId, "sessionId");
@@ -23,8 +27,10 @@ public record RunContext(
Objects.requireNonNull(deadline, "deadline must not be null");
Objects.requireNonNull(cancellation, "cancellation must not be null");
Objects.requireNonNull(budget, "budget must not be null");
Objects.requireNonNull(modelCalls, "modelCalls must not be null");
Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
Objects.requireNonNull(lifecycle, "lifecycle must not be null");
Objects.requireNonNull(progress, "progress must not be null");
}
private static void requireText(String value, String name) {
@@ -79,6 +79,40 @@ public final class EvidenceGuard {
: EvidenceGuardResult.invalid(violations);
}
public EvidenceGuardResult validateNoConclusionReferences(
RunContext context, DiagnosisDraft draft) {
Objects.requireNonNull(context, "context must not be null");
if (draft == null) {
return EvidenceGuardResult.invalid(List.of(
violation(EvidenceViolationCode.DRAFT_MISSING, "draft")));
}
if (draft.conclusion() != null) {
throw new IllegalArgumentException("no-conclusion validation requires conclusion=null");
}
List<EvidenceViolation> violations = new ArrayList<>();
for (int index = 0; index < draft.analysis().size(); index++) {
DiagnosisDraft.AnalysisItem analysis = draft.analysis().get(index);
if (analysis == null || analysis.toolCallIds().isEmpty()) {
continue;
}
if (analysis.kind() == null) {
violations.add(violation(
EvidenceViolationCode.ANALYSIS_KIND_MISSING,
"analysis[" + index + "].kind"));
continue;
}
List<VerifiedEvidence> ignoredEvidence = new ArrayList<>();
for (String toolCallId : analysis.toolCallIds()) {
verifyInvocation(
context, analysis, index, toolCallId, ignoredEvidence, violations);
}
}
return violations.isEmpty()
? EvidenceGuardResult.valid(VerifiedEvidenceSnapshot.empty())
: EvidenceGuardResult.invalid(violations);
}
private List<EvidenceViolation> validateDraft(DiagnosisDraft draft) {
List<EvidenceViolation> violations = new ArrayList<>();
if (draft == null) {
@@ -4,6 +4,9 @@ import com.superbiz.agent.harness.core.BudgetExceededException;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunAbortedException;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.audit.ModelCallAuditor;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.audit.ModelCallLedger;
import com.superbiz.agent.harness.retry.RetryFailure;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.metadata.Usage;
@@ -26,15 +29,23 @@ public final class GuardModelCall {
private final DiagnosisHarnessCore core;
private final ChatModel chatModel;
private final ExecutorService executor;
private final ModelCallAuditor auditor;
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
ExecutorService executor) {
this(core, chatModel, executor, new ModelCallAuditor(core));
}
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
ExecutorService executor, ModelCallAuditor auditor) {
this.core = Objects.requireNonNull(core, "core must not be null");
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
this.executor = Objects.requireNonNull(executor, "executor must not be null");
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
}
public String call(RunContext context, Prompt prompt, Duration timeout, long maxOutputBytes) {
public String call(RunContext context, ModelCallComponent component,
Prompt prompt, Duration timeout, long maxOutputBytes) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(prompt, "prompt must not be null");
Objects.requireNonNull(timeout, "timeout must not be null");
@@ -42,20 +53,24 @@ public final class GuardModelCall {
throw new IllegalArgumentException("timeout and output limit must be positive");
}
core.beforeModelCall(context);
Future<String> future = executor.submit(() -> invoke(context, prompt, maxOutputBytes));
ModelCallLedger.Call call = auditor.begin(context, component);
Future<String> future = executor.submit(() -> invoke(context, call, prompt, maxOutputBytes));
context.cancellation().onCancel(ignored -> future.cancel(true));
try {
return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
} catch (TimeoutException exception) {
future.cancel(true);
auditor.recordUsage(context, call, 0, 0, false);
throw new GuardModelCallException(
RetryFailure.TIMEOUT, "Guard model attempt timed out", exception);
} catch (CancellationException exception) {
auditor.recordUsage(context, call, 0, 0, false);
core.checkActive(context);
throw new GuardModelCallException(
RetryFailure.TRANSPORT, "Guard model attempt was cancelled", exception);
} catch (InterruptedException exception) {
future.cancel(true);
auditor.recordUsage(context, call, 0, 0, false);
Thread.currentThread().interrupt();
core.checkActive(context);
throw new GuardModelCallException(
@@ -76,15 +91,17 @@ public final class GuardModelCall {
}
}
private String invoke(RunContext context, Prompt prompt, long maxOutputBytes) {
private String invoke(RunContext context, ModelCallLedger.Call call,
Prompt prompt, long maxOutputBytes) {
ChatResponse response;
try {
response = chatModel.call(prompt);
} catch (RuntimeException exception) {
auditor.recordUsage(context, call, 0, 0, false);
throw new GuardModelCallException(
RetryFailure.TRANSPORT, "Guard model transport failed", exception);
}
recordUsage(context, response);
recordUsage(context, call, response);
core.checkActive(context);
AssistantMessage output = response == null || response.getResult() == null
? null : response.getResult().getOutput();
@@ -102,16 +119,18 @@ public final class GuardModelCall {
return output.getText();
}
private void recordUsage(RunContext context, ChatResponse response) {
private void recordUsage(RunContext context, ModelCallLedger.Call call, ChatResponse response) {
if (response == null || response.getMetadata() == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
Usage usage = response.getMetadata().getUsage();
if (usage == null) {
auditor.recordUsage(context, call, 0, 0, false);
return;
}
core.recordTokens(context, nonNegative(usage.getPromptTokens()),
nonNegative(usage.getCompletionTokens()));
auditor.recordUsage(context, call, nonNegative(usage.getPromptTokens()),
nonNegative(usage.getCompletionTokens()), true);
}
private static long nonNegative(Integer value) {
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.SemanticVerdict;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
@@ -81,7 +82,8 @@ public final class SemanticGuard {
context,
context.retryPolicies().semanticGuard(),
() -> parse(modelCall.call(
context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
context, ModelCallComponent.SEMANTIC_GUARD,
modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
this::classify,
attempt -> {
attemptRecorder.accept(attempt);
@@ -0,0 +1,19 @@
package com.superbiz.agent.harness.progress;
public record CompletedToolCall(
String toolCallId,
String toolName,
String normalizedScope) {
public CompletedToolCall {
requireText(toolCallId, "toolCallId");
requireText(toolName, "toolName");
requireText(normalizedScope, "normalizedScope");
}
private static void requireText(String value, String name) {
if (value == null || value.isBlank()) {
throw new IllegalArgumentException(name + " must not be blank");
}
}
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum DiagnosisCollectionState {
COLLECTING,
SATURATED
}
@@ -0,0 +1,13 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.core.RunContext;
@FunctionalInterface
public interface DiagnosisProgressProjection {
DiagnosisProgressSnapshot project(RunContext context);
static DiagnosisProgressProjection empty() {
return ignored -> DiagnosisProgressSnapshot.empty();
}
}
@@ -0,0 +1,228 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public final class DiagnosisProgressProjector implements DiagnosisProgressProjection {
private static final int MAX_FACTS = 12;
private static final int MAX_SUMMARY_CHARS = 320;
private static final int MAX_SCOPE_CHARS = 320;
private final CanonicalInvocationStore store;
private final ToolCallKeyFactory keyFactory;
private final ObjectMapper objectMapper;
public DiagnosisProgressProjector(CanonicalInvocationStore store,
ToolCallKeyFactory keyFactory,
ObjectMapper objectMapper) {
this.store = Objects.requireNonNull(store, "store must not be null");
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
@Override
public DiagnosisProgressSnapshot project(RunContext context) {
Objects.requireNonNull(context, "context must not be null");
DiagnosisProgressSnapshotState state = context.progress().snapshot();
Map<String, SafeFallback.VerifiedSource> sources = new LinkedHashMap<>();
Map<String, SafeFallback.ObservedFact> facts = new LinkedHashMap<>();
List<String> limitations = new ArrayList<>();
for (CompletedToolCall completed : state.completedToolCalls()) {
CanonicalToolInvocation invocation = resolve(context, completed, limitations);
if (invocation == null) {
continue;
}
try {
projectInvocation(completed, invocation, sources, facts);
} catch (RuntimeException exception) {
addLimitation(limitations, "部分已完成的工具结果格式无法验证,未纳入已检查事实");
}
if (facts.size() >= MAX_FACTS) {
addLimitation(limitations, "已检查事实较多,展示内容已截断");
break;
}
}
return new DiagnosisProgressSnapshot(
List.copyOf(sources.values()),
List.copyOf(facts.values()),
List.copyOf(limitations),
state.stopReason());
}
private CanonicalToolInvocation resolve(RunContext context,
CompletedToolCall completed,
List<String> limitations) {
try {
String key = keyFactory.create(context.runId(), completed.toolCallId());
CanonicalToolInvocation invocation = store.find(key).orElse(null);
if (invocation == null
|| !invocation.isReferencableBy(context.runId())
|| !completed.toolCallId().equals(invocation.toolCallId())
|| !completed.toolName().equals(invocation.toolName())) {
addLimitation(limitations, "部分已完成的工具记录无法验证,未纳入已检查事实");
return null;
}
return invocation;
} catch (RuntimeException exception) {
addLimitation(limitations, "部分已完成的工具记录暂时不可读取,未纳入已检查事实");
return null;
}
}
private void projectInvocation(CompletedToolCall completed,
CanonicalToolInvocation invocation,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
JsonNode root = readObject(invocation.agentResult());
String scope = publicScope(completed.toolName(), completed.normalizedScope(), root);
switch (completed.toolName()) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> projectRag(root, scope, sources, facts);
case AgentToolContracts.QUERY_LOGS -> projectLogs(root, scope, sources, facts);
case AgentToolContracts.QUERY_MYSQL -> projectMysql(root, scope, sources, facts);
default -> throw new IllegalArgumentException("Unsupported evidence Tool");
}
}
private void projectRag(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
JsonNode evidence = root.path("evidence");
if (!evidence.isArray() || evidence.isEmpty()) {
addFact(sources, facts, "RAG", "knowledge_base", scope,
"该知识检索范围内未发现可用文档证据");
return;
}
for (JsonNode item : evidence) {
String source = firstNonBlank(text(item, "source"), text(item, "title"),
text(item, "document_id"), "knowledge_base");
addFact(sources, facts, "RAG", source, scope,
firstNonBlank(text(item, "excerpt"), "已找到候选知识片段"));
}
}
private void projectLogs(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
String source = firstNonBlank(text(root, "source_kind"), "logs");
JsonNode events = root.path("events");
if (!events.isArray() || events.isEmpty()) {
addFact(sources, facts, "LOG", source, scope,
"该日志查询范围内未发现匹配事件");
return;
}
for (JsonNode event : events) {
addFact(sources, facts, "LOG", source, scope,
firstNonBlank(text(event, "message"), "已找到匹配日志事件"));
}
}
private void projectMysql(JsonNode root,
String scope,
Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts) {
String source = mysqlSource(scope);
JsonNode rows = root.path("rows");
if (!rows.isArray() || rows.isEmpty()) {
addFact(sources, facts, "MYSQL", source, scope,
"该只读数据查询范围内未发现匹配记录");
return;
}
for (JsonNode row : rows) {
addFact(sources, facts, "MYSQL", source, scope, row.toString());
}
}
private String publicScope(String toolName, String normalizedScope, JsonNode result) {
if (AgentToolContracts.LOOKUP_KNOWLEDGE.equals(toolName)) {
return bounded("query=" + text(result, "query"), MAX_SCOPE_CHARS);
}
if (AgentToolContracts.QUERY_LOGS.equals(toolName)) {
JsonNode scope = result.path("scope");
return bounded(scope.isObject() ? scope.toString() : normalizedScope, MAX_SCOPE_CHARS);
}
try {
JsonNode scope = objectMapper.readTree(normalizedScope);
return bounded("data_source=" + text(scope, "data_source"), MAX_SCOPE_CHARS);
} catch (JsonProcessingException exception) {
return "data_source=unknown";
}
}
private String mysqlSource(String scope) {
int separator = scope.indexOf('=');
return separator < 0 ? "mysql" : scope.substring(separator + 1);
}
private void addFact(Map<String, SafeFallback.VerifiedSource> sources,
Map<String, SafeFallback.ObservedFact> facts,
String sourceType,
String source,
String scope,
String summary) {
if (facts.size() >= MAX_FACTS) {
return;
}
String safeSource = bounded(source, 160);
String safeScope = bounded(scope, MAX_SCOPE_CHARS);
String safeSummary = bounded(summary, MAX_SUMMARY_CHARS);
String sourceKey = sourceType + '\u0000' + safeSource + '\u0000' + safeScope;
sources.putIfAbsent(sourceKey,
new SafeFallback.VerifiedSource(sourceType, safeSource, safeScope));
String factKey = sourceKey + '\u0000' + safeSummary;
facts.putIfAbsent(factKey,
new SafeFallback.ObservedFact(sourceType, safeSource, safeScope, safeSummary));
}
private JsonNode readObject(String value) {
try {
JsonNode root = objectMapper.readTree(value);
if (root == null || !root.isObject()) {
throw new IllegalArgumentException("Canonical Agent result must be an object");
}
return root;
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Canonical Agent result is invalid", exception);
}
}
private static void addLimitation(List<String> limitations, String value) {
if (!limitations.contains(value)) {
limitations.add(value);
}
}
private static String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.isBlank()) {
return value;
}
}
return "unknown";
}
private static String text(JsonNode node, String field) {
JsonNode value = node == null ? null : node.get(field);
return value == null || value.isNull() ? "" : value.asText("");
}
private static String bounded(String value, int max) {
String safe = value == null ? "" : value;
return safe.length() <= max ? safe : safe.substring(0, max);
}
}
@@ -0,0 +1,26 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.SafeFallback;
import java.util.List;
public record DiagnosisProgressSnapshot(
List<SafeFallback.VerifiedSource> verifiedSources,
List<SafeFallback.ObservedFact> observedFacts,
List<String> limitations,
DiagnosisStopReason stopReason) {
public DiagnosisProgressSnapshot {
verifiedSources = verifiedSources == null ? List.of() : List.copyOf(verifiedSources);
observedFacts = observedFacts == null ? List.of() : List.copyOf(observedFacts);
limitations = limitations == null ? List.of() : List.copyOf(limitations);
}
public static DiagnosisProgressSnapshot empty() {
return new DiagnosisProgressSnapshot(List.of(), List.of(), List.of(), null);
}
public boolean hasObservedFacts() {
return !observedFacts.isEmpty();
}
}
@@ -0,0 +1,16 @@
package com.superbiz.agent.harness.progress;
import java.util.List;
public record DiagnosisProgressSnapshotState(
int consecutiveNoGain,
DiagnosisCollectionState collectionState,
DiagnosisStopReason stopReason,
String pendingToolCallId,
boolean stopInstructionDelivered,
List<CompletedToolCall> completedToolCalls) {
public DiagnosisProgressSnapshotState {
completedToolCalls = completedToolCalls == null ? List.of() : List.copyOf(completedToolCalls);
}
}
@@ -0,0 +1,118 @@
package com.superbiz.agent.harness.progress;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public final class DiagnosisProgressTracker {
private final int stopAfterConsecutiveNoGain;
private final Set<ToolScopeIdentity> completedScopes = new LinkedHashSet<>();
private final List<CompletedToolCall> completedToolCalls = new ArrayList<>();
private int consecutiveNoGain;
private DiagnosisCollectionState collectionState = DiagnosisCollectionState.COLLECTING;
private DiagnosisStopReason stopReason;
private String pendingToolCallId;
private boolean stopInstructionDelivered;
public DiagnosisProgressTracker(int stopAfterConsecutiveNoGain) {
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
}
public synchronized void applyPreviousObservation(PreviousObservation observation) {
if (pendingToolCallId == null) {
if (observation != null) {
throw new IllegalArgumentException("No Tool observation is pending evaluation");
}
return;
}
if (observation == null) {
throw new IllegalArgumentException("Previous Tool observation must be evaluated");
}
if (!pendingToolCallId.equals(observation.toolCallId())) {
throw new IllegalArgumentException("Previous Tool observation ID is out of order");
}
pendingToolCallId = null;
applyGain(observation.informationGain());
}
public synchronized boolean isDuplicate(String toolName, String normalizedScope) {
return completedScopes.contains(new ToolScopeIdentity(toolName, normalizedScope));
}
public synchronized void recordDuplicateScope() {
applyGain(InformationGain.NO_GAIN);
}
public synchronized void recordCompleted(CompletedToolCall call, EvidenceStatus evidenceStatus) {
if (collectionState == DiagnosisCollectionState.SATURATED) {
throw new IllegalStateException("Cannot record Tool completion after saturation");
}
if (evidenceStatus != EvidenceStatus.EVIDENCE_FOUND
&& evidenceStatus != EvidenceStatus.NO_EVIDENCE) {
throw new IllegalArgumentException("Completed Tool requires a successful evidence status");
}
ToolScopeIdentity scope = new ToolScopeIdentity(call.toolName(), call.normalizedScope());
if (!completedScopes.add(scope)) {
throw new IllegalStateException("Completed Tool scope was already recorded");
}
completedToolCalls.add(call);
if (evidenceStatus == EvidenceStatus.NO_EVIDENCE) {
applyGain(InformationGain.NO_GAIN);
} else {
pendingToolCallId = call.toolCallId();
}
}
public synchronized boolean claimStopInstruction() {
if (collectionState != DiagnosisCollectionState.SATURATED) {
return false;
}
if (stopInstructionDelivered) {
return false;
}
stopInstructionDelivered = true;
return true;
}
public synchronized void markBudgetLimitReached() {
if (stopReason == null) {
stopReason = DiagnosisStopReason.BUDGET_LIMIT_REACHED;
}
}
public synchronized DiagnosisProgressSnapshotState snapshot() {
return new DiagnosisProgressSnapshotState(
consecutiveNoGain,
collectionState,
stopReason,
pendingToolCallId,
stopInstructionDelivered,
completedToolCalls);
}
public int stopAfterConsecutiveNoGain() {
return stopAfterConsecutiveNoGain;
}
private void applyGain(InformationGain gain) {
if (collectionState == DiagnosisCollectionState.SATURATED) {
throw new IllegalStateException("Collection is already saturated");
}
if (gain == InformationGain.GAINED) {
consecutiveNoGain = 0;
return;
}
consecutiveNoGain++;
if (consecutiveNoGain >= stopAfterConsecutiveNoGain) {
collectionState = DiagnosisCollectionState.SATURATED;
stopReason = DiagnosisStopReason.INFORMATION_SATURATED;
}
}
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum DiagnosisStopReason {
INFORMATION_SATURATED,
BUDGET_LIMIT_REACHED
}
@@ -0,0 +1,6 @@
package com.superbiz.agent.harness.progress;
public enum InformationGain {
GAINED,
NO_GAIN
}
@@ -0,0 +1,17 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.annotation.JsonProperty;
public record PreviousObservation(
@JsonProperty("tool_call_id") String toolCallId,
@JsonProperty("information_gain") InformationGain informationGain) {
public PreviousObservation {
if (toolCallId == null || toolCallId.isBlank()) {
throw new IllegalArgumentException("toolCallId must not be blank");
}
if (informationGain == null) {
throw new IllegalArgumentException("informationGain must not be null");
}
}
}
@@ -0,0 +1,13 @@
package com.superbiz.agent.harness.progress;
public record ToolScopeIdentity(String toolName, String normalizedScope) {
public ToolScopeIdentity {
if (toolName == null || toolName.isBlank()) {
throw new IllegalArgumentException("toolName must not be blank");
}
if (normalizedScope == null || normalizedScope.isBlank()) {
throw new IllegalArgumentException("normalizedScope must not be blank");
}
}
}
@@ -0,0 +1,76 @@
package com.superbiz.agent.harness.progress;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
public final class ToolScopeNormalizer {
private static final int DEFAULT_LOG_LOOKBACK_MINUTES = 30;
private final ObjectMapper objectMapper;
public ToolScopeNormalizer(ObjectMapper objectMapper) {
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
}
public String normalize(String toolName, Object input) {
Objects.requireNonNull(input, "input must not be null");
Map<String, Object> scope = switch (toolName) {
case AgentToolContracts.LOOKUP_KNOWLEDGE -> ragScope(requireType(input, RagToolRequest.class));
case AgentToolContracts.QUERY_LOGS -> logScope(requireType(input, QueryLogsRequest.class));
case AgentToolContracts.QUERY_MYSQL -> mysqlScope(requireType(input, MysqlToolRequest.class));
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
};
try {
return objectMapper.writeValueAsString(scope);
} catch (JsonProcessingException exception) {
throw new IllegalArgumentException("Tool scope is not serializable", exception);
}
}
private static Map<String, Object> ragScope(RagToolRequest request) {
return ordered("query", normalizeText(request.query()));
}
private static Map<String, Object> logScope(QueryLogsRequest request) {
Map<String, Object> scope = new LinkedHashMap<>();
scope.put("topic", request.topic() == null ? null : request.topic().name());
scope.put("query", normalizeText(request.query()));
scope.put("lookback_minutes", request.lookbackMinutes() == null
? DEFAULT_LOG_LOOKBACK_MINUTES : request.lookbackMinutes());
return scope;
}
private static Map<String, Object> mysqlScope(MysqlToolRequest request) {
Map<String, Object> scope = new LinkedHashMap<>();
scope.put("data_source", normalizeText(request.dataSource()));
scope.put("sql", normalizeText(request.sql()));
scope.put("params", request.params());
return scope;
}
private static Map<String, Object> ordered(String name, Object value) {
Map<String, Object> result = new LinkedHashMap<>();
result.put(name, value);
return result;
}
private static String normalizeText(String value) {
return value == null ? "" : value.trim();
}
private static <T> T requireType(Object input, Class<T> type) {
if (!type.isInstance(input)) {
throw new IllegalArgumentException("Unexpected Tool input type: " + input.getClass().getName());
}
return type.cast(input);
}
}
@@ -1,5 +1,6 @@
package com.superbiz.agent.harness.release;
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
@@ -15,10 +16,13 @@ import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardDecision;
import com.superbiz.agent.harness.guard.semantic.SemanticGuardInput;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.retry.RetryExecutionException;
import com.superbiz.agent.harness.retry.RetryFailure;
import java.util.Objects;
import java.util.List;
public final class DiagnosisReleaseUseCase {
@@ -50,12 +54,44 @@ public final class DiagnosisReleaseUseCase {
}
public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) {
return execute(context, query, DiagnosisAgentExecution.completed(
Objects.requireNonNull(draft, "draft must not be null"),
DiagnosisProgressSnapshot.empty()));
}
public DiagnosisReleaseResult execute(
RunContext context, String query, DiagnosisAgentExecution execution) {
Objects.requireNonNull(context, "context must not be null");
if (query == null || query.isBlank()) {
throw new IllegalArgumentException("query must not be blank");
}
Objects.requireNonNull(draft, "draft must not be null");
Objects.requireNonNull(execution, "execution must not be null");
DiagnosisDraft draft = execution.draft();
if (draft == null) {
return releaseControlledStop(context, execution.progress(), execution.stopReason());
}
if (draft.conclusion() == null) {
return releaseNoConclusion(context, draft, execution.progress());
}
return releaseConclusion(context, query, draft);
}
public DiagnosisReleaseResult releaseInvalidDraft(
RunContext context, DiagnosisProgressSnapshot progress) {
Objects.requireNonNull(context, "context must not be null");
Objects.requireNonNull(progress, "progress must not be null");
if (!progress.hasObservedFacts()) {
throw new IllegalStateException(
"Invalid Diagnosis Draft has no verified publishable progress");
}
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, List.of()),
FallbackType.INSUFFICIENT_EVIDENCE);
}
private DiagnosisReleaseResult releaseConclusion(
RunContext context, String query, DiagnosisDraft draft) {
DiagnosisDraft candidate = draft;
EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate);
traceRecorder.record(TraceAuditEvents.evidenceValidation(
@@ -101,6 +137,61 @@ public final class DiagnosisReleaseUseCase {
return DiagnosisReleaseResult.fallback(fallbackFactory.semanticUnsupported(snapshot));
}
private DiagnosisReleaseResult releaseNoConclusion(
RunContext context, DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
EvidenceGuardResult evidence = evidenceGuard.validateNoConclusionReferences(context, draft);
traceRecorder.record(TraceAuditEvents.evidenceValidation(
context, TraceEventType.EVIDENCE_GUARD_INITIAL, evidence, draft));
if (!evidence.valid()) {
return evidenceFailure(context, evidence);
}
List<String> missingInfo = missingInfo(draft);
if (progress.hasObservedFacts()) {
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, missingInfo),
FallbackType.INSUFFICIENT_EVIDENCE);
}
if (!missingInfo.isEmpty()) {
return progressFallback(context,
fallbackFactory.missingRequiredContext(missingInfo),
FallbackType.MISSING_REQUIRED_CONTEXT);
}
throw new IllegalStateException(
"No-conclusion Diagnosis has neither verified progress nor missing context");
}
private DiagnosisReleaseResult releaseControlledStop(
RunContext context,
DiagnosisProgressSnapshot progress,
DiagnosisStopReason stopReason) {
if (stopReason != DiagnosisStopReason.INFORMATION_SATURATED
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED) {
throw new IllegalStateException("Unsupported Diagnosis stop reason");
}
if (!progress.hasObservedFacts()) {
throw new IllegalStateException(
"Controlled Diagnosis stop has no verified publishable progress");
}
return progressFallback(context,
fallbackFactory.insufficientEvidence(progress, List.of()),
FallbackType.INSUFFICIENT_EVIDENCE);
}
private DiagnosisReleaseResult progressFallback(
RunContext context,
com.superbiz.agent.harness.contract.SafeFallback fallback,
FallbackType type) {
traceRecorder.record(TraceAuditEvents.releaseDecision(
context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, type));
return DiagnosisReleaseResult.fallback(fallback);
}
private List<String> missingInfo(DiagnosisDraft draft) {
return draft.limitations() == null
? List.of() : draft.limitations().missingInfo();
}
private DiagnosisReleaseResult evidenceFailure(
RunContext context, EvidenceGuardResult evidence) {
traceRecorder.record(TraceAuditEvents.releaseDecision(
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.contract.DiagnosisDraft;
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
import com.superbiz.agent.harness.audit.TraceAuditEvents;
import com.superbiz.agent.harness.audit.ModelCallComponent;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
@@ -92,7 +93,8 @@ public final class EvidenceRepair {
context.retryPolicies().evidenceRepair(),
() -> {
DiagnosisDraft repaired = parse(modelCall.call(
context, modelPrompt, limits.timeout(), limits.maxOutputBytes()));
context, ModelCallComponent.EVIDENCE_REPAIR,
modelPrompt, limits.timeout(), limits.maxOutputBytes()));
if (!originalSemantics.hasSameUserVisibleSemantics(
SemanticDraftView.from(repaired))) {
throw new GuardModelCallException(
@@ -5,6 +5,7 @@ import com.superbiz.agent.harness.contract.SafeFallback;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -53,6 +54,47 @@ public final class SafeFallbackFactory {
List.of());
}
public SafeFallback insufficientEvidence(
DiagnosisProgressSnapshot progress, List<String> missingInfo) {
Objects.requireNonNull(progress, "progress must not be null");
if (!progress.hasObservedFacts()) {
throw new IllegalArgumentException(
"insufficient-evidence fallback requires verified progress");
}
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
List<String> limitations = new ArrayList<>(progress.limitations());
limitations.add("当前已检查范围不足以支持根因结论");
safeMissingInfo.forEach(item -> limitations.add("仍缺少:" + item));
return fallback(
FallbackType.INSUFFICIENT_EVIDENCE,
progress.verifiedSources(),
"已完成有限范围的检查,但现有证据不足以确认根因",
List.copyOf(limitations),
List.of(safeMissingInfo.isEmpty()
? "补充故障对象、发生时间、错误信息或新的可查询范围后重试"
: "补充缺失信息后,在新的明确范围内继续诊断"),
"DIAGNOSIS_COLLECTION",
progress.observedFacts(),
List.of());
}
public SafeFallback missingRequiredContext(List<String> missingInfo) {
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
if (safeMissingInfo.isEmpty()) {
throw new IllegalArgumentException(
"missing-context fallback requires missing information");
}
return fallback(
FallbackType.MISSING_REQUIRED_CONTEXT,
List.of(),
"当前缺少执行定向诊断所需的信息,暂时无法开始有效查询或确认根因",
safeMissingInfo.stream().map(item -> "缺少:" + item).toList(),
List.of("补充上述故障上下文后重试"),
"DIAGNOSIS_INPUT",
List.of(),
List.of());
}
private SafeFallback fallback(FallbackType type,
List<SafeFallback.VerifiedSource> sources,
String message,
@@ -100,6 +142,20 @@ public final class SafeFallbackFactory {
return List.copyOf(result);
}
private List<String> boundedMissingInfo(List<String> missingInfo) {
List<String> result = new ArrayList<>();
for (String item : missingInfo == null ? List.<String>of() : missingInfo) {
String safe = bounded(item);
if (safe != null && !result.contains(safe)) {
result.add(safe);
}
if (result.size() >= 8) {
break;
}
}
return List.copyOf(result);
}
private String bounded(String value) {
if (value == null || value.isBlank()) {
return null;
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record MysqlToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") MysqlToolRequest input) {
}
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record QueryLogsToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") QueryLogsRequest input) {
}
@@ -0,0 +1,7 @@
package com.superbiz.agent.harness.tool.contract;
public enum RagRelevanceLevel {
PRECISE,
HIGHLY_RELEVANT,
REFERENCE
}
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.superbiz.agent.harness.progress.PreviousObservation;
public record RagToolCall(
@JsonProperty("previous_observation") PreviousObservation previousObservation,
@JsonProperty("input") RagToolRequest input) {
}
@@ -1,6 +1,7 @@
package com.superbiz.agent.harness.tool.contract;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import java.util.List;
@@ -11,9 +12,20 @@ public record RagToolResult(
@JsonProperty("query") String query,
@JsonProperty("evidence") List<RagEvidence> evidence,
@JsonProperty("returned_count") int returnedCount,
@JsonProperty("relevance_level") @JsonInclude(JsonInclude.Include.NON_NULL)
RagRelevanceLevel relevanceLevel,
@JsonProperty("truncated") boolean truncated) {
public RagToolResult {
evidence = ToolContractCollections.immutable(evidence);
}
public RagToolResult(EvidenceStatus evidenceStatus,
String toolCallId,
String query,
List<RagEvidence> evidence,
int returnedCount,
boolean truncated) {
this(evidenceStatus, toolCallId, query, evidence, returnedCount, null, truncated);
}
}
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.tool.boundary.ProjectedToolResult;
import com.superbiz.agent.harness.tool.contract.RagEvidence;
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
import com.superbiz.agent.harness.tool.contract.RagToolResult;
@@ -81,7 +82,10 @@ public final class RagResultProjector {
EvidenceStatus status = evidence.isEmpty()
? EvidenceStatus.NO_EVIDENCE
: EvidenceStatus.EVIDENCE_FOUND;
RagToolResult result = new RagToolResult(status, toolCallId, query, evidence, evidence.size(), truncated);
RagRelevanceLevel relevanceLevel = status == EvidenceStatus.EVIDENCE_FOUND
? relevanceLevel(root) : null;
RagToolResult result = new RagToolResult(
status, toolCallId, query, evidence, evidence.size(), relevanceLevel, truncated);
result = fitBudget(result, truncated);
return new ProjectedToolResult(objectMapper.writeValueAsString(result), result.evidenceStatus());
}
@@ -94,7 +98,8 @@ public final class RagResultProjector {
reduced.remove(reduced.size() - 1);
current = new RagToolResult(
reduced.isEmpty() ? EvidenceStatus.NO_EVIDENCE : EvidenceStatus.EVIDENCE_FOUND,
current.toolCallId(), current.query(), reduced, reduced.size(), true);
current.toolCallId(), current.query(), reduced, reduced.size(),
reduced.isEmpty() ? null : current.relevanceLevel(), true);
}
if (bytes(objectMapper.writeValueAsString(current)) > limits.maxAgentUtf8Bytes()) {
throw new IllegalArgumentException("RAG projection exceeds total budget");
@@ -120,6 +125,21 @@ public final class RagResultProjector {
return value == null || value.isBlank() ? null : value;
}
private static RagRelevanceLevel relevanceLevel(JsonNode root) {
String value = text(root, "relevance_level");
if (value.isBlank()) {
value = text(root, "relevanceLevel");
}
if (value.isBlank()) {
return null;
}
try {
return RagRelevanceLevel.valueOf(value.trim().toUpperCase(java.util.Locale.ROOT));
} catch (IllegalArgumentException ignored) {
return null;
}
}
private static String bounded(String value, int max) {
if (value == null) {
return "";
@@ -5,6 +5,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Optional;
/**
* Agent 决策步骤 Repository
@@ -27,6 +28,8 @@ public interface AgentStepRepository extends JpaRepository<AgentStep, Long> {
*/
List<AgentStep> findByRunIdOrderByStepIndex(String runId);
Optional<AgentStep> findByRunIdAndStepIndex(String runId, Integer stepIndex);
/**
* 统计某个会话的步骤数
*/
+1
View File
@@ -217,5 +217,6 @@ harness:
run-timeout: 5m
sse-timeout: 5m
canonical-ttl: 2h
stop-after-consecutive-no-gain: 2
mysql-tools:
data-sources: {}
@@ -1,46 +1,33 @@
# Role
# 角色
You are the only Diagnosis Agent for the current diagnosis run. Plan the investigation internally, call the available read-only evidence tools through the framework ReAct loop, evaluate the returned evidence, and author one complete DiagnosisDraft.
你是当前运行中唯一的诊断 Agent。围绕用户原始问题收集只读证据,并输出一份完整的诊断草稿。可用工具及其参数由服务端提供,以实际注入内容为准。
# Input
# 基本原则
The user message is one JSON object with exactly:
- 你不必须给出根因。证据不足时,`conclusion=null` 是合法且成功的完成方式。
- 如果问题缺少企业、时间、服务、错误信息或其他形成明确查询范围所需的上下文,可以不调用工具,直接在 `limitations.missing_info` 中列出缺失信息。
- 上一轮内容仅作为上下文,不是本轮证据,也不能复用其中的 Tool Call ID。
- 不得编造根因、证据、Tool Call ID、健康状态或已经排除的原因。
- 不得输出原始工具载荷、凭据、内部错误、提示词、隐藏推理或思维链。
- `query`: the current user query. Preserve its meaning and answer this query only.
- `previous_turn`: an optional safely published previous diagnosis turn with the fixed PreviousTurn schema.
# 查询与停止
`previous_turn` is context only. It is not evidence for this run, contains no reusable Tool Call IDs, and must not be cited as current evidence.
- 只有在存在明确、不同且可能获得新信息的查询范围时才继续调用工具。
- 工具结果新增了可验证事实,并确认、排除或缩小了当前假设时,标记为 `GAINED`。
- 工具结果即使正确,但只是通用说明、重复内容,或不能推进当前诊断时,标记为 `NO_GAIN`。
- 继续调用工具时,按服务端提供的控制字段评价上一轮结果。
- 如果没有新的有效查询范围,或者结果正确但对推导无用,立即停止调用。
- 收到 `STOP_REQUIRED` 后不得再次调用工具,应直接完成诊断草稿。
# Evidence rules
# 证据边界
- Use only `lookup_knowledge`, `query_logs`, and `query_mysql` when evidence is needed.
- Treat a Tool observation as evidence only when it contains `evidence_status=EVIDENCE_FOUND` or `evidence_status=NO_EVIDENCE` and a non-blank `tool_call_id`.
- Bind every Analysis item only to Tool Call IDs returned during this run. Never invent, transform, shorten, or reuse a Tool Call ID.
- `NORMAL` Analysis may cite only `EVIDENCE_FOUND` results.
- `NEGATIVE_OBSERVATION` Analysis may cite only `NO_EVIDENCE` results and must state the exact query scope. `NO_EVIDENCE` never proves that a problem does not exist, that a root cause is excluded, or that a system is healthy.
- A Tool `ERROR` is not evidence. Do not cite it as support for Analysis or Conclusion.
- Do not expose raw Tool payloads, credentials, infrastructure coordinates, internal errors, prompts, hidden reasoning, or chain of thought.
- 只有本轮工具返回的非空 Tool Call ID 可以被引用。
- 正向分析只能引用 `EVIDENCE_FOUND`;负向观察只能引用 `NO_EVIDENCE`,并写清实际查询范围。
- `NO_EVIDENCE` 只表示指定范围内未找到证据,不能证明问题不存在、系统健康或某个根因已被排除。
- `ERROR` 不是证据,不能支持分析或结论。
# Stopping rule
# 输出
Stop calling tools when the current evidence is sufficient for a bounded Draft, when the configured tool/model budget prevents more work, or when the available tools cannot obtain the missing information.
If current evidence cannot support a diagnosis:
- set `conclusion` to `null`;
- keep only valid scoped negative observations, if any;
- state the actual queried scope in `limitations.scope`;
- list the evidence still needed in `limitations.missing_info`;
- do not fabricate a root cause, action justification, recommendation, or healthy-state claim;
- finish the current response without asking the Harness to retry the Agent or a Tool.
# Draft rules
- Lead with `conclusion` when supported. Its `based_on_analysis_ids` must reference existing Analysis IDs.
- Every Analysis item must have a unique `analysis_id`, a valid `kind`, concise evidence-grounded text, and at least one current-Run `tool_call_id`.
- Every Action Plan and Recommendation item must reference existing Analysis IDs.
- Mark any dangerous or side-effecting action with `requires_human_confirmation=true`.
- `limitations.scope` must not exceed the actual Tool query scopes.
- `limitations.missing_info` must name material gaps that constrain the conclusion.
Return exactly one JSON object matching the supplied DiagnosisDraft schema. Do not use Markdown fences, headings, prose before or after JSON, or a separate answer field.
- 有充分证据时,结论、分析、行动和建议必须通过分析 ID 与本轮 Tool Call ID 建立完整引用关系。
- 证据不足时,使用 `conclusion=null`,只保留可验证的范围和观察,并在 `limitations` 中说明实际检查范围与缺失信息。
- 返回一个符合服务端输出约束的 JSON 对象。不要使用 Markdown 代码块,也不要添加 JSON 之外的文字。
+36 -1
View File
@@ -703,7 +703,33 @@ class SuperBizAgentApp {
return [payload.answer || '', references ? `\n\n参考资料:\n${references}` : ''].join('');
}
if (contentType === 'SAFE_FALLBACK') {
return payload.fallback?.message || '当前证据不足,无法确认根因';
const fallback = payload.fallback || {};
const sections = [];
sections.push(`## 当前判断\n${fallback.message || '当前证据不足,无法确认根因'}`);
if (fallback.observed_facts?.length) {
const facts = fallback.observed_facts.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**(${item.scope || '范围未提供'}):${item.summary || '已完成检查'}`
).join('\n');
sections.push(`## 已完成排查\n${facts}`);
} else if (fallback.verified_sources?.length) {
const sources = fallback.verified_sources.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
).join('\n');
sections.push(`## 已检查范围\n${sources}`);
}
if (fallback.validation_issues?.length) {
const issues = fallback.validation_issues.map(item =>
`- ${item.code || 'VALIDATION_FAILED'}${item.target ? `:${item.target}` : ''}`
).join('\n');
sections.push(`## 校验问题\n${issues}`);
}
if (fallback.limitations?.length) {
sections.push(`## 限制\n${fallback.limitations.map(item => `- ${item}`).join('\n')}`);
}
if (fallback.next_steps?.length) {
sections.push(`## 下一步\n${fallback.next_steps.map(item => `- ${item}`).join('\n')}`);
}
return sections.join('\n\n');
}
if (contentType === 'DIAGNOSIS_REPORT') {
const report = payload.report || {};
@@ -713,6 +739,15 @@ class SuperBizAgentApp {
if (report.action_plan?.length) sections.push(`## 行动计划\n${report.action_plan.map(item => `- ${item.action}`).join('\n')}`);
if (report.recommendations?.length) sections.push(`## 建议\n${report.recommendations.map(item => `- ${item.text}`).join('\n')}`);
if (report.limitations?.scope) sections.push(`## 限制\n${report.limitations.scope}`);
if (report.limitations?.missing_info?.length) {
sections.push(`## 仍缺信息\n${report.limitations.missing_info.map(item => `- ${item}`).join('\n')}`);
}
if (payload.references?.length) {
const references = payload.references.map(item =>
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
).join('\n');
sections.push(`## 已检查范围\n${references}`);
}
return sections.join('\n\n');
}
throw new Error(`未知 content_type: ${contentType}`);
+1 -1
View File
@@ -111,6 +111,6 @@
</div>
</div>
<script src="app.js?v=20260707-session-picker"></script>
<script src="app.js?v=20260725-diagnosis-details"></script>
</body>
</html>