feat(harness): add information gain stop and audit
This commit is contained in:
@@ -27,6 +27,7 @@ public class ChatHarnessProperties {
|
||||
private int maxModelCalls = 24;
|
||||
private int maxToolCalls = 24;
|
||||
private int maxCallsPerTool = 8;
|
||||
private int stopAfterConsecutiveNoGain = 2;
|
||||
private long maxInputTokens = 100_000;
|
||||
private long maxOutputTokens = 100_000;
|
||||
private long maxTotalTokens = 200_000;
|
||||
@@ -71,6 +72,7 @@ public class ChatHarnessProperties {
|
||||
positive(maxModelCalls, "maxModelCalls");
|
||||
positive(maxToolCalls, "maxToolCalls");
|
||||
positive(maxCallsPerTool, "maxCallsPerTool");
|
||||
positive(stopAfterConsecutiveNoGain, "stopAfterConsecutiveNoGain");
|
||||
positive(maxInputTokens, "maxInputTokens");
|
||||
positive(maxOutputTokens, "maxOutputTokens");
|
||||
positive(maxTotalTokens, "maxTotalTokens");
|
||||
|
||||
@@ -27,6 +27,7 @@ import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
|
||||
import com.superbiz.agent.harness.release.EvidenceRepair;
|
||||
import com.superbiz.agent.harness.release.EvidenceRepairLimits;
|
||||
import com.superbiz.agent.harness.release.SafeFallbackFactory;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressProjector;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
|
||||
@@ -50,6 +51,7 @@ import com.superbiz.agent.harness.application.KnowledgeQueryOperation;
|
||||
import com.superbiz.agent.harness.application.SystemChatOperation;
|
||||
import com.superbiz.agent.harness.application.IntentRouting;
|
||||
import com.superbiz.agent.harness.audit.HarnessAgentAuditHook;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
@@ -106,7 +108,8 @@ public class HarnessChatConfiguration {
|
||||
new RunBudgetLimits(properties.getMaxModelCalls(), properties.getMaxToolCalls(),
|
||||
properties.getMaxCallsPerTool(), properties.getMaxInputTokens(),
|
||||
properties.getMaxOutputTokens(), properties.getMaxTotalTokens(), properties.getMaxRunBytes()),
|
||||
HarnessRetryPolicies.strict());
|
||||
HarnessRetryPolicies.strict(),
|
||||
properties.getStopAfterConsecutiveNoGain());
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -116,8 +119,16 @@ public class HarnessChatConfiguration {
|
||||
|
||||
@Bean
|
||||
public GuardModelCall guardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
|
||||
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor) {
|
||||
return new GuardModelCall(core, chatModel, executor);
|
||||
@Qualifier("harnessModelExecutor") ThreadPoolExecutor executor,
|
||||
ModelCallAuditor modelCallAuditor) {
|
||||
return new GuardModelCall(core, chatModel, executor, modelCallAuditor);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public ModelCallAuditor modelCallAuditor(DiagnosisHarnessCore core,
|
||||
DiagnosisTraceRecorder traceRecorder,
|
||||
AgentStepRepository steps) {
|
||||
return new ModelCallAuditor(core, traceRecorder, steps);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -237,20 +248,30 @@ public class HarnessChatConfiguration {
|
||||
HarnessEvidenceTools tools, ObjectMapper mapper,
|
||||
AgentStepRepository steps,
|
||||
AgentReasoningAuditRepository reasoningAudits,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
DiagnosisTraceRecorder traceRecorder,
|
||||
ModelCallAuditor modelCallAuditor) {
|
||||
return new DiagnosisAgentFactory(chatModel, core, tools, mapper,
|
||||
List.of(new HarnessAgentAuditHook(
|
||||
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)));
|
||||
steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits)),
|
||||
traceRecorder, modelCallAuditor);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public DiagnosisAgentUseCase diagnosisAgentUseCase(DiagnosisHarnessCore core,
|
||||
DiagnosisAgentFactory factory,
|
||||
ObjectMapper mapper,
|
||||
ChatHarnessProperties properties) {
|
||||
ChatHarnessProperties properties,
|
||||
DiagnosisProgressProjector progressProjector) {
|
||||
return new DiagnosisAgentUseCase(core, factory, mapper, new DiagnosisAgentLimits(
|
||||
properties.getDiagnosisMaxQueryBytes(), properties.getDiagnosisMaxPreviousTurnBytes(),
|
||||
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()));
|
||||
properties.getDiagnosisMaxInputBytes(), properties.getDiagnosisMaxDraftBytes()),
|
||||
progressProjector);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public DiagnosisProgressProjector diagnosisProgressProjector(
|
||||
CanonicalInvocationStore store, ToolCallKeyFactory keyFactory, ObjectMapper mapper) {
|
||||
return new DiagnosisProgressProjector(store, keyFactory, mapper);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -281,13 +302,19 @@ public class HarnessChatConfiguration {
|
||||
attempt -> { }, traceRecorder);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public SafeFallbackFactory safeFallbackFactory() {
|
||||
return new SafeFallbackFactory();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public DiagnosisReleaseUseCase diagnosisReleaseUseCase(EvidenceGuard evidenceGuard,
|
||||
EvidenceRepair repair,
|
||||
SemanticGuard semanticGuard,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
DiagnosisTraceRecorder traceRecorder,
|
||||
SafeFallbackFactory fallbackFactory) {
|
||||
return new DiagnosisReleaseUseCase(
|
||||
evidenceGuard, repair, semanticGuard, new SafeFallbackFactory(), traceRecorder);
|
||||
evidenceGuard, repair, semanticGuard, fallbackFactory, traceRecorder);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@@ -327,9 +354,10 @@ public class HarnessChatConfiguration {
|
||||
|
||||
@Bean
|
||||
public DiagnosisOperation diagnosisOperation(DiagnosisAgentUseCase agent,
|
||||
DiagnosisReleaseUseCase release,
|
||||
PublishedResultPolicy policy) {
|
||||
return new DiagnosisChatExecutor(agent, release, policy);
|
||||
DiagnosisReleaseUseCase release,
|
||||
PublishedResultPolicy policy,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
return new DiagnosisChatExecutor(agent, release, policy, traceRecorder);
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public record DiagnosisAgentExecution(
|
||||
DiagnosisDraft draft,
|
||||
DiagnosisProgressSnapshot progress,
|
||||
DiagnosisStopReason stopReason) {
|
||||
|
||||
public DiagnosisAgentExecution {
|
||||
Objects.requireNonNull(progress, "progress must not be null");
|
||||
if (draft == null && stopReason == null) {
|
||||
throw new IllegalArgumentException("Execution without Draft requires a stop reason");
|
||||
}
|
||||
}
|
||||
|
||||
public static DiagnosisAgentExecution completed(
|
||||
DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
|
||||
return new DiagnosisAgentExecution(
|
||||
Objects.requireNonNull(draft, "draft must not be null"), progress, null);
|
||||
}
|
||||
|
||||
public static DiagnosisAgentExecution stopped(
|
||||
DiagnosisProgressSnapshot progress, DiagnosisStopReason stopReason) {
|
||||
return new DiagnosisAgentExecution(
|
||||
null, progress, Objects.requireNonNull(stopReason, "stopReason must not be null"));
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.agent;
|
||||
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
||||
import com.alibaba.cloud.ai.graph.agent.hook.Hook;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
@@ -19,6 +21,8 @@ public final class DiagnosisAgentFactory {
|
||||
private final HarnessEvidenceTools evidenceTools;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final List<Hook> auditHooks;
|
||||
private final DiagnosisTraceRecorder traceRecorder;
|
||||
private final ModelCallAuditor modelCallAuditor;
|
||||
private final String prompt;
|
||||
|
||||
public DiagnosisAgentFactory(ChatModel chatModel,
|
||||
@@ -26,11 +30,35 @@ public final class DiagnosisAgentFactory {
|
||||
HarnessEvidenceTools evidenceTools,
|
||||
ObjectMapper objectMapper,
|
||||
List<? extends Hook> auditHooks) {
|
||||
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
|
||||
DiagnosisTraceRecorder.noop(), new ModelCallAuditor(core));
|
||||
}
|
||||
|
||||
public DiagnosisAgentFactory(ChatModel chatModel,
|
||||
DiagnosisHarnessCore core,
|
||||
HarnessEvidenceTools evidenceTools,
|
||||
ObjectMapper objectMapper,
|
||||
List<? extends Hook> auditHooks,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
this(chatModel, core, evidenceTools, objectMapper, auditHooks,
|
||||
traceRecorder, new ModelCallAuditor(core));
|
||||
}
|
||||
|
||||
public DiagnosisAgentFactory(ChatModel chatModel,
|
||||
DiagnosisHarnessCore core,
|
||||
HarnessEvidenceTools evidenceTools,
|
||||
ObjectMapper objectMapper,
|
||||
List<? extends Hook> auditHooks,
|
||||
DiagnosisTraceRecorder traceRecorder,
|
||||
ModelCallAuditor modelCallAuditor) {
|
||||
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.auditHooks = List.copyOf(Objects.requireNonNull(auditHooks, "auditHooks must not be null"));
|
||||
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
|
||||
this.modelCallAuditor = Objects.requireNonNull(
|
||||
modelCallAuditor, "modelCallAuditor must not be null");
|
||||
this.prompt = DiagnosisAgentPrompt.load();
|
||||
}
|
||||
|
||||
@@ -43,8 +71,9 @@ public final class DiagnosisAgentFactory {
|
||||
.systemPrompt(prompt)
|
||||
.tools(evidenceTools.callbacks())
|
||||
.interceptors(
|
||||
new HarnessModelInterceptor(core, context),
|
||||
new HarnessToolInterceptor(context, evidenceTools, objectMapper))
|
||||
new HarnessModelInterceptor(core, context, modelCallAuditor),
|
||||
new HarnessToolInterceptor(
|
||||
context, evidenceTools, objectMapper, traceRecorder))
|
||||
.hooks(auditHooks)
|
||||
.outputSchema(new DiagnosisDraftOutputSchema(objectMapper).getFormat())
|
||||
.returnReasoningContents(true)
|
||||
|
||||
@@ -1,12 +1,59 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class DiagnosisAgentOutputException extends RuntimeException {
|
||||
|
||||
public enum Kind {
|
||||
EXECUTION_FAILED,
|
||||
EMPTY_DRAFT,
|
||||
INVALID_JSON,
|
||||
SCHEMA_INVALID
|
||||
}
|
||||
|
||||
private final Kind kind;
|
||||
private final long outputBytes;
|
||||
private final DiagnosisProgressSnapshot progress;
|
||||
|
||||
public DiagnosisAgentOutputException(String message) {
|
||||
super(message);
|
||||
this(message, null);
|
||||
}
|
||||
|
||||
public DiagnosisAgentOutputException(String message, Throwable cause) {
|
||||
this(message, cause, Kind.EXECUTION_FAILED, 0L, DiagnosisProgressSnapshot.empty());
|
||||
}
|
||||
|
||||
public DiagnosisAgentOutputException(String message,
|
||||
Throwable cause,
|
||||
Kind kind,
|
||||
long outputBytes,
|
||||
DiagnosisProgressSnapshot progress) {
|
||||
super(message, cause);
|
||||
if (outputBytes < 0) {
|
||||
throw new IllegalArgumentException("outputBytes must not be negative");
|
||||
}
|
||||
this.kind = Objects.requireNonNull(kind, "kind must not be null");
|
||||
this.outputBytes = outputBytes;
|
||||
this.progress = Objects.requireNonNull(progress, "progress must not be null");
|
||||
}
|
||||
|
||||
public Kind kind() {
|
||||
return kind;
|
||||
}
|
||||
|
||||
public long outputBytes() {
|
||||
return outputBytes;
|
||||
}
|
||||
|
||||
public DiagnosisProgressSnapshot progress() {
|
||||
return progress;
|
||||
}
|
||||
|
||||
public boolean isDraftContractFailure() {
|
||||
return kind == Kind.EMPTY_DRAFT
|
||||
|| kind == Kind.INVALID_JSON
|
||||
|| kind == Kind.SCHEMA_INVALID;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,13 +2,20 @@ package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.alibaba.cloud.ai.graph.agent.ReactAgent;
|
||||
import com.fasterxml.jackson.core.JsonParseException;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressProjection;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -23,11 +30,20 @@ public final class DiagnosisAgentUseCase {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ObjectReader draftReader;
|
||||
private final DiagnosisAgentLimits limits;
|
||||
private final DiagnosisProgressProjection progressProjection;
|
||||
|
||||
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
|
||||
DiagnosisAgentFactory agentFactory,
|
||||
ObjectMapper objectMapper,
|
||||
DiagnosisAgentLimits limits) {
|
||||
this(core, agentFactory, objectMapper, limits, DiagnosisProgressProjection.empty());
|
||||
}
|
||||
|
||||
public DiagnosisAgentUseCase(DiagnosisHarnessCore core,
|
||||
DiagnosisAgentFactory agentFactory,
|
||||
ObjectMapper objectMapper,
|
||||
DiagnosisAgentLimits limits,
|
||||
DiagnosisProgressProjection progressProjection) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.agentFactory = Objects.requireNonNull(agentFactory, "agentFactory must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
@@ -35,9 +51,11 @@ public final class DiagnosisAgentUseCase {
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
this.limits = Objects.requireNonNull(limits, "limits must not be null");
|
||||
this.progressProjection = Objects.requireNonNull(
|
||||
progressProjection, "progressProjection must not be null");
|
||||
}
|
||||
|
||||
public DiagnosisDraft execute(RunContext context, DiagnosisAgentInput input) {
|
||||
public DiagnosisAgentExecution execute(RunContext context, DiagnosisAgentInput input) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(input, "input must not be null");
|
||||
core.checkActive(context);
|
||||
@@ -64,23 +82,85 @@ public final class DiagnosisAgentUseCase {
|
||||
AssistantMessage response;
|
||||
try {
|
||||
response = agent.call(inputJson, config);
|
||||
core.checkActive(context);
|
||||
} catch (Exception e) {
|
||||
DiagnosisAgentExecution controlled = controlledExecution(context, e);
|
||||
if (controlled != null) {
|
||||
return controlled;
|
||||
}
|
||||
throw new DiagnosisAgentOutputException("Diagnosis Agent execution failed", e);
|
||||
}
|
||||
|
||||
core.checkActive(context);
|
||||
String output = response == null ? null : response.getText();
|
||||
if (output == null || output.isBlank()) {
|
||||
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an empty Draft");
|
||||
throw new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an empty Draft",
|
||||
null,
|
||||
DiagnosisAgentOutputException.Kind.EMPTY_DRAFT,
|
||||
0L,
|
||||
progressProjection.project(context));
|
||||
}
|
||||
long draftBytes = utf8Bytes(output);
|
||||
checkLimit("draft", draftBytes, limits.maxDraftBytes());
|
||||
core.reserveRunBytes(context, draftBytes);
|
||||
try {
|
||||
return draftReader.readValue(output);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new DiagnosisAgentOutputException("Diagnosis Agent returned an invalid Draft", e);
|
||||
core.reserveRunBytes(context, draftBytes);
|
||||
} catch (RuntimeException exception) {
|
||||
DiagnosisAgentExecution controlled = controlledExecution(context, exception);
|
||||
if (controlled != null) {
|
||||
return controlled;
|
||||
}
|
||||
throw exception;
|
||||
}
|
||||
try {
|
||||
DiagnosisDraft draft = draftReader.readValue(output);
|
||||
return DiagnosisAgentExecution.completed(draft, progressProjection.project(context));
|
||||
} catch (JsonProcessingException e) {
|
||||
DiagnosisAgentOutputException.Kind kind = e instanceof JsonParseException
|
||||
? DiagnosisAgentOutputException.Kind.INVALID_JSON
|
||||
: DiagnosisAgentOutputException.Kind.SCHEMA_INVALID;
|
||||
throw new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an invalid Draft",
|
||||
e,
|
||||
kind,
|
||||
draftBytes,
|
||||
progressProjection.project(context));
|
||||
}
|
||||
}
|
||||
|
||||
private DiagnosisAgentExecution controlledExecution(RunContext context, Throwable failure) {
|
||||
DiagnosisCollectionStoppedException stopped = findCause(
|
||||
failure, DiagnosisCollectionStoppedException.class);
|
||||
if (stopped != null) {
|
||||
return DiagnosisAgentExecution.stopped(
|
||||
progressProjection.project(context), stopped.stopReason());
|
||||
}
|
||||
RunAbortedException aborted = findCause(failure, RunAbortedException.class);
|
||||
if (aborted != null) {
|
||||
if (aborted.termination().state() == RunState.BUDGET_EXHAUSTED) {
|
||||
context.progress().markBudgetLimitReached();
|
||||
return DiagnosisAgentExecution.stopped(
|
||||
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
|
||||
}
|
||||
throw aborted;
|
||||
}
|
||||
BudgetExceededException budget = findCause(failure, BudgetExceededException.class);
|
||||
if (budget != null || context.lifecycle().state() == RunState.BUDGET_EXHAUSTED) {
|
||||
context.progress().markBudgetLimitReached();
|
||||
return DiagnosisAgentExecution.stopped(
|
||||
progressProjection.project(context), DiagnosisStopReason.BUDGET_LIMIT_REACHED);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static <T extends Throwable> T findCause(Throwable failure, Class<T> type) {
|
||||
Throwable current = failure;
|
||||
while (current != null) {
|
||||
if (type.isInstance(current)) {
|
||||
return type.cast(current);
|
||||
}
|
||||
current = current.getCause();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String writeJson(Object value) {
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
|
||||
public final class DiagnosisCollectionStoppedException extends RuntimeException {
|
||||
|
||||
private final DiagnosisStopReason stopReason;
|
||||
|
||||
public DiagnosisCollectionStoppedException(DiagnosisStopReason stopReason) {
|
||||
super("Diagnosis collection stopped: " + stopReason);
|
||||
this.stopReason = stopReason;
|
||||
}
|
||||
|
||||
public DiagnosisStopReason stopReason() {
|
||||
return stopReason;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
|
||||
@@ -8,8 +11,11 @@ import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolCallRequestEnvelope;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolCall;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsToolCall;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolCall;
|
||||
import org.springframework.ai.tool.ToolCallback;
|
||||
import org.springframework.ai.tool.function.FunctionToolCallback;
|
||||
|
||||
@@ -36,11 +42,11 @@ public final class HarnessEvidenceTools {
|
||||
this.invokers = Map.copyOf(registered);
|
||||
this.callbacks = List.of(
|
||||
definition(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolRequest.class),
|
||||
AgentToolContracts.LOOKUP_KNOWLEDGE_DESCRIPTION, RagToolCall.class),
|
||||
definition(AgentToolContracts.QUERY_LOGS,
|
||||
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsRequest.class),
|
||||
AgentToolContracts.QUERY_LOGS_DESCRIPTION, QueryLogsToolCall.class),
|
||||
definition(AgentToolContracts.QUERY_MYSQL,
|
||||
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolRequest.class));
|
||||
AgentToolContracts.QUERY_MYSQL_DESCRIPTION, MysqlToolCall.class));
|
||||
}
|
||||
|
||||
public static HarnessEvidenceTools fromAdapters(RagToolAdapter ragAdapter,
|
||||
@@ -72,6 +78,51 @@ public final class HarnessEvidenceTools {
|
||||
return invoker.invoke(context, toolCallId, arguments);
|
||||
}
|
||||
|
||||
public ParsedAgentToolCall parse(String toolName, String arguments, ObjectMapper objectMapper) {
|
||||
Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
try {
|
||||
ObjectReader reader = objectMapper.reader()
|
||||
.with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||
return switch (toolName) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
|
||||
RagToolCall call = reader.forType(RagToolCall.class).readValue(arguments);
|
||||
requireInput(call == null ? null : call.input());
|
||||
yield parsed(call.previousObservation(), call.input(), objectMapper);
|
||||
}
|
||||
case AgentToolContracts.QUERY_LOGS -> {
|
||||
QueryLogsToolCall call = reader.forType(QueryLogsToolCall.class).readValue(arguments);
|
||||
requireInput(call == null ? null : call.input());
|
||||
yield parsed(call.previousObservation(), call.input(), objectMapper);
|
||||
}
|
||||
case AgentToolContracts.QUERY_MYSQL -> {
|
||||
MysqlToolCall call = reader.forType(MysqlToolCall.class).readValue(arguments);
|
||||
requireInput(call == null ? null : call.input());
|
||||
yield parsed(call.previousObservation(), call.input(), objectMapper);
|
||||
}
|
||||
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
|
||||
};
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw exception;
|
||||
} catch (Exception exception) {
|
||||
throw new IllegalArgumentException("Tool Call Envelope is invalid", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static ParsedAgentToolCall parsed(
|
||||
com.superbiz.agent.harness.progress.PreviousObservation previousObservation,
|
||||
Object input,
|
||||
ObjectMapper objectMapper) throws Exception {
|
||||
return new ParsedAgentToolCall(
|
||||
previousObservation, input, objectMapper.writeValueAsString(input));
|
||||
}
|
||||
|
||||
private static void requireInput(Object input) {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("Tool Call Envelope input is required");
|
||||
}
|
||||
}
|
||||
|
||||
private static EvidenceToolInvoker bridge(String toolName, AdapterCall adapter) {
|
||||
return (context, toolCallId, arguments) -> adapter.execute(
|
||||
context,
|
||||
|
||||
@@ -6,6 +6,9 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.audit.ModelCallLedger;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
|
||||
@@ -15,10 +18,17 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final RunContext context;
|
||||
private final ModelCallAuditor auditor;
|
||||
|
||||
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context) {
|
||||
this(core, context, new ModelCallAuditor(core));
|
||||
}
|
||||
|
||||
public HarnessModelInterceptor(DiagnosisHarnessCore core, RunContext context,
|
||||
ModelCallAuditor auditor) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.context = Objects.requireNonNull(context, "context must not be null");
|
||||
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -31,23 +41,32 @@ public final class HarnessModelInterceptor extends ModelInterceptor {
|
||||
Objects.requireNonNull(request, "request must not be null");
|
||||
Objects.requireNonNull(handler, "handler must not be null");
|
||||
core.beforeModelCall(context);
|
||||
ModelResponse response = handler.call(request);
|
||||
recordUsage(response == null ? null : response.getChatResponse());
|
||||
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
ModelResponse response;
|
||||
try {
|
||||
response = handler.call(request);
|
||||
} catch (RuntimeException exception) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
throw exception;
|
||||
}
|
||||
recordUsage(call, response == null ? null : response.getChatResponse());
|
||||
core.checkActive(context);
|
||||
return response;
|
||||
}
|
||||
|
||||
private void recordUsage(ChatResponse response) {
|
||||
private void recordUsage(ModelCallLedger.Call call, ChatResponse response) {
|
||||
if (response == null || response.getMetadata() == null) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
return;
|
||||
}
|
||||
Usage usage = response.getMetadata().getUsage();
|
||||
if (usage == null) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
return;
|
||||
}
|
||||
long inputTokens = nonNegative(usage.getPromptTokens());
|
||||
long outputTokens = nonNegative(usage.getCompletionTokens());
|
||||
core.recordTokens(context, inputTokens, outputTokens);
|
||||
auditor.recordUsage(context, call, inputTokens, outputTokens, true);
|
||||
}
|
||||
|
||||
private static long nonNegative(Integer value) {
|
||||
|
||||
@@ -6,8 +6,16 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolInterceptor;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
import com.superbiz.agent.harness.contract.InvocationStatus;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.progress.CompletedToolCall;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisCollectionState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.progress.ToolScopeNormalizer;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -19,13 +27,26 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
private final RunContext context;
|
||||
private final HarnessEvidenceTools evidenceTools;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final ToolScopeNormalizer scopeNormalizer;
|
||||
private final ToolResultViewProjector viewProjector;
|
||||
private final DiagnosisTraceRecorder traceRecorder;
|
||||
|
||||
public HarnessToolInterceptor(RunContext context,
|
||||
HarnessEvidenceTools evidenceTools,
|
||||
ObjectMapper objectMapper) {
|
||||
this(context, evidenceTools, objectMapper, DiagnosisTraceRecorder.noop());
|
||||
}
|
||||
|
||||
public HarnessToolInterceptor(RunContext context,
|
||||
HarnessEvidenceTools evidenceTools,
|
||||
ObjectMapper objectMapper,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
this.context = Objects.requireNonNull(context, "context must not be null");
|
||||
this.evidenceTools = Objects.requireNonNull(evidenceTools, "evidenceTools must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
this.scopeNormalizer = new ToolScopeNormalizer(objectMapper);
|
||||
this.viewProjector = new ToolResultViewProjector(objectMapper);
|
||||
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -41,10 +62,76 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
return handler.call(request);
|
||||
}
|
||||
|
||||
DiagnosisProgressSnapshotState before = context.progress().snapshot();
|
||||
if (before.collectionState() == DiagnosisCollectionState.SATURATED
|
||||
&& before.stopInstructionDelivered()) {
|
||||
recordRejection(request, "COLLECTION_STOPPED");
|
||||
throw new DiagnosisCollectionStoppedException(before.stopReason());
|
||||
}
|
||||
|
||||
ParsedAgentToolCall call;
|
||||
String normalizedScope;
|
||||
try {
|
||||
call = evidenceTools.parse(request.getToolName(), request.getArguments(), objectMapper);
|
||||
context.progress().applyPreviousObservation(call.previousObservation());
|
||||
DiagnosisProgressSnapshotState evaluated = context.progress().snapshot();
|
||||
recordModelProgress(before, call, evaluated);
|
||||
if (evaluated.collectionState() == DiagnosisCollectionState.SATURATED) {
|
||||
recordRejection(request, "INFORMATION_SATURATED");
|
||||
return stopRequired(request, evaluated.stopReason());
|
||||
}
|
||||
normalizedScope = scopeNormalizer.normalize(request.getToolName(), call.businessInput());
|
||||
} catch (IllegalArgumentException | IllegalStateException exception) {
|
||||
recordRejection(request, "INVALID_PROGRESS_PROTOCOL");
|
||||
return safeError(request, "INVALID_PROGRESS_PROTOCOL");
|
||||
}
|
||||
|
||||
if (context.progress().isDuplicate(request.getToolName(), normalizedScope)) {
|
||||
recordRejection(request, "DUPLICATE_SCOPE");
|
||||
context.progress().recordDuplicateScope();
|
||||
DiagnosisProgressSnapshotState duplicate = context.progress().snapshot();
|
||||
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
|
||||
InformationGain.NO_GAIN, "HARNESS", duplicate);
|
||||
if (duplicate.collectionState() == DiagnosisCollectionState.SATURATED) {
|
||||
return stopRequired(request, duplicate.stopReason());
|
||||
}
|
||||
return duplicateScope(request);
|
||||
}
|
||||
|
||||
ToolBoundaryResult result = evidenceTools.invoke(
|
||||
context, request.getToolName(), request.getToolCallId(), request.getArguments());
|
||||
context, request.getToolName(), request.getToolCallId(), call.businessArguments());
|
||||
if (result.status() == InvocationStatus.READY) {
|
||||
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), result.agentResult());
|
||||
ToolControlView control = viewProjector.controlView(result.agentResult());
|
||||
if (control.evidenceStatus() != result.evidenceStatus()) {
|
||||
recordRejection(request, "OBSERVATION_CONTRACT_MISMATCH");
|
||||
return safeError(request, "OBSERVATION_CONTRACT_MISMATCH");
|
||||
}
|
||||
context.progress().recordCompleted(
|
||||
new CompletedToolCall(
|
||||
request.getToolCallId(), request.getToolName(), normalizedScope),
|
||||
result.evidenceStatus());
|
||||
DiagnosisProgressSnapshotState completed = context.progress().snapshot();
|
||||
if (result.evidenceStatus()
|
||||
== com.superbiz.agent.harness.contract.EvidenceStatus.NO_EVIDENCE) {
|
||||
recordProgress(request.getToolCallId(), request.getToolName(), normalizedScope,
|
||||
InformationGain.NO_GAIN, "HARNESS", completed);
|
||||
}
|
||||
boolean stopRequired = completed.collectionState() == DiagnosisCollectionState.SATURATED
|
||||
&& context.progress().claimStopInstruction();
|
||||
if (stopRequired) {
|
||||
traceRecorder.record(TraceAuditEvents.collectionStop(
|
||||
context, request.getToolCallId(), request.getToolName(), completed));
|
||||
}
|
||||
String observation = viewProjector.modelObservation(
|
||||
request.getToolName(), result.agentResult(), normalizedScope,
|
||||
stopRequired, completed.stopReason());
|
||||
return ToolCallResponse.of(request.getToolCallId(), request.getToolName(), observation);
|
||||
}
|
||||
if ("BUDGET_EXHAUSTED".equals(result.errorCode())) {
|
||||
context.progress().markBudgetLimitReached();
|
||||
traceRecorder.record(TraceAuditEvents.collectionStop(
|
||||
context, request.getToolCallId(), request.getToolName(),
|
||||
context.progress().snapshot()));
|
||||
}
|
||||
return ToolCallResponse.builder()
|
||||
.toolCallId(request.getToolCallId())
|
||||
@@ -55,6 +142,90 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
.build();
|
||||
}
|
||||
|
||||
private ToolCallResponse stopRequired(ToolCallRequest request, DiagnosisStopReason reason) {
|
||||
if (!context.progress().claimStopInstruction()) {
|
||||
throw new DiagnosisCollectionStoppedException(reason);
|
||||
}
|
||||
traceRecorder.record(TraceAuditEvents.collectionStop(
|
||||
context, request.getToolCallId(), request.getToolName(),
|
||||
context.progress().snapshot()));
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("tool_call_id", request.getToolCallId());
|
||||
observation.put("stop_required", true);
|
||||
observation.put("reason", reason.name());
|
||||
return ToolCallResponse.of(
|
||||
request.getToolCallId(), request.getToolName(), writeObservation(observation));
|
||||
}
|
||||
|
||||
private ToolCallResponse duplicateScope(ToolCallRequest request) {
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("tool_call_id", request.getToolCallId());
|
||||
observation.put("observation_status", "DUPLICATE_SCOPE");
|
||||
observation.put("information_gain", "NO_GAIN");
|
||||
observation.put("stop_required", false);
|
||||
return ToolCallResponse.of(
|
||||
request.getToolCallId(), request.getToolName(), writeObservation(observation));
|
||||
}
|
||||
|
||||
private ToolCallResponse safeError(ToolCallRequest request, String errorCode) {
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("evidence_status", "ERROR");
|
||||
observation.put("tool_call_id", request.getToolCallId());
|
||||
observation.put("error_code", errorCode);
|
||||
return ToolCallResponse.builder()
|
||||
.toolCallId(request.getToolCallId())
|
||||
.toolName(request.getToolName())
|
||||
.content(writeObservation(observation))
|
||||
.status("error")
|
||||
.metadata(Map.of("error", true))
|
||||
.build();
|
||||
}
|
||||
|
||||
private void recordRejection(ToolCallRequest request, String errorCode) {
|
||||
traceRecorder.record(TraceAuditEvents.toolRequestRejected(
|
||||
context, request.getToolCallId(), request.getToolName(), errorCode));
|
||||
}
|
||||
|
||||
private String writeObservation(Map<String, Object> observation) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(observation);
|
||||
} catch (JsonProcessingException exception) {
|
||||
return "{\"evidence_status\":\"ERROR\",\"error_code\":\"SERIALIZATION_ERROR\"}";
|
||||
}
|
||||
}
|
||||
|
||||
private void recordModelProgress(
|
||||
DiagnosisProgressSnapshotState before,
|
||||
ParsedAgentToolCall call,
|
||||
DiagnosisProgressSnapshotState after) {
|
||||
if (call.previousObservation() == null) {
|
||||
return;
|
||||
}
|
||||
before.completedToolCalls().stream()
|
||||
.filter(completed -> completed.toolCallId()
|
||||
.equals(call.previousObservation().toolCallId()))
|
||||
.findFirst()
|
||||
.ifPresent(completed -> recordProgress(
|
||||
completed.toolCallId(), completed.toolName(), completed.normalizedScope(),
|
||||
call.previousObservation().informationGain(), "MODEL", after));
|
||||
}
|
||||
|
||||
private void recordProgress(
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
String normalizedScope,
|
||||
InformationGain informationGain,
|
||||
String producer,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
traceRecorder.record(TraceAuditEvents.toolProgress(
|
||||
context, toolCallId, toolName, scopeSummary(toolName, normalizedScope),
|
||||
informationGain, producer, state));
|
||||
}
|
||||
|
||||
private String scopeSummary(String toolName, String normalizedScope) {
|
||||
return toolName + "#" + String.format("%08x", normalizedScope.hashCode());
|
||||
}
|
||||
|
||||
private String errorObservation(ToolBoundaryResult result) {
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("evidence_status", result.evidenceStatus());
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
|
||||
public record ParsedAgentToolCall(
|
||||
PreviousObservation previousObservation,
|
||||
Object businessInput,
|
||||
String businessArguments) {
|
||||
|
||||
public ParsedAgentToolCall {
|
||||
if (businessInput == null) {
|
||||
throw new IllegalArgumentException("businessInput must not be null");
|
||||
}
|
||||
if (businessArguments == null || businessArguments.isBlank()) {
|
||||
throw new IllegalArgumentException("businessArguments must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
|
||||
|
||||
public record ToolControlView(
|
||||
EvidenceStatus evidenceStatus,
|
||||
int returnedCount,
|
||||
RagRelevanceLevel relevanceLevel,
|
||||
boolean truncated) {
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class ToolResultViewProjector {
|
||||
|
||||
private static final List<String> COMMON_FIELDS = List.of(
|
||||
"evidence_status", "tool_call_id", "truncated");
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ToolResultViewProjector(ObjectMapper objectMapper) {
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
}
|
||||
|
||||
public ToolControlView controlView(String agentResult) {
|
||||
JsonNode root = readObject(agentResult);
|
||||
EvidenceStatus status;
|
||||
try {
|
||||
status = EvidenceStatus.valueOf(requiredText(root, "evidence_status"));
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IllegalArgumentException("Invalid evidence status", exception);
|
||||
}
|
||||
RagRelevanceLevel relevance = null;
|
||||
String relevanceText = text(root, "relevance_level");
|
||||
if (!relevanceText.isBlank()) {
|
||||
try {
|
||||
relevance = RagRelevanceLevel.valueOf(relevanceText);
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw new IllegalArgumentException("Invalid relevance level", exception);
|
||||
}
|
||||
}
|
||||
return new ToolControlView(
|
||||
status,
|
||||
Math.max(0, root.path("returned_count").asInt(0)),
|
||||
relevance,
|
||||
root.path("truncated").asBoolean(false));
|
||||
}
|
||||
|
||||
public String modelObservation(String toolName,
|
||||
String agentResult,
|
||||
String normalizedScope,
|
||||
boolean stopRequired,
|
||||
DiagnosisStopReason stopReason) {
|
||||
JsonNode root = readObject(agentResult);
|
||||
ObjectNode observation = objectMapper.createObjectNode();
|
||||
copy(root, observation, COMMON_FIELDS);
|
||||
switch (toolName) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE -> {
|
||||
ObjectNode scope = observation.putObject("scope");
|
||||
scope.put("query", text(root, "query"));
|
||||
copy(root, observation, List.of("evidence", "relevance_level"));
|
||||
}
|
||||
case AgentToolContracts.QUERY_LOGS ->
|
||||
copy(root, observation, List.of("source_kind", "scope", "patterns", "events"));
|
||||
case AgentToolContracts.QUERY_MYSQL -> {
|
||||
try {
|
||||
observation.set("scope", objectMapper.readTree(normalizedScope));
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalArgumentException("Normalized scope is invalid", exception);
|
||||
}
|
||||
copy(root, observation, List.of("columns", "rows"));
|
||||
}
|
||||
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
|
||||
}
|
||||
if (stopRequired) {
|
||||
observation.put("stop_required", true);
|
||||
observation.put("reason", Objects.requireNonNull(stopReason, "stopReason must not be null").name());
|
||||
}
|
||||
try {
|
||||
return objectMapper.writeValueAsString(observation);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalArgumentException("Model observation is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private JsonNode readObject(String value) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(value);
|
||||
if (root == null || !root.isObject()) {
|
||||
throw new IllegalArgumentException("Tool result must be a JSON object");
|
||||
}
|
||||
return root;
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalArgumentException("Tool result must be valid JSON", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static void copy(JsonNode source, ObjectNode target, List<String> fields) {
|
||||
for (String field : fields) {
|
||||
JsonNode value = source.get(field);
|
||||
if (value != null && !value.isNull()) {
|
||||
target.set(field, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String requiredText(JsonNode root, String field) {
|
||||
String value = text(root, field);
|
||||
if (value.isBlank()) {
|
||||
throw new IllegalArgumentException(field + " must not be blank");
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
private static String text(JsonNode root, String field) {
|
||||
JsonNode value = root.get(field);
|
||||
return value == null || value.isNull() ? "" : value.asText("");
|
||||
}
|
||||
}
|
||||
@@ -107,8 +107,7 @@ public final class ChatApplicationUseCase {
|
||||
|
||||
PathResult path = executePath(
|
||||
intent, context, request.query(), previousTurn.orElse(null), observer);
|
||||
core.checkActive(context);
|
||||
core.completeSuccess(context);
|
||||
completePath(context, intent, path);
|
||||
String safeJson = write(path.content());
|
||||
persistFinish(context, intent, path.outcome(), safeJson,
|
||||
path.publishedResult(), durationMillis(startedNanos));
|
||||
@@ -131,6 +130,20 @@ public final class ChatApplicationUseCase {
|
||||
}
|
||||
}
|
||||
|
||||
private void completePath(RunContext context, IntentType intent, PathResult path) {
|
||||
if (path.handledBudgetTermination()) {
|
||||
if (intent != IntentType.DIAGNOSIS
|
||||
|| path.outcome() != ReleaseOutcome.FALLBACK
|
||||
|| path.publishedResult() != null
|
||||
|| context.lifecycle().state() != RunState.BUDGET_EXHAUSTED) {
|
||||
throw new IllegalStateException("Invalid handled Diagnosis budget fallback");
|
||||
}
|
||||
return;
|
||||
}
|
||||
core.checkActive(context);
|
||||
core.completeSuccess(context);
|
||||
}
|
||||
|
||||
private PathResult executePath(IntentType intent,
|
||||
RunContext context,
|
||||
String query,
|
||||
@@ -140,18 +153,19 @@ public final class ChatApplicationUseCase {
|
||||
case SYSTEM_CHAT -> {
|
||||
observer.onStatus(ChatApplicationStatus.SYSTEM_RESPONDING);
|
||||
yield new PathResult(
|
||||
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null);
|
||||
ReleaseOutcome.SUCCESS, systemChat.execute(context, query), null, false);
|
||||
}
|
||||
case KNOWLEDGE_QUERY -> {
|
||||
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_SEARCHING);
|
||||
observer.onStatus(ChatApplicationStatus.KNOWLEDGE_ANSWERING);
|
||||
yield new PathResult(
|
||||
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null);
|
||||
ReleaseOutcome.SUCCESS, knowledgeQuery.execute(context, query), null, false);
|
||||
}
|
||||
case DIAGNOSIS -> {
|
||||
DiagnosisExecutionResult result = diagnosis.execute(
|
||||
context, query, previousTurn, observer::onStatus);
|
||||
yield new PathResult(result.outcome(), result.content(), result.publishedResult());
|
||||
yield new PathResult(result.outcome(), result.content(), result.publishedResult(),
|
||||
result.handledBudgetTermination());
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -251,7 +265,8 @@ public final class ChatApplicationUseCase {
|
||||
private record PathResult(
|
||||
ReleaseOutcome outcome,
|
||||
ChatApplicationContent content,
|
||||
PublishedResult publishedResult) {
|
||||
PublishedResult publishedResult,
|
||||
boolean handledBudgetTermination) {
|
||||
}
|
||||
|
||||
private static final class CoreRunControl implements ChatRunControl {
|
||||
|
||||
@@ -8,7 +8,15 @@ import java.util.Objects;
|
||||
public record DiagnosisExecutionResult(
|
||||
ReleaseOutcome outcome,
|
||||
ChatApplicationContent content,
|
||||
PublishedResult publishedResult) {
|
||||
PublishedResult publishedResult,
|
||||
boolean handledBudgetTermination) {
|
||||
|
||||
public DiagnosisExecutionResult(
|
||||
ReleaseOutcome outcome,
|
||||
ChatApplicationContent content,
|
||||
PublishedResult publishedResult) {
|
||||
this(outcome, content, publishedResult, false);
|
||||
}
|
||||
|
||||
public DiagnosisExecutionResult {
|
||||
Objects.requireNonNull(outcome, "outcome must not be null");
|
||||
@@ -25,5 +33,9 @@ public record DiagnosisExecutionResult(
|
||||
if (outcome != ReleaseOutcome.SUCCESS && publishedResult != null) {
|
||||
throw new IllegalArgumentException("only success can contain PublishedResult");
|
||||
}
|
||||
if (handledBudgetTermination && outcome != ReleaseOutcome.FALLBACK) {
|
||||
throw new IllegalArgumentException(
|
||||
"handled budget termination requires fallback release");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+46
-5
@@ -1,6 +1,8 @@
|
||||
package com.superbiz.agent.harness.application.executor;
|
||||
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
|
||||
import com.superbiz.agent.harness.application.ChatApplicationStatus;
|
||||
import com.superbiz.agent.harness.application.DiagnosisContent;
|
||||
@@ -8,7 +10,8 @@ import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
|
||||
import com.superbiz.agent.harness.application.DiagnosisOperation;
|
||||
import com.superbiz.agent.harness.application.FallbackContent;
|
||||
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
import com.superbiz.agent.harness.contract.PreviousTurn;
|
||||
import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
@@ -25,13 +28,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
|
||||
private final DiagnosisAgentUseCase diagnosisAgent;
|
||||
private final DiagnosisReleaseUseCase releaseUseCase;
|
||||
private final PublishedResultPolicy publishedPolicy;
|
||||
private final DiagnosisTraceRecorder traceRecorder;
|
||||
|
||||
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
|
||||
DiagnosisReleaseUseCase releaseUseCase,
|
||||
PublishedResultPolicy publishedPolicy) {
|
||||
this(diagnosisAgent, releaseUseCase, publishedPolicy, DiagnosisTraceRecorder.noop());
|
||||
}
|
||||
|
||||
public DiagnosisChatExecutor(DiagnosisAgentUseCase diagnosisAgent,
|
||||
DiagnosisReleaseUseCase releaseUseCase,
|
||||
PublishedResultPolicy publishedPolicy,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
this.diagnosisAgent = Objects.requireNonNull(diagnosisAgent, "diagnosisAgent must not be null");
|
||||
this.releaseUseCase = Objects.requireNonNull(releaseUseCase, "releaseUseCase must not be null");
|
||||
this.publishedPolicy = Objects.requireNonNull(publishedPolicy, "publishedPolicy must not be null");
|
||||
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -41,15 +53,22 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
|
||||
Consumer<ChatApplicationStatus> statusSink) {
|
||||
Objects.requireNonNull(statusSink, "statusSink must not be null");
|
||||
statusSink.accept(ChatApplicationStatus.DIAGNOSIS_RUNNING);
|
||||
DiagnosisDraft draft = diagnosisAgent.execute(
|
||||
context, new DiagnosisAgentInput(query, previousTurn));
|
||||
DiagnosisAgentExecution execution;
|
||||
try {
|
||||
execution = diagnosisAgent.execute(
|
||||
context, new DiagnosisAgentInput(query, previousTurn));
|
||||
} catch (DiagnosisAgentOutputException exception) {
|
||||
return recoverInvalidDraft(context, exception, statusSink);
|
||||
}
|
||||
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
|
||||
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, draft);
|
||||
DiagnosisReleaseResult released = releaseUseCase.execute(context, query, execution);
|
||||
if (released.outcome() == ReleaseOutcome.FALLBACK) {
|
||||
return new DiagnosisExecutionResult(
|
||||
ReleaseOutcome.FALLBACK,
|
||||
new FallbackContent(released.fallback()),
|
||||
null);
|
||||
null,
|
||||
execution.stopReason()
|
||||
== com.superbiz.agent.harness.progress.DiagnosisStopReason.BUDGET_LIMIT_REACHED);
|
||||
}
|
||||
PublishedResult published = publishedPolicy.create(
|
||||
query, released.draft(), released.verifiedEvidence()).orElse(null);
|
||||
@@ -60,4 +79,26 @@ public final class DiagnosisChatExecutor implements DiagnosisOperation {
|
||||
released.verifiedEvidence().verifiedSources()),
|
||||
published);
|
||||
}
|
||||
|
||||
private DiagnosisExecutionResult recoverInvalidDraft(
|
||||
RunContext context,
|
||||
DiagnosisAgentOutputException exception,
|
||||
Consumer<ChatApplicationStatus> statusSink) {
|
||||
if (!exception.isDraftContractFailure()) {
|
||||
throw exception;
|
||||
}
|
||||
boolean hasProgress = exception.progress().hasObservedFacts();
|
||||
traceRecorder.record(TraceAuditEvents.agentDraftInvalid(
|
||||
context, exception.kind(), exception.outputBytes(), hasProgress));
|
||||
if (!hasProgress) {
|
||||
throw exception;
|
||||
}
|
||||
statusSink.accept(ChatApplicationStatus.SAFETY_VALIDATING);
|
||||
DiagnosisReleaseResult released = releaseUseCase.releaseInvalidDraft(
|
||||
context, exception.progress());
|
||||
return new DiagnosisExecutionResult(
|
||||
ReleaseOutcome.FALLBACK,
|
||||
new FallbackContent(released.fallback()),
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -16,6 +16,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.RagEvidence;
|
||||
@@ -106,7 +107,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
|
||||
}
|
||||
core.reserveRunBytes(context, bytes);
|
||||
String output = modelCall.call(
|
||||
context,
|
||||
context, ModelCallComponent.KNOWLEDGE_ANSWER,
|
||||
new Prompt(List.of(new SystemMessage(prompt), new UserMessage(modelInput))),
|
||||
limits.modelTimeout(),
|
||||
limits.maxModelOutputBytes());
|
||||
|
||||
+2
-1
@@ -7,6 +7,7 @@ import com.superbiz.agent.harness.application.SystemChatOperation;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
@@ -47,7 +48,7 @@ public final class SystemChatExecutor implements SystemChatOperation {
|
||||
}
|
||||
core.reserveRunBytes(context, bytes);
|
||||
String answer = modelCall.call(
|
||||
context,
|
||||
context, ModelCallComponent.SYSTEM_CHAT,
|
||||
new Prompt(List.of(new SystemMessage(SYSTEM_PROMPT), new UserMessage(query))),
|
||||
limits.timeout(),
|
||||
limits.maxOutputBytes());
|
||||
|
||||
@@ -12,6 +12,7 @@ import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.core.RunBudgetUsage;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.repository.ChatSessionRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import org.springframework.stereotype.Component;
|
||||
@@ -119,6 +120,8 @@ public class JpaChatRunStore implements ChatRunStore {
|
||||
run.setTotalDurationMs(Math.max(0, durationMs));
|
||||
RunBudgetUsage usage = context.budget().snapshot();
|
||||
run.setTotalTokenCount(saturatingInt(usage.totalTokens()));
|
||||
run.setStepCount(context.modelCalls().componentCallCount(
|
||||
ModelCallComponent.DIAGNOSIS_AGENT));
|
||||
run.setToolCallCount(saturatingInt(usage.toolCalls()));
|
||||
runs.save(run);
|
||||
if (outcome == ReleaseOutcome.SUCCESS || outcome == ReleaseOutcome.FALLBACK) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.application.IntentRouting;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
@@ -86,7 +87,8 @@ public final class IntentRouter implements IntentRouting {
|
||||
context,
|
||||
context.retryPolicies().intentRouter(),
|
||||
() -> parse(modelCall.call(
|
||||
context, prompt, remaining(started), limits.maxOutputBytes())),
|
||||
context, ModelCallComponent.INTENT_ROUTER,
|
||||
prompt, remaining(started), limits.maxOutputBytes())),
|
||||
this::classify,
|
||||
attempt -> {
|
||||
attemptRecorder.accept(attempt);
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public final class ModelCallAuditor {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(ModelCallAuditor.class);
|
||||
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final DiagnosisTraceRecorder traceRecorder;
|
||||
private final AgentStepRepository agentSteps;
|
||||
|
||||
public ModelCallAuditor(DiagnosisHarnessCore core) {
|
||||
this(core, DiagnosisTraceRecorder.noop(), null);
|
||||
}
|
||||
|
||||
public ModelCallAuditor(DiagnosisHarnessCore core,
|
||||
DiagnosisTraceRecorder traceRecorder,
|
||||
AgentStepRepository agentSteps) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
|
||||
this.agentSteps = agentSteps;
|
||||
}
|
||||
|
||||
public ModelCallLedger.Call begin(RunContext context, ModelCallComponent component) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
return context.modelCalls().begin(component);
|
||||
}
|
||||
|
||||
public void recordUsage(RunContext context, ModelCallLedger.Call call,
|
||||
long inputTokens, long outputTokens, boolean usageAvailable) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (!context.modelCalls().record(call, inputTokens, outputTokens, usageAvailable)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (usageAvailable) {
|
||||
core.recordTokens(context, inputTokens, outputTokens);
|
||||
}
|
||||
} finally {
|
||||
traceRecorder.record(TraceAuditEvents.modelTokenUsage(
|
||||
context, call, inputTokens, outputTokens, usageAvailable));
|
||||
updateAgentStep(context, call, inputTokens, outputTokens, usageAvailable);
|
||||
}
|
||||
}
|
||||
|
||||
private void updateAgentStep(RunContext context, ModelCallLedger.Call call,
|
||||
long inputTokens, long outputTokens, boolean usageAvailable) {
|
||||
if (!usageAvailable || agentSteps == null
|
||||
|| call.component() != ModelCallComponent.DIAGNOSIS_AGENT) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
agentSteps.findByRunIdAndStepIndex(context.runId(), call.componentRound() - 1)
|
||||
.ifPresent(step -> saveTokens(step, inputTokens, outputTokens));
|
||||
} catch (RuntimeException exception) {
|
||||
log.warn("Failed to persist AgentStep token audit: runId={}, round={}",
|
||||
context.runId(), call.componentRound());
|
||||
}
|
||||
}
|
||||
|
||||
private void saveTokens(AgentStep step, long inputTokens, long outputTokens) {
|
||||
step.setTokenCount(saturatingInt(safeAdd(inputTokens, outputTokens)));
|
||||
agentSteps.save(step);
|
||||
}
|
||||
|
||||
private static long safeAdd(long left, long right) {
|
||||
try {
|
||||
return Math.addExact(left, right);
|
||||
} catch (ArithmeticException exception) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
private static int saturatingInt(long value) {
|
||||
return value >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) Math.max(0, value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
public enum ModelCallComponent {
|
||||
INTENT_ROUTER(TracePhase.ROUTING),
|
||||
SYSTEM_CHAT(TracePhase.AGENT),
|
||||
KNOWLEDGE_ANSWER(TracePhase.AGENT),
|
||||
DIAGNOSIS_AGENT(TracePhase.AGENT),
|
||||
EVIDENCE_REPAIR(TracePhase.EVIDENCE),
|
||||
SEMANTIC_GUARD(TracePhase.SEMANTIC);
|
||||
|
||||
private final TracePhase tracePhase;
|
||||
|
||||
ModelCallComponent(TracePhase tracePhase) {
|
||||
this.tracePhase = tracePhase;
|
||||
}
|
||||
|
||||
public TracePhase tracePhase() {
|
||||
return tracePhase;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import java.util.EnumMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public final class ModelCallLedger {
|
||||
|
||||
private final Map<ModelCallComponent, Integer> componentRounds =
|
||||
new EnumMap<>(ModelCallComponent.class);
|
||||
private final Set<Call> auditedCalls = new HashSet<>();
|
||||
|
||||
private int startedCallCount;
|
||||
private int usageUnavailableCount;
|
||||
private long inputTokens;
|
||||
private long outputTokens;
|
||||
private long totalTokens;
|
||||
|
||||
public synchronized Call begin(ModelCallComponent component) {
|
||||
Objects.requireNonNull(component, "component must not be null");
|
||||
int round = componentRounds.merge(component, 1, Integer::sum);
|
||||
startedCallCount++;
|
||||
return new Call(component, round);
|
||||
}
|
||||
|
||||
public synchronized boolean record(Call call, long input, long output, boolean usageAvailable) {
|
||||
Objects.requireNonNull(call, "call must not be null");
|
||||
if (input < 0 || output < 0) {
|
||||
throw new IllegalArgumentException("token counts must not be negative");
|
||||
}
|
||||
if (!auditedCalls.add(call)) {
|
||||
return false;
|
||||
}
|
||||
if (!usageAvailable) {
|
||||
usageUnavailableCount++;
|
||||
return true;
|
||||
}
|
||||
inputTokens = safeAdd(inputTokens, input);
|
||||
outputTokens = safeAdd(outputTokens, output);
|
||||
totalTokens = safeAdd(totalTokens, safeAdd(input, output));
|
||||
return true;
|
||||
}
|
||||
|
||||
public synchronized int componentCallCount(ModelCallComponent component) {
|
||||
Objects.requireNonNull(component, "component must not be null");
|
||||
return componentRounds.getOrDefault(component, 0);
|
||||
}
|
||||
|
||||
public synchronized Snapshot snapshot() {
|
||||
return new Snapshot(
|
||||
startedCallCount,
|
||||
auditedCalls.size(),
|
||||
usageUnavailableCount,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens);
|
||||
}
|
||||
|
||||
private static long safeAdd(long left, long right) {
|
||||
try {
|
||||
return Math.addExact(left, right);
|
||||
} catch (ArithmeticException exception) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
public record Call(ModelCallComponent component, int componentRound) {
|
||||
public Call {
|
||||
Objects.requireNonNull(component, "component must not be null");
|
||||
if (componentRound < 1) {
|
||||
throw new IllegalArgumentException("componentRound must be positive");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public record Snapshot(
|
||||
int startedCallCount,
|
||||
int auditedCallCount,
|
||||
int usageUnavailableCount,
|
||||
long inputTokens,
|
||||
long outputTokens,
|
||||
long totalTokens) {
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
@@ -8,6 +9,8 @@ import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -33,11 +36,27 @@ public final class TraceAuditEvents {
|
||||
|
||||
public static DiagnosisTraceAuditEvent runFinished(
|
||||
RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) {
|
||||
var budget = context.budget().snapshot();
|
||||
ModelCallLedger.Snapshot modelCalls = context.modelCalls().snapshot();
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
if (intent != null) {
|
||||
details.put("intent", intent.name());
|
||||
}
|
||||
details.put("release_outcome", outcome.name());
|
||||
details.put("model_call_count", budget.modelCalls());
|
||||
details.put("run_input_tokens", budget.inputTokens());
|
||||
details.put("run_output_tokens", budget.outputTokens());
|
||||
details.put("run_total_tokens", budget.totalTokens());
|
||||
details.put("audited_model_call_count", modelCalls.auditedCallCount());
|
||||
details.put("audited_input_tokens", modelCalls.inputTokens());
|
||||
details.put("audited_output_tokens", modelCalls.outputTokens());
|
||||
details.put("audited_total_tokens", modelCalls.totalTokens());
|
||||
details.put("usage_unavailable_count", modelCalls.usageUnavailableCount());
|
||||
details.put("tokens_reconciled", budget.modelCalls() == modelCalls.auditedCallCount()
|
||||
&& modelCalls.usageUnavailableCount() == 0
|
||||
&& budget.inputTokens() == modelCalls.inputTokens()
|
||||
&& budget.outputTokens() == modelCalls.outputTokens()
|
||||
&& budget.totalTokens() == modelCalls.totalTokens());
|
||||
return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED,
|
||||
terminalStatus(outcome), null, durationMs, details);
|
||||
}
|
||||
@@ -66,6 +85,41 @@ public final class TraceAuditEvents {
|
||||
TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent modelTokenUsage(
|
||||
RunContext context,
|
||||
ModelCallLedger.Call call,
|
||||
long inputTokens,
|
||||
long outputTokens,
|
||||
boolean usageAvailable) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("component", call.component().name());
|
||||
details.put("component_round", call.componentRound());
|
||||
details.put("usage_available", usageAvailable);
|
||||
if (usageAvailable) {
|
||||
details.put("input_tokens", inputTokens);
|
||||
details.put("output_tokens", outputTokens);
|
||||
details.put("total_tokens", safeAdd(inputTokens, outputTokens));
|
||||
}
|
||||
return new DiagnosisTraceAuditEvent(
|
||||
context.sessionId(), context.runId(), call.component().tracePhase(),
|
||||
TraceEventType.MODEL_TOKEN_USAGE,
|
||||
usageAvailable ? TraceEventStatus.SUCCEEDED : TraceEventStatus.UNAVAILABLE,
|
||||
call.componentRound(), null, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent agentDraftInvalid(
|
||||
RunContext context,
|
||||
DiagnosisAgentOutputException.Kind kind,
|
||||
long outputBytes,
|
||||
boolean hasPublishableProgress) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("failure_kind", kind.name());
|
||||
details.put("output_bytes", outputBytes);
|
||||
details.put("has_publishable_progress", hasPublishableProgress);
|
||||
return event(context, TracePhase.AGENT, TraceEventType.AGENT_DRAFT_INVALID,
|
||||
TraceEventStatus.REJECTED, null, null, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("tool_call_id", tool.toolCallId());
|
||||
@@ -84,6 +138,55 @@ public final class TraceAuditEvents {
|
||||
null, tool.durationMs(), details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent toolRequestRejected(
|
||||
RunContext context, String toolCallId, String toolName, String errorCode) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("tool_call_id", safeIdentifier(toolCallId));
|
||||
details.put("tool_name", safeIdentifier(toolName));
|
||||
details.put("error_code", safeIdentifier(errorCode));
|
||||
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
|
||||
TraceEventStatus.REJECTED, null, null, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent toolProgress(
|
||||
RunContext context,
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
String scopeSummary,
|
||||
InformationGain informationGain,
|
||||
String producer,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("tool_call_id", toolCallId);
|
||||
details.put("tool_name", toolName);
|
||||
details.put("scope_summary", scopeSummary);
|
||||
details.put("information_gain", informationGain.name());
|
||||
details.put("producer", producer);
|
||||
details.put("consecutive_no_gain", state.consecutiveNoGain());
|
||||
details.put("collection_state", state.collectionState().name());
|
||||
if (state.stopReason() != null) {
|
||||
details.put("stop_reason", state.stopReason().name());
|
||||
}
|
||||
return event(context, TracePhase.TOOL, TraceEventType.TOOL_PROGRESS,
|
||||
TraceEventStatus.SUCCEEDED, null, null, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent collectionStop(
|
||||
RunContext context,
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("tool_call_id", toolCallId);
|
||||
details.put("tool_name", toolName);
|
||||
details.put("collection_state", state.collectionState().name());
|
||||
if (state.stopReason() != null) {
|
||||
details.put("stop_reason", state.stopReason().name());
|
||||
}
|
||||
return event(context, TracePhase.TOOL, TraceEventType.COLLECTION_STOP,
|
||||
TraceEventStatus.SUCCEEDED, null, null, details);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent evidenceValidation(
|
||||
RunContext context, TraceEventType type,
|
||||
EvidenceGuardResult result, DiagnosisDraft draft) {
|
||||
@@ -198,6 +301,18 @@ public final class TraceAuditEvents {
|
||||
return new ToolReferences(List.copyOf(safeIds), invalid);
|
||||
}
|
||||
|
||||
private static String safeIdentifier(String value) {
|
||||
return value != null && SAFE_TOOL_CALL_ID.matcher(value).matches() ? value : "INVALID";
|
||||
}
|
||||
|
||||
private static long safeAdd(long left, long right) {
|
||||
try {
|
||||
return Math.addExact(left, right);
|
||||
} catch (ArithmeticException exception) {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
private record ToolReferences(List<String> safeIds, int invalidCount) {
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,13 @@ public enum TraceEventType {
|
||||
RUN_STARTED,
|
||||
ROUTING_ATTEMPT,
|
||||
ROUTING_DECISION,
|
||||
MODEL_TOKEN_USAGE,
|
||||
AGENT_MODEL_STEP,
|
||||
AGENT_DRAFT_INVALID,
|
||||
TOOL_INVOCATION,
|
||||
TOOL_REQUEST_REJECTED,
|
||||
TOOL_PROGRESS,
|
||||
COLLECTION_STOP,
|
||||
EVIDENCE_GUARD_INITIAL,
|
||||
EVIDENCE_REPAIR_ATTEMPT,
|
||||
EVIDENCE_GUARD_RECHECK,
|
||||
|
||||
@@ -3,5 +3,8 @@ package com.superbiz.agent.harness.contract;
|
||||
public enum FallbackType {
|
||||
EVIDENCE_VALIDATION_FAILED,
|
||||
SEMANTIC_UNSUPPORTED,
|
||||
SEMANTIC_UNAVAILABLE
|
||||
SEMANTIC_UNAVAILABLE,
|
||||
BUDGET_EXHAUSTED,
|
||||
INSUFFICIENT_EVIDENCE,
|
||||
MISSING_REQUIRED_CONTEXT
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package com.superbiz.agent.harness.core;
|
||||
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
|
||||
import com.superbiz.agent.harness.audit.ModelCallLedger;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
@@ -15,17 +17,31 @@ public final class DiagnosisHarnessCore {
|
||||
private final Duration maxRunDuration;
|
||||
private final RunBudgetLimits budgetLimits;
|
||||
private final HarnessRetryPolicies retryPolicies;
|
||||
private final int stopAfterConsecutiveNoGain;
|
||||
|
||||
public DiagnosisHarnessCore(Clock clock,
|
||||
Supplier<String> runIdSupplier,
|
||||
Duration maxRunDuration,
|
||||
RunBudgetLimits budgetLimits,
|
||||
HarnessRetryPolicies retryPolicies) {
|
||||
this(clock, runIdSupplier, maxRunDuration, budgetLimits, retryPolicies, 2);
|
||||
}
|
||||
|
||||
public DiagnosisHarnessCore(Clock clock,
|
||||
Supplier<String> runIdSupplier,
|
||||
Duration maxRunDuration,
|
||||
RunBudgetLimits budgetLimits,
|
||||
HarnessRetryPolicies retryPolicies,
|
||||
int stopAfterConsecutiveNoGain) {
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
this.runIdSupplier = Objects.requireNonNull(runIdSupplier, "runIdSupplier must not be null");
|
||||
this.maxRunDuration = requirePositive(maxRunDuration, "maxRunDuration");
|
||||
this.budgetLimits = Objects.requireNonNull(budgetLimits, "budgetLimits must not be null");
|
||||
this.retryPolicies = Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
|
||||
if (stopAfterConsecutiveNoGain <= 0) {
|
||||
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
|
||||
}
|
||||
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
|
||||
}
|
||||
|
||||
public RunContext startRun(String sessionId) {
|
||||
@@ -42,8 +58,10 @@ public final class DiagnosisHarnessCore {
|
||||
deadline,
|
||||
cancellation,
|
||||
new RunBudget(budgetLimits),
|
||||
new ModelCallLedger(),
|
||||
retryPolicies,
|
||||
lifecycle);
|
||||
lifecycle,
|
||||
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain));
|
||||
cancellation.onCancel(reason -> lifecycle.finish(terminalState(reason), reason.name()));
|
||||
return context;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package com.superbiz.agent.harness.core;
|
||||
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressTracker;
|
||||
import com.superbiz.agent.harness.audit.ModelCallLedger;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Objects;
|
||||
@@ -14,8 +16,10 @@ public record RunContext(
|
||||
Instant deadline,
|
||||
RunCancellation cancellation,
|
||||
RunBudget budget,
|
||||
ModelCallLedger modelCalls,
|
||||
HarnessRetryPolicies retryPolicies,
|
||||
RunLifecycle lifecycle) {
|
||||
RunLifecycle lifecycle,
|
||||
DiagnosisProgressTracker progress) {
|
||||
|
||||
public RunContext {
|
||||
requireText(sessionId, "sessionId");
|
||||
@@ -23,8 +27,10 @@ public record RunContext(
|
||||
Objects.requireNonNull(deadline, "deadline must not be null");
|
||||
Objects.requireNonNull(cancellation, "cancellation must not be null");
|
||||
Objects.requireNonNull(budget, "budget must not be null");
|
||||
Objects.requireNonNull(modelCalls, "modelCalls must not be null");
|
||||
Objects.requireNonNull(retryPolicies, "retryPolicies must not be null");
|
||||
Objects.requireNonNull(lifecycle, "lifecycle must not be null");
|
||||
Objects.requireNonNull(progress, "progress must not be null");
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
|
||||
@@ -79,6 +79,40 @@ public final class EvidenceGuard {
|
||||
: EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
public EvidenceGuardResult validateNoConclusionReferences(
|
||||
RunContext context, DiagnosisDraft draft) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (draft == null) {
|
||||
return EvidenceGuardResult.invalid(List.of(
|
||||
violation(EvidenceViolationCode.DRAFT_MISSING, "draft")));
|
||||
}
|
||||
if (draft.conclusion() != null) {
|
||||
throw new IllegalArgumentException("no-conclusion validation requires conclusion=null");
|
||||
}
|
||||
|
||||
List<EvidenceViolation> violations = new ArrayList<>();
|
||||
for (int index = 0; index < draft.analysis().size(); index++) {
|
||||
DiagnosisDraft.AnalysisItem analysis = draft.analysis().get(index);
|
||||
if (analysis == null || analysis.toolCallIds().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
if (analysis.kind() == null) {
|
||||
violations.add(violation(
|
||||
EvidenceViolationCode.ANALYSIS_KIND_MISSING,
|
||||
"analysis[" + index + "].kind"));
|
||||
continue;
|
||||
}
|
||||
List<VerifiedEvidence> ignoredEvidence = new ArrayList<>();
|
||||
for (String toolCallId : analysis.toolCallIds()) {
|
||||
verifyInvocation(
|
||||
context, analysis, index, toolCallId, ignoredEvidence, violations);
|
||||
}
|
||||
}
|
||||
return violations.isEmpty()
|
||||
? EvidenceGuardResult.valid(VerifiedEvidenceSnapshot.empty())
|
||||
: EvidenceGuardResult.invalid(violations);
|
||||
}
|
||||
|
||||
private List<EvidenceViolation> validateDraft(DiagnosisDraft draft) {
|
||||
List<EvidenceViolation> violations = new ArrayList<>();
|
||||
if (draft == null) {
|
||||
|
||||
@@ -4,6 +4,9 @@ import com.superbiz.agent.harness.core.BudgetExceededException;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunAbortedException;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.audit.ModelCallLedger;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.Usage;
|
||||
@@ -26,15 +29,23 @@ public final class GuardModelCall {
|
||||
private final DiagnosisHarnessCore core;
|
||||
private final ChatModel chatModel;
|
||||
private final ExecutorService executor;
|
||||
private final ModelCallAuditor auditor;
|
||||
|
||||
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
|
||||
ExecutorService executor) {
|
||||
this(core, chatModel, executor, new ModelCallAuditor(core));
|
||||
}
|
||||
|
||||
public GuardModelCall(DiagnosisHarnessCore core, ChatModel chatModel,
|
||||
ExecutorService executor, ModelCallAuditor auditor) {
|
||||
this.core = Objects.requireNonNull(core, "core must not be null");
|
||||
this.chatModel = Objects.requireNonNull(chatModel, "chatModel must not be null");
|
||||
this.executor = Objects.requireNonNull(executor, "executor must not be null");
|
||||
this.auditor = Objects.requireNonNull(auditor, "auditor must not be null");
|
||||
}
|
||||
|
||||
public String call(RunContext context, Prompt prompt, Duration timeout, long maxOutputBytes) {
|
||||
public String call(RunContext context, ModelCallComponent component,
|
||||
Prompt prompt, Duration timeout, long maxOutputBytes) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(prompt, "prompt must not be null");
|
||||
Objects.requireNonNull(timeout, "timeout must not be null");
|
||||
@@ -42,20 +53,24 @@ public final class GuardModelCall {
|
||||
throw new IllegalArgumentException("timeout and output limit must be positive");
|
||||
}
|
||||
core.beforeModelCall(context);
|
||||
Future<String> future = executor.submit(() -> invoke(context, prompt, maxOutputBytes));
|
||||
ModelCallLedger.Call call = auditor.begin(context, component);
|
||||
Future<String> future = executor.submit(() -> invoke(context, call, prompt, maxOutputBytes));
|
||||
context.cancellation().onCancel(ignored -> future.cancel(true));
|
||||
try {
|
||||
return future.get(timeout.toNanos(), TimeUnit.NANOSECONDS);
|
||||
} catch (TimeoutException exception) {
|
||||
future.cancel(true);
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TIMEOUT, "Guard model attempt timed out", exception);
|
||||
} catch (CancellationException exception) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model attempt was cancelled", exception);
|
||||
} catch (InterruptedException exception) {
|
||||
future.cancel(true);
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
Thread.currentThread().interrupt();
|
||||
core.checkActive(context);
|
||||
throw new GuardModelCallException(
|
||||
@@ -76,15 +91,17 @@ public final class GuardModelCall {
|
||||
}
|
||||
}
|
||||
|
||||
private String invoke(RunContext context, Prompt prompt, long maxOutputBytes) {
|
||||
private String invoke(RunContext context, ModelCallLedger.Call call,
|
||||
Prompt prompt, long maxOutputBytes) {
|
||||
ChatResponse response;
|
||||
try {
|
||||
response = chatModel.call(prompt);
|
||||
} catch (RuntimeException exception) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
throw new GuardModelCallException(
|
||||
RetryFailure.TRANSPORT, "Guard model transport failed", exception);
|
||||
}
|
||||
recordUsage(context, response);
|
||||
recordUsage(context, call, response);
|
||||
core.checkActive(context);
|
||||
AssistantMessage output = response == null || response.getResult() == null
|
||||
? null : response.getResult().getOutput();
|
||||
@@ -102,16 +119,18 @@ public final class GuardModelCall {
|
||||
return output.getText();
|
||||
}
|
||||
|
||||
private void recordUsage(RunContext context, ChatResponse response) {
|
||||
private void recordUsage(RunContext context, ModelCallLedger.Call call, ChatResponse response) {
|
||||
if (response == null || response.getMetadata() == null) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
return;
|
||||
}
|
||||
Usage usage = response.getMetadata().getUsage();
|
||||
if (usage == null) {
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
return;
|
||||
}
|
||||
core.recordTokens(context, nonNegative(usage.getPromptTokens()),
|
||||
nonNegative(usage.getCompletionTokens()));
|
||||
auditor.recordUsage(context, call, nonNegative(usage.getPromptTokens()),
|
||||
nonNegative(usage.getCompletionTokens()), true);
|
||||
}
|
||||
|
||||
private static long nonNegative(Integer value) {
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
@@ -81,7 +82,8 @@ public final class SemanticGuard {
|
||||
context,
|
||||
context.retryPolicies().semanticGuard(),
|
||||
() -> parse(modelCall.call(
|
||||
context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
|
||||
context, ModelCallComponent.SEMANTIC_GUARD,
|
||||
modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())),
|
||||
this::classify,
|
||||
attempt -> {
|
||||
attemptRecorder.accept(attempt);
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public record CompletedToolCall(
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
String normalizedScope) {
|
||||
|
||||
public CompletedToolCall {
|
||||
requireText(toolCallId, "toolCallId");
|
||||
requireText(toolName, "toolName");
|
||||
requireText(normalizedScope, "normalizedScope");
|
||||
}
|
||||
|
||||
private static void requireText(String value, String name) {
|
||||
if (value == null || value.isBlank()) {
|
||||
throw new IllegalArgumentException(name + " must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public enum DiagnosisCollectionState {
|
||||
COLLECTING,
|
||||
SATURATED
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
|
||||
@FunctionalInterface
|
||||
public interface DiagnosisProgressProjection {
|
||||
|
||||
DiagnosisProgressSnapshot project(RunContext context);
|
||||
|
||||
static DiagnosisProgressProjection empty() {
|
||||
return ignored -> DiagnosisProgressSnapshot.empty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class DiagnosisProgressProjector implements DiagnosisProgressProjection {
|
||||
|
||||
private static final int MAX_FACTS = 12;
|
||||
private static final int MAX_SUMMARY_CHARS = 320;
|
||||
private static final int MAX_SCOPE_CHARS = 320;
|
||||
|
||||
private final CanonicalInvocationStore store;
|
||||
private final ToolCallKeyFactory keyFactory;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public DiagnosisProgressProjector(CanonicalInvocationStore store,
|
||||
ToolCallKeyFactory keyFactory,
|
||||
ObjectMapper objectMapper) {
|
||||
this.store = Objects.requireNonNull(store, "store must not be null");
|
||||
this.keyFactory = Objects.requireNonNull(keyFactory, "keyFactory must not be null");
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public DiagnosisProgressSnapshot project(RunContext context) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
DiagnosisProgressSnapshotState state = context.progress().snapshot();
|
||||
Map<String, SafeFallback.VerifiedSource> sources = new LinkedHashMap<>();
|
||||
Map<String, SafeFallback.ObservedFact> facts = new LinkedHashMap<>();
|
||||
List<String> limitations = new ArrayList<>();
|
||||
for (CompletedToolCall completed : state.completedToolCalls()) {
|
||||
CanonicalToolInvocation invocation = resolve(context, completed, limitations);
|
||||
if (invocation == null) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
projectInvocation(completed, invocation, sources, facts);
|
||||
} catch (RuntimeException exception) {
|
||||
addLimitation(limitations, "部分已完成的工具结果格式无法验证,未纳入已检查事实");
|
||||
}
|
||||
if (facts.size() >= MAX_FACTS) {
|
||||
addLimitation(limitations, "已检查事实较多,展示内容已截断");
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new DiagnosisProgressSnapshot(
|
||||
List.copyOf(sources.values()),
|
||||
List.copyOf(facts.values()),
|
||||
List.copyOf(limitations),
|
||||
state.stopReason());
|
||||
}
|
||||
|
||||
private CanonicalToolInvocation resolve(RunContext context,
|
||||
CompletedToolCall completed,
|
||||
List<String> limitations) {
|
||||
try {
|
||||
String key = keyFactory.create(context.runId(), completed.toolCallId());
|
||||
CanonicalToolInvocation invocation = store.find(key).orElse(null);
|
||||
if (invocation == null
|
||||
|| !invocation.isReferencableBy(context.runId())
|
||||
|| !completed.toolCallId().equals(invocation.toolCallId())
|
||||
|| !completed.toolName().equals(invocation.toolName())) {
|
||||
addLimitation(limitations, "部分已完成的工具记录无法验证,未纳入已检查事实");
|
||||
return null;
|
||||
}
|
||||
return invocation;
|
||||
} catch (RuntimeException exception) {
|
||||
addLimitation(limitations, "部分已完成的工具记录暂时不可读取,未纳入已检查事实");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void projectInvocation(CompletedToolCall completed,
|
||||
CanonicalToolInvocation invocation,
|
||||
Map<String, SafeFallback.VerifiedSource> sources,
|
||||
Map<String, SafeFallback.ObservedFact> facts) {
|
||||
JsonNode root = readObject(invocation.agentResult());
|
||||
String scope = publicScope(completed.toolName(), completed.normalizedScope(), root);
|
||||
switch (completed.toolName()) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE -> projectRag(root, scope, sources, facts);
|
||||
case AgentToolContracts.QUERY_LOGS -> projectLogs(root, scope, sources, facts);
|
||||
case AgentToolContracts.QUERY_MYSQL -> projectMysql(root, scope, sources, facts);
|
||||
default -> throw new IllegalArgumentException("Unsupported evidence Tool");
|
||||
}
|
||||
}
|
||||
|
||||
private void projectRag(JsonNode root,
|
||||
String scope,
|
||||
Map<String, SafeFallback.VerifiedSource> sources,
|
||||
Map<String, SafeFallback.ObservedFact> facts) {
|
||||
JsonNode evidence = root.path("evidence");
|
||||
if (!evidence.isArray() || evidence.isEmpty()) {
|
||||
addFact(sources, facts, "RAG", "knowledge_base", scope,
|
||||
"该知识检索范围内未发现可用文档证据");
|
||||
return;
|
||||
}
|
||||
for (JsonNode item : evidence) {
|
||||
String source = firstNonBlank(text(item, "source"), text(item, "title"),
|
||||
text(item, "document_id"), "knowledge_base");
|
||||
addFact(sources, facts, "RAG", source, scope,
|
||||
firstNonBlank(text(item, "excerpt"), "已找到候选知识片段"));
|
||||
}
|
||||
}
|
||||
|
||||
private void projectLogs(JsonNode root,
|
||||
String scope,
|
||||
Map<String, SafeFallback.VerifiedSource> sources,
|
||||
Map<String, SafeFallback.ObservedFact> facts) {
|
||||
String source = firstNonBlank(text(root, "source_kind"), "logs");
|
||||
JsonNode events = root.path("events");
|
||||
if (!events.isArray() || events.isEmpty()) {
|
||||
addFact(sources, facts, "LOG", source, scope,
|
||||
"该日志查询范围内未发现匹配事件");
|
||||
return;
|
||||
}
|
||||
for (JsonNode event : events) {
|
||||
addFact(sources, facts, "LOG", source, scope,
|
||||
firstNonBlank(text(event, "message"), "已找到匹配日志事件"));
|
||||
}
|
||||
}
|
||||
|
||||
private void projectMysql(JsonNode root,
|
||||
String scope,
|
||||
Map<String, SafeFallback.VerifiedSource> sources,
|
||||
Map<String, SafeFallback.ObservedFact> facts) {
|
||||
String source = mysqlSource(scope);
|
||||
JsonNode rows = root.path("rows");
|
||||
if (!rows.isArray() || rows.isEmpty()) {
|
||||
addFact(sources, facts, "MYSQL", source, scope,
|
||||
"该只读数据查询范围内未发现匹配记录");
|
||||
return;
|
||||
}
|
||||
for (JsonNode row : rows) {
|
||||
addFact(sources, facts, "MYSQL", source, scope, row.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private String publicScope(String toolName, String normalizedScope, JsonNode result) {
|
||||
if (AgentToolContracts.LOOKUP_KNOWLEDGE.equals(toolName)) {
|
||||
return bounded("query=" + text(result, "query"), MAX_SCOPE_CHARS);
|
||||
}
|
||||
if (AgentToolContracts.QUERY_LOGS.equals(toolName)) {
|
||||
JsonNode scope = result.path("scope");
|
||||
return bounded(scope.isObject() ? scope.toString() : normalizedScope, MAX_SCOPE_CHARS);
|
||||
}
|
||||
try {
|
||||
JsonNode scope = objectMapper.readTree(normalizedScope);
|
||||
return bounded("data_source=" + text(scope, "data_source"), MAX_SCOPE_CHARS);
|
||||
} catch (JsonProcessingException exception) {
|
||||
return "data_source=unknown";
|
||||
}
|
||||
}
|
||||
|
||||
private String mysqlSource(String scope) {
|
||||
int separator = scope.indexOf('=');
|
||||
return separator < 0 ? "mysql" : scope.substring(separator + 1);
|
||||
}
|
||||
|
||||
private void addFact(Map<String, SafeFallback.VerifiedSource> sources,
|
||||
Map<String, SafeFallback.ObservedFact> facts,
|
||||
String sourceType,
|
||||
String source,
|
||||
String scope,
|
||||
String summary) {
|
||||
if (facts.size() >= MAX_FACTS) {
|
||||
return;
|
||||
}
|
||||
String safeSource = bounded(source, 160);
|
||||
String safeScope = bounded(scope, MAX_SCOPE_CHARS);
|
||||
String safeSummary = bounded(summary, MAX_SUMMARY_CHARS);
|
||||
String sourceKey = sourceType + '\u0000' + safeSource + '\u0000' + safeScope;
|
||||
sources.putIfAbsent(sourceKey,
|
||||
new SafeFallback.VerifiedSource(sourceType, safeSource, safeScope));
|
||||
String factKey = sourceKey + '\u0000' + safeSummary;
|
||||
facts.putIfAbsent(factKey,
|
||||
new SafeFallback.ObservedFact(sourceType, safeSource, safeScope, safeSummary));
|
||||
}
|
||||
|
||||
private JsonNode readObject(String value) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(value);
|
||||
if (root == null || !root.isObject()) {
|
||||
throw new IllegalArgumentException("Canonical Agent result must be an object");
|
||||
}
|
||||
return root;
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalArgumentException("Canonical Agent result is invalid", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addLimitation(List<String> limitations, String value) {
|
||||
if (!limitations.contains(value)) {
|
||||
limitations.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private static String firstNonBlank(String... values) {
|
||||
for (String value : values) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
private static String text(JsonNode node, String field) {
|
||||
JsonNode value = node == null ? null : node.get(field);
|
||||
return value == null || value.isNull() ? "" : value.asText("");
|
||||
}
|
||||
|
||||
private static String bounded(String value, int max) {
|
||||
String safe = value == null ? "" : value;
|
||||
return safe.length() <= max ? safe : safe.substring(0, max);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record DiagnosisProgressSnapshot(
|
||||
List<SafeFallback.VerifiedSource> verifiedSources,
|
||||
List<SafeFallback.ObservedFact> observedFacts,
|
||||
List<String> limitations,
|
||||
DiagnosisStopReason stopReason) {
|
||||
|
||||
public DiagnosisProgressSnapshot {
|
||||
verifiedSources = verifiedSources == null ? List.of() : List.copyOf(verifiedSources);
|
||||
observedFacts = observedFacts == null ? List.of() : List.copyOf(observedFacts);
|
||||
limitations = limitations == null ? List.of() : List.copyOf(limitations);
|
||||
}
|
||||
|
||||
public static DiagnosisProgressSnapshot empty() {
|
||||
return new DiagnosisProgressSnapshot(List.of(), List.of(), List.of(), null);
|
||||
}
|
||||
|
||||
public boolean hasObservedFacts() {
|
||||
return !observedFacts.isEmpty();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public record DiagnosisProgressSnapshotState(
|
||||
int consecutiveNoGain,
|
||||
DiagnosisCollectionState collectionState,
|
||||
DiagnosisStopReason stopReason,
|
||||
String pendingToolCallId,
|
||||
boolean stopInstructionDelivered,
|
||||
List<CompletedToolCall> completedToolCalls) {
|
||||
|
||||
public DiagnosisProgressSnapshotState {
|
||||
completedToolCalls = completedToolCalls == null ? List.of() : List.copyOf(completedToolCalls);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
public final class DiagnosisProgressTracker {
|
||||
|
||||
private final int stopAfterConsecutiveNoGain;
|
||||
private final Set<ToolScopeIdentity> completedScopes = new LinkedHashSet<>();
|
||||
private final List<CompletedToolCall> completedToolCalls = new ArrayList<>();
|
||||
private int consecutiveNoGain;
|
||||
private DiagnosisCollectionState collectionState = DiagnosisCollectionState.COLLECTING;
|
||||
private DiagnosisStopReason stopReason;
|
||||
private String pendingToolCallId;
|
||||
private boolean stopInstructionDelivered;
|
||||
|
||||
public DiagnosisProgressTracker(int stopAfterConsecutiveNoGain) {
|
||||
if (stopAfterConsecutiveNoGain <= 0) {
|
||||
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
|
||||
}
|
||||
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
|
||||
}
|
||||
|
||||
public synchronized void applyPreviousObservation(PreviousObservation observation) {
|
||||
if (pendingToolCallId == null) {
|
||||
if (observation != null) {
|
||||
throw new IllegalArgumentException("No Tool observation is pending evaluation");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (observation == null) {
|
||||
throw new IllegalArgumentException("Previous Tool observation must be evaluated");
|
||||
}
|
||||
if (!pendingToolCallId.equals(observation.toolCallId())) {
|
||||
throw new IllegalArgumentException("Previous Tool observation ID is out of order");
|
||||
}
|
||||
pendingToolCallId = null;
|
||||
applyGain(observation.informationGain());
|
||||
}
|
||||
|
||||
public synchronized boolean isDuplicate(String toolName, String normalizedScope) {
|
||||
return completedScopes.contains(new ToolScopeIdentity(toolName, normalizedScope));
|
||||
}
|
||||
|
||||
public synchronized void recordDuplicateScope() {
|
||||
applyGain(InformationGain.NO_GAIN);
|
||||
}
|
||||
|
||||
public synchronized void recordCompleted(CompletedToolCall call, EvidenceStatus evidenceStatus) {
|
||||
if (collectionState == DiagnosisCollectionState.SATURATED) {
|
||||
throw new IllegalStateException("Cannot record Tool completion after saturation");
|
||||
}
|
||||
if (evidenceStatus != EvidenceStatus.EVIDENCE_FOUND
|
||||
&& evidenceStatus != EvidenceStatus.NO_EVIDENCE) {
|
||||
throw new IllegalArgumentException("Completed Tool requires a successful evidence status");
|
||||
}
|
||||
ToolScopeIdentity scope = new ToolScopeIdentity(call.toolName(), call.normalizedScope());
|
||||
if (!completedScopes.add(scope)) {
|
||||
throw new IllegalStateException("Completed Tool scope was already recorded");
|
||||
}
|
||||
completedToolCalls.add(call);
|
||||
if (evidenceStatus == EvidenceStatus.NO_EVIDENCE) {
|
||||
applyGain(InformationGain.NO_GAIN);
|
||||
} else {
|
||||
pendingToolCallId = call.toolCallId();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized boolean claimStopInstruction() {
|
||||
if (collectionState != DiagnosisCollectionState.SATURATED) {
|
||||
return false;
|
||||
}
|
||||
if (stopInstructionDelivered) {
|
||||
return false;
|
||||
}
|
||||
stopInstructionDelivered = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
public synchronized void markBudgetLimitReached() {
|
||||
if (stopReason == null) {
|
||||
stopReason = DiagnosisStopReason.BUDGET_LIMIT_REACHED;
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized DiagnosisProgressSnapshotState snapshot() {
|
||||
return new DiagnosisProgressSnapshotState(
|
||||
consecutiveNoGain,
|
||||
collectionState,
|
||||
stopReason,
|
||||
pendingToolCallId,
|
||||
stopInstructionDelivered,
|
||||
completedToolCalls);
|
||||
}
|
||||
|
||||
public int stopAfterConsecutiveNoGain() {
|
||||
return stopAfterConsecutiveNoGain;
|
||||
}
|
||||
|
||||
private void applyGain(InformationGain gain) {
|
||||
if (collectionState == DiagnosisCollectionState.SATURATED) {
|
||||
throw new IllegalStateException("Collection is already saturated");
|
||||
}
|
||||
if (gain == InformationGain.GAINED) {
|
||||
consecutiveNoGain = 0;
|
||||
return;
|
||||
}
|
||||
consecutiveNoGain++;
|
||||
if (consecutiveNoGain >= stopAfterConsecutiveNoGain) {
|
||||
collectionState = DiagnosisCollectionState.SATURATED;
|
||||
stopReason = DiagnosisStopReason.INFORMATION_SATURATED;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public enum DiagnosisStopReason {
|
||||
INFORMATION_SATURATED,
|
||||
BUDGET_LIMIT_REACHED
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public enum InformationGain {
|
||||
GAINED,
|
||||
NO_GAIN
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public record PreviousObservation(
|
||||
@JsonProperty("tool_call_id") String toolCallId,
|
||||
@JsonProperty("information_gain") InformationGain informationGain) {
|
||||
|
||||
public PreviousObservation {
|
||||
if (toolCallId == null || toolCallId.isBlank()) {
|
||||
throw new IllegalArgumentException("toolCallId must not be blank");
|
||||
}
|
||||
if (informationGain == null) {
|
||||
throw new IllegalArgumentException("informationGain must not be null");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public record ToolScopeIdentity(String toolName, String normalizedScope) {
|
||||
|
||||
public ToolScopeIdentity {
|
||||
if (toolName == null || toolName.isBlank()) {
|
||||
throw new IllegalArgumentException("toolName must not be blank");
|
||||
}
|
||||
if (normalizedScope == null || normalizedScope.isBlank()) {
|
||||
throw new IllegalArgumentException("normalizedScope must not be blank");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
public final class ToolScopeNormalizer {
|
||||
|
||||
private static final int DEFAULT_LOG_LOOKBACK_MINUTES = 30;
|
||||
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
public ToolScopeNormalizer(ObjectMapper objectMapper) {
|
||||
this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
|
||||
}
|
||||
|
||||
public String normalize(String toolName, Object input) {
|
||||
Objects.requireNonNull(input, "input must not be null");
|
||||
Map<String, Object> scope = switch (toolName) {
|
||||
case AgentToolContracts.LOOKUP_KNOWLEDGE -> ragScope(requireType(input, RagToolRequest.class));
|
||||
case AgentToolContracts.QUERY_LOGS -> logScope(requireType(input, QueryLogsRequest.class));
|
||||
case AgentToolContracts.QUERY_MYSQL -> mysqlScope(requireType(input, MysqlToolRequest.class));
|
||||
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
|
||||
};
|
||||
try {
|
||||
return objectMapper.writeValueAsString(scope);
|
||||
} catch (JsonProcessingException exception) {
|
||||
throw new IllegalArgumentException("Tool scope is not serializable", exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, Object> ragScope(RagToolRequest request) {
|
||||
return ordered("query", normalizeText(request.query()));
|
||||
}
|
||||
|
||||
private static Map<String, Object> logScope(QueryLogsRequest request) {
|
||||
Map<String, Object> scope = new LinkedHashMap<>();
|
||||
scope.put("topic", request.topic() == null ? null : request.topic().name());
|
||||
scope.put("query", normalizeText(request.query()));
|
||||
scope.put("lookback_minutes", request.lookbackMinutes() == null
|
||||
? DEFAULT_LOG_LOOKBACK_MINUTES : request.lookbackMinutes());
|
||||
return scope;
|
||||
}
|
||||
|
||||
private static Map<String, Object> mysqlScope(MysqlToolRequest request) {
|
||||
Map<String, Object> scope = new LinkedHashMap<>();
|
||||
scope.put("data_source", normalizeText(request.dataSource()));
|
||||
scope.put("sql", normalizeText(request.sql()));
|
||||
scope.put("params", request.params());
|
||||
return scope;
|
||||
}
|
||||
|
||||
private static Map<String, Object> ordered(String name, Object value) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put(name, value);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String normalizeText(String value) {
|
||||
return value == null ? "" : value.trim();
|
||||
}
|
||||
|
||||
private static <T> T requireType(Object input, Class<T> type) {
|
||||
if (!type.isInstance(input)) {
|
||||
throw new IllegalArgumentException("Unexpected Tool input type: " + input.getClass().getName());
|
||||
}
|
||||
return type.cast(input);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
@@ -15,10 +16,13 @@ import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardDecision;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardInput;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.retry.RetryExecutionException;
|
||||
import com.superbiz.agent.harness.retry.RetryFailure;
|
||||
|
||||
import java.util.Objects;
|
||||
import java.util.List;
|
||||
|
||||
public final class DiagnosisReleaseUseCase {
|
||||
|
||||
@@ -50,12 +54,44 @@ public final class DiagnosisReleaseUseCase {
|
||||
}
|
||||
|
||||
public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) {
|
||||
return execute(context, query, DiagnosisAgentExecution.completed(
|
||||
Objects.requireNonNull(draft, "draft must not be null"),
|
||||
DiagnosisProgressSnapshot.empty()));
|
||||
}
|
||||
|
||||
public DiagnosisReleaseResult execute(
|
||||
RunContext context, String query, DiagnosisAgentExecution execution) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
if (query == null || query.isBlank()) {
|
||||
throw new IllegalArgumentException("query must not be blank");
|
||||
}
|
||||
Objects.requireNonNull(draft, "draft must not be null");
|
||||
Objects.requireNonNull(execution, "execution must not be null");
|
||||
|
||||
DiagnosisDraft draft = execution.draft();
|
||||
if (draft == null) {
|
||||
return releaseControlledStop(context, execution.progress(), execution.stopReason());
|
||||
}
|
||||
if (draft.conclusion() == null) {
|
||||
return releaseNoConclusion(context, draft, execution.progress());
|
||||
}
|
||||
return releaseConclusion(context, query, draft);
|
||||
}
|
||||
|
||||
public DiagnosisReleaseResult releaseInvalidDraft(
|
||||
RunContext context, DiagnosisProgressSnapshot progress) {
|
||||
Objects.requireNonNull(context, "context must not be null");
|
||||
Objects.requireNonNull(progress, "progress must not be null");
|
||||
if (!progress.hasObservedFacts()) {
|
||||
throw new IllegalStateException(
|
||||
"Invalid Diagnosis Draft has no verified publishable progress");
|
||||
}
|
||||
return progressFallback(context,
|
||||
fallbackFactory.insufficientEvidence(progress, List.of()),
|
||||
FallbackType.INSUFFICIENT_EVIDENCE);
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult releaseConclusion(
|
||||
RunContext context, String query, DiagnosisDraft draft) {
|
||||
DiagnosisDraft candidate = draft;
|
||||
EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate);
|
||||
traceRecorder.record(TraceAuditEvents.evidenceValidation(
|
||||
@@ -101,6 +137,61 @@ public final class DiagnosisReleaseUseCase {
|
||||
return DiagnosisReleaseResult.fallback(fallbackFactory.semanticUnsupported(snapshot));
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult releaseNoConclusion(
|
||||
RunContext context, DiagnosisDraft draft, DiagnosisProgressSnapshot progress) {
|
||||
EvidenceGuardResult evidence = evidenceGuard.validateNoConclusionReferences(context, draft);
|
||||
traceRecorder.record(TraceAuditEvents.evidenceValidation(
|
||||
context, TraceEventType.EVIDENCE_GUARD_INITIAL, evidence, draft));
|
||||
if (!evidence.valid()) {
|
||||
return evidenceFailure(context, evidence);
|
||||
}
|
||||
|
||||
List<String> missingInfo = missingInfo(draft);
|
||||
if (progress.hasObservedFacts()) {
|
||||
return progressFallback(context,
|
||||
fallbackFactory.insufficientEvidence(progress, missingInfo),
|
||||
FallbackType.INSUFFICIENT_EVIDENCE);
|
||||
}
|
||||
if (!missingInfo.isEmpty()) {
|
||||
return progressFallback(context,
|
||||
fallbackFactory.missingRequiredContext(missingInfo),
|
||||
FallbackType.MISSING_REQUIRED_CONTEXT);
|
||||
}
|
||||
throw new IllegalStateException(
|
||||
"No-conclusion Diagnosis has neither verified progress nor missing context");
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult releaseControlledStop(
|
||||
RunContext context,
|
||||
DiagnosisProgressSnapshot progress,
|
||||
DiagnosisStopReason stopReason) {
|
||||
if (stopReason != DiagnosisStopReason.INFORMATION_SATURATED
|
||||
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED) {
|
||||
throw new IllegalStateException("Unsupported Diagnosis stop reason");
|
||||
}
|
||||
if (!progress.hasObservedFacts()) {
|
||||
throw new IllegalStateException(
|
||||
"Controlled Diagnosis stop has no verified publishable progress");
|
||||
}
|
||||
return progressFallback(context,
|
||||
fallbackFactory.insufficientEvidence(progress, List.of()),
|
||||
FallbackType.INSUFFICIENT_EVIDENCE);
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult progressFallback(
|
||||
RunContext context,
|
||||
com.superbiz.agent.harness.contract.SafeFallback fallback,
|
||||
FallbackType type) {
|
||||
traceRecorder.record(TraceAuditEvents.releaseDecision(
|
||||
context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, type));
|
||||
return DiagnosisReleaseResult.fallback(fallback);
|
||||
}
|
||||
|
||||
private List<String> missingInfo(DiagnosisDraft draft) {
|
||||
return draft.limitations() == null
|
||||
? List.of() : draft.limitations().missingInfo();
|
||||
}
|
||||
|
||||
private DiagnosisReleaseResult evidenceFailure(
|
||||
RunContext context, EvidenceGuardResult evidence) {
|
||||
traceRecorder.record(TraceAuditEvents.releaseDecision(
|
||||
|
||||
@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceAuditEvents;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
@@ -92,7 +93,8 @@ public final class EvidenceRepair {
|
||||
context.retryPolicies().evidenceRepair(),
|
||||
() -> {
|
||||
DiagnosisDraft repaired = parse(modelCall.call(
|
||||
context, modelPrompt, limits.timeout(), limits.maxOutputBytes()));
|
||||
context, ModelCallComponent.EVIDENCE_REPAIR,
|
||||
modelPrompt, limits.timeout(), limits.maxOutputBytes()));
|
||||
if (!originalSemantics.hasSameUserVisibleSemantics(
|
||||
SemanticDraftView.from(repaired))) {
|
||||
throw new GuardModelCallException(
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@@ -53,6 +54,47 @@ public final class SafeFallbackFactory {
|
||||
List.of());
|
||||
}
|
||||
|
||||
public SafeFallback insufficientEvidence(
|
||||
DiagnosisProgressSnapshot progress, List<String> missingInfo) {
|
||||
Objects.requireNonNull(progress, "progress must not be null");
|
||||
if (!progress.hasObservedFacts()) {
|
||||
throw new IllegalArgumentException(
|
||||
"insufficient-evidence fallback requires verified progress");
|
||||
}
|
||||
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
|
||||
List<String> limitations = new ArrayList<>(progress.limitations());
|
||||
limitations.add("当前已检查范围不足以支持根因结论");
|
||||
safeMissingInfo.forEach(item -> limitations.add("仍缺少:" + item));
|
||||
return fallback(
|
||||
FallbackType.INSUFFICIENT_EVIDENCE,
|
||||
progress.verifiedSources(),
|
||||
"已完成有限范围的检查,但现有证据不足以确认根因",
|
||||
List.copyOf(limitations),
|
||||
List.of(safeMissingInfo.isEmpty()
|
||||
? "补充故障对象、发生时间、错误信息或新的可查询范围后重试"
|
||||
: "补充缺失信息后,在新的明确范围内继续诊断"),
|
||||
"DIAGNOSIS_COLLECTION",
|
||||
progress.observedFacts(),
|
||||
List.of());
|
||||
}
|
||||
|
||||
public SafeFallback missingRequiredContext(List<String> missingInfo) {
|
||||
List<String> safeMissingInfo = boundedMissingInfo(missingInfo);
|
||||
if (safeMissingInfo.isEmpty()) {
|
||||
throw new IllegalArgumentException(
|
||||
"missing-context fallback requires missing information");
|
||||
}
|
||||
return fallback(
|
||||
FallbackType.MISSING_REQUIRED_CONTEXT,
|
||||
List.of(),
|
||||
"当前缺少执行定向诊断所需的信息,暂时无法开始有效查询或确认根因",
|
||||
safeMissingInfo.stream().map(item -> "缺少:" + item).toList(),
|
||||
List.of("补充上述故障上下文后重试"),
|
||||
"DIAGNOSIS_INPUT",
|
||||
List.of(),
|
||||
List.of());
|
||||
}
|
||||
|
||||
private SafeFallback fallback(FallbackType type,
|
||||
List<SafeFallback.VerifiedSource> sources,
|
||||
String message,
|
||||
@@ -100,6 +142,20 @@ public final class SafeFallbackFactory {
|
||||
return List.copyOf(result);
|
||||
}
|
||||
|
||||
private List<String> boundedMissingInfo(List<String> missingInfo) {
|
||||
List<String> result = new ArrayList<>();
|
||||
for (String item : missingInfo == null ? List.<String>of() : missingInfo) {
|
||||
String safe = bounded(item);
|
||||
if (safe != null && !result.contains(safe)) {
|
||||
result.add(safe);
|
||||
}
|
||||
if (result.size() >= 8) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return List.copyOf(result);
|
||||
}
|
||||
|
||||
private String bounded(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return null;
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
|
||||
public record MysqlToolCall(
|
||||
@JsonProperty("previous_observation") PreviousObservation previousObservation,
|
||||
@JsonProperty("input") MysqlToolRequest input) {
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
|
||||
public record QueryLogsToolCall(
|
||||
@JsonProperty("previous_observation") PreviousObservation previousObservation,
|
||||
@JsonProperty("input") QueryLogsRequest input) {
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
public enum RagRelevanceLevel {
|
||||
PRECISE,
|
||||
HIGHLY_RELEVANT,
|
||||
REFERENCE
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
|
||||
public record RagToolCall(
|
||||
@JsonProperty("previous_observation") PreviousObservation previousObservation,
|
||||
@JsonProperty("input") RagToolRequest input) {
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
|
||||
import java.util.List;
|
||||
@@ -11,9 +12,20 @@ public record RagToolResult(
|
||||
@JsonProperty("query") String query,
|
||||
@JsonProperty("evidence") List<RagEvidence> evidence,
|
||||
@JsonProperty("returned_count") int returnedCount,
|
||||
@JsonProperty("relevance_level") @JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
RagRelevanceLevel relevanceLevel,
|
||||
@JsonProperty("truncated") boolean truncated) {
|
||||
|
||||
public RagToolResult {
|
||||
evidence = ToolContractCollections.immutable(evidence);
|
||||
}
|
||||
|
||||
public RagToolResult(EvidenceStatus evidenceStatus,
|
||||
String toolCallId,
|
||||
String query,
|
||||
List<RagEvidence> evidence,
|
||||
int returnedCount,
|
||||
boolean truncated) {
|
||||
this(evidenceStatus, toolCallId, query, evidence, returnedCount, null, truncated);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.tool.boundary.ProjectedToolResult;
|
||||
import com.superbiz.agent.harness.tool.contract.RagEvidence;
|
||||
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolResult;
|
||||
|
||||
@@ -81,7 +82,10 @@ public final class RagResultProjector {
|
||||
EvidenceStatus status = evidence.isEmpty()
|
||||
? EvidenceStatus.NO_EVIDENCE
|
||||
: EvidenceStatus.EVIDENCE_FOUND;
|
||||
RagToolResult result = new RagToolResult(status, toolCallId, query, evidence, evidence.size(), truncated);
|
||||
RagRelevanceLevel relevanceLevel = status == EvidenceStatus.EVIDENCE_FOUND
|
||||
? relevanceLevel(root) : null;
|
||||
RagToolResult result = new RagToolResult(
|
||||
status, toolCallId, query, evidence, evidence.size(), relevanceLevel, truncated);
|
||||
result = fitBudget(result, truncated);
|
||||
return new ProjectedToolResult(objectMapper.writeValueAsString(result), result.evidenceStatus());
|
||||
}
|
||||
@@ -94,7 +98,8 @@ public final class RagResultProjector {
|
||||
reduced.remove(reduced.size() - 1);
|
||||
current = new RagToolResult(
|
||||
reduced.isEmpty() ? EvidenceStatus.NO_EVIDENCE : EvidenceStatus.EVIDENCE_FOUND,
|
||||
current.toolCallId(), current.query(), reduced, reduced.size(), true);
|
||||
current.toolCallId(), current.query(), reduced, reduced.size(),
|
||||
reduced.isEmpty() ? null : current.relevanceLevel(), true);
|
||||
}
|
||||
if (bytes(objectMapper.writeValueAsString(current)) > limits.maxAgentUtf8Bytes()) {
|
||||
throw new IllegalArgumentException("RAG projection exceeds total budget");
|
||||
@@ -120,6 +125,21 @@ public final class RagResultProjector {
|
||||
return value == null || value.isBlank() ? null : value;
|
||||
}
|
||||
|
||||
private static RagRelevanceLevel relevanceLevel(JsonNode root) {
|
||||
String value = text(root, "relevance_level");
|
||||
if (value.isBlank()) {
|
||||
value = text(root, "relevanceLevel");
|
||||
}
|
||||
if (value.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return RagRelevanceLevel.valueOf(value.trim().toUpperCase(java.util.Locale.ROOT));
|
||||
} catch (IllegalArgumentException ignored) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String bounded(String value, int max) {
|
||||
if (value == null) {
|
||||
return "";
|
||||
|
||||
@@ -5,6 +5,7 @@ import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Agent 决策步骤 Repository
|
||||
@@ -27,6 +28,8 @@ public interface AgentStepRepository extends JpaRepository<AgentStep, Long> {
|
||||
*/
|
||||
List<AgentStep> findByRunIdOrderByStepIndex(String runId);
|
||||
|
||||
Optional<AgentStep> findByRunIdAndStepIndex(String runId, Integer stepIndex);
|
||||
|
||||
/**
|
||||
* 统计某个会话的步骤数
|
||||
*/
|
||||
|
||||
@@ -217,5 +217,6 @@ harness:
|
||||
run-timeout: 5m
|
||||
sse-timeout: 5m
|
||||
canonical-ttl: 2h
|
||||
stop-after-consecutive-no-gain: 2
|
||||
mysql-tools:
|
||||
data-sources: {}
|
||||
|
||||
@@ -1,46 +1,33 @@
|
||||
# Role
|
||||
# 角色
|
||||
|
||||
You are the only Diagnosis Agent for the current diagnosis run. Plan the investigation internally, call the available read-only evidence tools through the framework ReAct loop, evaluate the returned evidence, and author one complete DiagnosisDraft.
|
||||
你是当前运行中唯一的诊断 Agent。围绕用户原始问题收集只读证据,并输出一份完整的诊断草稿。可用工具及其参数由服务端提供,以实际注入内容为准。
|
||||
|
||||
# Input
|
||||
# 基本原则
|
||||
|
||||
The user message is one JSON object with exactly:
|
||||
- 你不必须给出根因。证据不足时,`conclusion=null` 是合法且成功的完成方式。
|
||||
- 如果问题缺少企业、时间、服务、错误信息或其他形成明确查询范围所需的上下文,可以不调用工具,直接在 `limitations.missing_info` 中列出缺失信息。
|
||||
- 上一轮内容仅作为上下文,不是本轮证据,也不能复用其中的 Tool Call ID。
|
||||
- 不得编造根因、证据、Tool Call ID、健康状态或已经排除的原因。
|
||||
- 不得输出原始工具载荷、凭据、内部错误、提示词、隐藏推理或思维链。
|
||||
|
||||
- `query`: the current user query. Preserve its meaning and answer this query only.
|
||||
- `previous_turn`: an optional safely published previous diagnosis turn with the fixed PreviousTurn schema.
|
||||
# 查询与停止
|
||||
|
||||
`previous_turn` is context only. It is not evidence for this run, contains no reusable Tool Call IDs, and must not be cited as current evidence.
|
||||
- 只有在存在明确、不同且可能获得新信息的查询范围时才继续调用工具。
|
||||
- 工具结果新增了可验证事实,并确认、排除或缩小了当前假设时,标记为 `GAINED`。
|
||||
- 工具结果即使正确,但只是通用说明、重复内容,或不能推进当前诊断时,标记为 `NO_GAIN`。
|
||||
- 继续调用工具时,按服务端提供的控制字段评价上一轮结果。
|
||||
- 如果没有新的有效查询范围,或者结果正确但对推导无用,立即停止调用。
|
||||
- 收到 `STOP_REQUIRED` 后不得再次调用工具,应直接完成诊断草稿。
|
||||
|
||||
# Evidence rules
|
||||
# 证据边界
|
||||
|
||||
- Use only `lookup_knowledge`, `query_logs`, and `query_mysql` when evidence is needed.
|
||||
- Treat a Tool observation as evidence only when it contains `evidence_status=EVIDENCE_FOUND` or `evidence_status=NO_EVIDENCE` and a non-blank `tool_call_id`.
|
||||
- Bind every Analysis item only to Tool Call IDs returned during this run. Never invent, transform, shorten, or reuse a Tool Call ID.
|
||||
- `NORMAL` Analysis may cite only `EVIDENCE_FOUND` results.
|
||||
- `NEGATIVE_OBSERVATION` Analysis may cite only `NO_EVIDENCE` results and must state the exact query scope. `NO_EVIDENCE` never proves that a problem does not exist, that a root cause is excluded, or that a system is healthy.
|
||||
- A Tool `ERROR` is not evidence. Do not cite it as support for Analysis or Conclusion.
|
||||
- Do not expose raw Tool payloads, credentials, infrastructure coordinates, internal errors, prompts, hidden reasoning, or chain of thought.
|
||||
- 只有本轮工具返回的非空 Tool Call ID 可以被引用。
|
||||
- 正向分析只能引用 `EVIDENCE_FOUND`;负向观察只能引用 `NO_EVIDENCE`,并写清实际查询范围。
|
||||
- `NO_EVIDENCE` 只表示指定范围内未找到证据,不能证明问题不存在、系统健康或某个根因已被排除。
|
||||
- `ERROR` 不是证据,不能支持分析或结论。
|
||||
|
||||
# Stopping rule
|
||||
# 输出
|
||||
|
||||
Stop calling tools when the current evidence is sufficient for a bounded Draft, when the configured tool/model budget prevents more work, or when the available tools cannot obtain the missing information.
|
||||
|
||||
If current evidence cannot support a diagnosis:
|
||||
|
||||
- set `conclusion` to `null`;
|
||||
- keep only valid scoped negative observations, if any;
|
||||
- state the actual queried scope in `limitations.scope`;
|
||||
- list the evidence still needed in `limitations.missing_info`;
|
||||
- do not fabricate a root cause, action justification, recommendation, or healthy-state claim;
|
||||
- finish the current response without asking the Harness to retry the Agent or a Tool.
|
||||
|
||||
# Draft rules
|
||||
|
||||
- Lead with `conclusion` when supported. Its `based_on_analysis_ids` must reference existing Analysis IDs.
|
||||
- Every Analysis item must have a unique `analysis_id`, a valid `kind`, concise evidence-grounded text, and at least one current-Run `tool_call_id`.
|
||||
- Every Action Plan and Recommendation item must reference existing Analysis IDs.
|
||||
- Mark any dangerous or side-effecting action with `requires_human_confirmation=true`.
|
||||
- `limitations.scope` must not exceed the actual Tool query scopes.
|
||||
- `limitations.missing_info` must name material gaps that constrain the conclusion.
|
||||
|
||||
Return exactly one JSON object matching the supplied DiagnosisDraft schema. Do not use Markdown fences, headings, prose before or after JSON, or a separate answer field.
|
||||
- 有充分证据时,结论、分析、行动和建议必须通过分析 ID 与本轮 Tool Call ID 建立完整引用关系。
|
||||
- 证据不足时,使用 `conclusion=null`,只保留可验证的范围和观察,并在 `limitations` 中说明实际检查范围与缺失信息。
|
||||
- 返回一个符合服务端输出约束的 JSON 对象。不要使用 Markdown 代码块,也不要添加 JSON 之外的文字。
|
||||
|
||||
@@ -703,7 +703,33 @@ class SuperBizAgentApp {
|
||||
return [payload.answer || '', references ? `\n\n参考资料:\n${references}` : ''].join('');
|
||||
}
|
||||
if (contentType === 'SAFE_FALLBACK') {
|
||||
return payload.fallback?.message || '当前证据不足,无法确认根因';
|
||||
const fallback = payload.fallback || {};
|
||||
const sections = [];
|
||||
sections.push(`## 当前判断\n${fallback.message || '当前证据不足,无法确认根因'}`);
|
||||
if (fallback.observed_facts?.length) {
|
||||
const facts = fallback.observed_facts.map(item =>
|
||||
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**(${item.scope || '范围未提供'}):${item.summary || '已完成检查'}`
|
||||
).join('\n');
|
||||
sections.push(`## 已完成排查\n${facts}`);
|
||||
} else if (fallback.verified_sources?.length) {
|
||||
const sources = fallback.verified_sources.map(item =>
|
||||
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
|
||||
).join('\n');
|
||||
sections.push(`## 已检查范围\n${sources}`);
|
||||
}
|
||||
if (fallback.validation_issues?.length) {
|
||||
const issues = fallback.validation_issues.map(item =>
|
||||
`- ${item.code || 'VALIDATION_FAILED'}${item.target ? `:${item.target}` : ''}`
|
||||
).join('\n');
|
||||
sections.push(`## 校验问题\n${issues}`);
|
||||
}
|
||||
if (fallback.limitations?.length) {
|
||||
sections.push(`## 限制\n${fallback.limitations.map(item => `- ${item}`).join('\n')}`);
|
||||
}
|
||||
if (fallback.next_steps?.length) {
|
||||
sections.push(`## 下一步\n${fallback.next_steps.map(item => `- ${item}`).join('\n')}`);
|
||||
}
|
||||
return sections.join('\n\n');
|
||||
}
|
||||
if (contentType === 'DIAGNOSIS_REPORT') {
|
||||
const report = payload.report || {};
|
||||
@@ -713,6 +739,15 @@ class SuperBizAgentApp {
|
||||
if (report.action_plan?.length) sections.push(`## 行动计划\n${report.action_plan.map(item => `- ${item.action}`).join('\n')}`);
|
||||
if (report.recommendations?.length) sections.push(`## 建议\n${report.recommendations.map(item => `- ${item.text}`).join('\n')}`);
|
||||
if (report.limitations?.scope) sections.push(`## 限制\n${report.limitations.scope}`);
|
||||
if (report.limitations?.missing_info?.length) {
|
||||
sections.push(`## 仍缺信息\n${report.limitations.missing_info.map(item => `- ${item}`).join('\n')}`);
|
||||
}
|
||||
if (payload.references?.length) {
|
||||
const references = payload.references.map(item =>
|
||||
`- **${item.source_type || '证据'} / ${item.source || '未知来源'}**:${item.scope || '范围未提供'}`
|
||||
).join('\n');
|
||||
sections.push(`## 已检查范围\n${references}`);
|
||||
}
|
||||
return sections.join('\n\n');
|
||||
}
|
||||
throw new Error(`未知 content_type: ${contentType}`);
|
||||
|
||||
@@ -111,6 +111,6 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="app.js?v=20260707-session-picker"></script>
|
||||
<script src="app.js?v=20260725-diagnosis-details"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -31,6 +31,15 @@ import static org.mockito.Mockito.mock;
|
||||
|
||||
class HarnessChatConfigurationTest {
|
||||
|
||||
@Test
|
||||
void diagnosisNoGainThresholdDefaultsAndValidates() {
|
||||
ChatHarnessProperties properties = new ChatHarnessProperties();
|
||||
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
|
||||
|
||||
properties.setStopAfterConsecutiveNoGain(0);
|
||||
assertThrows(IllegalArgumentException.class, properties::validate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void defaultsArePositiveAndExecutorsAreBounded() {
|
||||
ChatHarnessProperties properties = new ChatHarnessProperties();
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisAgentPromptTest {
|
||||
|
||||
@Test
|
||||
void promptLicensesAbandonmentWithoutDuplicatingToolSchema() {
|
||||
String prompt = DiagnosisAgentPrompt.load();
|
||||
|
||||
assertTrue(prompt.contains("你不必须给出根因"));
|
||||
assertTrue(prompt.contains("conclusion=null"));
|
||||
assertTrue(prompt.contains("可以不调用工具"));
|
||||
assertTrue(prompt.contains("GAINED"));
|
||||
assertTrue(prompt.contains("NO_GAIN"));
|
||||
assertTrue(prompt.contains("正确,但只是通用说明、重复内容"));
|
||||
assertTrue(prompt.contains("STOP_REQUIRED"));
|
||||
assertFalse(prompt.contains("lookup_knowledge"));
|
||||
assertFalse(prompt.contains("query_logs"));
|
||||
assertFalse(prompt.contains("query_mysql"));
|
||||
assertFalse(prompt.contains("previous_observation"));
|
||||
assertFalse(prompt.contains("next_action"));
|
||||
assertFalse(prompt.contains("阈值"));
|
||||
assertFalse(prompt.contains("计数器"));
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,8 @@ import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
@@ -71,8 +73,9 @@ class DiagnosisAgentUseCaseTest {
|
||||
"上一轮支付服务为什么超时?", "支付服务连接池已耗尽", "payment-service",
|
||||
List.of("未覆盖退款服务"), List.of(new SourceDocument("doc-1", "连接池手册")));
|
||||
|
||||
DiagnosisDraft draft = useCase.execute(
|
||||
DiagnosisAgentExecution execution = useCase.execute(
|
||||
context, new DiagnosisAgentInput("那退款服务呢?", previous));
|
||||
DiagnosisDraft draft = execution.draft();
|
||||
|
||||
assertEquals("退款服务出现连接池等待", draft.conclusion().text());
|
||||
assertEquals(AnalysisKind.NORMAL, draft.analysis().get(0).kind());
|
||||
@@ -109,8 +112,9 @@ class DiagnosisAgentUseCaseTest {
|
||||
toolCall("call-empty-1", "{\"query\":\"refund timeout\"}"),
|
||||
new AssistantMessage(noEvidenceDraft("call-empty-1")));
|
||||
|
||||
DiagnosisDraft draft = useCase(core, model, tools, LARGE_LIMITS)
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("退款为什么超时?", null));
|
||||
DiagnosisDraft draft = execution.draft();
|
||||
|
||||
assertNull(draft.conclusion());
|
||||
assertEquals(AnalysisKind.NEGATIVE_OBSERVATION, draft.analysis().get(0).kind());
|
||||
@@ -119,6 +123,32 @@ class DiagnosisAgentUseCaseTest {
|
||||
assertEquals(1, toolCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void ignoredStopInstructionCannotExecuteAnotherTool() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(6, 6, 6,
|
||||
100, 100, 200, 100_000));
|
||||
RunContext context = core.startRun("session-stop", "run-stop");
|
||||
AtomicInteger toolCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
toolCalls.incrementAndGet();
|
||||
return ToolBoundaryResult.ready(id, noEvidence(id), EvidenceStatus.NO_EVIDENCE);
|
||||
});
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
toolCall("call-empty-1", "{\"query\":\"first\"}"),
|
||||
toolCall("call-empty-2", "{\"query\":\"second\"}"),
|
||||
toolCall("call-ignored-stop", "{\"query\":\"third\"}"));
|
||||
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, execution.stopReason());
|
||||
assertEquals(3, model.calls());
|
||||
assertEquals(2, toolCalls.get());
|
||||
assertEquals(2, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fencedOutputFailsClosedWithoutAgentRetry() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
@@ -126,14 +156,45 @@ class DiagnosisAgentUseCaseTest {
|
||||
new AssistantMessage("```json\n" + supportedDraft("call-1") + "\n```"));
|
||||
RunContext context = core.startRun("session-fenced", "run-fenced");
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
DiagnosisAgentOutputException failure = assertThrows(
|
||||
DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
|
||||
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
|
||||
assertFalse(failure.progress().hasObservedFacts());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(1, context.budget().snapshot().modelCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftCarriesOnlyProjectedProgressForOuterReleaseBoundary() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
new AssistantMessage("not-json"));
|
||||
RunContext context = core.startRun("session-invalid-progress", "run-invalid-progress");
|
||||
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
|
||||
List.of(new com.superbiz.agent.harness.contract.SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new com.superbiz.agent.harness.contract.SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown", "已完成知识范围检查")),
|
||||
List.of(), null);
|
||||
DiagnosisAgentFactory factory = new DiagnosisAgentFactory(
|
||||
model, core, tools(errorInvoker()), objectMapper, List.of());
|
||||
DiagnosisAgentUseCase useCase = new DiagnosisAgentUseCase(
|
||||
core, factory, objectMapper, LARGE_LIMITS, ignored -> progress);
|
||||
|
||||
DiagnosisAgentOutputException failure = assertThrows(
|
||||
DiagnosisAgentOutputException.class,
|
||||
() -> useCase.execute(
|
||||
context, new DiagnosisAgentInput("诊断未知故障", null)));
|
||||
|
||||
assertEquals(DiagnosisAgentOutputException.Kind.INVALID_JSON, failure.kind());
|
||||
assertEquals("已完成知识范围检查",
|
||||
failure.progress().observedFacts().get(0).summary());
|
||||
assertFalse(failure.getMessage().contains("not-json"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void modelCallBudgetStopsNextReactRoundBeforeChatModel() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(1, 2, 2,
|
||||
@@ -147,10 +208,11 @@ class DiagnosisAgentUseCaseTest {
|
||||
toolCall("call-budget-1", "{\"query\":\"timeout\"}"),
|
||||
new AssistantMessage(supportedDraft("call-budget-1")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
}
|
||||
@@ -163,10 +225,11 @@ class DiagnosisAgentUseCaseTest {
|
||||
ScriptedChatModel model = new ScriptedChatModel(6, 6,
|
||||
new AssistantMessage(supportedDraft("call-unused")));
|
||||
|
||||
assertThrows(DiagnosisAgentOutputException.class, () ->
|
||||
useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null)));
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools(errorInvoker()), LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断超时", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.BUDGET_LIMIT_REACHED, execution.stopReason());
|
||||
assertEquals(1, model.calls());
|
||||
assertEquals(12, context.budget().snapshot().totalTokens());
|
||||
assertEquals(RunState.BUDGET_EXHAUSTED, context.lifecycle().state());
|
||||
@@ -261,7 +324,8 @@ class DiagnosisAgentUseCaseTest {
|
||||
return AssistantMessage.builder()
|
||||
.content("")
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE, arguments)))
|
||||
id, "function", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"input\":" + arguments + "}")))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ModelResponse;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.ModelCallAuditor;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
class HarnessModelInterceptorTest {
|
||||
|
||||
@Test
|
||||
void extractsProviderUsageWithoutPersistingModelContent() {
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
RunContext context = core.startRun("session-agent-token", "run-agent-token");
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessModelInterceptor interceptor = new HarnessModelInterceptor(
|
||||
core, context,
|
||||
new ModelCallAuditor(core, trace::add, mock(AgentStepRepository.class)));
|
||||
AssistantMessage assistant = new AssistantMessage("secret-model-content");
|
||||
ChatResponse response = new ChatResponse(
|
||||
List.of(new Generation(assistant)),
|
||||
ChatResponseMetadata.builder().usage(new DefaultUsage(21, 4)).build());
|
||||
|
||||
ModelResponse actual = interceptor.interceptModel(
|
||||
ModelRequest.builder().messages(List.of()).build(),
|
||||
ignored -> ModelResponse.of(assistant, response));
|
||||
|
||||
assertEquals(response, actual.getChatResponse());
|
||||
assertEquals(25, context.budget().snapshot().totalTokens());
|
||||
assertEquals(25, context.modelCalls().snapshot().totalTokens());
|
||||
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, trace.get(0).eventType());
|
||||
assertEquals("DIAGNOSIS_AGENT", trace.get(0).details().get("component"));
|
||||
assertFalse(trace.get(0).details().toString().contains("secret-model-content"));
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallRequest;
|
||||
import com.alibaba.cloud.ai.graph.agent.interceptor.ToolCallResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
@@ -23,6 +25,7 @@ import org.springframework.ai.tool.execution.ToolExecutionException;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@@ -52,7 +55,9 @@ class HarnessToolInterceptorTest {
|
||||
AgentToolContracts.QUERY_MYSQL),
|
||||
callbacks.stream().map(callback -> callback.getToolDefinition().name()).toList());
|
||||
callbacks.forEach(callback -> {
|
||||
assertFalse(callback.getToolDefinition().inputSchema().contains("tool_call_id"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("previous_observation"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("information_gain"));
|
||||
assertTrue(callback.getToolDefinition().inputSchema().contains("\"input\""));
|
||||
assertFalse(callback.getToolDefinition().description().isBlank());
|
||||
assertThrows(ToolExecutionException.class, () -> callback.call("{}"));
|
||||
});
|
||||
@@ -72,7 +77,8 @@ class HarnessToolInterceptorTest {
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7", "{\"query\":\"timeout\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "framework-call-7",
|
||||
"{\"input\":{\"query\":\"timeout\"}}"),
|
||||
ignored -> {
|
||||
throw new AssertionError("registered Tool must not bypass Harness interceptor");
|
||||
});
|
||||
@@ -82,6 +88,7 @@ class HarnessToolInterceptorTest {
|
||||
assertEquals("framework-call-7", envelope.getValue().toolCallId());
|
||||
assertEquals(context.runId(), envelope.getValue().runId());
|
||||
assertEquals(AgentToolContracts.LOOKUP_KNOWLEDGE, envelope.getValue().toolName());
|
||||
assertEquals("{\"query\":\"timeout\"}", envelope.getValue().requestJson());
|
||||
assertEquals("framework-call-7", response.getToolCallId());
|
||||
assertTrue(response.getResult().contains("framework-call-7"));
|
||||
}
|
||||
@@ -94,7 +101,8 @@ class HarnessToolInterceptorTest {
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error", "{\"query\":\"secret\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-error",
|
||||
"{\"input\":{\"query\":\"secret\"}}"),
|
||||
ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(response.getResult());
|
||||
@@ -142,13 +150,164 @@ class HarnessToolInterceptorTest {
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(context, tools, objectMapper);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1", "{\"query\":\"one\"}"),
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"),
|
||||
ignored -> null);
|
||||
|
||||
assertEquals(1, invocations.get());
|
||||
assertEquals(1, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void requiresExactPreviousEvaluationBeforeAnotherToolExecutes() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
|
||||
ToolCallResponse missing = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
|
||||
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
|
||||
ToolCallResponse accepted = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
|
||||
"input":{"query":"three"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
assertTrue(missing.isError());
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL",
|
||||
objectMapper.readTree(missing.getResult()).path("error_code").asText());
|
||||
assertFalse(accepted.isError());
|
||||
assertEquals(2, invocations.get());
|
||||
DiagnosisTraceAuditEvent rejected = trace.stream()
|
||||
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("call-2", rejected.details().get("tool_call_id"));
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
|
||||
String details = rejected.details().toString();
|
||||
assertFalse(details.contains("two"));
|
||||
assertFalse(details.contains("previous_observation"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void duplicateSuccessfulScopeIsNotExecutedAndCanSaturateRun() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"same\"}}"), ignored -> null);
|
||||
ToolCallResponse duplicate = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"NO_GAIN"},
|
||||
"input":{"query":"same"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(duplicate.getResult());
|
||||
assertEquals(1, invocations.get());
|
||||
assertTrue(observation.path("stop_required").asBoolean());
|
||||
assertEquals("INFORMATION_SATURATED", observation.path("reason").asText());
|
||||
assertTrue(trace.stream().anyMatch(event ->
|
||||
event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED
|
||||
&& "DUPLICATE_SCOPE".equals(event.details().get("error_code"))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void observationContractRejectionIsTracedWithoutPayload() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) ->
|
||||
ToolBoundaryResult.ready(id, """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-contract",
|
||||
"query":"secret-query","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND));
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper, trace::add);
|
||||
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-contract",
|
||||
"{\"input\":{\"query\":\"secret-query\"}}"), ignored -> null);
|
||||
|
||||
assertTrue(response.isError());
|
||||
DiagnosisTraceAuditEvent rejected = trace.stream()
|
||||
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("OBSERVATION_CONTRACT_MISMATCH", rejected.details().get("error_code"));
|
||||
assertFalse(rejected.details().toString().contains("secret-query"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void modelInformationGainIsRecordedWithoutRawScope() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> ready(id));
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"secret-query-value\"}}"), ignored -> null);
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"GAINED"},
|
||||
"input":{"query":"different-secret-query"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
assertEquals(1, trace.size());
|
||||
DiagnosisTraceAuditEvent event = trace.get(0);
|
||||
assertEquals(TraceEventType.TOOL_PROGRESS, event.eventType());
|
||||
assertEquals("MODEL", event.details().get("producer"));
|
||||
assertEquals("GAINED", event.details().get("information_gain"));
|
||||
assertTrue(event.details().get("scope_summary").toString()
|
||||
.startsWith(AgentToolContracts.LOOKUP_KNOWLEDGE + "#"));
|
||||
assertFalse(event.details().toString().contains("secret-query-value"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void harnessProgressTraceDoesNotContainSqlParamsPayloadOrBudget() {
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
EvidenceToolInvoker unused = (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
EvidenceToolInvoker mysql = (context, id, arguments) -> ToolBoundaryResult.ready(
|
||||
id,
|
||||
"{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"columns\":[],\"rows\":[],\"returned_count\":0,"
|
||||
+ "\"truncated\":false}",
|
||||
EvidenceStatus.NO_EVIDENCE);
|
||||
HarnessEvidenceTools tools = new HarnessEvidenceTools(unused, unused, mysql);
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(4, 4, 4), tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.QUERY_MYSQL, "call-sql", """
|
||||
{"input":{"data_source":"orders","sql":"select * from secret_table where tenant_id = ?",
|
||||
"params":["tenant-secret"]}}
|
||||
"""), ignored -> null);
|
||||
|
||||
String recorded = trace.toString();
|
||||
assertEquals(List.of(TraceEventType.TOOL_PROGRESS),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertTrue(recorded.contains("HARNESS"));
|
||||
assertTrue(recorded.contains("NO_GAIN"));
|
||||
assertFalse(recorded.contains("secret_table"));
|
||||
assertFalse(recorded.contains("tenant-secret"));
|
||||
assertFalse(recorded.contains("raw_response"));
|
||||
assertFalse(recorded.contains("remaining"));
|
||||
assertFalse(recorded.contains("prompt"));
|
||||
assertFalse(recorded.contains("thought"));
|
||||
}
|
||||
|
||||
private HarnessEvidenceTools fakeTools(EvidenceToolInvoker ragInvoker) {
|
||||
EvidenceToolInvoker unused = (context, id, arguments) ->
|
||||
ToolBoundaryResult.error(id, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
@@ -162,7 +321,8 @@ class HarnessToolInterceptorTest {
|
||||
private ToolBoundaryResult ready(String toolCallId) {
|
||||
return ToolBoundaryResult.ready(toolCallId,
|
||||
"{\"evidence_status\":\"EVIDENCE_FOUND\",\"tool_call_id\":\""
|
||||
+ toolCallId + "\",\"evidence\":[]}",
|
||||
+ toolCallId + "\",\"query\":\"scope\",\"evidence\":[],"
|
||||
+ "\"returned_count\":1,\"truncated\":false}",
|
||||
EvidenceStatus.EVIDENCE_FOUND);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.superbiz.agent.harness.agent;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.RagRelevanceLevel;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ToolResultViewProjectorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolResultViewProjector projector = new ToolResultViewProjector(objectMapper);
|
||||
|
||||
@Test
|
||||
void separatesRagControlDataFromModelObservation() throws Exception {
|
||||
String canonical = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
|
||||
"evidence":[{"document_id":"doc-1","excerpt":"generic"}],"returned_count":1,
|
||||
"relevance_level":"REFERENCE","truncated":false,"raw_score":0.42}
|
||||
""";
|
||||
|
||||
ToolControlView control = projector.controlView(canonical);
|
||||
JsonNode model = objectMapper.readTree(projector.modelObservation(
|
||||
AgentToolContracts.LOOKUP_KNOWLEDGE, canonical, "{\"query\":\"unknown\"}",
|
||||
true, DiagnosisStopReason.INFORMATION_SATURATED));
|
||||
|
||||
assertEquals(1, control.returnedCount());
|
||||
assertEquals(RagRelevanceLevel.REFERENCE, control.relevanceLevel());
|
||||
assertEquals("REFERENCE", model.path("relevance_level").asText());
|
||||
assertEquals("unknown", model.path("scope").path("query").asText());
|
||||
assertTrue(model.path("stop_required").asBoolean());
|
||||
assertFalse(model.has("returned_count"));
|
||||
assertFalse(model.has("raw_score"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mysqlObservationUsesNormalizedScopeWithoutInternalFields() throws Exception {
|
||||
String canonical = """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-2","columns":["id"],
|
||||
"rows":[{"id":1}],"returned_count":1,"truncated":false}
|
||||
""";
|
||||
|
||||
JsonNode model = objectMapper.readTree(projector.modelObservation(
|
||||
AgentToolContracts.QUERY_MYSQL, canonical,
|
||||
"{\"data_source\":\"orders\",\"sql\":\"SELECT id FROM orders\",\"params\":[]}",
|
||||
false, null));
|
||||
|
||||
assertEquals("orders", model.path("scope").path("data_source").asText());
|
||||
assertEquals(1, model.path("rows").size());
|
||||
assertFalse(model.has("returned_count"));
|
||||
}
|
||||
}
|
||||
@@ -9,14 +9,17 @@ import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.PreviousTurn;
|
||||
import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.core.RunState;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -157,6 +160,88 @@ class ChatApplicationUseCaseTest {
|
||||
store.startedContext.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void handledDiagnosisBudgetFallbackIsPersistedWithoutReclassification() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
|
||||
SafeFallback fallback = new SafeFallback(
|
||||
FallbackType.INSUFFICIENT_EVIDENCE,
|
||||
null,
|
||||
"已完成有限范围的检查,但现有证据不足以确认根因",
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of("当前已检查范围不足以支持根因结论"),
|
||||
List.of("补充故障上下文后重试"),
|
||||
"DIAGNOSIS_COLLECTION",
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"该知识检索范围内未发现可用文档证据")),
|
||||
List.of());
|
||||
return new DiagnosisExecutionResult(
|
||||
ReleaseOutcome.FALLBACK, new FallbackContent(fallback), null, true);
|
||||
});
|
||||
|
||||
ChatApplicationResult result = useCase.execute(
|
||||
new ChatApplicationRequest("diagnose unknown issue", "session-budget-fallback"));
|
||||
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertEquals(ChatContentType.SAFE_FALLBACK, result.contentType());
|
||||
FallbackContent content = (FallbackContent) result.content();
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, content.fallback().type());
|
||||
assertEquals(1, content.fallback().observedFacts().size());
|
||||
assertEquals(ReleaseOutcome.FALLBACK, store.finishedOutcome);
|
||||
assertTrue(store.finishedSafeContentJson.contains("knowledge_base"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unhandledDiagnosisBudgetFailureDoesNotBuildFallbackInApplication() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.budget().reserveToolCall("lookup_knowledge");
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "tool budget exhausted");
|
||||
throw new IllegalStateException("unhandled diagnosis failure");
|
||||
});
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest(
|
||||
"diagnose unknown issue", "session-budget-unhandled")));
|
||||
|
||||
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
|
||||
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
|
||||
}
|
||||
|
||||
@Test
|
||||
void diagnosisBudgetExhaustionWithoutToolCallsRemainsFailure() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.DIAGNOSIS,
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
context.lifecycle().finish(RunState.BUDGET_EXHAUSTED, "model budget exhausted");
|
||||
throw new IllegalStateException("diagnosis failed before evidence collection");
|
||||
});
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest(
|
||||
"diagnose unavailable model", "session-budget-no-tools")));
|
||||
|
||||
assertEquals(ChatFailureCode.INTERNAL_FAILURE, failure.code());
|
||||
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientDisconnectControlCancelsSameRunAndPersistsCancelled() {
|
||||
FakeStore store = new FakeStore();
|
||||
@@ -252,6 +337,7 @@ class ChatApplicationUseCaseTest {
|
||||
private IntentType finishedIntent;
|
||||
private ReleaseOutcome finishedOutcome;
|
||||
private PublishedResult finishedPublishedResult;
|
||||
private String finishedSafeContentJson;
|
||||
|
||||
@Override
|
||||
public Optional<RoutingHistory> findLatestRoutingHistory(String sessionId) {
|
||||
@@ -283,6 +369,7 @@ class ChatApplicationUseCaseTest {
|
||||
finishedIntent = intent;
|
||||
finishedOutcome = outcome;
|
||||
finishedPublishedResult = publishedResult;
|
||||
finishedSafeContentJson = safeContentJson;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
@@ -14,6 +14,7 @@ import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.audit.ModelCallComponent;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
@@ -101,6 +102,8 @@ class PublishedResultPersistenceTest {
|
||||
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
|
||||
RunContext context = core().startRun("session-1", "run-fallback");
|
||||
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
context.modelCalls().begin(ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
|
||||
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
|
||||
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
|
||||
@@ -111,6 +114,7 @@ class PublishedResultPersistenceTest {
|
||||
assertEquals(ReleaseOutcome.FALLBACK, entity.getReleaseOutcome());
|
||||
assertNull(entity.getPublishedResult());
|
||||
assertEquals(12, entity.getTotalDurationMs());
|
||||
assertEquals(2, entity.getStepCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package com.superbiz.agent.harness.application.executor;
|
||||
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentInput;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentUseCase;
|
||||
import com.superbiz.agent.harness.application.ChatApplicationStatus;
|
||||
import com.superbiz.agent.harness.application.DiagnosisExecutionResult;
|
||||
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
|
||||
import com.superbiz.agent.harness.application.persistence.PreviousTurnLimits;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.release.DiagnosisReleaseResult;
|
||||
import com.superbiz.agent.harness.release.DiagnosisReleaseUseCase;
|
||||
import com.superbiz.agent.harness.release.SafeFallbackFactory;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verifyNoInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class DiagnosisChatExecutorTest {
|
||||
|
||||
@Test
|
||||
void invalidDraftWithVerifiedProgressUsesDeterministicReleaseFallback() {
|
||||
RunContext context = context("run-invalid-progress");
|
||||
DiagnosisProgressSnapshot progress = progress();
|
||||
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an invalid Draft",
|
||||
new IllegalArgumentException("redacted"),
|
||||
DiagnosisAgentOutputException.Kind.SCHEMA_INVALID,
|
||||
321L,
|
||||
progress);
|
||||
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
|
||||
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
|
||||
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
|
||||
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(progress, List.of());
|
||||
when(release.releaseInvalidDraft(context, progress))
|
||||
.thenReturn(DiagnosisReleaseResult.fallback(fallback));
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
|
||||
agent,
|
||||
release,
|
||||
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
|
||||
trace::add);
|
||||
List<ChatApplicationStatus> statuses = new ArrayList<>();
|
||||
|
||||
DiagnosisExecutionResult result = executor.execute(
|
||||
context, "诊断未知故障", null, statuses::add);
|
||||
|
||||
assertEquals(ReleaseOutcome.FALLBACK, result.outcome());
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE,
|
||||
((com.superbiz.agent.harness.application.FallbackContent) result.content())
|
||||
.fallback().type());
|
||||
assertEquals(List.of(
|
||||
ChatApplicationStatus.DIAGNOSIS_RUNNING,
|
||||
ChatApplicationStatus.SAFETY_VALIDATING), statuses);
|
||||
assertEquals(List.of(TraceEventType.AGENT_DRAFT_INVALID),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertEquals("SCHEMA_INVALID", trace.get(0).details().get("failure_kind"));
|
||||
assertEquals(321L, trace.get(0).details().get("output_bytes"));
|
||||
assertEquals(true, trace.get(0).details().get("has_publishable_progress"));
|
||||
assertEquals(3, trace.get(0).details().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftWithoutVerifiedProgressStillFailsClosed() {
|
||||
RunContext context = context("run-invalid-empty");
|
||||
DiagnosisAgentOutputException failure = new DiagnosisAgentOutputException(
|
||||
"Diagnosis Agent returned an invalid Draft",
|
||||
null,
|
||||
DiagnosisAgentOutputException.Kind.INVALID_JSON,
|
||||
12L,
|
||||
DiagnosisProgressSnapshot.empty());
|
||||
DiagnosisAgentUseCase agent = mock(DiagnosisAgentUseCase.class);
|
||||
DiagnosisReleaseUseCase release = mock(DiagnosisReleaseUseCase.class);
|
||||
when(agent.execute(eq(context), any(DiagnosisAgentInput.class))).thenThrow(failure);
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
DiagnosisChatExecutor executor = new DiagnosisChatExecutor(
|
||||
agent,
|
||||
release,
|
||||
new PublishedResultPolicy(PreviousTurnLimits.defaults()),
|
||||
trace::add);
|
||||
|
||||
DiagnosisAgentOutputException actual = assertThrows(
|
||||
DiagnosisAgentOutputException.class,
|
||||
() -> executor.execute(context, "诊断未知故障", null, ignored -> { }));
|
||||
|
||||
assertSame(failure, actual);
|
||||
verifyNoInteractions(release);
|
||||
assertEquals(false, trace.get(0).details().get("has_publishable_progress"));
|
||||
}
|
||||
|
||||
private RunContext context(String runId) {
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused-run", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(4, 3, 2, 100, 100, 200, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
return core.startRun("session-executor", runId);
|
||||
}
|
||||
|
||||
private DiagnosisProgressSnapshot progress() {
|
||||
return new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"已完成知识范围检查")),
|
||||
List.of(), null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class ModelCallAuditorTest {
|
||||
|
||||
@Test
|
||||
void recordsComponentUsageIntoRunBudgetTraceAndAgentStep() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-token", "run-token");
|
||||
AgentStepRepository steps = mock(AgentStepRepository.class);
|
||||
AgentStep step = AgentStep.builder().id(7L).runId(context.runId()).stepIndex(0).build();
|
||||
when(steps.findByRunIdAndStepIndex(context.runId(), 0)).thenReturn(Optional.of(step));
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(core, trace::add, steps);
|
||||
|
||||
core.beforeModelCall(context);
|
||||
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.DIAGNOSIS_AGENT);
|
||||
auditor.recordUsage(context, call, 13, 5, true);
|
||||
|
||||
assertEquals(18, context.budget().snapshot().totalTokens());
|
||||
assertEquals(18, context.modelCalls().snapshot().totalTokens());
|
||||
assertEquals(18, step.getTokenCount());
|
||||
verify(steps).save(step);
|
||||
DiagnosisTraceAuditEvent event = trace.get(0);
|
||||
assertEquals(TraceEventType.MODEL_TOKEN_USAGE, event.eventType());
|
||||
assertEquals("DIAGNOSIS_AGENT", event.details().get("component"));
|
||||
assertEquals(1, event.details().get("component_round"));
|
||||
assertEquals(13L, event.details().get("input_tokens"));
|
||||
assertEquals(5L, event.details().get("output_tokens"));
|
||||
assertEquals(18L, event.details().get("total_tokens"));
|
||||
assertEquals(true, event.details().get("usage_available"));
|
||||
String details = event.details().toString().toLowerCase();
|
||||
assertFalse(details.contains("prompt"));
|
||||
assertFalse(details.contains("content"));
|
||||
assertFalse(details.contains("response"));
|
||||
assertFalse(details.contains("argument"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unavailableUsageIsVisibleWithoutFabricatingTokens() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-missing", "run-missing");
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(
|
||||
core, trace::add, mock(AgentStepRepository.class));
|
||||
|
||||
core.beforeModelCall(context);
|
||||
ModelCallLedger.Call call = auditor.begin(context, ModelCallComponent.INTENT_ROUTER);
|
||||
auditor.recordUsage(context, call, 0, 0, false);
|
||||
|
||||
ModelCallLedger.Snapshot snapshot = context.modelCalls().snapshot();
|
||||
assertEquals(1, snapshot.auditedCallCount());
|
||||
assertEquals(1, snapshot.usageUnavailableCount());
|
||||
assertEquals(0, snapshot.totalTokens());
|
||||
assertEquals(0, context.budget().snapshot().totalTokens());
|
||||
assertEquals(false, trace.get(0).details().get("usage_available"));
|
||||
assertFalse(trace.get(0).details().containsKey("input_tokens"));
|
||||
assertFalse(trace.get(0).details().containsKey("output_tokens"));
|
||||
assertFalse(trace.get(0).details().containsKey("total_tokens"));
|
||||
DiagnosisTraceAuditEvent finished = TraceAuditEvents.runFinished(
|
||||
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
|
||||
com.superbiz.agent.harness.contract.ReleaseOutcome.FAILED, 1);
|
||||
assertEquals(false, finished.details().get("tokens_reconciled"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void runFinishedExposesTokenReconciliationWithoutContent() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-finish", "run-finish");
|
||||
ModelCallAuditor auditor = new ModelCallAuditor(
|
||||
core, DiagnosisTraceRecorder.noop(), mock(AgentStepRepository.class));
|
||||
core.beforeModelCall(context);
|
||||
auditor.recordUsage(context,
|
||||
auditor.begin(context, ModelCallComponent.INTENT_ROUTER), 8, 2, true);
|
||||
|
||||
DiagnosisTraceAuditEvent event = TraceAuditEvents.runFinished(
|
||||
context, com.superbiz.agent.harness.contract.IntentType.DIAGNOSIS,
|
||||
com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, 50);
|
||||
|
||||
assertEquals(10L, event.details().get("run_total_tokens"));
|
||||
assertEquals(10L, event.details().get("audited_total_tokens"));
|
||||
assertEquals(true, event.details().get("tokens_reconciled"));
|
||||
assertEquals(0, event.details().get("usage_unavailable_count"));
|
||||
assertTrue(event.details().containsKey("audited_model_call_count"));
|
||||
assertFalse(event.details().toString().contains("secret"));
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(20, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationStore;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalToolInvocation;
|
||||
import com.superbiz.agent.harness.tool.store.ToolCallKeyFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisProgressProjectorTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ToolCallKeyFactory keys = new ToolCallKeyFactory("test:tool");
|
||||
|
||||
@Test
|
||||
void projectsReadyCurrentRunRecordsInExecutionOrder() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(
|
||||
new CompletedToolCall("call-1", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}"),
|
||||
EvidenceStatus.EVIDENCE_FOUND);
|
||||
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
|
||||
records.put(keys.create(context.runId(), "call-1"), ready(
|
||||
"call-1", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"""
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"call-1","query":"unknown",
|
||||
"evidence":[{"document_id":"doc-1","source":"guide.md","excerpt":"generic guidance"}],
|
||||
"returned_count":1,"truncated":false}
|
||||
""", EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertEquals(1, snapshot.verifiedSources().size());
|
||||
assertEquals("guide.md", snapshot.verifiedSources().get(0).source());
|
||||
assertEquals(1, snapshot.observedFacts().size());
|
||||
assertEquals("generic guidance", snapshot.observedFacts().get(0).summary());
|
||||
assertTrue(snapshot.limitations().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void excludesMissingExpiredProjectingCrossRunAndErrorRecords() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(call("missing"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("expired"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("projecting"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("cross"), EvidenceStatus.NO_EVIDENCE);
|
||||
context.progress().recordCompleted(call("error"), EvidenceStatus.NO_EVIDENCE);
|
||||
Map<String, CanonicalToolInvocation> records = new LinkedHashMap<>();
|
||||
records.put(keys.create(context.runId(), "projecting"),
|
||||
CanonicalToolInvocation.projecting(
|
||||
"projecting", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}", Instant.now()));
|
||||
records.put(keys.create(context.runId(), "cross"), ready(
|
||||
"cross", "other-run", AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
noEvidence("cross"), EvidenceStatus.NO_EVIDENCE));
|
||||
records.put(keys.create(context.runId(), "error"),
|
||||
CanonicalToolInvocation.projecting(
|
||||
"error", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
"{\"query\":\"unknown\"}", Instant.now())
|
||||
.markError(null, "TOOL_EXECUTION_ERROR", Instant.now()));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertTrue(snapshot.observedFacts().isEmpty());
|
||||
assertFalse(snapshot.limitations().isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void boundsProjectedFactsAndReportsTruncation() {
|
||||
RunContext context = context();
|
||||
context.progress().recordCompleted(call("many"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
StringBuilder evidence = new StringBuilder();
|
||||
for (int index = 0; index < 13; index++) {
|
||||
if (index > 0) {
|
||||
evidence.append(',');
|
||||
}
|
||||
evidence.append("{\"document_id\":\"doc-")
|
||||
.append(index)
|
||||
.append("\",\"source\":\"guide-")
|
||||
.append(index)
|
||||
.append("\",\"excerpt\":\"")
|
||||
.append("x".repeat(400))
|
||||
.append("\"}");
|
||||
}
|
||||
String result = "{\"evidence_status\":\"EVIDENCE_FOUND\","
|
||||
+ "\"tool_call_id\":\"many\",\"query\":\"unknown\",\"evidence\":["
|
||||
+ evidence + "],\"returned_count\":13,\"truncated\":false}";
|
||||
Map<String, CanonicalToolInvocation> records = Map.of(
|
||||
keys.create(context.runId(), "many"),
|
||||
ready("many", context.runId(), AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
result, EvidenceStatus.EVIDENCE_FOUND));
|
||||
|
||||
DiagnosisProgressSnapshot snapshot = projector(records).project(context);
|
||||
|
||||
assertEquals(12, snapshot.observedFacts().size());
|
||||
assertTrue(snapshot.observedFacts().stream()
|
||||
.allMatch(fact -> fact.summary().length() <= 320));
|
||||
assertTrue(snapshot.limitations().contains("已检查事实较多,展示内容已截断"));
|
||||
}
|
||||
|
||||
private DiagnosisProgressProjector projector(Map<String, CanonicalToolInvocation> records) {
|
||||
return new DiagnosisProgressProjector(new MapStore(records), keys, objectMapper);
|
||||
}
|
||||
|
||||
private CompletedToolCall call(String id) {
|
||||
return new CompletedToolCall(
|
||||
id, AgentToolContracts.LOOKUP_KNOWLEDGE, "{\"query\":\"" + id + "\"}");
|
||||
}
|
||||
|
||||
private CanonicalToolInvocation ready(String id, String runId, String toolName,
|
||||
String agentResult, EvidenceStatus status) {
|
||||
return CanonicalToolInvocation.projecting(
|
||||
id, runId, toolName, "{\"query\":\"unknown\"}", Instant.now())
|
||||
.markReady("{}", agentResult, status, Instant.now());
|
||||
}
|
||||
|
||||
private String noEvidence(String id) {
|
||||
return "{\"evidence_status\":\"NO_EVIDENCE\",\"tool_call_id\":\"" + id
|
||||
+ "\",\"query\":\"unknown\",\"evidence\":[],\"returned_count\":0,"
|
||||
+ "\"truncated\":false}";
|
||||
}
|
||||
|
||||
private RunContext context() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "run-1", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 1000, 1000, 2000, 100_000),
|
||||
HarnessRetryPolicies.strict(), 10)
|
||||
.startRun("session-1", "run-1");
|
||||
}
|
||||
|
||||
private static final class MapStore implements CanonicalInvocationStore {
|
||||
|
||||
private final Map<String, CanonicalToolInvocation> records;
|
||||
private final CanonicalInvocationLimits limits =
|
||||
new CanonicalInvocationLimits(Duration.ofHours(1), 1_000_000, 100_000);
|
||||
|
||||
private MapStore(Map<String, CanonicalToolInvocation> records) {
|
||||
this.records = records;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalInvocationLimits limits() {
|
||||
return limits;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void begin(String key, CanonicalToolInvocation invocation) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CanonicalToolInvocation> find(String key) {
|
||||
return Optional.ofNullable(records.get(key));
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markReady(String key, String rawResponse, String agentResult,
|
||||
EvidenceStatus evidenceStatus, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CanonicalToolInvocation markError(String key, String rawResponse,
|
||||
String errorCode, Instant completedAt) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisProgressTrackerTest {
|
||||
|
||||
@Test
|
||||
void gainedResetsConsecutiveNoGainAndConsumesPendingCallOnce() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordDuplicateScope();
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
|
||||
|
||||
DiagnosisProgressSnapshotState state = tracker.snapshot();
|
||||
assertEquals(0, state.consecutiveNoGain());
|
||||
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
|
||||
new PreviousObservation("call-1", InformationGain.NO_GAIN)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsMissingOrOutOfOrderEvaluation() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(null));
|
||||
assertThrows(IllegalArgumentException.class, () -> tracker.applyPreviousObservation(
|
||||
new PreviousObservation("call-other", InformationGain.GAINED)));
|
||||
assertEquals("call-1", tracker.snapshot().pendingToolCallId());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deterministicNoEvidenceSaturatesAndStopInstructionIsClaimedOnce() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2);
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.NO_EVIDENCE);
|
||||
tracker.recordCompleted(call("call-2", "scope-2"), EvidenceStatus.NO_EVIDENCE);
|
||||
|
||||
DiagnosisProgressSnapshotState state = tracker.snapshot();
|
||||
assertEquals(DiagnosisCollectionState.SATURATED, state.collectionState());
|
||||
assertEquals(DiagnosisStopReason.INFORMATION_SATURATED, state.stopReason());
|
||||
assertTrue(tracker.claimStopInstruction());
|
||||
assertFalse(tracker.claimStopInstruction());
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlySuccessfulCompletedScopesParticipateInDuplicateDetection() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3);
|
||||
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-1"));
|
||||
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
|
||||
assertTrue(tracker.isDuplicate("lookup_knowledge", "scope-1"));
|
||||
assertFalse(tracker.isDuplicate("query_logs", "scope-1"));
|
||||
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
|
||||
}
|
||||
|
||||
private CompletedToolCall call(String id, String scope) {
|
||||
return new CompletedToolCall(id, "lookup_knowledge", scope);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.tool.contract.AgentToolContracts;
|
||||
import com.superbiz.agent.harness.tool.contract.LogTopic;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.QueryLogsRequest;
|
||||
import com.superbiz.agent.harness.tool.contract.RagToolRequest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||
|
||||
class ToolScopeNormalizerTest {
|
||||
|
||||
private final ToolScopeNormalizer normalizer = new ToolScopeNormalizer(new ObjectMapper());
|
||||
|
||||
@Test
|
||||
void trimsRagParametersWithoutSemanticRewriting() {
|
||||
String first = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest(" refund timeout "));
|
||||
String same = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest("refund timeout"));
|
||||
String rewritten = normalizer.normalize(AgentToolContracts.LOOKUP_KNOWLEDGE,
|
||||
new RagToolRequest("refund request is slow"));
|
||||
|
||||
assertEquals(first, same);
|
||||
assertNotEquals(first, rewritten);
|
||||
}
|
||||
|
||||
@Test
|
||||
void normalizesMissingLogLookbackToAdapterDefault() {
|
||||
String implicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
|
||||
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", null));
|
||||
String explicit = normalizer.normalize(AgentToolContracts.QUERY_LOGS,
|
||||
new QueryLogsRequest(LogTopic.APPLICATION, "timeout", 30));
|
||||
|
||||
assertEquals(implicit, explicit);
|
||||
}
|
||||
|
||||
@Test
|
||||
void mysqlScopeIncludesLogicalSourceSqlAndParameters() {
|
||||
String first = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
|
||||
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(1)));
|
||||
String otherParam = normalizer.normalize(AgentToolContracts.QUERY_MYSQL,
|
||||
new MysqlToolRequest("orders", "SELECT id FROM orders WHERE id = ?", List.of(2)));
|
||||
|
||||
assertNotEquals(first, otherParam);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentExecution;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
@@ -9,6 +10,7 @@ import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
@@ -16,6 +18,8 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuard;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticGuardLimits;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.store.CanonicalInvocationLimits;
|
||||
@@ -91,6 +95,101 @@ class DiagnosisReleaseUseCaseTest {
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noConclusionWithVerifiedNegativeReferenceSkipsRepairAndSemanticGuard() {
|
||||
Fixture fixture = fixture();
|
||||
readyNoEvidence(fixture.store, fixture.context, "call-empty-1");
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null,
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
|
||||
"知识库范围内未找到匹配证据", List.of("call-empty-1"))),
|
||||
List.of(),
|
||||
List.of(),
|
||||
new DiagnosisDraft.Limitations(
|
||||
"query=unknown", List.of("故障服务运行日志")));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.completed(draft, progress(null)));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals(1, result.fallback().observedFacts().size());
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
assertEquals(List.of(
|
||||
TraceEventType.EVIDENCE_GUARD_INITIAL,
|
||||
TraceEventType.RELEASE_DECISION),
|
||||
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void noConclusionWithoutToolCallsPublishesMissingContext() {
|
||||
Fixture fixture = fixture();
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null, List.of(), List.of(), List.of(),
|
||||
new DiagnosisDraft.Limitations(
|
||||
"未指定故障范围", List.of("企业标识", "故障发生时间")));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断切换企业失败的问题",
|
||||
DiagnosisAgentExecution.completed(draft, DiagnosisProgressSnapshot.empty()));
|
||||
|
||||
assertFallback(result, FallbackType.MISSING_REQUIRED_CONTEXT, 0);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fabricatedNoConclusionReferenceFailsWithoutModelRepair() {
|
||||
Fixture fixture = fixture();
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
null,
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NEGATIVE_OBSERVATION,
|
||||
"声称未找到证据", List.of("missing-call"))),
|
||||
List.of(), List.of(),
|
||||
new DiagnosisDraft.Limitations("query=unknown", List.of()));
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.completed(draft, progress(null)));
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void controlledSaturationPublishesProgressWithoutAnotherModelCall() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.stopped(
|
||||
progress(DiagnosisStopReason.INFORMATION_SATURATED),
|
||||
DiagnosisStopReason.INFORMATION_SATURATED));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.releaseInvalidDraft(
|
||||
fixture.context, progress(null));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals("该知识检索范围内未发现可用文档证据",
|
||||
result.fallback().observedFacts().get(0).summary());
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
assertEquals(List.of(TraceEventType.RELEASE_DECISION),
|
||||
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneStructuralRepairCanFixIdsWithoutChangingReportSemantics() {
|
||||
Fixture fixture = fixture(
|
||||
@@ -204,7 +303,7 @@ class DiagnosisReleaseUseCaseTest {
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)),
|
||||
ignored -> { }, trace),
|
||||
new SafeFallbackFactory(), trace);
|
||||
return new Fixture(context, model, useCase, trace);
|
||||
return new Fixture(context, store, model, useCase, trace);
|
||||
}
|
||||
|
||||
private DiagnosisDraft duplicateDraft() {
|
||||
@@ -274,6 +373,29 @@ class DiagnosisReleaseUseCaseTest {
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private void readyNoEvidence(InMemoryStore store, RunContext context, String callId) {
|
||||
CanonicalToolInvocation invocation = CanonicalToolInvocation.projecting(
|
||||
callId, context.runId(), "lookup_knowledge", "{\"query\":\"unknown\"}",
|
||||
Instant.parse("2026-07-21T10:00:00Z"))
|
||||
.markReady("raw-must-not-be-read", """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"call-empty-1",
|
||||
"query":"unknown","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", EvidenceStatus.NO_EVIDENCE,
|
||||
Instant.parse("2026-07-21T10:00:01Z"));
|
||||
store.records.put(PREFIX + ":" + context.runId() + ":" + callId, invocation);
|
||||
}
|
||||
|
||||
private DiagnosisProgressSnapshot progress(DiagnosisStopReason stopReason) {
|
||||
return new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "knowledge_base", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "knowledge_base", "query=unknown",
|
||||
"该知识检索范围内未发现可用文档证据")),
|
||||
List.of(),
|
||||
stopReason);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
@@ -337,6 +459,7 @@ class DiagnosisReleaseUseCaseTest {
|
||||
|
||||
private record Fixture(
|
||||
RunContext context,
|
||||
InMemoryStore store,
|
||||
ScriptedChatModel model,
|
||||
DiagnosisReleaseUseCase useCase,
|
||||
RecordingTraceRecorder trace) {
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
import com.superbiz.agent.harness.contract.SafeFallback;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshot;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class SafeFallbackFactoryTest {
|
||||
|
||||
@Test
|
||||
void insufficientEvidenceReusesVerifiedProgress() {
|
||||
DiagnosisProgressSnapshot progress = new DiagnosisProgressSnapshot(
|
||||
List.of(new SafeFallback.VerifiedSource(
|
||||
"RAG", "guide.md", "query=unknown")),
|
||||
List.of(new SafeFallback.ObservedFact(
|
||||
"RAG", "guide.md", "query=unknown", "未找到匹配的故障证据")),
|
||||
List.of("部分记录已过期"),
|
||||
DiagnosisStopReason.BUDGET_LIMIT_REACHED);
|
||||
|
||||
SafeFallback fallback = new SafeFallbackFactory().insufficientEvidence(
|
||||
progress, List.of("企业标识"));
|
||||
|
||||
assertEquals(FallbackType.INSUFFICIENT_EVIDENCE, fallback.type());
|
||||
assertEquals("DIAGNOSIS_COLLECTION", fallback.failureStage());
|
||||
assertEquals(progress.verifiedSources(), fallback.verifiedSources());
|
||||
assertEquals(progress.observedFacts(), fallback.observedFacts());
|
||||
assertTrue(fallback.limitations().contains("仍缺少:企业标识"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingRequiredContextContainsNoFabricatedProgress() {
|
||||
SafeFallback fallback = new SafeFallbackFactory().missingRequiredContext(
|
||||
List.of("企业标识", "故障发生时间"));
|
||||
|
||||
assertEquals(FallbackType.MISSING_REQUIRED_CONTEXT, fallback.type());
|
||||
assertEquals("DIAGNOSIS_INPUT", fallback.failureStage());
|
||||
assertTrue(fallback.verifiedSources().isEmpty());
|
||||
assertTrue(fallback.observedFacts().isEmpty());
|
||||
assertEquals(2, fallback.limitations().size());
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.superbiz.agent.harness.tool.contract;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.progress.PreviousObservation;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
class ToolCallEnvelopeContractTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void serializesSharedControlFieldsInSnakeCase() throws Exception {
|
||||
RagToolCall call = new RagToolCall(
|
||||
new PreviousObservation("call-1", InformationGain.NO_GAIN),
|
||||
new RagToolRequest("refund timeout"));
|
||||
|
||||
JsonNode json = objectMapper.readTree(objectMapper.writeValueAsString(call));
|
||||
|
||||
assertEquals("call-1", json.path("previous_observation").path("tool_call_id").asText());
|
||||
assertEquals("NO_GAIN", json.path("previous_observation").path("information_gain").asText());
|
||||
assertEquals("refund timeout", json.path("input").path("query").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void deserializesAllTypedBusinessInputs() throws Exception {
|
||||
QueryLogsToolCall logs = objectMapper.readValue("""
|
||||
{"input":{"topic":"APPLICATION","query":"timeout","lookback_minutes":30}}
|
||||
""", QueryLogsToolCall.class);
|
||||
MysqlToolCall mysql = objectMapper.readValue("""
|
||||
{"input":{"data_source":"orders","sql":"SELECT id FROM orders WHERE id = ?","params":[1]}}
|
||||
""", MysqlToolCall.class);
|
||||
|
||||
assertEquals(LogTopic.APPLICATION, logs.input().topic());
|
||||
assertEquals(List.of(1), mysql.input().params());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnknownInformationGain() {
|
||||
assertThrows(Exception.class, () -> objectMapper.readValue("""
|
||||
{"previous_observation":{"tool_call_id":"call-1","information_gain":"UNKNOWN"},
|
||||
"input":{"query":"timeout"}}
|
||||
""", RagToolCall.class));
|
||||
}
|
||||
}
|
||||
+22
-1
@@ -40,6 +40,25 @@ class RagResultProjectorTest {
|
||||
assertFalse(projected.agentResult().contains("score"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void preservesCamelAndSnakeCaseReferenceLevel() throws Exception {
|
||||
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
|
||||
String camel = """
|
||||
{"found":true,"relevanceLevel":"REFERENCE",
|
||||
"evidenceBlocks":[{"source":"doc","content":"generic guidance"}]}
|
||||
""";
|
||||
String snake = camel.replace("relevanceLevel", "relevance_level");
|
||||
|
||||
JsonNode camelResult = objectMapper.readTree(
|
||||
projector.project(new RagToolRequest("unknown"), "call-1", camel).agentResult());
|
||||
JsonNode snakeResult = objectMapper.readTree(
|
||||
projector.project(new RagToolRequest("unknown"), "call-2", snake).agentResult());
|
||||
|
||||
assertEquals("REFERENCE", camelResult.path("relevance_level").asText());
|
||||
assertEquals("REFERENCE", snakeResult.path("relevance_level").asText());
|
||||
assertEquals("EVIDENCE_FOUND", camelResult.path("evidence_status").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
void returnsNoEvidenceForEmptyUsableBlocks() throws Exception {
|
||||
RagResultProjector projector = new RagResultProjector(objectMapper, ToolProjectionLimits.defaults());
|
||||
@@ -48,6 +67,8 @@ class RagResultProjectorTest {
|
||||
"{\"found\":false,\"evidenceBlocks\":[{\"source\":\"doc\",\"content\":\"\"}]}");
|
||||
|
||||
assertEquals(EvidenceStatus.NO_EVIDENCE, projected.evidenceStatus());
|
||||
assertEquals(0, objectMapper.readTree(projected.agentResult()).path("evidence").size());
|
||||
JsonNode json = objectMapper.readTree(projected.agentResult());
|
||||
assertEquals(0, json.path("evidence").size());
|
||||
assertFalse(json.has("relevance_level"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user