feat(harness): complete protocol repair stop and archive ISS-016
Add repairable INVALID_PROGRESS_PROTOCOL observations, independent PROGRESS_PROTOCOL_VIOLATED saturation, and controlled release paths. Archive the OpenSpec change after syncing main specs and devflow.
This commit is contained in:
@@ -28,6 +28,7 @@ public class ChatHarnessProperties {
|
||||
private int maxToolCalls = 24;
|
||||
private int maxCallsPerTool = 8;
|
||||
private int stopAfterConsecutiveNoGain = 2;
|
||||
private int stopAfterConsecutiveProgressProtocolViolations = 2;
|
||||
private long maxInputTokens = 100_000;
|
||||
private long maxOutputTokens = 100_000;
|
||||
private long maxTotalTokens = 200_000;
|
||||
@@ -73,6 +74,8 @@ public class ChatHarnessProperties {
|
||||
positive(maxToolCalls, "maxToolCalls");
|
||||
positive(maxCallsPerTool, "maxCallsPerTool");
|
||||
positive(stopAfterConsecutiveNoGain, "stopAfterConsecutiveNoGain");
|
||||
positive(stopAfterConsecutiveProgressProtocolViolations,
|
||||
"stopAfterConsecutiveProgressProtocolViolations");
|
||||
positive(maxInputTokens, "maxInputTokens");
|
||||
positive(maxOutputTokens, "maxOutputTokens");
|
||||
positive(maxTotalTokens, "maxTotalTokens");
|
||||
|
||||
@@ -109,7 +109,8 @@ public class HarnessChatConfiguration {
|
||||
properties.getMaxCallsPerTool(), properties.getMaxInputTokens(),
|
||||
properties.getMaxOutputTokens(), properties.getMaxTotalTokens(), properties.getMaxRunBytes()),
|
||||
HarnessRetryPolicies.strict(),
|
||||
properties.getStopAfterConsecutiveNoGain());
|
||||
properties.getStopAfterConsecutiveNoGain(),
|
||||
properties.getStopAfterConsecutiveProgressProtocolViolations());
|
||||
}
|
||||
|
||||
@Bean
|
||||
|
||||
@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.ObjectReader;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.progress.ProgressProtocolViolationException;
|
||||
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
|
||||
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.RagToolAdapter;
|
||||
@@ -102,10 +104,17 @@ public final class HarnessEvidenceTools {
|
||||
}
|
||||
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
|
||||
};
|
||||
} catch (ProgressProtocolViolationException exception) {
|
||||
throw exception;
|
||||
} catch (IllegalArgumentException exception) {
|
||||
throw exception;
|
||||
} catch (Exception exception) {
|
||||
throw new IllegalArgumentException("Tool Call Envelope is invalid", exception);
|
||||
throw new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.INVALID_ENVELOPE,
|
||||
"Tool Call Envelope is invalid",
|
||||
null,
|
||||
null,
|
||||
exception);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +128,11 @@ public final class HarnessEvidenceTools {
|
||||
|
||||
private static void requireInput(Object input) {
|
||||
if (input == null) {
|
||||
throw new IllegalArgumentException("Tool Call Envelope input is required");
|
||||
throw new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.MISSING_INPUT,
|
||||
"Tool Call Envelope input is required",
|
||||
"input",
|
||||
null);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,10 +15,13 @@ import com.superbiz.agent.harness.progress.DiagnosisCollectionState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.progress.ProgressProtocolViolationException;
|
||||
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
|
||||
import com.superbiz.agent.harness.progress.ToolScopeNormalizer;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
@@ -81,9 +84,13 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
return stopRequired(request, evaluated.stopReason());
|
||||
}
|
||||
normalizedScope = scopeNormalizer.normalize(request.getToolName(), call.businessInput());
|
||||
} catch (ProgressProtocolViolationException exception) {
|
||||
return handleProgressProtocolViolation(request, exception);
|
||||
} catch (IllegalArgumentException | IllegalStateException exception) {
|
||||
recordRejection(request, "INVALID_PROGRESS_PROTOCOL");
|
||||
return safeError(request, "INVALID_PROGRESS_PROTOCOL");
|
||||
return handleProgressProtocolViolation(request,
|
||||
new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.INVALID_ENVELOPE,
|
||||
"Tool Call Envelope is invalid", null, null, exception));
|
||||
}
|
||||
|
||||
if (context.progress().isDuplicate(request.getToolName(), normalizedScope)) {
|
||||
@@ -167,6 +174,53 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
request.getToolCallId(), request.getToolName(), writeObservation(observation));
|
||||
}
|
||||
|
||||
private ToolCallResponse handleProgressProtocolViolation(
|
||||
ToolCallRequest request,
|
||||
ProgressProtocolViolationException exception) {
|
||||
DiagnosisProgressSnapshotState state =
|
||||
context.progress().recordProgressProtocolViolation();
|
||||
boolean stopRequired = state.collectionState() == DiagnosisCollectionState.SATURATED;
|
||||
recordRejection(request, "INVALID_PROGRESS_PROTOCOL", exception.violationType(),
|
||||
!stopRequired, state);
|
||||
if (stopRequired) {
|
||||
return stopRequired(request, state.stopReason());
|
||||
}
|
||||
return repairableProtocolError(request, exception, state);
|
||||
}
|
||||
|
||||
private ToolCallResponse repairableProtocolError(
|
||||
ToolCallRequest request,
|
||||
ProgressProtocolViolationException exception,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("evidence_status", "ERROR");
|
||||
observation.put("tool_call_id", request.getToolCallId());
|
||||
observation.put("error_code", "INVALID_PROGRESS_PROTOCOL");
|
||||
observation.put("repair_required", true);
|
||||
observation.put("violation_type", exception.violationType().name());
|
||||
if (exception.missingField() != null && !exception.missingField().isBlank()) {
|
||||
observation.put("missing_field", exception.missingField());
|
||||
}
|
||||
if (exception.expectedPreviousToolCallId() != null
|
||||
&& !exception.expectedPreviousToolCallId().isBlank()) {
|
||||
observation.put("expected_previous_tool_call_id",
|
||||
exception.expectedPreviousToolCallId());
|
||||
}
|
||||
observation.put("allowed_information_gain", List.of(
|
||||
InformationGain.GAINED.name(), InformationGain.NO_GAIN.name()));
|
||||
observation.put("instruction",
|
||||
"Before calling another tool, evaluate the previous observation.");
|
||||
observation.put("consecutive_protocol_violations",
|
||||
state.consecutiveProgressProtocolViolations());
|
||||
return ToolCallResponse.builder()
|
||||
.toolCallId(request.getToolCallId())
|
||||
.toolName(request.getToolName())
|
||||
.content(writeObservation(observation))
|
||||
.status("error")
|
||||
.metadata(Map.of("error", true))
|
||||
.build();
|
||||
}
|
||||
|
||||
private ToolCallResponse safeError(ToolCallRequest request, String errorCode) {
|
||||
Map<String, Object> observation = new LinkedHashMap<>();
|
||||
observation.put("evidence_status", "ERROR");
|
||||
@@ -182,8 +236,18 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
|
||||
}
|
||||
|
||||
private void recordRejection(ToolCallRequest request, String errorCode) {
|
||||
recordRejection(request, errorCode, null, false, context.progress().snapshot());
|
||||
}
|
||||
|
||||
private void recordRejection(
|
||||
ToolCallRequest request,
|
||||
String errorCode,
|
||||
ProgressProtocolViolationType violationType,
|
||||
boolean repairPromptDelivered,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
traceRecorder.record(TraceAuditEvents.toolRequestRejected(
|
||||
context, request.getToolCallId(), request.getToolName(), errorCode));
|
||||
context, request.getToolCallId(), request.getToolName(), errorCode,
|
||||
violationType, repairPromptDelivered, state));
|
||||
}
|
||||
|
||||
private String writeObservation(Map<String, Object> observation) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
|
||||
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
|
||||
import com.superbiz.agent.harness.progress.InformationGain;
|
||||
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -140,10 +141,33 @@ public final class TraceAuditEvents {
|
||||
|
||||
public static DiagnosisTraceAuditEvent toolRequestRejected(
|
||||
RunContext context, String toolCallId, String toolName, String errorCode) {
|
||||
return toolRequestRejected(context, toolCallId, toolName, errorCode,
|
||||
null, false, null);
|
||||
}
|
||||
|
||||
public static DiagnosisTraceAuditEvent toolRequestRejected(
|
||||
RunContext context,
|
||||
String toolCallId,
|
||||
String toolName,
|
||||
String errorCode,
|
||||
ProgressProtocolViolationType violationType,
|
||||
boolean repairPromptDelivered,
|
||||
DiagnosisProgressSnapshotState state) {
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
details.put("tool_call_id", safeIdentifier(toolCallId));
|
||||
details.put("tool_name", safeIdentifier(toolName));
|
||||
details.put("error_code", safeIdentifier(errorCode));
|
||||
if (violationType != null) {
|
||||
details.put("violation_type", violationType.name());
|
||||
details.put("repair_prompt_delivered", repairPromptDelivered);
|
||||
}
|
||||
if (state != null && state.consecutiveProgressProtocolViolations() > 0) {
|
||||
details.put("consecutive_protocol_violations",
|
||||
state.consecutiveProgressProtocolViolations());
|
||||
}
|
||||
if (state != null && state.stopReason() != null) {
|
||||
details.put("stop_reason", state.stopReason().name());
|
||||
}
|
||||
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
|
||||
TraceEventStatus.REJECTED, null, null, details);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ public final class DiagnosisHarnessCore {
|
||||
private final RunBudgetLimits budgetLimits;
|
||||
private final HarnessRetryPolicies retryPolicies;
|
||||
private final int stopAfterConsecutiveNoGain;
|
||||
private final int stopAfterConsecutiveProgressProtocolViolations;
|
||||
|
||||
public DiagnosisHarnessCore(Clock clock,
|
||||
Supplier<String> runIdSupplier,
|
||||
@@ -33,6 +34,17 @@ public final class DiagnosisHarnessCore {
|
||||
RunBudgetLimits budgetLimits,
|
||||
HarnessRetryPolicies retryPolicies,
|
||||
int stopAfterConsecutiveNoGain) {
|
||||
this(clock, runIdSupplier, maxRunDuration, budgetLimits, retryPolicies,
|
||||
stopAfterConsecutiveNoGain, 2);
|
||||
}
|
||||
|
||||
public DiagnosisHarnessCore(Clock clock,
|
||||
Supplier<String> runIdSupplier,
|
||||
Duration maxRunDuration,
|
||||
RunBudgetLimits budgetLimits,
|
||||
HarnessRetryPolicies retryPolicies,
|
||||
int stopAfterConsecutiveNoGain,
|
||||
int stopAfterConsecutiveProgressProtocolViolations) {
|
||||
this.clock = Objects.requireNonNull(clock, "clock must not be null");
|
||||
this.runIdSupplier = Objects.requireNonNull(runIdSupplier, "runIdSupplier must not be null");
|
||||
this.maxRunDuration = requirePositive(maxRunDuration, "maxRunDuration");
|
||||
@@ -41,7 +53,13 @@ public final class DiagnosisHarnessCore {
|
||||
if (stopAfterConsecutiveNoGain <= 0) {
|
||||
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
|
||||
}
|
||||
if (stopAfterConsecutiveProgressProtocolViolations <= 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"stopAfterConsecutiveProgressProtocolViolations must be positive");
|
||||
}
|
||||
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
|
||||
this.stopAfterConsecutiveProgressProtocolViolations =
|
||||
stopAfterConsecutiveProgressProtocolViolations;
|
||||
}
|
||||
|
||||
public RunContext startRun(String sessionId) {
|
||||
@@ -61,7 +79,8 @@ public final class DiagnosisHarnessCore {
|
||||
new ModelCallLedger(),
|
||||
retryPolicies,
|
||||
lifecycle,
|
||||
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain));
|
||||
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain,
|
||||
stopAfterConsecutiveProgressProtocolViolations));
|
||||
cancellation.onCancel(reason -> lifecycle.finish(terminalState(reason), reason.name()));
|
||||
return context;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.util.List;
|
||||
|
||||
public record DiagnosisProgressSnapshotState(
|
||||
int consecutiveNoGain,
|
||||
int consecutiveProgressProtocolViolations,
|
||||
DiagnosisCollectionState collectionState,
|
||||
DiagnosisStopReason stopReason,
|
||||
String pendingToolCallId,
|
||||
|
||||
@@ -10,35 +10,63 @@ import java.util.Set;
|
||||
public final class DiagnosisProgressTracker {
|
||||
|
||||
private final int stopAfterConsecutiveNoGain;
|
||||
private final int stopAfterConsecutiveProgressProtocolViolations;
|
||||
private final Set<ToolScopeIdentity> completedScopes = new LinkedHashSet<>();
|
||||
private final List<CompletedToolCall> completedToolCalls = new ArrayList<>();
|
||||
private int consecutiveNoGain;
|
||||
private int consecutiveProgressProtocolViolations;
|
||||
private DiagnosisCollectionState collectionState = DiagnosisCollectionState.COLLECTING;
|
||||
private DiagnosisStopReason stopReason;
|
||||
private String pendingToolCallId;
|
||||
private boolean stopInstructionDelivered;
|
||||
|
||||
public DiagnosisProgressTracker(int stopAfterConsecutiveNoGain) {
|
||||
this(stopAfterConsecutiveNoGain, 2);
|
||||
}
|
||||
|
||||
public DiagnosisProgressTracker(
|
||||
int stopAfterConsecutiveNoGain,
|
||||
int stopAfterConsecutiveProgressProtocolViolations) {
|
||||
if (stopAfterConsecutiveNoGain <= 0) {
|
||||
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
|
||||
}
|
||||
if (stopAfterConsecutiveProgressProtocolViolations <= 0) {
|
||||
throw new IllegalArgumentException(
|
||||
"stopAfterConsecutiveProgressProtocolViolations must be positive");
|
||||
}
|
||||
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
|
||||
this.stopAfterConsecutiveProgressProtocolViolations =
|
||||
stopAfterConsecutiveProgressProtocolViolations;
|
||||
}
|
||||
|
||||
public synchronized void applyPreviousObservation(PreviousObservation observation) {
|
||||
if (pendingToolCallId == null) {
|
||||
if (observation != null) {
|
||||
throw new IllegalArgumentException("No Tool observation is pending evaluation");
|
||||
throw new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.UNEXPECTED_PREVIOUS_OBSERVATION,
|
||||
"No Tool observation is pending evaluation",
|
||||
"previous_observation",
|
||||
null);
|
||||
}
|
||||
clearProtocolViolations();
|
||||
return;
|
||||
}
|
||||
if (observation == null) {
|
||||
throw new IllegalArgumentException("Previous Tool observation must be evaluated");
|
||||
throw new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.MISSING_PREVIOUS_OBSERVATION,
|
||||
"Previous Tool observation must be evaluated",
|
||||
"previous_observation",
|
||||
pendingToolCallId);
|
||||
}
|
||||
if (!pendingToolCallId.equals(observation.toolCallId())) {
|
||||
throw new IllegalArgumentException("Previous Tool observation ID is out of order");
|
||||
throw new ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType.OUT_OF_ORDER_PREVIOUS_OBSERVATION,
|
||||
"Previous Tool observation ID is out of order",
|
||||
"previous_observation.tool_call_id",
|
||||
pendingToolCallId);
|
||||
}
|
||||
pendingToolCallId = null;
|
||||
clearProtocolViolations();
|
||||
applyGain(observation.informationGain());
|
||||
}
|
||||
|
||||
@@ -47,6 +75,7 @@ public final class DiagnosisProgressTracker {
|
||||
}
|
||||
|
||||
public synchronized void recordDuplicateScope() {
|
||||
clearProtocolViolations();
|
||||
applyGain(InformationGain.NO_GAIN);
|
||||
}
|
||||
|
||||
@@ -64,12 +93,26 @@ public final class DiagnosisProgressTracker {
|
||||
}
|
||||
completedToolCalls.add(call);
|
||||
if (evidenceStatus == EvidenceStatus.NO_EVIDENCE) {
|
||||
clearProtocolViolations();
|
||||
applyGain(InformationGain.NO_GAIN);
|
||||
} else {
|
||||
pendingToolCallId = call.toolCallId();
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized DiagnosisProgressSnapshotState recordProgressProtocolViolation() {
|
||||
if (collectionState == DiagnosisCollectionState.SATURATED) {
|
||||
return snapshot();
|
||||
}
|
||||
consecutiveProgressProtocolViolations++;
|
||||
if (consecutiveProgressProtocolViolations
|
||||
>= stopAfterConsecutiveProgressProtocolViolations) {
|
||||
collectionState = DiagnosisCollectionState.SATURATED;
|
||||
stopReason = DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED;
|
||||
}
|
||||
return snapshot();
|
||||
}
|
||||
|
||||
public synchronized boolean claimStopInstruction() {
|
||||
if (collectionState != DiagnosisCollectionState.SATURATED) {
|
||||
return false;
|
||||
@@ -90,6 +133,7 @@ public final class DiagnosisProgressTracker {
|
||||
public synchronized DiagnosisProgressSnapshotState snapshot() {
|
||||
return new DiagnosisProgressSnapshotState(
|
||||
consecutiveNoGain,
|
||||
consecutiveProgressProtocolViolations,
|
||||
collectionState,
|
||||
stopReason,
|
||||
pendingToolCallId,
|
||||
@@ -101,6 +145,10 @@ public final class DiagnosisProgressTracker {
|
||||
return stopAfterConsecutiveNoGain;
|
||||
}
|
||||
|
||||
public int stopAfterConsecutiveProgressProtocolViolations() {
|
||||
return stopAfterConsecutiveProgressProtocolViolations;
|
||||
}
|
||||
|
||||
private void applyGain(InformationGain gain) {
|
||||
if (collectionState == DiagnosisCollectionState.SATURATED) {
|
||||
throw new IllegalStateException("Collection is already saturated");
|
||||
@@ -115,4 +163,8 @@ public final class DiagnosisProgressTracker {
|
||||
stopReason = DiagnosisStopReason.INFORMATION_SATURATED;
|
||||
}
|
||||
}
|
||||
|
||||
private void clearProtocolViolations() {
|
||||
consecutiveProgressProtocolViolations = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,5 +2,6 @@ package com.superbiz.agent.harness.progress;
|
||||
|
||||
public enum DiagnosisStopReason {
|
||||
INFORMATION_SATURATED,
|
||||
BUDGET_LIMIT_REACHED
|
||||
BUDGET_LIMIT_REACHED,
|
||||
PROGRESS_PROTOCOL_VIOLATED
|
||||
}
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public final class ProgressProtocolViolationException extends IllegalArgumentException {
|
||||
|
||||
private final ProgressProtocolViolationType violationType;
|
||||
private final String missingField;
|
||||
private final String expectedPreviousToolCallId;
|
||||
|
||||
public ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType violationType,
|
||||
String message,
|
||||
String missingField,
|
||||
String expectedPreviousToolCallId) {
|
||||
super(message);
|
||||
this.violationType = violationType;
|
||||
this.missingField = missingField;
|
||||
this.expectedPreviousToolCallId = expectedPreviousToolCallId;
|
||||
}
|
||||
|
||||
public ProgressProtocolViolationException(
|
||||
ProgressProtocolViolationType violationType,
|
||||
String message,
|
||||
String missingField,
|
||||
String expectedPreviousToolCallId,
|
||||
Throwable cause) {
|
||||
super(message, cause);
|
||||
this.violationType = violationType;
|
||||
this.missingField = missingField;
|
||||
this.expectedPreviousToolCallId = expectedPreviousToolCallId;
|
||||
}
|
||||
|
||||
public ProgressProtocolViolationType violationType() {
|
||||
return violationType;
|
||||
}
|
||||
|
||||
public String missingField() {
|
||||
return missingField;
|
||||
}
|
||||
|
||||
public String expectedPreviousToolCallId() {
|
||||
return expectedPreviousToolCallId;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.superbiz.agent.harness.progress;
|
||||
|
||||
public enum ProgressProtocolViolationType {
|
||||
MISSING_PREVIOUS_OBSERVATION,
|
||||
OUT_OF_ORDER_PREVIOUS_OBSERVATION,
|
||||
UNEXPECTED_PREVIOUS_OBSERVATION,
|
||||
MISSING_INPUT,
|
||||
INVALID_ENVELOPE
|
||||
}
|
||||
@@ -166,7 +166,8 @@ public final class DiagnosisReleaseUseCase {
|
||||
DiagnosisProgressSnapshot progress,
|
||||
DiagnosisStopReason stopReason) {
|
||||
if (stopReason != DiagnosisStopReason.INFORMATION_SATURATED
|
||||
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED) {
|
||||
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED
|
||||
&& stopReason != DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED) {
|
||||
throw new IllegalStateException("Unsupported Diagnosis stop reason");
|
||||
}
|
||||
if (!progress.hasObservedFacts()) {
|
||||
|
||||
@@ -35,9 +35,13 @@ class HarnessChatConfigurationTest {
|
||||
void diagnosisNoGainThresholdDefaultsAndValidates() {
|
||||
ChatHarnessProperties properties = new ChatHarnessProperties();
|
||||
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
|
||||
assertEquals(2, properties.getStopAfterConsecutiveProgressProtocolViolations());
|
||||
|
||||
properties.setStopAfterConsecutiveNoGain(0);
|
||||
assertThrows(IllegalArgumentException.class, properties::validate);
|
||||
properties.setStopAfterConsecutiveNoGain(2);
|
||||
properties.setStopAfterConsecutiveProgressProtocolViolations(0);
|
||||
assertThrows(IllegalArgumentException.class, properties::validate);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -149,6 +149,41 @@ class DiagnosisAgentUseCaseTest {
|
||||
assertEquals(2, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void consecutiveProgressProtocolViolationsStopWithoutConsumingToolBudget() {
|
||||
DiagnosisHarnessCore core = core(new RunBudgetLimits(8, 8, 8,
|
||||
100, 100, 200, 100_000));
|
||||
RunContext context = core.startRun("session-protocol-stop", "run-protocol-stop");
|
||||
AtomicInteger toolCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
|
||||
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
|
||||
toolCalls.incrementAndGet();
|
||||
return ToolBoundaryResult.ready(id, evidence(id), EvidenceStatus.EVIDENCE_FOUND);
|
||||
});
|
||||
// First Tool succeeds and becomes pending evaluation. The next two Tool Calls omit
|
||||
// previous_observation, hit the protocol-violation threshold, and force a controlled stop
|
||||
// when the model still ignores STOP_REQUIRED.
|
||||
ScriptedChatModel model = new ScriptedChatModel(1, 1,
|
||||
toolCall("call-evidence-1", "{\"query\":\"first\"}"),
|
||||
toolCall("call-missing-eval-1", "{\"query\":\"second\"}"),
|
||||
toolCall("call-missing-eval-2", "{\"query\":\"third\"}"),
|
||||
toolCall("call-ignored-protocol-stop", "{\"query\":\"fourth\"}"));
|
||||
|
||||
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
|
||||
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
|
||||
|
||||
assertNull(execution.draft());
|
||||
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, execution.stopReason());
|
||||
// Default empty projector is used here; publishable facts are covered by Release tests.
|
||||
// Tracker still retains the completed Tool identity that a real projector would read.
|
||||
assertEquals(1, context.progress().snapshot().completedToolCalls().size());
|
||||
assertEquals("call-evidence-1",
|
||||
context.progress().snapshot().completedToolCalls().get(0).toolCallId());
|
||||
assertEquals(4, model.calls());
|
||||
assertEquals(1, toolCalls.get());
|
||||
assertEquals(1, context.budget().snapshot().toolCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
void fencedOutputFailsClosedWithoutAgentRetry() {
|
||||
DiagnosisHarnessCore core = core(defaultBudget());
|
||||
|
||||
@@ -10,6 +10,7 @@ import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
|
||||
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
|
||||
@@ -182,8 +183,16 @@ class HarnessToolInterceptorTest {
|
||||
"""), ignored -> null);
|
||||
|
||||
assertTrue(missing.isError());
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL",
|
||||
objectMapper.readTree(missing.getResult()).path("error_code").asText());
|
||||
JsonNode missingObservation = objectMapper.readTree(missing.getResult());
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL", missingObservation.path("error_code").asText());
|
||||
assertTrue(missingObservation.path("repair_required").asBoolean());
|
||||
assertEquals("MISSING_PREVIOUS_OBSERVATION",
|
||||
missingObservation.path("violation_type").asText());
|
||||
assertEquals("previous_observation", missingObservation.path("missing_field").asText());
|
||||
assertEquals("call-1", missingObservation.path("expected_previous_tool_call_id").asText());
|
||||
assertEquals(List.of("GAINED", "NO_GAIN"), objectMapper.convertValue(
|
||||
missingObservation.path("allowed_information_gain"), List.class));
|
||||
assertFalse(missing.getResult().contains("two"));
|
||||
assertFalse(accepted.isError());
|
||||
assertEquals(2, invocations.get());
|
||||
DiagnosisTraceAuditEvent rejected = trace.stream()
|
||||
@@ -191,9 +200,124 @@ class HarnessToolInterceptorTest {
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("call-2", rejected.details().get("tool_call_id"));
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
|
||||
assertEquals("MISSING_PREVIOUS_OBSERVATION", rejected.details().get("violation_type"));
|
||||
assertEquals(true, rejected.details().get("repair_prompt_delivered"));
|
||||
assertEquals(1, rejected.details().get("consecutive_protocol_violations"));
|
||||
String details = rejected.details().toString();
|
||||
assertFalse(details.contains("two"));
|
||||
assertFalse(details.contains("previous_observation"));
|
||||
assertFalse(details.contains("secret"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void outOfOrderPreviousObservationIsRepairableWithoutExecutingTool() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
|
||||
ToolCallResponse response = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
|
||||
{"previous_observation":{"tool_call_id":"call-other","information_gain":"GAINED"},
|
||||
"input":{"query":"two-secret"}}
|
||||
"""), ignored -> null);
|
||||
|
||||
JsonNode observation = objectMapper.readTree(response.getResult());
|
||||
assertTrue(response.isError());
|
||||
assertEquals(1, invocations.get());
|
||||
assertEquals("OUT_OF_ORDER_PREVIOUS_OBSERVATION",
|
||||
observation.path("violation_type").asText());
|
||||
assertEquals("call-1", observation.path("expected_previous_tool_call_id").asText());
|
||||
assertFalse(response.getResult().contains("two-secret"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void consecutiveProtocolViolationsDeliverStopRequiredOnce() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
RunContext run = context(6, 6, 6);
|
||||
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
|
||||
run, tools, objectMapper, trace::add);
|
||||
|
||||
interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
|
||||
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
|
||||
ToolCallResponse first = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
|
||||
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
|
||||
ToolCallResponse stop = interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3",
|
||||
"{\"input\":{\"query\":\"three\"}}"), ignored -> null);
|
||||
|
||||
JsonNode firstObservation = objectMapper.readTree(first.getResult());
|
||||
JsonNode stopObservation = objectMapper.readTree(stop.getResult());
|
||||
assertTrue(firstObservation.path("repair_required").asBoolean());
|
||||
assertTrue(stopObservation.path("stop_required").asBoolean());
|
||||
assertEquals("PROGRESS_PROTOCOL_VIOLATED", stopObservation.path("reason").asText());
|
||||
assertEquals(1, invocations.get());
|
||||
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED,
|
||||
run.progress().snapshot().stopReason());
|
||||
|
||||
DiagnosisCollectionStoppedException ignoredStop = assertThrows(
|
||||
DiagnosisCollectionStoppedException.class,
|
||||
() -> interceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-4",
|
||||
"{\"input\":{\"query\":\"four\"}}"), ignored -> null));
|
||||
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, ignoredStop.stopReason());
|
||||
assertEquals(1, invocations.get());
|
||||
|
||||
DiagnosisTraceAuditEvent stopRejection = trace.stream()
|
||||
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
|
||||
.filter(event -> "call-3".equals(event.details().get("tool_call_id")))
|
||||
.findFirst().orElseThrow();
|
||||
assertEquals("INVALID_PROGRESS_PROTOCOL", stopRejection.details().get("error_code"));
|
||||
assertEquals("MISSING_PREVIOUS_OBSERVATION", stopRejection.details().get("violation_type"));
|
||||
assertEquals(false, stopRejection.details().get("repair_prompt_delivered"));
|
||||
assertEquals(2, stopRejection.details().get("consecutive_protocol_violations"));
|
||||
assertEquals("PROGRESS_PROTOCOL_VIOLATED", stopRejection.details().get("stop_reason"));
|
||||
String details = stopRejection.details().toString();
|
||||
assertFalse(details.contains("three"));
|
||||
assertFalse(details.contains("exception"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void missingInputAndInvalidEnvelopeProduceTypedProtocolErrors() throws Exception {
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
|
||||
invocations.incrementAndGet();
|
||||
return ready(id);
|
||||
});
|
||||
HarnessToolInterceptor missingInterceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
HarnessToolInterceptor invalidInterceptor = new HarnessToolInterceptor(
|
||||
context(3, 3, 3), tools, objectMapper);
|
||||
|
||||
ToolCallResponse missingInput = missingInterceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-missing-input",
|
||||
"{\"previous_observation\":null}"), ignored -> null);
|
||||
ToolCallResponse invalidEnvelope = invalidInterceptor.interceptToolCall(
|
||||
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-invalid",
|
||||
"{not-json"), ignored -> null);
|
||||
|
||||
JsonNode missing = objectMapper.readTree(missingInput.getResult());
|
||||
JsonNode invalid = objectMapper.readTree(invalidEnvelope.getResult());
|
||||
assertEquals("MISSING_INPUT", missing.path("violation_type").asText());
|
||||
assertEquals("input", missing.path("missing_field").asText());
|
||||
assertEquals("INVALID_ENVELOPE", invalid.path("violation_type").asText());
|
||||
assertTrue(missing.path("repair_required").asBoolean());
|
||||
assertTrue(invalid.path("repair_required").asBoolean());
|
||||
assertEquals(0, invocations.get());
|
||||
assertFalse(missingInput.getResult().contains("not-json"));
|
||||
assertFalse(invalidEnvelope.getResult().contains("not-json"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -61,6 +61,53 @@ class DiagnosisProgressTrackerTest {
|
||||
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void progressProtocolViolationsAccumulateIndependentlyFromNoGain() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3, 2);
|
||||
tracker.recordDuplicateScope();
|
||||
|
||||
DiagnosisProgressSnapshotState first = tracker.recordProgressProtocolViolation();
|
||||
assertEquals(1, first.consecutiveProgressProtocolViolations());
|
||||
assertEquals(1, first.consecutiveNoGain());
|
||||
assertEquals(DiagnosisCollectionState.COLLECTING, first.collectionState());
|
||||
|
||||
DiagnosisProgressSnapshotState second = tracker.recordProgressProtocolViolation();
|
||||
assertEquals(2, second.consecutiveProgressProtocolViolations());
|
||||
assertEquals(1, second.consecutiveNoGain());
|
||||
assertEquals(DiagnosisCollectionState.SATURATED, second.collectionState());
|
||||
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, second.stopReason());
|
||||
assertTrue(tracker.claimStopInstruction());
|
||||
assertFalse(tracker.claimStopInstruction());
|
||||
}
|
||||
|
||||
@Test
|
||||
void validEvaluationClearsProtocolViolationCount() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2, 3);
|
||||
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
|
||||
tracker.recordProgressProtocolViolation();
|
||||
|
||||
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
|
||||
|
||||
DiagnosisProgressSnapshotState state = tracker.snapshot();
|
||||
assertEquals(0, state.consecutiveProgressProtocolViolations());
|
||||
assertEquals(0, state.consecutiveNoGain());
|
||||
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsUnexpectedPreviousObservationWithTypedViolation() {
|
||||
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2, 2);
|
||||
|
||||
ProgressProtocolViolationException failure = assertThrows(
|
||||
ProgressProtocolViolationException.class,
|
||||
() -> tracker.applyPreviousObservation(
|
||||
new PreviousObservation("call-x", InformationGain.GAINED)));
|
||||
|
||||
assertEquals(ProgressProtocolViolationType.UNEXPECTED_PREVIOUS_OBSERVATION,
|
||||
failure.violationType());
|
||||
assertEquals("previous_observation", failure.missingField());
|
||||
}
|
||||
|
||||
private CompletedToolCall call(String id, String scope) {
|
||||
return new CompletedToolCall(id, "lookup_knowledge", scope);
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisReleaseUseCaseTest {
|
||||
@@ -175,6 +176,37 @@ class DiagnosisReleaseUseCaseTest {
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void controlledProtocolViolationPublishesProgressWithoutAnotherModelCall() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
DiagnosisReleaseResult result = fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.stopped(
|
||||
progress(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED),
|
||||
DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED));
|
||||
|
||||
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void controlledProtocolViolationWithoutProgressFailsClosed() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
IllegalStateException failure = assertThrows(IllegalStateException.class, () ->
|
||||
fixture.useCase.execute(
|
||||
fixture.context,
|
||||
"诊断未知故障",
|
||||
DiagnosisAgentExecution.stopped(
|
||||
DiagnosisProgressSnapshot.empty(),
|
||||
DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED)));
|
||||
|
||||
assertTrue(failure.getMessage().contains("no verified publishable progress"));
|
||||
assertEquals(0, fixture.model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
|
||||
Fixture fixture = fixture();
|
||||
|
||||
Reference in New Issue
Block a user