feat(harness): complete protocol repair stop and archive ISS-016

Add repairable INVALID_PROGRESS_PROTOCOL observations, independent
PROGRESS_PROTOCOL_VIOLATED saturation, and controlled release paths.
Archive the OpenSpec change after syncing main specs and devflow.
This commit is contained in:
zhuyongxin
2026-07-27 19:10:07 +08:00
parent 5c369f3b6c
commit 38f781b157
44 changed files with 977 additions and 80 deletions
@@ -28,6 +28,7 @@ public class ChatHarnessProperties {
private int maxToolCalls = 24;
private int maxCallsPerTool = 8;
private int stopAfterConsecutiveNoGain = 2;
private int stopAfterConsecutiveProgressProtocolViolations = 2;
private long maxInputTokens = 100_000;
private long maxOutputTokens = 100_000;
private long maxTotalTokens = 200_000;
@@ -73,6 +74,8 @@ public class ChatHarnessProperties {
positive(maxToolCalls, "maxToolCalls");
positive(maxCallsPerTool, "maxCallsPerTool");
positive(stopAfterConsecutiveNoGain, "stopAfterConsecutiveNoGain");
positive(stopAfterConsecutiveProgressProtocolViolations,
"stopAfterConsecutiveProgressProtocolViolations");
positive(maxInputTokens, "maxInputTokens");
positive(maxOutputTokens, "maxOutputTokens");
positive(maxTotalTokens, "maxTotalTokens");
@@ -109,7 +109,8 @@ public class HarnessChatConfiguration {
properties.getMaxCallsPerTool(), properties.getMaxInputTokens(),
properties.getMaxOutputTokens(), properties.getMaxTotalTokens(), properties.getMaxRunBytes()),
HarnessRetryPolicies.strict(),
properties.getStopAfterConsecutiveNoGain());
properties.getStopAfterConsecutiveNoGain(),
properties.getStopAfterConsecutiveProgressProtocolViolations());
}
@Bean
@@ -4,6 +4,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.ProgressProtocolViolationException;
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
import com.superbiz.agent.harness.tool.adapter.RagToolAdapter;
@@ -102,10 +104,17 @@ public final class HarnessEvidenceTools {
}
default -> throw new IllegalArgumentException("Unsupported evidence Tool: " + toolName);
};
} catch (ProgressProtocolViolationException exception) {
throw exception;
} catch (IllegalArgumentException exception) {
throw exception;
} catch (Exception exception) {
throw new IllegalArgumentException("Tool Call Envelope is invalid", exception);
throw new ProgressProtocolViolationException(
ProgressProtocolViolationType.INVALID_ENVELOPE,
"Tool Call Envelope is invalid",
null,
null,
exception);
}
}
@@ -119,7 +128,11 @@ public final class HarnessEvidenceTools {
private static void requireInput(Object input) {
if (input == null) {
throw new IllegalArgumentException("Tool Call Envelope input is required");
throw new ProgressProtocolViolationException(
ProgressProtocolViolationType.MISSING_INPUT,
"Tool Call Envelope input is required",
"input",
null);
}
}
@@ -15,10 +15,13 @@ import com.superbiz.agent.harness.progress.DiagnosisCollectionState;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.ProgressProtocolViolationException;
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
import com.superbiz.agent.harness.progress.ToolScopeNormalizer;
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -81,9 +84,13 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
return stopRequired(request, evaluated.stopReason());
}
normalizedScope = scopeNormalizer.normalize(request.getToolName(), call.businessInput());
} catch (ProgressProtocolViolationException exception) {
return handleProgressProtocolViolation(request, exception);
} catch (IllegalArgumentException | IllegalStateException exception) {
recordRejection(request, "INVALID_PROGRESS_PROTOCOL");
return safeError(request, "INVALID_PROGRESS_PROTOCOL");
return handleProgressProtocolViolation(request,
new ProgressProtocolViolationException(
ProgressProtocolViolationType.INVALID_ENVELOPE,
"Tool Call Envelope is invalid", null, null, exception));
}
if (context.progress().isDuplicate(request.getToolName(), normalizedScope)) {
@@ -167,6 +174,53 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
request.getToolCallId(), request.getToolName(), writeObservation(observation));
}
private ToolCallResponse handleProgressProtocolViolation(
ToolCallRequest request,
ProgressProtocolViolationException exception) {
DiagnosisProgressSnapshotState state =
context.progress().recordProgressProtocolViolation();
boolean stopRequired = state.collectionState() == DiagnosisCollectionState.SATURATED;
recordRejection(request, "INVALID_PROGRESS_PROTOCOL", exception.violationType(),
!stopRequired, state);
if (stopRequired) {
return stopRequired(request, state.stopReason());
}
return repairableProtocolError(request, exception, state);
}
private ToolCallResponse repairableProtocolError(
ToolCallRequest request,
ProgressProtocolViolationException exception,
DiagnosisProgressSnapshotState state) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", "ERROR");
observation.put("tool_call_id", request.getToolCallId());
observation.put("error_code", "INVALID_PROGRESS_PROTOCOL");
observation.put("repair_required", true);
observation.put("violation_type", exception.violationType().name());
if (exception.missingField() != null && !exception.missingField().isBlank()) {
observation.put("missing_field", exception.missingField());
}
if (exception.expectedPreviousToolCallId() != null
&& !exception.expectedPreviousToolCallId().isBlank()) {
observation.put("expected_previous_tool_call_id",
exception.expectedPreviousToolCallId());
}
observation.put("allowed_information_gain", List.of(
InformationGain.GAINED.name(), InformationGain.NO_GAIN.name()));
observation.put("instruction",
"Before calling another tool, evaluate the previous observation.");
observation.put("consecutive_protocol_violations",
state.consecutiveProgressProtocolViolations());
return ToolCallResponse.builder()
.toolCallId(request.getToolCallId())
.toolName(request.getToolName())
.content(writeObservation(observation))
.status("error")
.metadata(Map.of("error", true))
.build();
}
private ToolCallResponse safeError(ToolCallRequest request, String errorCode) {
Map<String, Object> observation = new LinkedHashMap<>();
observation.put("evidence_status", "ERROR");
@@ -182,8 +236,18 @@ public final class HarnessToolInterceptor extends ToolInterceptor {
}
private void recordRejection(ToolCallRequest request, String errorCode) {
recordRejection(request, errorCode, null, false, context.progress().snapshot());
}
private void recordRejection(
ToolCallRequest request,
String errorCode,
ProgressProtocolViolationType violationType,
boolean repairPromptDelivered,
DiagnosisProgressSnapshotState state) {
traceRecorder.record(TraceAuditEvents.toolRequestRejected(
context, request.getToolCallId(), request.getToolName(), errorCode));
context, request.getToolCallId(), request.getToolName(), errorCode,
violationType, repairPromptDelivered, state));
}
private String writeObservation(Map<String, Object> observation) {
@@ -11,6 +11,7 @@ import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
import com.superbiz.agent.harness.progress.InformationGain;
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
import com.superbiz.agent.harness.retry.RetryAttempt;
import java.util.ArrayList;
@@ -140,10 +141,33 @@ public final class TraceAuditEvents {
public static DiagnosisTraceAuditEvent toolRequestRejected(
RunContext context, String toolCallId, String toolName, String errorCode) {
return toolRequestRejected(context, toolCallId, toolName, errorCode,
null, false, null);
}
public static DiagnosisTraceAuditEvent toolRequestRejected(
RunContext context,
String toolCallId,
String toolName,
String errorCode,
ProgressProtocolViolationType violationType,
boolean repairPromptDelivered,
DiagnosisProgressSnapshotState state) {
Map<String, Object> details = new LinkedHashMap<>();
details.put("tool_call_id", safeIdentifier(toolCallId));
details.put("tool_name", safeIdentifier(toolName));
details.put("error_code", safeIdentifier(errorCode));
if (violationType != null) {
details.put("violation_type", violationType.name());
details.put("repair_prompt_delivered", repairPromptDelivered);
}
if (state != null && state.consecutiveProgressProtocolViolations() > 0) {
details.put("consecutive_protocol_violations",
state.consecutiveProgressProtocolViolations());
}
if (state != null && state.stopReason() != null) {
details.put("stop_reason", state.stopReason().name());
}
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
TraceEventStatus.REJECTED, null, null, details);
}
@@ -18,6 +18,7 @@ public final class DiagnosisHarnessCore {
private final RunBudgetLimits budgetLimits;
private final HarnessRetryPolicies retryPolicies;
private final int stopAfterConsecutiveNoGain;
private final int stopAfterConsecutiveProgressProtocolViolations;
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
@@ -33,6 +34,17 @@ public final class DiagnosisHarnessCore {
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies,
int stopAfterConsecutiveNoGain) {
this(clock, runIdSupplier, maxRunDuration, budgetLimits, retryPolicies,
stopAfterConsecutiveNoGain, 2);
}
public DiagnosisHarnessCore(Clock clock,
Supplier<String> runIdSupplier,
Duration maxRunDuration,
RunBudgetLimits budgetLimits,
HarnessRetryPolicies retryPolicies,
int stopAfterConsecutiveNoGain,
int stopAfterConsecutiveProgressProtocolViolations) {
this.clock = Objects.requireNonNull(clock, "clock must not be null");
this.runIdSupplier = Objects.requireNonNull(runIdSupplier, "runIdSupplier must not be null");
this.maxRunDuration = requirePositive(maxRunDuration, "maxRunDuration");
@@ -41,7 +53,13 @@ public final class DiagnosisHarnessCore {
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
if (stopAfterConsecutiveProgressProtocolViolations <= 0) {
throw new IllegalArgumentException(
"stopAfterConsecutiveProgressProtocolViolations must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
this.stopAfterConsecutiveProgressProtocolViolations =
stopAfterConsecutiveProgressProtocolViolations;
}
public RunContext startRun(String sessionId) {
@@ -61,7 +79,8 @@ public final class DiagnosisHarnessCore {
new ModelCallLedger(),
retryPolicies,
lifecycle,
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain));
new DiagnosisProgressTracker(stopAfterConsecutiveNoGain,
stopAfterConsecutiveProgressProtocolViolations));
cancellation.onCancel(reason -> lifecycle.finish(terminalState(reason), reason.name()));
return context;
}
@@ -4,6 +4,7 @@ import java.util.List;
public record DiagnosisProgressSnapshotState(
int consecutiveNoGain,
int consecutiveProgressProtocolViolations,
DiagnosisCollectionState collectionState,
DiagnosisStopReason stopReason,
String pendingToolCallId,
@@ -10,35 +10,63 @@ import java.util.Set;
public final class DiagnosisProgressTracker {
private final int stopAfterConsecutiveNoGain;
private final int stopAfterConsecutiveProgressProtocolViolations;
private final Set<ToolScopeIdentity> completedScopes = new LinkedHashSet<>();
private final List<CompletedToolCall> completedToolCalls = new ArrayList<>();
private int consecutiveNoGain;
private int consecutiveProgressProtocolViolations;
private DiagnosisCollectionState collectionState = DiagnosisCollectionState.COLLECTING;
private DiagnosisStopReason stopReason;
private String pendingToolCallId;
private boolean stopInstructionDelivered;
public DiagnosisProgressTracker(int stopAfterConsecutiveNoGain) {
this(stopAfterConsecutiveNoGain, 2);
}
public DiagnosisProgressTracker(
int stopAfterConsecutiveNoGain,
int stopAfterConsecutiveProgressProtocolViolations) {
if (stopAfterConsecutiveNoGain <= 0) {
throw new IllegalArgumentException("stopAfterConsecutiveNoGain must be positive");
}
if (stopAfterConsecutiveProgressProtocolViolations <= 0) {
throw new IllegalArgumentException(
"stopAfterConsecutiveProgressProtocolViolations must be positive");
}
this.stopAfterConsecutiveNoGain = stopAfterConsecutiveNoGain;
this.stopAfterConsecutiveProgressProtocolViolations =
stopAfterConsecutiveProgressProtocolViolations;
}
public synchronized void applyPreviousObservation(PreviousObservation observation) {
if (pendingToolCallId == null) {
if (observation != null) {
throw new IllegalArgumentException("No Tool observation is pending evaluation");
throw new ProgressProtocolViolationException(
ProgressProtocolViolationType.UNEXPECTED_PREVIOUS_OBSERVATION,
"No Tool observation is pending evaluation",
"previous_observation",
null);
}
clearProtocolViolations();
return;
}
if (observation == null) {
throw new IllegalArgumentException("Previous Tool observation must be evaluated");
throw new ProgressProtocolViolationException(
ProgressProtocolViolationType.MISSING_PREVIOUS_OBSERVATION,
"Previous Tool observation must be evaluated",
"previous_observation",
pendingToolCallId);
}
if (!pendingToolCallId.equals(observation.toolCallId())) {
throw new IllegalArgumentException("Previous Tool observation ID is out of order");
throw new ProgressProtocolViolationException(
ProgressProtocolViolationType.OUT_OF_ORDER_PREVIOUS_OBSERVATION,
"Previous Tool observation ID is out of order",
"previous_observation.tool_call_id",
pendingToolCallId);
}
pendingToolCallId = null;
clearProtocolViolations();
applyGain(observation.informationGain());
}
@@ -47,6 +75,7 @@ public final class DiagnosisProgressTracker {
}
public synchronized void recordDuplicateScope() {
clearProtocolViolations();
applyGain(InformationGain.NO_GAIN);
}
@@ -64,12 +93,26 @@ public final class DiagnosisProgressTracker {
}
completedToolCalls.add(call);
if (evidenceStatus == EvidenceStatus.NO_EVIDENCE) {
clearProtocolViolations();
applyGain(InformationGain.NO_GAIN);
} else {
pendingToolCallId = call.toolCallId();
}
}
public synchronized DiagnosisProgressSnapshotState recordProgressProtocolViolation() {
if (collectionState == DiagnosisCollectionState.SATURATED) {
return snapshot();
}
consecutiveProgressProtocolViolations++;
if (consecutiveProgressProtocolViolations
>= stopAfterConsecutiveProgressProtocolViolations) {
collectionState = DiagnosisCollectionState.SATURATED;
stopReason = DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED;
}
return snapshot();
}
public synchronized boolean claimStopInstruction() {
if (collectionState != DiagnosisCollectionState.SATURATED) {
return false;
@@ -90,6 +133,7 @@ public final class DiagnosisProgressTracker {
public synchronized DiagnosisProgressSnapshotState snapshot() {
return new DiagnosisProgressSnapshotState(
consecutiveNoGain,
consecutiveProgressProtocolViolations,
collectionState,
stopReason,
pendingToolCallId,
@@ -101,6 +145,10 @@ public final class DiagnosisProgressTracker {
return stopAfterConsecutiveNoGain;
}
public int stopAfterConsecutiveProgressProtocolViolations() {
return stopAfterConsecutiveProgressProtocolViolations;
}
private void applyGain(InformationGain gain) {
if (collectionState == DiagnosisCollectionState.SATURATED) {
throw new IllegalStateException("Collection is already saturated");
@@ -115,4 +163,8 @@ public final class DiagnosisProgressTracker {
stopReason = DiagnosisStopReason.INFORMATION_SATURATED;
}
}
private void clearProtocolViolations() {
consecutiveProgressProtocolViolations = 0;
}
}
@@ -2,5 +2,6 @@ package com.superbiz.agent.harness.progress;
public enum DiagnosisStopReason {
INFORMATION_SATURATED,
BUDGET_LIMIT_REACHED
BUDGET_LIMIT_REACHED,
PROGRESS_PROTOCOL_VIOLATED
}
@@ -0,0 +1,43 @@
package com.superbiz.agent.harness.progress;
public final class ProgressProtocolViolationException extends IllegalArgumentException {
private final ProgressProtocolViolationType violationType;
private final String missingField;
private final String expectedPreviousToolCallId;
public ProgressProtocolViolationException(
ProgressProtocolViolationType violationType,
String message,
String missingField,
String expectedPreviousToolCallId) {
super(message);
this.violationType = violationType;
this.missingField = missingField;
this.expectedPreviousToolCallId = expectedPreviousToolCallId;
}
public ProgressProtocolViolationException(
ProgressProtocolViolationType violationType,
String message,
String missingField,
String expectedPreviousToolCallId,
Throwable cause) {
super(message, cause);
this.violationType = violationType;
this.missingField = missingField;
this.expectedPreviousToolCallId = expectedPreviousToolCallId;
}
public ProgressProtocolViolationType violationType() {
return violationType;
}
public String missingField() {
return missingField;
}
public String expectedPreviousToolCallId() {
return expectedPreviousToolCallId;
}
}
@@ -0,0 +1,9 @@
package com.superbiz.agent.harness.progress;
public enum ProgressProtocolViolationType {
MISSING_PREVIOUS_OBSERVATION,
OUT_OF_ORDER_PREVIOUS_OBSERVATION,
UNEXPECTED_PREVIOUS_OBSERVATION,
MISSING_INPUT,
INVALID_ENVELOPE
}
@@ -166,7 +166,8 @@ public final class DiagnosisReleaseUseCase {
DiagnosisProgressSnapshot progress,
DiagnosisStopReason stopReason) {
if (stopReason != DiagnosisStopReason.INFORMATION_SATURATED
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED) {
&& stopReason != DiagnosisStopReason.BUDGET_LIMIT_REACHED
&& stopReason != DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED) {
throw new IllegalStateException("Unsupported Diagnosis stop reason");
}
if (!progress.hasObservedFacts()) {
@@ -35,9 +35,13 @@ class HarnessChatConfigurationTest {
void diagnosisNoGainThresholdDefaultsAndValidates() {
ChatHarnessProperties properties = new ChatHarnessProperties();
assertEquals(2, properties.getStopAfterConsecutiveNoGain());
assertEquals(2, properties.getStopAfterConsecutiveProgressProtocolViolations());
properties.setStopAfterConsecutiveNoGain(0);
assertThrows(IllegalArgumentException.class, properties::validate);
properties.setStopAfterConsecutiveNoGain(2);
properties.setStopAfterConsecutiveProgressProtocolViolations(0);
assertThrows(IllegalArgumentException.class, properties::validate);
}
@Test
@@ -149,6 +149,41 @@ class DiagnosisAgentUseCaseTest {
assertEquals(2, context.budget().snapshot().toolCalls());
}
@Test
void consecutiveProgressProtocolViolationsStopWithoutConsumingToolBudget() {
DiagnosisHarnessCore core = core(new RunBudgetLimits(8, 8, 8,
100, 100, 200, 100_000));
RunContext context = core.startRun("session-protocol-stop", "run-protocol-stop");
AtomicInteger toolCalls = new AtomicInteger();
HarnessEvidenceTools tools = tools((runContext, id, arguments) -> {
core.beforeToolCall(runContext, AgentToolContracts.LOOKUP_KNOWLEDGE);
toolCalls.incrementAndGet();
return ToolBoundaryResult.ready(id, evidence(id), EvidenceStatus.EVIDENCE_FOUND);
});
// First Tool succeeds and becomes pending evaluation. The next two Tool Calls omit
// previous_observation, hit the protocol-violation threshold, and force a controlled stop
// when the model still ignores STOP_REQUIRED.
ScriptedChatModel model = new ScriptedChatModel(1, 1,
toolCall("call-evidence-1", "{\"query\":\"first\"}"),
toolCall("call-missing-eval-1", "{\"query\":\"second\"}"),
toolCall("call-missing-eval-2", "{\"query\":\"third\"}"),
toolCall("call-ignored-protocol-stop", "{\"query\":\"fourth\"}"));
DiagnosisAgentExecution execution = useCase(core, model, tools, LARGE_LIMITS)
.execute(context, new DiagnosisAgentInput("诊断未知故障", null));
assertNull(execution.draft());
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, execution.stopReason());
// Default empty projector is used here; publishable facts are covered by Release tests.
// Tracker still retains the completed Tool identity that a real projector would read.
assertEquals(1, context.progress().snapshot().completedToolCalls().size());
assertEquals("call-evidence-1",
context.progress().snapshot().completedToolCalls().get(0).toolCallId());
assertEquals(4, model.calls());
assertEquals(1, toolCalls.get());
assertEquals(1, context.budget().snapshot().toolCalls());
}
@Test
void fencedOutputFailsClosedWithoutAgentRetry() {
DiagnosisHarnessCore core = core(defaultBudget());
@@ -10,6 +10,7 @@ import com.superbiz.agent.harness.contract.EvidenceStatus;
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
import com.superbiz.agent.harness.core.RunBudgetLimits;
import com.superbiz.agent.harness.core.RunContext;
import com.superbiz.agent.harness.progress.DiagnosisStopReason;
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
import com.superbiz.agent.harness.tool.adapter.MysqlToolAdapter;
import com.superbiz.agent.harness.tool.adapter.QueryLogsToolAdapter;
@@ -182,8 +183,16 @@ class HarnessToolInterceptorTest {
"""), ignored -> null);
assertTrue(missing.isError());
assertEquals("INVALID_PROGRESS_PROTOCOL",
objectMapper.readTree(missing.getResult()).path("error_code").asText());
JsonNode missingObservation = objectMapper.readTree(missing.getResult());
assertEquals("INVALID_PROGRESS_PROTOCOL", missingObservation.path("error_code").asText());
assertTrue(missingObservation.path("repair_required").asBoolean());
assertEquals("MISSING_PREVIOUS_OBSERVATION",
missingObservation.path("violation_type").asText());
assertEquals("previous_observation", missingObservation.path("missing_field").asText());
assertEquals("call-1", missingObservation.path("expected_previous_tool_call_id").asText());
assertEquals(List.of("GAINED", "NO_GAIN"), objectMapper.convertValue(
missingObservation.path("allowed_information_gain"), List.class));
assertFalse(missing.getResult().contains("two"));
assertFalse(accepted.isError());
assertEquals(2, invocations.get());
DiagnosisTraceAuditEvent rejected = trace.stream()
@@ -191,9 +200,124 @@ class HarnessToolInterceptorTest {
.findFirst().orElseThrow();
assertEquals("call-2", rejected.details().get("tool_call_id"));
assertEquals("INVALID_PROGRESS_PROTOCOL", rejected.details().get("error_code"));
assertEquals("MISSING_PREVIOUS_OBSERVATION", rejected.details().get("violation_type"));
assertEquals(true, rejected.details().get("repair_prompt_delivered"));
assertEquals(1, rejected.details().get("consecutive_protocol_violations"));
String details = rejected.details().toString();
assertFalse(details.contains("two"));
assertFalse(details.contains("previous_observation"));
assertFalse(details.contains("secret"));
}
@Test
void outOfOrderPreviousObservationIsRepairableWithoutExecutingTool() throws Exception {
AtomicInteger invocations = new AtomicInteger();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
ToolCallResponse response = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2", """
{"previous_observation":{"tool_call_id":"call-other","information_gain":"GAINED"},
"input":{"query":"two-secret"}}
"""), ignored -> null);
JsonNode observation = objectMapper.readTree(response.getResult());
assertTrue(response.isError());
assertEquals(1, invocations.get());
assertEquals("OUT_OF_ORDER_PREVIOUS_OBSERVATION",
observation.path("violation_type").asText());
assertEquals("call-1", observation.path("expected_previous_tool_call_id").asText());
assertFalse(response.getResult().contains("two-secret"));
}
@Test
void consecutiveProtocolViolationsDeliverStopRequiredOnce() throws Exception {
AtomicInteger invocations = new AtomicInteger();
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
RunContext run = context(6, 6, 6);
HarnessToolInterceptor interceptor = new HarnessToolInterceptor(
run, tools, objectMapper, trace::add);
interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-1",
"{\"input\":{\"query\":\"one\"}}"), ignored -> null);
ToolCallResponse first = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-2",
"{\"input\":{\"query\":\"two\"}}"), ignored -> null);
ToolCallResponse stop = interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-3",
"{\"input\":{\"query\":\"three\"}}"), ignored -> null);
JsonNode firstObservation = objectMapper.readTree(first.getResult());
JsonNode stopObservation = objectMapper.readTree(stop.getResult());
assertTrue(firstObservation.path("repair_required").asBoolean());
assertTrue(stopObservation.path("stop_required").asBoolean());
assertEquals("PROGRESS_PROTOCOL_VIOLATED", stopObservation.path("reason").asText());
assertEquals(1, invocations.get());
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED,
run.progress().snapshot().stopReason());
DiagnosisCollectionStoppedException ignoredStop = assertThrows(
DiagnosisCollectionStoppedException.class,
() -> interceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-4",
"{\"input\":{\"query\":\"four\"}}"), ignored -> null));
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, ignoredStop.stopReason());
assertEquals(1, invocations.get());
DiagnosisTraceAuditEvent stopRejection = trace.stream()
.filter(event -> event.eventType() == TraceEventType.TOOL_REQUEST_REJECTED)
.filter(event -> "call-3".equals(event.details().get("tool_call_id")))
.findFirst().orElseThrow();
assertEquals("INVALID_PROGRESS_PROTOCOL", stopRejection.details().get("error_code"));
assertEquals("MISSING_PREVIOUS_OBSERVATION", stopRejection.details().get("violation_type"));
assertEquals(false, stopRejection.details().get("repair_prompt_delivered"));
assertEquals(2, stopRejection.details().get("consecutive_protocol_violations"));
assertEquals("PROGRESS_PROTOCOL_VIOLATED", stopRejection.details().get("stop_reason"));
String details = stopRejection.details().toString();
assertFalse(details.contains("three"));
assertFalse(details.contains("exception"));
}
@Test
void missingInputAndInvalidEnvelopeProduceTypedProtocolErrors() throws Exception {
AtomicInteger invocations = new AtomicInteger();
HarnessEvidenceTools tools = fakeTools((context, id, arguments) -> {
invocations.incrementAndGet();
return ready(id);
});
HarnessToolInterceptor missingInterceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper);
HarnessToolInterceptor invalidInterceptor = new HarnessToolInterceptor(
context(3, 3, 3), tools, objectMapper);
ToolCallResponse missingInput = missingInterceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-missing-input",
"{\"previous_observation\":null}"), ignored -> null);
ToolCallResponse invalidEnvelope = invalidInterceptor.interceptToolCall(
request(AgentToolContracts.LOOKUP_KNOWLEDGE, "call-invalid",
"{not-json"), ignored -> null);
JsonNode missing = objectMapper.readTree(missingInput.getResult());
JsonNode invalid = objectMapper.readTree(invalidEnvelope.getResult());
assertEquals("MISSING_INPUT", missing.path("violation_type").asText());
assertEquals("input", missing.path("missing_field").asText());
assertEquals("INVALID_ENVELOPE", invalid.path("violation_type").asText());
assertTrue(missing.path("repair_required").asBoolean());
assertTrue(invalid.path("repair_required").asBoolean());
assertEquals(0, invocations.get());
assertFalse(missingInput.getResult().contains("not-json"));
assertFalse(invalidEnvelope.getResult().contains("not-json"));
}
@Test
@@ -61,6 +61,53 @@ class DiagnosisProgressTrackerTest {
assertFalse(tracker.isDuplicate("lookup_knowledge", "scope-2"));
}
@Test
void progressProtocolViolationsAccumulateIndependentlyFromNoGain() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(3, 2);
tracker.recordDuplicateScope();
DiagnosisProgressSnapshotState first = tracker.recordProgressProtocolViolation();
assertEquals(1, first.consecutiveProgressProtocolViolations());
assertEquals(1, first.consecutiveNoGain());
assertEquals(DiagnosisCollectionState.COLLECTING, first.collectionState());
DiagnosisProgressSnapshotState second = tracker.recordProgressProtocolViolation();
assertEquals(2, second.consecutiveProgressProtocolViolations());
assertEquals(1, second.consecutiveNoGain());
assertEquals(DiagnosisCollectionState.SATURATED, second.collectionState());
assertEquals(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED, second.stopReason());
assertTrue(tracker.claimStopInstruction());
assertFalse(tracker.claimStopInstruction());
}
@Test
void validEvaluationClearsProtocolViolationCount() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2, 3);
tracker.recordCompleted(call("call-1", "scope-1"), EvidenceStatus.EVIDENCE_FOUND);
tracker.recordProgressProtocolViolation();
tracker.applyPreviousObservation(new PreviousObservation("call-1", InformationGain.GAINED));
DiagnosisProgressSnapshotState state = tracker.snapshot();
assertEquals(0, state.consecutiveProgressProtocolViolations());
assertEquals(0, state.consecutiveNoGain());
assertEquals(DiagnosisCollectionState.COLLECTING, state.collectionState());
}
@Test
void rejectsUnexpectedPreviousObservationWithTypedViolation() {
DiagnosisProgressTracker tracker = new DiagnosisProgressTracker(2, 2);
ProgressProtocolViolationException failure = assertThrows(
ProgressProtocolViolationException.class,
() -> tracker.applyPreviousObservation(
new PreviousObservation("call-x", InformationGain.GAINED)));
assertEquals(ProgressProtocolViolationType.UNEXPECTED_PREVIOUS_OBSERVATION,
failure.violationType());
assertEquals("previous_observation", failure.missingField());
}
private CompletedToolCall call(String id, String scope) {
return new CompletedToolCall(id, "lookup_knowledge", scope);
}
@@ -53,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
class DiagnosisReleaseUseCaseTest {
@@ -175,6 +176,37 @@ class DiagnosisReleaseUseCaseTest {
assertEquals(0, fixture.model.calls.get());
}
@Test
void controlledProtocolViolationPublishesProgressWithoutAnotherModelCall() {
Fixture fixture = fixture();
DiagnosisReleaseResult result = fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.stopped(
progress(DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED),
DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED));
assertFallback(result, FallbackType.INSUFFICIENT_EVIDENCE, 1);
assertEquals(0, fixture.model.calls.get());
}
@Test
void controlledProtocolViolationWithoutProgressFailsClosed() {
Fixture fixture = fixture();
IllegalStateException failure = assertThrows(IllegalStateException.class, () ->
fixture.useCase.execute(
fixture.context,
"诊断未知故障",
DiagnosisAgentExecution.stopped(
DiagnosisProgressSnapshot.empty(),
DiagnosisStopReason.PROGRESS_PROTOCOL_VIOLATED)));
assertTrue(failure.getMessage().contains("no verified publishable progress"));
assertEquals(0, fixture.model.calls.get());
}
@Test
void invalidDraftWithVerifiedProgressPublishesOnlyProgressFallback() {
Fixture fixture = fixture();