zhuyongxin
38f781b157
Add repairable INVALID_PROGRESS_PROTOCOL observations, independent PROGRESS_PROTOCOL_VIOLATED saturation, and controlled release paths. Archive the OpenSpec change after syncing main specs and devflow.
343 lines
16 KiB
Java
343 lines
16 KiB
Java
package com.superbiz.agent.harness.audit;
|
|
|
|
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
|
import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException;
|
|
import com.superbiz.agent.harness.contract.FallbackType;
|
|
import com.superbiz.agent.harness.contract.IntentType;
|
|
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
|
import com.superbiz.agent.harness.contract.SemanticVerdict;
|
|
import com.superbiz.agent.harness.core.RunContext;
|
|
import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult;
|
|
import com.superbiz.agent.harness.guard.evidence.EvidenceViolation;
|
|
import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState;
|
|
import com.superbiz.agent.harness.progress.InformationGain;
|
|
import com.superbiz.agent.harness.progress.ProgressProtocolViolationType;
|
|
import com.superbiz.agent.harness.retry.RetryAttempt;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.LinkedHashMap;
|
|
import java.util.LinkedHashSet;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.regex.Pattern;
|
|
|
|
public final class TraceAuditEvents {
|
|
|
|
private static final Pattern SAFE_TOOL_CALL_ID =
|
|
Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}");
|
|
|
|
private TraceAuditEvents() {
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent runStarted(RunContext context) {
|
|
return event(context, TracePhase.RUN, TraceEventType.RUN_STARTED,
|
|
TraceEventStatus.STARTED, null, null, Map.of());
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent runFinished(
|
|
RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) {
|
|
var budget = context.budget().snapshot();
|
|
ModelCallLedger.Snapshot modelCalls = context.modelCalls().snapshot();
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
if (intent != null) {
|
|
details.put("intent", intent.name());
|
|
}
|
|
details.put("release_outcome", outcome.name());
|
|
details.put("model_call_count", budget.modelCalls());
|
|
details.put("run_input_tokens", budget.inputTokens());
|
|
details.put("run_output_tokens", budget.outputTokens());
|
|
details.put("run_total_tokens", budget.totalTokens());
|
|
details.put("audited_model_call_count", modelCalls.auditedCallCount());
|
|
details.put("audited_input_tokens", modelCalls.inputTokens());
|
|
details.put("audited_output_tokens", modelCalls.outputTokens());
|
|
details.put("audited_total_tokens", modelCalls.totalTokens());
|
|
details.put("usage_unavailable_count", modelCalls.usageUnavailableCount());
|
|
details.put("tokens_reconciled", budget.modelCalls() == modelCalls.auditedCallCount()
|
|
&& modelCalls.usageUnavailableCount() == 0
|
|
&& budget.inputTokens() == modelCalls.inputTokens()
|
|
&& budget.outputTokens() == modelCalls.outputTokens()
|
|
&& budget.totalTokens() == modelCalls.totalTokens());
|
|
return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED,
|
|
terminalStatus(outcome), null, durationMs, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent routingAttempt(
|
|
RunContext context, RetryAttempt attempt) {
|
|
return retry(context, TracePhase.ROUTING, TraceEventType.ROUTING_ATTEMPT, attempt);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent routingDecision(
|
|
RunContext context, IntentType intent) {
|
|
return event(context, TracePhase.ROUTING, TraceEventType.ROUTING_DECISION,
|
|
TraceEventStatus.SUCCEEDED, null, null, Map.of("intent", intent.name()));
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent agentModelStep(
|
|
String sessionId, String runId, String agentName, int stepIndex,
|
|
int durationMs, Map<String, Object> input, Map<String, Object> output) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("agent_name", agentName);
|
|
details.put("step_index", stepIndex);
|
|
details.put("input", input);
|
|
details.put("output", output);
|
|
return new DiagnosisTraceAuditEvent(
|
|
sessionId, runId, TracePhase.AGENT, TraceEventType.AGENT_MODEL_STEP,
|
|
TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent modelTokenUsage(
|
|
RunContext context,
|
|
ModelCallLedger.Call call,
|
|
long inputTokens,
|
|
long outputTokens,
|
|
boolean usageAvailable) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("component", call.component().name());
|
|
details.put("component_round", call.componentRound());
|
|
details.put("usage_available", usageAvailable);
|
|
if (usageAvailable) {
|
|
details.put("input_tokens", inputTokens);
|
|
details.put("output_tokens", outputTokens);
|
|
details.put("total_tokens", safeAdd(inputTokens, outputTokens));
|
|
}
|
|
return new DiagnosisTraceAuditEvent(
|
|
context.sessionId(), context.runId(), call.component().tracePhase(),
|
|
TraceEventType.MODEL_TOKEN_USAGE,
|
|
usageAvailable ? TraceEventStatus.SUCCEEDED : TraceEventStatus.UNAVAILABLE,
|
|
call.componentRound(), null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent agentDraftInvalid(
|
|
RunContext context,
|
|
DiagnosisAgentOutputException.Kind kind,
|
|
long outputBytes,
|
|
boolean hasPublishableProgress) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("failure_kind", kind.name());
|
|
details.put("output_bytes", outputBytes);
|
|
details.put("has_publishable_progress", hasPublishableProgress);
|
|
return event(context, TracePhase.AGENT, TraceEventType.AGENT_DRAFT_INVALID,
|
|
TraceEventStatus.REJECTED, null, null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("tool_call_id", tool.toolCallId());
|
|
details.put("tool_name", tool.toolName());
|
|
details.put("invocation_status", tool.status().name());
|
|
details.put("evidence_status", tool.evidenceStatus().name());
|
|
details.put("request_bytes", tool.requestBytes());
|
|
details.put("agent_result_bytes", tool.agentResultBytes());
|
|
if (tool.errorCode() != null) {
|
|
details.put("error_code", tool.errorCode());
|
|
}
|
|
return new DiagnosisTraceAuditEvent(
|
|
tool.sessionId(), tool.runId(), TracePhase.TOOL, TraceEventType.TOOL_INVOCATION,
|
|
tool.status() == com.superbiz.agent.harness.contract.InvocationStatus.READY
|
|
? TraceEventStatus.READY : TraceEventStatus.ERROR,
|
|
null, tool.durationMs(), details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent toolRequestRejected(
|
|
RunContext context, String toolCallId, String toolName, String errorCode) {
|
|
return toolRequestRejected(context, toolCallId, toolName, errorCode,
|
|
null, false, null);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent toolRequestRejected(
|
|
RunContext context,
|
|
String toolCallId,
|
|
String toolName,
|
|
String errorCode,
|
|
ProgressProtocolViolationType violationType,
|
|
boolean repairPromptDelivered,
|
|
DiagnosisProgressSnapshotState state) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("tool_call_id", safeIdentifier(toolCallId));
|
|
details.put("tool_name", safeIdentifier(toolName));
|
|
details.put("error_code", safeIdentifier(errorCode));
|
|
if (violationType != null) {
|
|
details.put("violation_type", violationType.name());
|
|
details.put("repair_prompt_delivered", repairPromptDelivered);
|
|
}
|
|
if (state != null && state.consecutiveProgressProtocolViolations() > 0) {
|
|
details.put("consecutive_protocol_violations",
|
|
state.consecutiveProgressProtocolViolations());
|
|
}
|
|
if (state != null && state.stopReason() != null) {
|
|
details.put("stop_reason", state.stopReason().name());
|
|
}
|
|
return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED,
|
|
TraceEventStatus.REJECTED, null, null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent toolProgress(
|
|
RunContext context,
|
|
String toolCallId,
|
|
String toolName,
|
|
String scopeSummary,
|
|
InformationGain informationGain,
|
|
String producer,
|
|
DiagnosisProgressSnapshotState state) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("tool_call_id", toolCallId);
|
|
details.put("tool_name", toolName);
|
|
details.put("scope_summary", scopeSummary);
|
|
details.put("information_gain", informationGain.name());
|
|
details.put("producer", producer);
|
|
details.put("consecutive_no_gain", state.consecutiveNoGain());
|
|
details.put("collection_state", state.collectionState().name());
|
|
if (state.stopReason() != null) {
|
|
details.put("stop_reason", state.stopReason().name());
|
|
}
|
|
return event(context, TracePhase.TOOL, TraceEventType.TOOL_PROGRESS,
|
|
TraceEventStatus.SUCCEEDED, null, null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent collectionStop(
|
|
RunContext context,
|
|
String toolCallId,
|
|
String toolName,
|
|
DiagnosisProgressSnapshotState state) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("tool_call_id", toolCallId);
|
|
details.put("tool_name", toolName);
|
|
details.put("collection_state", state.collectionState().name());
|
|
if (state.stopReason() != null) {
|
|
details.put("stop_reason", state.stopReason().name());
|
|
}
|
|
return event(context, TracePhase.TOOL, TraceEventType.COLLECTION_STOP,
|
|
TraceEventStatus.SUCCEEDED, null, null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent evidenceValidation(
|
|
RunContext context, TraceEventType type,
|
|
EvidenceGuardResult result, DiagnosisDraft draft) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("violation_count", result.violations().size());
|
|
details.put("violations", violations(result.violations()));
|
|
ToolReferences references = toolReferences(draft);
|
|
details.put("referenced_tool_call_ids", references.safeIds());
|
|
details.put("invalid_tool_reference_count", references.invalidCount());
|
|
result.verifiedSnapshot().ifPresent(snapshot -> {
|
|
details.put("verified_analysis_count", snapshot.analyses().size());
|
|
details.put("verified_source_count", snapshot.verifiedSources().size());
|
|
});
|
|
return event(context, TracePhase.EVIDENCE, type,
|
|
result.valid() ? TraceEventStatus.PASSED : TraceEventStatus.REJECTED,
|
|
null, null, details);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent evidenceRepairAttempt(
|
|
RunContext context, RetryAttempt attempt) {
|
|
return retry(context, TracePhase.EVIDENCE,
|
|
TraceEventType.EVIDENCE_REPAIR_ATTEMPT, attempt);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent semanticAttempt(
|
|
RunContext context, RetryAttempt attempt) {
|
|
return retry(context, TracePhase.SEMANTIC,
|
|
TraceEventType.SEMANTIC_GUARD_ATTEMPT, attempt);
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent semanticDecision(
|
|
RunContext context, SemanticVerdict verdict) {
|
|
TraceEventStatus status = verdict == SemanticVerdict.SUPPORTED
|
|
? TraceEventStatus.SUPPORTED : TraceEventStatus.UNSUPPORTED;
|
|
return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION,
|
|
status, null, null, Map.of("verdict", verdict.name()));
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent semanticUnavailable(RunContext context) {
|
|
return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION,
|
|
TraceEventStatus.UNAVAILABLE, null, null, Map.of());
|
|
}
|
|
|
|
public static DiagnosisTraceAuditEvent releaseDecision(
|
|
RunContext context, ReleaseOutcome outcome, FallbackType fallbackType) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
details.put("release_outcome", outcome.name());
|
|
if (fallbackType != null) {
|
|
details.put("fallback_type", fallbackType.name());
|
|
}
|
|
return event(context, TracePhase.RELEASE, TraceEventType.RELEASE_DECISION,
|
|
terminalStatus(outcome), null, null, details);
|
|
}
|
|
|
|
private static DiagnosisTraceAuditEvent retry(
|
|
RunContext context, TracePhase phase, TraceEventType type, RetryAttempt attempt) {
|
|
Map<String, Object> details = new LinkedHashMap<>();
|
|
if (attempt.failure() != null) {
|
|
details.put("failure", attempt.failure().name());
|
|
}
|
|
return event(context, phase, type,
|
|
attempt.success() ? TraceEventStatus.SUCCEEDED : TraceEventStatus.FAILED,
|
|
attempt.attemptNumber(), null, details);
|
|
}
|
|
|
|
private static DiagnosisTraceAuditEvent event(
|
|
RunContext context, TracePhase phase, TraceEventType type, TraceEventStatus status,
|
|
Integer attemptNo, Integer durationMs, Map<String, Object> details) {
|
|
return new DiagnosisTraceAuditEvent(
|
|
context.sessionId(), context.runId(), phase, type, status,
|
|
attemptNo, durationMs, details);
|
|
}
|
|
|
|
private static TraceEventStatus terminalStatus(ReleaseOutcome outcome) {
|
|
return switch (outcome) {
|
|
case SUCCESS -> TraceEventStatus.SUCCEEDED;
|
|
case FALLBACK -> TraceEventStatus.FALLBACK;
|
|
case FAILED -> TraceEventStatus.FAILED;
|
|
case CANCELLED -> TraceEventStatus.CANCELLED;
|
|
};
|
|
}
|
|
|
|
private static List<Map<String, String>> violations(List<EvidenceViolation> violations) {
|
|
List<Map<String, String>> result = new ArrayList<>();
|
|
for (EvidenceViolation violation : violations) {
|
|
Map<String, String> item = new LinkedHashMap<>();
|
|
item.put("code", violation.code().name());
|
|
item.put("target", violation.target());
|
|
result.add(Map.copyOf(item));
|
|
}
|
|
return List.copyOf(result);
|
|
}
|
|
|
|
private static ToolReferences toolReferences(DiagnosisDraft draft) {
|
|
if (draft == null) {
|
|
return new ToolReferences(List.of(), 0);
|
|
}
|
|
Set<String> safeIds = new LinkedHashSet<>();
|
|
int invalid = 0;
|
|
for (DiagnosisDraft.AnalysisItem item : draft.analysis()) {
|
|
if (item == null) {
|
|
continue;
|
|
}
|
|
for (String id : item.toolCallIds()) {
|
|
if (id != null && SAFE_TOOL_CALL_ID.matcher(id).matches()) {
|
|
safeIds.add(id);
|
|
} else {
|
|
invalid++;
|
|
}
|
|
}
|
|
}
|
|
return new ToolReferences(List.copyOf(safeIds), invalid);
|
|
}
|
|
|
|
private static String safeIdentifier(String value) {
|
|
return value != null && SAFE_TOOL_CALL_ID.matcher(value).matches() ? value : "INVALID";
|
|
}
|
|
|
|
private static long safeAdd(long left, long right) {
|
|
try {
|
|
return Math.addExact(left, right);
|
|
} catch (ArithmeticException exception) {
|
|
return Long.MAX_VALUE;
|
|
}
|
|
}
|
|
|
|
private record ToolReferences(List<String> safeIds, int invalidCount) {
|
|
}
|
|
}
|