package com.superbiz.agent.harness.audit; import com.superbiz.agent.harness.contract.DiagnosisDraft; import com.superbiz.agent.harness.agent.DiagnosisAgentOutputException; import com.superbiz.agent.harness.contract.FallbackType; import com.superbiz.agent.harness.contract.IntentType; import com.superbiz.agent.harness.contract.ReleaseOutcome; import com.superbiz.agent.harness.contract.SemanticVerdict; import com.superbiz.agent.harness.core.RunContext; import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult; import com.superbiz.agent.harness.guard.evidence.EvidenceViolation; import com.superbiz.agent.harness.progress.DiagnosisProgressSnapshotState; import com.superbiz.agent.harness.progress.InformationGain; import com.superbiz.agent.harness.progress.ProgressProtocolViolationType; import com.superbiz.agent.harness.retry.RetryAttempt; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; public final class TraceAuditEvents { private static final Pattern SAFE_TOOL_CALL_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}"); private TraceAuditEvents() { } public static DiagnosisTraceAuditEvent runStarted(RunContext context) { return event(context, TracePhase.RUN, TraceEventType.RUN_STARTED, TraceEventStatus.STARTED, null, null, Map.of()); } public static DiagnosisTraceAuditEvent runFinished( RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) { var budget = context.budget().snapshot(); ModelCallLedger.Snapshot modelCalls = context.modelCalls().snapshot(); Map details = new LinkedHashMap<>(); if (intent != null) { details.put("intent", intent.name()); } details.put("release_outcome", outcome.name()); details.put("model_call_count", budget.modelCalls()); details.put("run_input_tokens", budget.inputTokens()); details.put("run_output_tokens", budget.outputTokens()); details.put("run_total_tokens", budget.totalTokens()); details.put("audited_model_call_count", modelCalls.auditedCallCount()); details.put("audited_input_tokens", modelCalls.inputTokens()); details.put("audited_output_tokens", modelCalls.outputTokens()); details.put("audited_total_tokens", modelCalls.totalTokens()); details.put("usage_unavailable_count", modelCalls.usageUnavailableCount()); details.put("tokens_reconciled", budget.modelCalls() == modelCalls.auditedCallCount() && modelCalls.usageUnavailableCount() == 0 && budget.inputTokens() == modelCalls.inputTokens() && budget.outputTokens() == modelCalls.outputTokens() && budget.totalTokens() == modelCalls.totalTokens()); return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED, terminalStatus(outcome), null, durationMs, details); } public static DiagnosisTraceAuditEvent routingAttempt( RunContext context, RetryAttempt attempt) { return retry(context, TracePhase.ROUTING, TraceEventType.ROUTING_ATTEMPT, attempt); } public static DiagnosisTraceAuditEvent routingDecision( RunContext context, IntentType intent) { return event(context, TracePhase.ROUTING, TraceEventType.ROUTING_DECISION, TraceEventStatus.SUCCEEDED, null, null, Map.of("intent", intent.name())); } public static DiagnosisTraceAuditEvent agentModelStep( String sessionId, String runId, String agentName, int stepIndex, int durationMs, Map input, Map output) { Map details = new LinkedHashMap<>(); details.put("agent_name", agentName); details.put("step_index", stepIndex); details.put("input", input); details.put("output", output); return new DiagnosisTraceAuditEvent( sessionId, runId, TracePhase.AGENT, TraceEventType.AGENT_MODEL_STEP, TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details); } public static DiagnosisTraceAuditEvent modelTokenUsage( RunContext context, ModelCallLedger.Call call, long inputTokens, long outputTokens, boolean usageAvailable) { Map details = new LinkedHashMap<>(); details.put("component", call.component().name()); details.put("component_round", call.componentRound()); details.put("usage_available", usageAvailable); if (usageAvailable) { details.put("input_tokens", inputTokens); details.put("output_tokens", outputTokens); details.put("total_tokens", safeAdd(inputTokens, outputTokens)); } return new DiagnosisTraceAuditEvent( context.sessionId(), context.runId(), call.component().tracePhase(), TraceEventType.MODEL_TOKEN_USAGE, usageAvailable ? TraceEventStatus.SUCCEEDED : TraceEventStatus.UNAVAILABLE, call.componentRound(), null, details); } public static DiagnosisTraceAuditEvent agentDraftInvalid( RunContext context, DiagnosisAgentOutputException.Kind kind, long outputBytes, boolean hasPublishableProgress) { Map details = new LinkedHashMap<>(); details.put("failure_kind", kind.name()); details.put("output_bytes", outputBytes); details.put("has_publishable_progress", hasPublishableProgress); return event(context, TracePhase.AGENT, TraceEventType.AGENT_DRAFT_INVALID, TraceEventStatus.REJECTED, null, null, details); } public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) { Map details = new LinkedHashMap<>(); details.put("tool_call_id", tool.toolCallId()); details.put("tool_name", tool.toolName()); details.put("invocation_status", tool.status().name()); details.put("evidence_status", tool.evidenceStatus().name()); details.put("request_bytes", tool.requestBytes()); details.put("agent_result_bytes", tool.agentResultBytes()); if (tool.errorCode() != null) { details.put("error_code", tool.errorCode()); } return new DiagnosisTraceAuditEvent( tool.sessionId(), tool.runId(), TracePhase.TOOL, TraceEventType.TOOL_INVOCATION, tool.status() == com.superbiz.agent.harness.contract.InvocationStatus.READY ? TraceEventStatus.READY : TraceEventStatus.ERROR, null, tool.durationMs(), details); } public static DiagnosisTraceAuditEvent toolRequestRejected( RunContext context, String toolCallId, String toolName, String errorCode) { return toolRequestRejected(context, toolCallId, toolName, errorCode, null, false, null); } public static DiagnosisTraceAuditEvent toolRequestRejected( RunContext context, String toolCallId, String toolName, String errorCode, ProgressProtocolViolationType violationType, boolean repairPromptDelivered, DiagnosisProgressSnapshotState state) { Map details = new LinkedHashMap<>(); details.put("tool_call_id", safeIdentifier(toolCallId)); details.put("tool_name", safeIdentifier(toolName)); details.put("error_code", safeIdentifier(errorCode)); if (violationType != null) { details.put("violation_type", violationType.name()); details.put("repair_prompt_delivered", repairPromptDelivered); } if (state != null && state.consecutiveProgressProtocolViolations() > 0) { details.put("consecutive_protocol_violations", state.consecutiveProgressProtocolViolations()); } if (state != null && state.stopReason() != null) { details.put("stop_reason", state.stopReason().name()); } return event(context, TracePhase.TOOL, TraceEventType.TOOL_REQUEST_REJECTED, TraceEventStatus.REJECTED, null, null, details); } public static DiagnosisTraceAuditEvent toolProgress( RunContext context, String toolCallId, String toolName, String scopeSummary, InformationGain informationGain, String producer, DiagnosisProgressSnapshotState state) { Map details = new LinkedHashMap<>(); details.put("tool_call_id", toolCallId); details.put("tool_name", toolName); details.put("scope_summary", scopeSummary); details.put("information_gain", informationGain.name()); details.put("producer", producer); details.put("consecutive_no_gain", state.consecutiveNoGain()); details.put("collection_state", state.collectionState().name()); if (state.stopReason() != null) { details.put("stop_reason", state.stopReason().name()); } return event(context, TracePhase.TOOL, TraceEventType.TOOL_PROGRESS, TraceEventStatus.SUCCEEDED, null, null, details); } public static DiagnosisTraceAuditEvent collectionStop( RunContext context, String toolCallId, String toolName, DiagnosisProgressSnapshotState state) { Map details = new LinkedHashMap<>(); details.put("tool_call_id", toolCallId); details.put("tool_name", toolName); details.put("collection_state", state.collectionState().name()); if (state.stopReason() != null) { details.put("stop_reason", state.stopReason().name()); } return event(context, TracePhase.TOOL, TraceEventType.COLLECTION_STOP, TraceEventStatus.SUCCEEDED, null, null, details); } public static DiagnosisTraceAuditEvent evidenceValidation( RunContext context, TraceEventType type, EvidenceGuardResult result, DiagnosisDraft draft) { Map details = new LinkedHashMap<>(); details.put("violation_count", result.violations().size()); details.put("violations", violations(result.violations())); ToolReferences references = toolReferences(draft); details.put("referenced_tool_call_ids", references.safeIds()); details.put("invalid_tool_reference_count", references.invalidCount()); result.verifiedSnapshot().ifPresent(snapshot -> { details.put("verified_analysis_count", snapshot.analyses().size()); details.put("verified_source_count", snapshot.verifiedSources().size()); }); return event(context, TracePhase.EVIDENCE, type, result.valid() ? TraceEventStatus.PASSED : TraceEventStatus.REJECTED, null, null, details); } public static DiagnosisTraceAuditEvent evidenceRepairAttempt( RunContext context, RetryAttempt attempt) { return retry(context, TracePhase.EVIDENCE, TraceEventType.EVIDENCE_REPAIR_ATTEMPT, attempt); } public static DiagnosisTraceAuditEvent semanticAttempt( RunContext context, RetryAttempt attempt) { return retry(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_ATTEMPT, attempt); } public static DiagnosisTraceAuditEvent semanticDecision( RunContext context, SemanticVerdict verdict) { TraceEventStatus status = verdict == SemanticVerdict.SUPPORTED ? TraceEventStatus.SUPPORTED : TraceEventStatus.UNSUPPORTED; return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION, status, null, null, Map.of("verdict", verdict.name())); } public static DiagnosisTraceAuditEvent semanticUnavailable(RunContext context) { return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION, TraceEventStatus.UNAVAILABLE, null, null, Map.of()); } public static DiagnosisTraceAuditEvent releaseDecision( RunContext context, ReleaseOutcome outcome, FallbackType fallbackType) { Map details = new LinkedHashMap<>(); details.put("release_outcome", outcome.name()); if (fallbackType != null) { details.put("fallback_type", fallbackType.name()); } return event(context, TracePhase.RELEASE, TraceEventType.RELEASE_DECISION, terminalStatus(outcome), null, null, details); } private static DiagnosisTraceAuditEvent retry( RunContext context, TracePhase phase, TraceEventType type, RetryAttempt attempt) { Map details = new LinkedHashMap<>(); if (attempt.failure() != null) { details.put("failure", attempt.failure().name()); } return event(context, phase, type, attempt.success() ? TraceEventStatus.SUCCEEDED : TraceEventStatus.FAILED, attempt.attemptNumber(), null, details); } private static DiagnosisTraceAuditEvent event( RunContext context, TracePhase phase, TraceEventType type, TraceEventStatus status, Integer attemptNo, Integer durationMs, Map details) { return new DiagnosisTraceAuditEvent( context.sessionId(), context.runId(), phase, type, status, attemptNo, durationMs, details); } private static TraceEventStatus terminalStatus(ReleaseOutcome outcome) { return switch (outcome) { case SUCCESS -> TraceEventStatus.SUCCEEDED; case FALLBACK -> TraceEventStatus.FALLBACK; case FAILED -> TraceEventStatus.FAILED; case CANCELLED -> TraceEventStatus.CANCELLED; }; } private static List> violations(List violations) { List> result = new ArrayList<>(); for (EvidenceViolation violation : violations) { Map item = new LinkedHashMap<>(); item.put("code", violation.code().name()); item.put("target", violation.target()); result.add(Map.copyOf(item)); } return List.copyOf(result); } private static ToolReferences toolReferences(DiagnosisDraft draft) { if (draft == null) { return new ToolReferences(List.of(), 0); } Set safeIds = new LinkedHashSet<>(); int invalid = 0; for (DiagnosisDraft.AnalysisItem item : draft.analysis()) { if (item == null) { continue; } for (String id : item.toolCallIds()) { if (id != null && SAFE_TOOL_CALL_ID.matcher(id).matches()) { safeIds.add(id); } else { invalid++; } } } return new ToolReferences(List.copyOf(safeIds), invalid); } private static String safeIdentifier(String value) { return value != null && SAFE_TOOL_CALL_ID.matcher(value).matches() ? value : "INVALID"; } private static long safeAdd(long left, long right) { try { return Math.addExact(left, right); } catch (ArithmeticException exception) { return Long.MAX_VALUE; } } private record ToolReferences(List safeIds, int invalidCount) { } }