diff --git a/mvp/issues/active/ISS-014-single-react-agent-harness-aci-ptk-refactor.md b/mvp/issues/active/ISS-014-single-react-agent-harness-aci-ptk-refactor.md index 4b2362c..877ac6a 100644 --- a/mvp/issues/active/ISS-014-single-react-agent-harness-aci-ptk-refactor.md +++ b/mvp/issues/active/ISS-014-single-react-agent-harness-aci-ptk-refactor.md @@ -1,6 +1,6 @@ # ISS-014 单体 ReAct Agent、Harness 与 ACI 工具瘦身 -**状态**：已完成（阶段 0-7 已验收并归档） +**状态**：阶段 0-7 已验收并归档；运行质量改进项进行中 **严重程度**：高 **发现时间**：2026-07-20 **目标分支**：`refactor/chat-single-react-harness` @@ -137,7 +137,7 @@ DIAGNOSIS - 调用其他 Agent； - 外层重试状态机。 -不要求 Agent 输出 Thought 或 Chain of Thought。内部规划不形成对外协议，也不持久化为业务事实。 +不要求 Agent 输出 Thought 或 Chain of Thought。内部规划不形成对外协议，也不持久化为业务事实。若模型供应商实际返回 reasoning 元数据，可按独立审计数据保留受限副本；该副本不进入普通 Trace、证据快照或业务结果。 ### 4.2 Harness @@ -509,7 +509,7 @@ EvidenceGuard 或 SemanticGuard 降级使用 `content + done(FALLBACK)`；Router 禁止输出： -- Chain of Thought 和内部规划； +- Chain of Thought 和内部规划（普通 SSE/Trace 不输出；独立审计存储仅限供应商实际返回的 reasoning 元数据）； - System Prompt、模型 Prompt 和完整上下文； - 未脱敏 Tool 参数； - 原始日志、完整数据库行和完整检索 Trace； @@ -1335,7 +1335,7 @@ ISS-014 是总设计 Issue，不创建跨阶段共享的 OpenSpec change。以 - 不引入业务 StateGraph； - 不实现 Harness 插件市场、通用 Pipeline DSL 或策略语言； - 不保留旧同步 Chat、`/chat_stream` 或旧 Tool Contract； -- 不输出或持久化 Chain of Thought； +- 不向普通 SSE、Trace、证据快照或业务结果输出 Chain of Thought；若供应商实际返回 reasoning 元数据，只允许写入独立的受限审计表，不将其作为业务事实或证据。 - 不使用本地 `refs/*.md` 作为 Tool 原始结果主存储； - 不向 Agent 暴露 Redis Client、Redis Tool、key、连接信息或完整调用记录； - 不提供历史来源召回 Tool、通用 Memory Tool 或完整上下文恢复能力； @@ -1386,7 +1386,33 @@ ISS-014 是总设计 Issue，不创建跨阶段共享的 OpenSpec change。以 实现中不得为这些风险创建 HarnessPlugin、Graph、通用 Tool DSL 或大量预留接口；按各阶段最小范围实现并通过门禁验证。 -## 17. 相关文件 +## 17. 阶段 7 后续运行质量改进 + +本节记录阶段 7 归档后的实际验证结果和后续开发边界。当前不恢复完整端到端验证，待以下代码项完成后再安排一次受控的成功/失败用例验证。 + +### 已完成并纳入本次提交 + +- 新增独立追加式 `diagnosis_trace_event` 表，按 `sessionId + runId` 记录 Run、Routing、Agent、Tool、Evidence、Semantic 和 Release 生命周期事件；普通 Trace 查询不再依赖其他业务表拼接完整时间线。 +- Knowledge Query 的 `KnowledgeAnswerDraft` 使用框架 `BeanOutputConverter` 注入实际 JSON Schema，避免模型因缺少结构约束返回 `Knowledge answer is invalid`。 +- Fallback 增加 `failure_stage`、`observed_facts` 和 `validation_issues`，在不暴露 Prompt、原始 Draft 或 Tool 载荷的前提下，为用户提供可审计的失败阶段、观察事实和校验问题。 +- 新增独立 `agent_reasoning_audit` 表和 reasoning 审计查询接口。Agent Hook 只保存供应商实际返回的 reasoning 元数据并限制长度；普通 Trace 仅记录是否存在及字节数，不返回原文。 + +### 已验证证据 + +- Diagnosis 成功用例：`sessionId=mvp-demo-success-e2e-20260722-2032`，`runId=e72d9c7b-2f78-44c8-9086-f2e9d6987f24`，`release_outcome=SUCCESS`，总 Token 8455，Tool 调用 1 次。 +- Knowledge Query 成功用例：`sessionId=mvp-demo-order-timeout-fixed-20260723`，`runId=2b704d1b-8f11-40ee-a4c6-5e5e04d3847c`，`release_outcome=SUCCESS`，总 Token 1998。 +- 相关 focused tests 和编译已通过；本轮不宣称 V015 数据库迁移和 reasoning endpoint 已完成真实数据库 E2E 验证。 + +### 下一阶段必须补齐 + +1. Diagnosis Agent 的硬停止策略：限制证据轮次，预算耗尽前强制生成最终 Draft，阻止重复 `lookup_knowledge`。 +2. Evidence Repair 的实际 `DiagnosisDraft` Schema 注入，消除真实运行中的 `PARSE_ERROR`。 +3. 使用真实 Provider 验证 reasoning 元数据：Provider 不返回时仍需写入 `reasoning_available=false` 的审计记录。 +4. 对 V015 迁移和 reasoning 查询接口执行一次受控真实数据库/E2E 验证，并核对 `sessionId + runId`。 +5. 明确 reasoning 审计数据的访问控制、保留期限和加密策略。 +6. 当工具成功但无法构造 verified snapshot 时，继续细化 Evidence 验证失败的 `observed_facts` 和 `validation_issues`。 + +## 18. 相关文件 - `mvp/issues/active/ISS-012-executor-token-budget-and-context-growth.md` - `mvp/issues/active/ISS-013-chat-entry-decoupling-and-sse.md` diff --git a/src/main/java/com/superbiz/agent/config/HarnessChatConfiguration.java b/src/main/java/com/superbiz/agent/config/HarnessChatConfiguration.java index 0e1972f..5c4cb9f 100644 --- a/src/main/java/com/superbiz/agent/config/HarnessChatConfiguration.java +++ b/src/main/java/com/superbiz/agent/config/HarnessChatConfiguration.java @@ -50,8 +50,10 @@ import com.superbiz.agent.harness.application.KnowledgeQueryOperation; import com.superbiz.agent.harness.application.SystemChatOperation; import com.superbiz.agent.harness.application.IntentRouting; import com.superbiz.agent.harness.audit.HarnessAgentAuditHook; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; import com.superbiz.agent.harness.audit.ToolInvocationAuditSink; import com.superbiz.agent.repository.AgentStepRepository; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import com.superbiz.agent.tool.LookupKnowledgeTool; import com.superbiz.agent.agent.tool.QueryLogsTools; import org.springframework.ai.chat.model.ChatModel; @@ -232,11 +234,13 @@ public class HarnessChatConfiguration { @Bean public DiagnosisAgentFactory diagnosisAgentFactory(ChatModel chatModel, DiagnosisHarnessCore core, - HarnessEvidenceTools tools, ObjectMapper mapper, - AgentStepRepository steps) { + HarnessEvidenceTools tools, ObjectMapper mapper, + AgentStepRepository steps, + AgentReasoningAuditRepository reasoningAudits, + DiagnosisTraceRecorder traceRecorder) { return new DiagnosisAgentFactory(chatModel, core, tools, mapper, List.of(new HarnessAgentAuditHook( - steps, mapper, DiagnosisAgentFactory.AGENT_NAME))); + steps, mapper, DiagnosisAgentFactory.AGENT_NAME, traceRecorder, reasoningAudits))); } @Bean @@ -258,28 +262,32 @@ public class HarnessChatConfiguration { @Bean public EvidenceRepair evidenceRepair(DiagnosisHarnessCore core, HarnessRetryExecutor retries, GuardModelCall modelCall, ObjectMapper mapper, - ChatHarnessProperties properties) { + ChatHarnessProperties properties, + DiagnosisTraceRecorder traceRecorder) { return new EvidenceRepair(core, retries, modelCall, mapper, new EvidenceRepairLimits(properties.getRepairMaxInputBytes(), properties.getRepairMaxOutputBytes(), properties.getRepairTimeout()), - attempt -> { }); + attempt -> { }, traceRecorder); } @Bean public SemanticGuard semanticGuard(DiagnosisHarnessCore core, HarnessRetryExecutor retries, GuardModelCall modelCall, ObjectMapper mapper, - ChatHarnessProperties properties) { + ChatHarnessProperties properties, + DiagnosisTraceRecorder traceRecorder) { return new SemanticGuard(core, retries, modelCall, mapper, new SemanticGuardLimits(properties.getSemanticMaxInputBytes(), properties.getSemanticMaxOutputBytes(), properties.getSemanticAttemptTimeout(), properties.getSemanticTotalTimeout()), - attempt -> { }); + attempt -> { }, traceRecorder); } @Bean public DiagnosisReleaseUseCase diagnosisReleaseUseCase(EvidenceGuard evidenceGuard, EvidenceRepair repair, - SemanticGuard semanticGuard) { - return new DiagnosisReleaseUseCase(evidenceGuard, repair, semanticGuard, new SafeFallbackFactory()); + SemanticGuard semanticGuard, + DiagnosisTraceRecorder traceRecorder) { + return new DiagnosisReleaseUseCase( + evidenceGuard, repair, semanticGuard, new SafeFallbackFactory(), traceRecorder); } @Bean @@ -290,11 +298,12 @@ public class HarnessChatConfiguration { @Bean public IntentRouting intentRouter(DiagnosisHarnessCore core, HarnessRetryExecutor retries, GuardModelCall modelCall, ObjectMapper mapper, - ChatHarnessProperties properties) { + ChatHarnessProperties properties, + DiagnosisTraceRecorder traceRecorder) { return new IntentRouter(core, retries, modelCall, mapper, new IntentRouterLimits(properties.getRouterMaxInputBytes(), properties.getRouterMaxOutputBytes(), properties.getRouterAttemptTimeout(), properties.getRouterTotalTimeout()), - attempt -> { }); + attempt -> { }, traceRecorder); } @Bean @@ -330,9 +339,10 @@ public class HarnessChatConfiguration { SystemChatOperation system, KnowledgeQueryOperation knowledge, DiagnosisOperation diagnosis, - ObjectMapper mapper) { + ObjectMapper mapper, + DiagnosisTraceRecorder traceRecorder) { return new ChatApplicationUseCase(core, () -> UUID.randomUUID().toString(), runStore, - router, system, knowledge, diagnosis, mapper); + router, system, knowledge, diagnosis, mapper, traceRecorder); } private static ThreadPoolExecutor boundedExecutor(int core, int max, int queue, String prefix) { diff --git a/src/main/java/com/superbiz/agent/controller/DiagnosisTraceController.java b/src/main/java/com/superbiz/agent/controller/DiagnosisTraceController.java index c7d9474..ce4364c 100644 --- a/src/main/java/com/superbiz/agent/controller/DiagnosisTraceController.java +++ b/src/main/java/com/superbiz/agent/controller/DiagnosisTraceController.java @@ -11,6 +11,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController @RequestMapping("/api/diagnosis") @RequiredArgsConstructor @@ -25,4 +27,13 @@ public class DiagnosisTraceController { ) { return ResponseEntity.ok(Result.success(diagnosisTraceService.getTrace(sessionId, runId))); } + + @GetMapping("/{sessionId}/trace/reasoning") + public ResponseEntity>> getReasoning( + @PathVariable String sessionId, + @RequestParam String runId + ) { + return ResponseEntity.ok(Result.success( + diagnosisTraceService.getReasoning(sessionId, runId))); + } } diff --git a/src/main/java/com/superbiz/agent/domain/entity/AgentReasoningAudit.java b/src/main/java/com/superbiz/agent/domain/entity/AgentReasoningAudit.java new file mode 100644 index 0000000..08f71bb --- /dev/null +++ b/src/main/java/com/superbiz/agent/domain/entity/AgentReasoningAudit.java @@ -0,0 +1,61 @@ +package com.superbiz.agent.domain.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "agent_reasoning_audit", indexes = { + @Index(name = "idx_reasoning_run_step", columnList = "run_id, step_index"), + @Index(name = "idx_reasoning_session_created", columnList = "session_id, created_at") +}) +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AgentReasoningAudit { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "session_id", nullable = false, length = 64) + private String sessionId; + + @Column(name = "run_id", nullable = false, length = 64) + private String runId; + + @Column(name = "step_index", nullable = false) + private Integer stepIndex; + + @Column(name = "agent_name", nullable = false, length = 64) + private String agentName; + + @Column(name = "reasoning_available", nullable = false) + private Boolean reasoningAvailable; + + @Column(name = "reasoning_content", columnDefinition = "LONGTEXT") + private String reasoningContent; + + @Column(name = "content_bytes", nullable = false) + private Integer contentBytes; + + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + @PrePersist + protected void onCreate() { + createdAt = LocalDateTime.now(); + } +} diff --git a/src/main/java/com/superbiz/agent/domain/entity/DiagnosisTraceEvent.java b/src/main/java/com/superbiz/agent/domain/entity/DiagnosisTraceEvent.java new file mode 100644 index 0000000..2ddb49b --- /dev/null +++ b/src/main/java/com/superbiz/agent/domain/entity/DiagnosisTraceEvent.java @@ -0,0 +1,73 @@ +package com.superbiz.agent.domain.entity; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.PrePersist; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; + +import java.time.LocalDateTime; + +@Entity +@Table(name = "diagnosis_trace_event", indexes = { + @Index(name = "idx_trace_event_run_sequence", columnList = "run_id, sequence_no, id"), + @Index(name = "idx_trace_event_session_created", columnList = "session_id, created_at, id"), + @Index(name = "idx_trace_event_phase", columnList = "phase, event_type") +}) +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DiagnosisTraceEvent { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(name = "session_id", nullable = false, length = 64) + private String sessionId; + + @Column(name = "run_id", nullable = false, length = 64) + private String runId; + + @Column(name = "sequence_no", nullable = false) + private Integer sequenceNo; + + @Column(name = "phase", nullable = false, length = 32) + private String phase; + + @Column(name = "event_type", nullable = false, length = 48) + private String eventType; + + @Column(name = "status", nullable = false, length = 32) + private String status; + + @Column(name = "attempt_no") + private Integer attemptNo; + + @Column(name = "duration_ms") + private Integer durationMs; + + @JdbcTypeCode(SqlTypes.JSON) + @Column(name = "details", nullable = false, columnDefinition = "JSON") + private String details; + + @Column(name = "created_at", nullable = false, updatable = false) + private LocalDateTime createdAt; + + @PrePersist + protected void onCreate() { + if (createdAt == null) { + createdAt = LocalDateTime.now(); + } + } +} diff --git a/src/main/java/com/superbiz/agent/dto/DiagnosisTraceResponse.java b/src/main/java/com/superbiz/agent/dto/DiagnosisTraceResponse.java index 67c0dd4..ed97c11 100644 --- a/src/main/java/com/superbiz/agent/dto/DiagnosisTraceResponse.java +++ b/src/main/java/com/superbiz/agent/dto/DiagnosisTraceResponse.java @@ -21,6 +21,7 @@ public class DiagnosisTraceResponse { private RunTrace run; private List steps; private List toolInvocations; + private List timeline; private TraceSummary summary; @Data @@ -47,6 +48,8 @@ public class DiagnosisTraceResponse { private String query; private String status; private String agentFlow; + private String intent; + private String releaseOutcome; private Integer totalDurationMs; private Integer totalTokenCount; private Integer stepCount; @@ -70,6 +73,8 @@ public class DiagnosisTraceResponse { private String query; private String status; private String agentFlow; + private String intent; + private String releaseOutcome; private Integer totalDurationMs; private Integer totalTokenCount; private Integer stepCount; @@ -129,6 +134,41 @@ public class DiagnosisTraceResponse { private LocalDateTime createdAt; } + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class TraceEventTrace { + private Long id; + private String sessionId; + private String runId; + private Integer sequenceNo; + private String phase; + private String eventType; + private String status; + private Integer attemptNo; + private Integer durationMs; + private String detailsRaw; + private Map details; + private LocalDateTime createdAt; + } + + @Data + @Builder + @NoArgsConstructor + @AllArgsConstructor + public static class AgentReasoningTrace { + private Long id; + private String sessionId; + private String runId; + private Integer stepIndex; + private String agentName; + private Boolean reasoningAvailable; + private String reasoningContent; + private Integer contentBytes; + private LocalDateTime createdAt; + } + @Data @Builder @NoArgsConstructor @@ -139,6 +179,8 @@ public class DiagnosisTraceResponse { private int returnedStepCount; private int persistedToolCallCount; private int returnedToolCallCount; + private int persistedTraceEventCount; + private int returnedTraceEventCount; private boolean hasVerifierEvaluation; private boolean hasFeedback; } @@ -153,6 +195,8 @@ public class DiagnosisTraceResponse { private String query; private String status; private String agentFlow; + private String intent; + private String releaseOutcome; private String answerPreview; private Integer stepCount; private Integer toolCallCount; diff --git a/src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentFactory.java b/src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentFactory.java index d1fd51c..d26ca18 100644 --- a/src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentFactory.java +++ b/src/main/java/com/superbiz/agent/harness/agent/DiagnosisAgentFactory.java @@ -47,6 +47,7 @@ public final class DiagnosisAgentFactory { new HarnessToolInterceptor(context, evidenceTools, objectMapper)) .hooks(auditHooks) .outputSchema(new DiagnosisDraftOutputSchema(objectMapper).getFormat()) + .returnReasoningContents(true) .parallelToolExecution(false) .releaseThread(true) .build(); diff --git a/src/main/java/com/superbiz/agent/harness/application/ChatApplicationUseCase.java b/src/main/java/com/superbiz/agent/harness/application/ChatApplicationUseCase.java index 012b408..2d5320a 100644 --- a/src/main/java/com/superbiz/agent/harness/application/ChatApplicationUseCase.java +++ b/src/main/java/com/superbiz/agent/harness/application/ChatApplicationUseCase.java @@ -3,6 +3,8 @@ package com.superbiz.agent.harness.application; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.harness.application.persistence.ChatRunStore; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceAuditEvents; import com.superbiz.agent.harness.application.persistence.RoutingHistory; import com.superbiz.agent.harness.application.routing.IntentRouterInput; import com.superbiz.agent.harness.application.routing.IntentRoutingException; @@ -33,6 +35,7 @@ public final class ChatApplicationUseCase { private final KnowledgeQueryOperation knowledgeQuery; private final DiagnosisOperation diagnosis; private final ObjectMapper objectMapper; + private final DiagnosisTraceRecorder traceRecorder; public ChatApplicationUseCase(DiagnosisHarnessCore core, Supplier sessionIdSupplier, @@ -42,6 +45,19 @@ public final class ChatApplicationUseCase { KnowledgeQueryOperation knowledgeQuery, DiagnosisOperation diagnosis, ObjectMapper objectMapper) { + this(core, sessionIdSupplier, runStore, router, systemChat, knowledgeQuery, + diagnosis, objectMapper, DiagnosisTraceRecorder.noop()); + } + + public ChatApplicationUseCase(DiagnosisHarnessCore core, + Supplier sessionIdSupplier, + ChatRunStore runStore, + IntentRouting router, + SystemChatOperation systemChat, + KnowledgeQueryOperation knowledgeQuery, + DiagnosisOperation diagnosis, + ObjectMapper objectMapper, + DiagnosisTraceRecorder traceRecorder) { this.core = Objects.requireNonNull(core, "core must not be null"); this.sessionIdSupplier = Objects.requireNonNull( sessionIdSupplier, "sessionIdSupplier must not be null"); @@ -51,6 +67,7 @@ public final class ChatApplicationUseCase { this.knowledgeQuery = Objects.requireNonNull(knowledgeQuery, "knowledgeQuery must not be null"); this.diagnosis = Objects.requireNonNull(diagnosis, "diagnosis must not be null"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); } public ChatApplicationResult execute(ChatApplicationRequest request) { @@ -78,12 +95,14 @@ public final class ChatApplicationUseCase { IntentType intent = null; try { persistStart(context, request.query()); + traceRecorder.record(TraceAuditEvents.runStarted(context)); observer.onStarted(new CoreRunControl(core, context)); observer.onStatus(ChatApplicationStatus.ROUTING); intent = router.route(context, new IntentRouterInput( request.query(), history.map(RoutingHistory::intent).orElse(null), history.map(RoutingHistory::userQuery).orElse(null))); + traceRecorder.record(TraceAuditEvents.routingDecision(context, intent)); persistIntent(context.runId(), intent); PathResult path = executePath( @@ -93,6 +112,8 @@ public final class ChatApplicationUseCase { String safeJson = write(path.content()); persistFinish(context, intent, path.outcome(), safeJson, path.publishedResult(), durationMillis(startedNanos)); + traceRecorder.record(TraceAuditEvents.runFinished( + context, intent, path.outcome(), durationMillis(startedNanos))); return new ChatApplicationResult( context.sessionId(), context.runId(), intent, path.outcome(), path.content().contentType(), path.content()); @@ -104,6 +125,8 @@ public final class ChatApplicationUseCase { } catch (RuntimeException persistenceFailure) { exception.addSuppressed(persistenceFailure); } + traceRecorder.record(TraceAuditEvents.runFinished( + context, intent, terminal, durationMillis(startedNanos))); throw safeFailure(exception, terminal); } } diff --git a/src/main/java/com/superbiz/agent/harness/application/executor/KnowledgeQueryExecutor.java b/src/main/java/com/superbiz/agent/harness/application/executor/KnowledgeQueryExecutor.java index 7d5d59a..f3be88c 100644 --- a/src/main/java/com/superbiz/agent/harness/application/executor/KnowledgeQueryExecutor.java +++ b/src/main/java/com/superbiz/agent/harness/application/executor/KnowledgeQueryExecutor.java @@ -24,6 +24,7 @@ import com.superbiz.agent.harness.tool.contract.RagToolResult; import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.converter.BeanOutputConverter; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -39,7 +40,8 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation { private static final String SYSTEM_PROMPT = """ 你只根据提供的有界知识库证据回答原始问题。每个 answer item 必须引用 exact tool_call_id 和实际 document_ids。 - 不要使用外部知识，不要改写问题，不要输出 markdown fence。返回严格 KnowledgeAnswerDraft JSON。 + 不要使用外部知识，不要改写问题。answer_items 必须是非空数组；每个 item 必须包含非空 text、 + exact tool_call_id 和至少一个 actual document_id。limitations 必须是字符串数组。 """; private final DiagnosisHarnessCore core; @@ -48,6 +50,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation { private final ObjectMapper objectMapper; private final ObjectReader ragReader; private final ObjectReader answerReader; + private final String prompt; private final Supplier callIdSupplier; private final KnowledgeQueryLimits limits; @@ -67,6 +70,8 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation { this.answerReader = objectMapper.readerFor(KnowledgeAnswerDraft.class) .with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + this.prompt = SYSTEM_PROMPT + System.lineSeparator() + + new BeanOutputConverter<>(KnowledgeAnswerDraft.class, objectMapper).getFormat(); this.callIdSupplier = Objects.requireNonNull(callIdSupplier, "callIdSupplier must not be null"); this.limits = Objects.requireNonNull(limits, "limits must not be null"); } @@ -102,7 +107,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation { core.reserveRunBytes(context, bytes); String output = modelCall.call( context, - new Prompt(List.of(new SystemMessage(SYSTEM_PROMPT), new UserMessage(modelInput))), + new Prompt(List.of(new SystemMessage(prompt), new UserMessage(modelInput))), limits.modelTimeout(), limits.maxModelOutputBytes()); KnowledgeAnswerDraft draft = readAnswer(output); diff --git a/src/main/java/com/superbiz/agent/harness/application/routing/IntentRouter.java b/src/main/java/com/superbiz/agent/harness/application/routing/IntentRouter.java index 48cca0b..00c0a98 100644 --- a/src/main/java/com/superbiz/agent/harness/application/routing/IntentRouter.java +++ b/src/main/java/com/superbiz/agent/harness/application/routing/IntentRouter.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.harness.contract.IntentType; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceAuditEvents; import com.superbiz.agent.harness.application.IntentRouting; import com.superbiz.agent.harness.core.DiagnosisHarnessCore; import com.superbiz.agent.harness.core.RunContext; @@ -36,6 +38,7 @@ public final class IntentRouter implements IntentRouting { private final ObjectMapper objectMapper; private final IntentRouterLimits limits; private final Consumer attemptRecorder; + private final DiagnosisTraceRecorder traceRecorder; private final String systemPrompt; public IntentRouter(DiagnosisHarnessCore core, @@ -44,12 +47,24 @@ public final class IntentRouter implements IntentRouting { ObjectMapper objectMapper, IntentRouterLimits limits, Consumer attemptRecorder) { + this(core, retryExecutor, modelCall, objectMapper, limits, + attemptRecorder, DiagnosisTraceRecorder.noop()); + } + + public IntentRouter(DiagnosisHarnessCore core, + HarnessRetryExecutor retryExecutor, + GuardModelCall modelCall, + ObjectMapper objectMapper, + IntentRouterLimits limits, + Consumer attemptRecorder, + DiagnosisTraceRecorder traceRecorder) { this.core = Objects.requireNonNull(core, "core must not be null"); this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null"); this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null"); this.limits = Objects.requireNonNull(limits, "limits must not be null"); this.attemptRecorder = Objects.requireNonNull(attemptRecorder, "attemptRecorder must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); this.systemPrompt = IntentRouterPrompt.load(); } @@ -73,7 +88,10 @@ public final class IntentRouter implements IntentRouting { () -> parse(modelCall.call( context, prompt, remaining(started), limits.maxOutputBytes())), this::classify, - attemptRecorder); + attempt -> { + attemptRecorder.accept(attempt); + traceRecorder.record(TraceAuditEvents.routingAttempt(context, attempt)); + }); } catch (RetryExecutionException exception) { if (exception.failure() == RetryFailure.CANCELLED || exception.failure() == RetryFailure.BUDGET_EXHAUSTED) { diff --git a/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceAuditEvent.java b/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceAuditEvent.java new file mode 100644 index 0000000..126e437 --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceAuditEvent.java @@ -0,0 +1,37 @@ +package com.superbiz.agent.harness.audit; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +public record DiagnosisTraceAuditEvent( + String sessionId, + String runId, + TracePhase phase, + TraceEventType eventType, + TraceEventStatus status, + Integer attemptNo, + Integer durationMs, + Map details) { + + public DiagnosisTraceAuditEvent { + requireText(sessionId, "sessionId"); + requireText(runId, "runId"); + Objects.requireNonNull(phase, "phase must not be null"); + Objects.requireNonNull(eventType, "eventType must not be null"); + Objects.requireNonNull(status, "status must not be null"); + if (attemptNo != null && attemptNo < 1) { + throw new IllegalArgumentException("attemptNo must be positive"); + } + if (durationMs != null && durationMs < 0) { + throw new IllegalArgumentException("durationMs must not be negative"); + } + details = details == null ? Map.of() : Map.copyOf(new LinkedHashMap<>(details)); + } + + private static void requireText(String value, String name) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(name + " must not be blank"); + } + } +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceRecorder.java b/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceRecorder.java new file mode 100644 index 0000000..4b1a238 --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/DiagnosisTraceRecorder.java @@ -0,0 +1,12 @@ +package com.superbiz.agent.harness.audit; + +@FunctionalInterface +public interface DiagnosisTraceRecorder { + + void record(DiagnosisTraceAuditEvent event); + + static DiagnosisTraceRecorder noop() { + return ignored -> { + }; + } +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHook.java b/src/main/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHook.java index 389eb0f..2802fe2 100644 --- a/src/main/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHook.java +++ b/src/main/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHook.java @@ -8,6 +8,8 @@ import com.alibaba.cloud.ai.graph.agent.hook.messages.MessagesModelHook; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.AgentStep; +import com.superbiz.agent.domain.entity.AgentReasoningAudit; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import com.superbiz.agent.repository.AgentStepRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +19,7 @@ import org.springframework.ai.chat.messages.Message; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.nio.charset.StandardCharsets; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -28,16 +31,31 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { private final AgentStepRepository repository; private final ObjectMapper objectMapper; private final String agentName; + private final DiagnosisTraceRecorder traceRecorder; + private final AgentReasoningAuditRepository reasoningRepository; private final ConcurrentHashMap stepCounters = new ConcurrentHashMap<>(); private final ConcurrentHashMap pendingSteps = new ConcurrentHashMap<>(); public HarnessAgentAuditHook(AgentStepRepository repository, ObjectMapper objectMapper, String agentName) { + this(repository, objectMapper, agentName, DiagnosisTraceRecorder.noop()); + } + + public HarnessAgentAuditHook(AgentStepRepository repository, ObjectMapper objectMapper, + String agentName, DiagnosisTraceRecorder traceRecorder) { + this(repository, objectMapper, agentName, traceRecorder, null); + } + + public HarnessAgentAuditHook(AgentStepRepository repository, ObjectMapper objectMapper, + String agentName, DiagnosisTraceRecorder traceRecorder, + AgentReasoningAuditRepository reasoningRepository) { this.repository = Objects.requireNonNull(repository, "repository must not be null"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null"); if (agentName == null || agentName.isBlank()) { throw new IllegalArgumentException("agentName must not be blank"); } this.agentName = agentName; + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); + this.reasoningRepository = reasoningRepository; } @Override @@ -53,6 +71,7 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { return new AgentCommand(messages); } int stepIndex = stepCounters.merge(identity.runId(), 0, (current, ignored) -> current + 1); + PendingStep pending = new PendingStep(null, System.nanoTime(), inputMetadata(messages)); try { AgentStep saved = repository.save(AgentStep.builder() .sessionId(identity.sessionId()) @@ -62,11 +81,11 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { .modelInput(write(inputMetadata(messages))) .hasToolCall(false) .build()); - pendingSteps.put(stepKey(identity.runId(), stepIndex), - new PendingStep(saved.getId(), System.nanoTime())); + pending = new PendingStep(saved.getId(), pending.startedNanos(), pending.input()); } catch (RuntimeException exception) { log.warn("Failed to persist AgentStep audit before model: agent={}", agentName); } + pendingSteps.put(stepKey(identity.runId(), stepIndex), pending); return new AgentCommand(messages); } @@ -79,26 +98,33 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { } int stepIndex = stepCounters.getOrDefault(identity.runId(), 0); PendingStep pending = pendingSteps.remove(stepKey(identity.runId(), stepIndex)); - if (pending == null || pending.id() == null) { + if (pending == null) { return new AgentCommand(messages); } + AssistantMessage assistant = lastAssistant(messages); + List toolNames = assistant == null || assistant.getToolCalls() == null + ? List.of() + : assistant.getToolCalls().stream().map(AssistantMessage.ToolCall::name) + .distinct().sorted().toList(); + ReasoningContent reasoning = reasoningContent(assistant); + Map output = outputMetadata(assistant, toolNames, reasoning); + int durationMs = durationMillis(pending.startedNanos()); try { - AgentStep step = repository.findById(pending.id()).orElse(null); + AgentStep step = pending.id() == null ? null : repository.findById(pending.id()).orElse(null); if (step != null) { - AssistantMessage assistant = lastAssistant(messages); - List toolNames = assistant == null || assistant.getToolCalls() == null - ? List.of() - : assistant.getToolCalls().stream().map(AssistantMessage.ToolCall::name) - .distinct().sorted().toList(); - step.setModelOutput(write(outputMetadata(assistant, toolNames))); + step.setModelOutput(write(output)); step.setThought(null); step.setHasToolCall(!toolNames.isEmpty()); - step.setDurationMs(durationMillis(pending.startedNanos())); + step.setDurationMs(durationMs); repository.save(step); } } catch (RuntimeException exception) { log.warn("Failed to complete AgentStep audit: agent={}", agentName); } + persistReasoning(identity, stepIndex, reasoning); + traceRecorder.record(TraceAuditEvents.agentModelStep( + identity.sessionId(), identity.runId(), agentName, stepIndex, + durationMs, pending.input(), output)); return new AgentCommand(messages); } @@ -112,14 +138,56 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { return metadata; } - private Map outputMetadata(AssistantMessage assistant, List toolNames) { + private Map outputMetadata(AssistantMessage assistant, List toolNames, + ReasoningContent reasoning) { Map metadata = new LinkedHashMap<>(); metadata.put("has_text", assistant != null && assistant.getText() != null && !assistant.getText().isBlank()); metadata.put("tool_names", toolNames); + metadata.put("reasoning_available", reasoning.available()); + metadata.put("reasoning_bytes", reasoning.bytes()); return metadata; } + private ReasoningContent reasoningContent(AssistantMessage assistant) { + if (assistant == null || assistant.getMetadata() == null) { + return ReasoningContent.empty(); + } + for (String key : List.of("reasoning_content", "reasoningContent", "reasoning", "thinking")) { + Object value = assistant.getMetadata().get(key); + if (value instanceof CharSequence text && !text.toString().isBlank()) { + String content = bounded(text.toString()); + return new ReasoningContent(content, true, + content.getBytes(StandardCharsets.UTF_8).length); + } + } + return ReasoningContent.empty(); + } + + private void persistReasoning(AuditIdentity identity, int stepIndex, ReasoningContent reasoning) { + if (reasoningRepository == null) { + return; + } + try { + reasoningRepository.save(AgentReasoningAudit.builder() + .sessionId(identity.sessionId()) + .runId(identity.runId()) + .stepIndex(stepIndex) + .agentName(agentName) + .reasoningAvailable(reasoning.available()) + .reasoningContent(reasoning.content()) + .contentBytes(reasoning.bytes()) + .build()); + } catch (RuntimeException exception) { + log.warn("Failed to persist reasoning audit: agent={}, step={}", agentName, stepIndex); + } + } + + private static String bounded(String value) { + int maxChars = 32_000; + return value.length() <= maxChars ? value : value.substring(0, maxChars); + } + private AuditIdentity identity(RunnableConfig config) { if (config == null) { return null; @@ -165,6 +233,12 @@ public final class HarnessAgentAuditHook extends MessagesModelHook { private record AuditIdentity(String sessionId, String runId) { } - private record PendingStep(Long id, long startedNanos) { + private record ReasoningContent(String content, boolean available, int bytes) { + private static ReasoningContent empty() { + return new ReasoningContent(null, false, 0); + } + } + + private record PendingStep(Long id, long startedNanos, Map input) { } } diff --git a/src/main/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorder.java b/src/main/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorder.java new file mode 100644 index 0000000..d56bf0e --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorder.java @@ -0,0 +1,67 @@ +package com.superbiz.agent.harness.audit; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.domain.entity.DiagnosisTraceEvent; +import com.superbiz.agent.repository.DiagnosisTraceEventRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Component +public final class JpaDiagnosisTraceRecorder implements DiagnosisTraceRecorder { + + private static final Logger log = LoggerFactory.getLogger(JpaDiagnosisTraceRecorder.class); + + private final DiagnosisTraceEventRepository repository; + private final ObjectMapper objectMapper; + private final ConcurrentHashMap sequences = new ConcurrentHashMap<>(); + + public JpaDiagnosisTraceRecorder(DiagnosisTraceEventRepository repository, ObjectMapper objectMapper) { + this.repository = Objects.requireNonNull(repository, "repository must not be null"); + this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null"); + } + + @Override + public void record(DiagnosisTraceAuditEvent event) { + Objects.requireNonNull(event, "event must not be null"); + try { + int sequence = sequences.computeIfAbsent(event.runId(), this::loadSequence) + .incrementAndGet(); + repository.save(DiagnosisTraceEvent.builder() + .sessionId(event.sessionId()) + .runId(event.runId()) + .sequenceNo(sequence) + .phase(event.phase().name()) + .eventType(event.eventType().name()) + .status(event.status().name()) + .attemptNo(event.attemptNo()) + .durationMs(event.durationMs()) + .details(writeDetails(event)) + .build()); + } catch (RuntimeException exception) { + log.warn("Failed to persist diagnosis trace event: phase={}, type={}", + event.phase(), event.eventType()); + } finally { + if (event.eventType() == TraceEventType.RUN_FINISHED) { + sequences.remove(event.runId()); + } + } + } + + private AtomicInteger loadSequence(String runId) { + return new AtomicInteger(repository.findMaxSequenceNoByRunId(runId)); + } + + private String writeDetails(DiagnosisTraceAuditEvent event) { + try { + return objectMapper.writeValueAsString(event.details()); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("Trace event details are not serializable", exception); + } + } +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSink.java b/src/main/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSink.java index 810a77c..5f3e4eb 100644 --- a/src/main/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSink.java +++ b/src/main/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSink.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.ToolInvocation; import com.superbiz.agent.harness.contract.InvocationStatus; import com.superbiz.agent.repository.ToolInvocationRepository; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.LinkedHashMap; @@ -16,15 +17,24 @@ public final class JpaToolInvocationAuditSink implements ToolInvocationAuditSink private final ToolInvocationRepository repository; private final ObjectMapper objectMapper; + private final DiagnosisTraceRecorder traceRecorder; public JpaToolInvocationAuditSink(ToolInvocationRepository repository, ObjectMapper objectMapper) { + this(repository, objectMapper, DiagnosisTraceRecorder.noop()); + } + + @Autowired + public JpaToolInvocationAuditSink(ToolInvocationRepository repository, ObjectMapper objectMapper, + DiagnosisTraceRecorder traceRecorder) { this.repository = Objects.requireNonNull(repository, "repository must not be null"); this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); } @Override public void record(ToolInvocationAuditEvent event) { Objects.requireNonNull(event, "event must not be null"); + traceRecorder.record(TraceAuditEvents.toolInvocation(event)); repository.save(ToolInvocation.builder() .sessionId(event.sessionId()) .runId(event.runId()) diff --git a/src/main/java/com/superbiz/agent/harness/audit/TraceAuditEvents.java b/src/main/java/com/superbiz/agent/harness/audit/TraceAuditEvents.java new file mode 100644 index 0000000..39ba100 --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/TraceAuditEvents.java @@ -0,0 +1,203 @@ +package com.superbiz.agent.harness.audit; + +import com.superbiz.agent.harness.contract.DiagnosisDraft; +import com.superbiz.agent.harness.contract.FallbackType; +import com.superbiz.agent.harness.contract.IntentType; +import com.superbiz.agent.harness.contract.ReleaseOutcome; +import com.superbiz.agent.harness.contract.SemanticVerdict; +import com.superbiz.agent.harness.core.RunContext; +import com.superbiz.agent.harness.guard.evidence.EvidenceGuardResult; +import com.superbiz.agent.harness.guard.evidence.EvidenceViolation; +import com.superbiz.agent.harness.retry.RetryAttempt; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public final class TraceAuditEvents { + + private static final Pattern SAFE_TOOL_CALL_ID = + Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}"); + + private TraceAuditEvents() { + } + + public static DiagnosisTraceAuditEvent runStarted(RunContext context) { + return event(context, TracePhase.RUN, TraceEventType.RUN_STARTED, + TraceEventStatus.STARTED, null, null, Map.of()); + } + + public static DiagnosisTraceAuditEvent runFinished( + RunContext context, IntentType intent, ReleaseOutcome outcome, int durationMs) { + Map details = new LinkedHashMap<>(); + if (intent != null) { + details.put("intent", intent.name()); + } + details.put("release_outcome", outcome.name()); + return event(context, TracePhase.RUN, TraceEventType.RUN_FINISHED, + terminalStatus(outcome), null, durationMs, details); + } + + public static DiagnosisTraceAuditEvent routingAttempt( + RunContext context, RetryAttempt attempt) { + return retry(context, TracePhase.ROUTING, TraceEventType.ROUTING_ATTEMPT, attempt); + } + + public static DiagnosisTraceAuditEvent routingDecision( + RunContext context, IntentType intent) { + return event(context, TracePhase.ROUTING, TraceEventType.ROUTING_DECISION, + TraceEventStatus.SUCCEEDED, null, null, Map.of("intent", intent.name())); + } + + public static DiagnosisTraceAuditEvent agentModelStep( + String sessionId, String runId, String agentName, int stepIndex, + int durationMs, Map input, Map output) { + Map details = new LinkedHashMap<>(); + details.put("agent_name", agentName); + details.put("step_index", stepIndex); + details.put("input", input); + details.put("output", output); + return new DiagnosisTraceAuditEvent( + sessionId, runId, TracePhase.AGENT, TraceEventType.AGENT_MODEL_STEP, + TraceEventStatus.SUCCEEDED, stepIndex + 1, durationMs, details); + } + + public static DiagnosisTraceAuditEvent toolInvocation(ToolInvocationAuditEvent tool) { + Map details = new LinkedHashMap<>(); + details.put("tool_call_id", tool.toolCallId()); + details.put("tool_name", tool.toolName()); + details.put("invocation_status", tool.status().name()); + details.put("evidence_status", tool.evidenceStatus().name()); + details.put("request_bytes", tool.requestBytes()); + details.put("agent_result_bytes", tool.agentResultBytes()); + if (tool.errorCode() != null) { + details.put("error_code", tool.errorCode()); + } + return new DiagnosisTraceAuditEvent( + tool.sessionId(), tool.runId(), TracePhase.TOOL, TraceEventType.TOOL_INVOCATION, + tool.status() == com.superbiz.agent.harness.contract.InvocationStatus.READY + ? TraceEventStatus.READY : TraceEventStatus.ERROR, + null, tool.durationMs(), details); + } + + public static DiagnosisTraceAuditEvent evidenceValidation( + RunContext context, TraceEventType type, + EvidenceGuardResult result, DiagnosisDraft draft) { + Map details = new LinkedHashMap<>(); + details.put("violation_count", result.violations().size()); + details.put("violations", violations(result.violations())); + ToolReferences references = toolReferences(draft); + details.put("referenced_tool_call_ids", references.safeIds()); + details.put("invalid_tool_reference_count", references.invalidCount()); + result.verifiedSnapshot().ifPresent(snapshot -> { + details.put("verified_analysis_count", snapshot.analyses().size()); + details.put("verified_source_count", snapshot.verifiedSources().size()); + }); + return event(context, TracePhase.EVIDENCE, type, + result.valid() ? TraceEventStatus.PASSED : TraceEventStatus.REJECTED, + null, null, details); + } + + public static DiagnosisTraceAuditEvent evidenceRepairAttempt( + RunContext context, RetryAttempt attempt) { + return retry(context, TracePhase.EVIDENCE, + TraceEventType.EVIDENCE_REPAIR_ATTEMPT, attempt); + } + + public static DiagnosisTraceAuditEvent semanticAttempt( + RunContext context, RetryAttempt attempt) { + return retry(context, TracePhase.SEMANTIC, + TraceEventType.SEMANTIC_GUARD_ATTEMPT, attempt); + } + + public static DiagnosisTraceAuditEvent semanticDecision( + RunContext context, SemanticVerdict verdict) { + TraceEventStatus status = verdict == SemanticVerdict.SUPPORTED + ? TraceEventStatus.SUPPORTED : TraceEventStatus.UNSUPPORTED; + return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION, + status, null, null, Map.of("verdict", verdict.name())); + } + + public static DiagnosisTraceAuditEvent semanticUnavailable(RunContext context) { + return event(context, TracePhase.SEMANTIC, TraceEventType.SEMANTIC_GUARD_DECISION, + TraceEventStatus.UNAVAILABLE, null, null, Map.of()); + } + + public static DiagnosisTraceAuditEvent releaseDecision( + RunContext context, ReleaseOutcome outcome, FallbackType fallbackType) { + Map details = new LinkedHashMap<>(); + details.put("release_outcome", outcome.name()); + if (fallbackType != null) { + details.put("fallback_type", fallbackType.name()); + } + return event(context, TracePhase.RELEASE, TraceEventType.RELEASE_DECISION, + terminalStatus(outcome), null, null, details); + } + + private static DiagnosisTraceAuditEvent retry( + RunContext context, TracePhase phase, TraceEventType type, RetryAttempt attempt) { + Map details = new LinkedHashMap<>(); + if (attempt.failure() != null) { + details.put("failure", attempt.failure().name()); + } + return event(context, phase, type, + attempt.success() ? TraceEventStatus.SUCCEEDED : TraceEventStatus.FAILED, + attempt.attemptNumber(), null, details); + } + + private static DiagnosisTraceAuditEvent event( + RunContext context, TracePhase phase, TraceEventType type, TraceEventStatus status, + Integer attemptNo, Integer durationMs, Map details) { + return new DiagnosisTraceAuditEvent( + context.sessionId(), context.runId(), phase, type, status, + attemptNo, durationMs, details); + } + + private static TraceEventStatus terminalStatus(ReleaseOutcome outcome) { + return switch (outcome) { + case SUCCESS -> TraceEventStatus.SUCCEEDED; + case FALLBACK -> TraceEventStatus.FALLBACK; + case FAILED -> TraceEventStatus.FAILED; + case CANCELLED -> TraceEventStatus.CANCELLED; + }; + } + + private static List> violations(List violations) { + List> result = new ArrayList<>(); + for (EvidenceViolation violation : violations) { + Map item = new LinkedHashMap<>(); + item.put("code", violation.code().name()); + item.put("target", violation.target()); + result.add(Map.copyOf(item)); + } + return List.copyOf(result); + } + + private static ToolReferences toolReferences(DiagnosisDraft draft) { + if (draft == null) { + return new ToolReferences(List.of(), 0); + } + Set safeIds = new LinkedHashSet<>(); + int invalid = 0; + for (DiagnosisDraft.AnalysisItem item : draft.analysis()) { + if (item == null) { + continue; + } + for (String id : item.toolCallIds()) { + if (id != null && SAFE_TOOL_CALL_ID.matcher(id).matches()) { + safeIds.add(id); + } else { + invalid++; + } + } + } + return new ToolReferences(List.copyOf(safeIds), invalid); + } + + private record ToolReferences(List safeIds, int invalidCount) { + } +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/TraceEventStatus.java b/src/main/java/com/superbiz/agent/harness/audit/TraceEventStatus.java new file mode 100644 index 0000000..f26ff5d --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/TraceEventStatus.java @@ -0,0 +1,16 @@ +package com.superbiz.agent.harness.audit; + +public enum TraceEventStatus { + STARTED, + SUCCEEDED, + FAILED, + READY, + ERROR, + PASSED, + REJECTED, + SUPPORTED, + UNSUPPORTED, + UNAVAILABLE, + FALLBACK, + CANCELLED +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/TraceEventType.java b/src/main/java/com/superbiz/agent/harness/audit/TraceEventType.java new file mode 100644 index 0000000..129e393 --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/TraceEventType.java @@ -0,0 +1,16 @@ +package com.superbiz.agent.harness.audit; + +public enum TraceEventType { + RUN_STARTED, + ROUTING_ATTEMPT, + ROUTING_DECISION, + AGENT_MODEL_STEP, + TOOL_INVOCATION, + EVIDENCE_GUARD_INITIAL, + EVIDENCE_REPAIR_ATTEMPT, + EVIDENCE_GUARD_RECHECK, + SEMANTIC_GUARD_ATTEMPT, + SEMANTIC_GUARD_DECISION, + RELEASE_DECISION, + RUN_FINISHED +} diff --git a/src/main/java/com/superbiz/agent/harness/audit/TracePhase.java b/src/main/java/com/superbiz/agent/harness/audit/TracePhase.java new file mode 100644 index 0000000..a3f9402 --- /dev/null +++ b/src/main/java/com/superbiz/agent/harness/audit/TracePhase.java @@ -0,0 +1,11 @@ +package com.superbiz.agent.harness.audit; + +public enum TracePhase { + RUN, + ROUTING, + AGENT, + TOOL, + EVIDENCE, + SEMANTIC, + RELEASE +} diff --git a/src/main/java/com/superbiz/agent/harness/contract/SafeFallback.java b/src/main/java/com/superbiz/agent/harness/contract/SafeFallback.java index bd07b2a..47be2f8 100644 --- a/src/main/java/com/superbiz/agent/harness/contract/SafeFallback.java +++ b/src/main/java/com/superbiz/agent/harness/contract/SafeFallback.java @@ -10,12 +10,27 @@ public record SafeFallback( @JsonProperty("message") String message, @JsonProperty("verified_sources") List verifiedSources, @JsonProperty("limitations") List limitations, - @JsonProperty("next_steps") List nextSteps) { + @JsonProperty("next_steps") List nextSteps, + @JsonProperty("failure_stage") String failureStage, + @JsonProperty("observed_facts") List observedFacts, + @JsonProperty("validation_issues") List validationIssues) { public SafeFallback { verifiedSources = ContractCollections.immutable(verifiedSources); limitations = ContractCollections.immutable(limitations); nextSteps = ContractCollections.immutable(nextSteps); + observedFacts = ContractCollections.immutable(observedFacts); + validationIssues = ContractCollections.immutable(validationIssues); + } + + public SafeFallback(FallbackType type, + String conclusion, + String message, + List verifiedSources, + List limitations, + List nextSteps) { + this(type, conclusion, message, verifiedSources, limitations, nextSteps, + null, List.of(), List.of()); } public record VerifiedSource( @@ -23,4 +38,16 @@ public record SafeFallback( @JsonProperty("source") String source, @JsonProperty("scope") String scope) { } + + public record ObservedFact( + @JsonProperty("source_type") String sourceType, + @JsonProperty("source") String source, + @JsonProperty("scope") String scope, + @JsonProperty("summary") String summary) { + } + + public record ValidationIssue( + @JsonProperty("code") String code, + @JsonProperty("target") String target) { + } } diff --git a/src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticGuard.java b/src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticGuard.java index 707424d..f684004 100644 --- a/src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticGuard.java +++ b/src/main/java/com/superbiz/agent/harness/guard/semantic/SemanticGuard.java @@ -4,6 +4,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.harness.contract.SemanticVerdict; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceAuditEvents; import com.superbiz.agent.harness.core.DiagnosisHarnessCore; import com.superbiz.agent.harness.core.RunContext; import com.superbiz.agent.harness.retry.HarnessRetryExecutor; @@ -31,6 +33,7 @@ public final class SemanticGuard { private final ObjectMapper objectMapper; private final SemanticGuardLimits limits; private final Consumer attemptRecorder; + private final DiagnosisTraceRecorder traceRecorder; private final String prompt; public SemanticGuard(DiagnosisHarnessCore core, @@ -39,6 +42,17 @@ public final class SemanticGuard { ObjectMapper objectMapper, SemanticGuardLimits limits, Consumer attemptRecorder) { + this(core, retryExecutor, modelCall, objectMapper, limits, + attemptRecorder, DiagnosisTraceRecorder.noop()); + } + + public SemanticGuard(DiagnosisHarnessCore core, + HarnessRetryExecutor retryExecutor, + GuardModelCall modelCall, + ObjectMapper objectMapper, + SemanticGuardLimits limits, + Consumer attemptRecorder, + DiagnosisTraceRecorder traceRecorder) { this.core = Objects.requireNonNull(core, "core must not be null"); this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null"); this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null"); @@ -46,6 +60,7 @@ public final class SemanticGuard { this.limits = Objects.requireNonNull(limits, "limits must not be null"); this.attemptRecorder = Objects.requireNonNull( attemptRecorder, "attemptRecorder must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); this.prompt = SemanticGuardPrompt.load(); } @@ -68,7 +83,10 @@ public final class SemanticGuard { () -> parse(modelCall.call( context, modelPrompt, remainingTimeout(startedNanos), limits.maxOutputBytes())), this::classify, - attemptRecorder); + attempt -> { + attemptRecorder.accept(attempt); + traceRecorder.record(TraceAuditEvents.semanticAttempt(context, attempt)); + }); } private SemanticGuardDecision parse(String output) { diff --git a/src/main/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCase.java b/src/main/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCase.java index 28d7b3d..d7eaa84 100644 --- a/src/main/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCase.java +++ b/src/main/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCase.java @@ -1,6 +1,10 @@ package com.superbiz.agent.harness.release; import com.superbiz.agent.harness.contract.DiagnosisDraft; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceAuditEvents; +import com.superbiz.agent.harness.audit.TraceEventType; +import com.superbiz.agent.harness.contract.FallbackType; import com.superbiz.agent.harness.contract.SemanticVerdict; import com.superbiz.agent.harness.core.BudgetExceededException; import com.superbiz.agent.harness.core.RunAbortedException; @@ -22,16 +26,27 @@ public final class DiagnosisReleaseUseCase { private final EvidenceRepair evidenceRepair; private final SemanticGuard semanticGuard; private final SafeFallbackFactory fallbackFactory; + private final DiagnosisTraceRecorder traceRecorder; public DiagnosisReleaseUseCase(EvidenceGuard evidenceGuard, EvidenceRepair evidenceRepair, SemanticGuard semanticGuard, SafeFallbackFactory fallbackFactory) { + this(evidenceGuard, evidenceRepair, semanticGuard, fallbackFactory, + DiagnosisTraceRecorder.noop()); + } + + public DiagnosisReleaseUseCase(EvidenceGuard evidenceGuard, + EvidenceRepair evidenceRepair, + SemanticGuard semanticGuard, + SafeFallbackFactory fallbackFactory, + DiagnosisTraceRecorder traceRecorder) { this.evidenceGuard = Objects.requireNonNull(evidenceGuard, "evidenceGuard must not be null"); this.evidenceRepair = Objects.requireNonNull(evidenceRepair, "evidenceRepair must not be null"); this.semanticGuard = Objects.requireNonNull(semanticGuard, "semanticGuard must not be null"); this.fallbackFactory = Objects.requireNonNull( fallbackFactory, "fallbackFactory must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); } public DiagnosisReleaseResult execute(RunContext context, String query, DiagnosisDraft draft) { @@ -43,16 +58,20 @@ public final class DiagnosisReleaseUseCase { DiagnosisDraft candidate = draft; EvidenceGuardResult evidence = evidenceGuard.validate(context, candidate); + traceRecorder.record(TraceAuditEvents.evidenceValidation( + context, TraceEventType.EVIDENCE_GUARD_INITIAL, evidence, candidate)); if (!evidence.valid()) { try { candidate = evidenceRepair.repair(context, query, draft, evidence.violations()); evidence = evidenceGuard.validate(context, candidate); + traceRecorder.record(TraceAuditEvents.evidenceValidation( + context, TraceEventType.EVIDENCE_GUARD_RECHECK, evidence, candidate)); } catch (RuntimeException exception) { propagateTerminal(exception); - return evidenceFailure(); + return evidenceFailure(context, evidence); } if (!evidence.valid()) { - return evidenceFailure(); + return evidenceFailure(context, evidence); } } @@ -63,18 +82,32 @@ public final class DiagnosisReleaseUseCase { context, SemanticGuardInput.from(query, candidate, snapshot)); } catch (RuntimeException exception) { propagateTerminal(exception); + traceRecorder.record(TraceAuditEvents.semanticUnavailable(context)); + traceRecorder.record(TraceAuditEvents.releaseDecision( + context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, + FallbackType.SEMANTIC_UNAVAILABLE)); return DiagnosisReleaseResult.fallback( fallbackFactory.semanticUnavailable(snapshot)); } - return decision.verdict() == SemanticVerdict.SUPPORTED - ? DiagnosisReleaseResult.success(candidate, snapshot) - : DiagnosisReleaseResult.fallback( - fallbackFactory.semanticUnsupported(snapshot)); + traceRecorder.record(TraceAuditEvents.semanticDecision(context, decision.verdict())); + if (decision.verdict() == SemanticVerdict.SUPPORTED) { + traceRecorder.record(TraceAuditEvents.releaseDecision( + context, com.superbiz.agent.harness.contract.ReleaseOutcome.SUCCESS, null)); + return DiagnosisReleaseResult.success(candidate, snapshot); + } + traceRecorder.record(TraceAuditEvents.releaseDecision( + context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, + FallbackType.SEMANTIC_UNSUPPORTED)); + return DiagnosisReleaseResult.fallback(fallbackFactory.semanticUnsupported(snapshot)); } - private DiagnosisReleaseResult evidenceFailure() { + private DiagnosisReleaseResult evidenceFailure( + RunContext context, EvidenceGuardResult evidence) { + traceRecorder.record(TraceAuditEvents.releaseDecision( + context, com.superbiz.agent.harness.contract.ReleaseOutcome.FALLBACK, + FallbackType.EVIDENCE_VALIDATION_FAILED)); return DiagnosisReleaseResult.fallback( - fallbackFactory.evidenceValidationFailed()); + fallbackFactory.evidenceValidationFailed(evidence.violations())); } private void propagateTerminal(RuntimeException exception) { diff --git a/src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java b/src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java index 8df5855..396ee33 100644 --- a/src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java +++ b/src/main/java/com/superbiz/agent/harness/release/EvidenceRepair.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.superbiz.agent.harness.contract.DiagnosisDraft; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceAuditEvents; import com.superbiz.agent.harness.core.DiagnosisHarnessCore; import com.superbiz.agent.harness.core.RunContext; import com.superbiz.agent.harness.guard.evidence.EvidenceViolation; @@ -32,6 +34,7 @@ public final class EvidenceRepair { private final ObjectReader draftReader; private final EvidenceRepairLimits limits; private final Consumer attemptRecorder; + private final DiagnosisTraceRecorder traceRecorder; private final String prompt; public EvidenceRepair(DiagnosisHarnessCore core, @@ -40,6 +43,17 @@ public final class EvidenceRepair { ObjectMapper objectMapper, EvidenceRepairLimits limits, Consumer attemptRecorder) { + this(core, retryExecutor, modelCall, objectMapper, limits, + attemptRecorder, DiagnosisTraceRecorder.noop()); + } + + public EvidenceRepair(DiagnosisHarnessCore core, + HarnessRetryExecutor retryExecutor, + GuardModelCall modelCall, + ObjectMapper objectMapper, + EvidenceRepairLimits limits, + Consumer attemptRecorder, + DiagnosisTraceRecorder traceRecorder) { this.core = Objects.requireNonNull(core, "core must not be null"); this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null"); this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null"); @@ -50,6 +64,7 @@ public final class EvidenceRepair { this.limits = Objects.requireNonNull(limits, "limits must not be null"); this.attemptRecorder = Objects.requireNonNull( attemptRecorder, "attemptRecorder must not be null"); + this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null"); this.prompt = EvidenceRepairPrompt.load(); } @@ -87,7 +102,10 @@ public final class EvidenceRepair { return repaired; }, this::classify, - attemptRecorder); + attempt -> { + attemptRecorder.accept(attempt); + traceRecorder.record(TraceAuditEvents.evidenceRepairAttempt(context, attempt)); + }); } private DiagnosisDraft parse(String output) { diff --git a/src/main/java/com/superbiz/agent/harness/release/SafeFallbackFactory.java b/src/main/java/com/superbiz/agent/harness/release/SafeFallbackFactory.java index ed37cf9..604951a 100644 --- a/src/main/java/com/superbiz/agent/harness/release/SafeFallbackFactory.java +++ b/src/main/java/com/superbiz/agent/harness/release/SafeFallbackFactory.java @@ -2,50 +2,109 @@ package com.superbiz.agent.harness.release; import com.superbiz.agent.harness.contract.FallbackType; import com.superbiz.agent.harness.contract.SafeFallback; +import com.superbiz.agent.harness.guard.evidence.EvidenceViolation; +import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence; import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; public final class SafeFallbackFactory { - public SafeFallback evidenceValidationFailed() { + private static final int MAX_OBSERVED_FACTS = 12; + private static final int MAX_SUMMARY_CHARS = 320; + + public SafeFallback evidenceValidationFailed(List violations) { return fallback( FallbackType.EVIDENCE_VALIDATION_FAILED, List.of(), - "当前证据无法完成真实性校验，无法确认根因", - "证据引用校验未通过", - "重新收集当前诊断范围内的证据后再发起诊断"); + "已完成证据引用检查，但当前报告无法通过真实性校验，因此不能确认根因", + List.of("证据引用或报告结构校验未通过"), + List.of("根据 validation_issues 修正报告结构，或补充对应范围的证据后重试"), + "EVIDENCE_VALIDATION", + List.of(), + issues(violations)); } public SafeFallback semanticUnsupported(VerifiedEvidenceSnapshot snapshot) { return fallback( FallbackType.SEMANTIC_UNSUPPORTED, sources(snapshot), - "当前证据不足，无法确认根因", - "语义校验未通过", - "补充当前缺失的数据后重新发起诊断"); + "已收集到可验证事实，但这些事实不足以支持当前根因结论", + List.of("语义校验未通过，已确认事实仍可用于后续排查"), + List.of("围绕 observed_facts 补充缺失的实时日志、指标或数据库证据后重试"), + "SEMANTIC_VALIDATION", + facts(snapshot), + List.of()); } public SafeFallback semanticUnavailable(VerifiedEvidenceSnapshot snapshot) { return fallback( FallbackType.SEMANTIC_UNAVAILABLE, sources(snapshot), - "当前证据暂时无法完成语义校验，无法确认根因", - "语义校验暂不可用", - "稍后重试或补充当前缺失的数据"); + "已收集到可验证事实，但当前无法完成语义校验，因此暂不发布根因结论", + List.of("语义校验暂不可用"), + List.of("稍后重试；已确认事实可继续用于人工排查"), + "SEMANTIC_VALIDATION", + facts(snapshot), + List.of()); } private SafeFallback fallback(FallbackType type, List sources, String message, - String limitation, - String nextStep) { + List limitations, + List nextSteps, + String failureStage, + List observedFacts, + List validationIssues) { return new SafeFallback( - type, null, message, sources, List.of(limitation), List.of(nextStep)); + type, null, message, sources, limitations, nextSteps, + failureStage, observedFacts, validationIssues); } private List sources(VerifiedEvidenceSnapshot snapshot) { return Objects.requireNonNull(snapshot, "snapshot must not be null").verifiedSources(); } + + private List facts(VerifiedEvidenceSnapshot snapshot) { + Objects.requireNonNull(snapshot, "snapshot must not be null"); + Map unique = new LinkedHashMap<>(); + for (var analysis : snapshot.analyses()) { + for (VerifiedEvidence evidence : analysis.evidence()) { + if (unique.size() >= MAX_OBSERVED_FACTS) { + return List.copyOf(unique.values()); + } + String summary = bounded(evidence.excerpt()); + if (summary == null) { + summary = "已验证结构化证据，具体值保留在受限审计记录中"; + } + String key = evidence.sourceType() + '\u0000' + evidence.source() + + '\u0000' + evidence.scope() + '\u0000' + summary; + unique.putIfAbsent(key, new SafeFallback.ObservedFact( + evidence.sourceType(), evidence.source(), evidence.scope(), summary)); + } + } + return List.copyOf(unique.values()); + } + + private List issues(List violations) { + List result = new ArrayList<>(); + for (EvidenceViolation violation : violations == null ? List.of() : violations) { + result.add(new SafeFallback.ValidationIssue( + violation.code().name(), violation.target())); + } + return List.copyOf(result); + } + + private String bounded(String value) { + if (value == null || value.isBlank()) { + return null; + } + return value.length() <= MAX_SUMMARY_CHARS + ? value : value.substring(0, MAX_SUMMARY_CHARS); + } } diff --git a/src/main/java/com/superbiz/agent/repository/AgentReasoningAuditRepository.java b/src/main/java/com/superbiz/agent/repository/AgentReasoningAuditRepository.java new file mode 100644 index 0000000..527fcb4 --- /dev/null +++ b/src/main/java/com/superbiz/agent/repository/AgentReasoningAuditRepository.java @@ -0,0 +1,13 @@ +package com.superbiz.agent.repository; + +import com.superbiz.agent.domain.entity.AgentReasoningAudit; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface AgentReasoningAuditRepository extends JpaRepository { + + List findByRunIdOrderByStepIndex(String runId); +} diff --git a/src/main/java/com/superbiz/agent/repository/DiagnosisTraceEventRepository.java b/src/main/java/com/superbiz/agent/repository/DiagnosisTraceEventRepository.java new file mode 100644 index 0000000..5c27a60 --- /dev/null +++ b/src/main/java/com/superbiz/agent/repository/DiagnosisTraceEventRepository.java @@ -0,0 +1,18 @@ +package com.superbiz.agent.repository; + +import com.superbiz.agent.domain.entity.DiagnosisTraceEvent; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface DiagnosisTraceEventRepository extends JpaRepository { + + List findByRunIdOrderBySequenceNoAscIdAsc(String runId); + + @Query("select coalesce(max(event.sequenceNo), 0) from DiagnosisTraceEvent event where event.runId = :runId") + int findMaxSequenceNoByRunId(@Param("runId") String runId); +} diff --git a/src/main/java/com/superbiz/agent/service/DiagnosisTraceService.java b/src/main/java/com/superbiz/agent/service/DiagnosisTraceService.java index 65265ac..104e73b 100644 --- a/src/main/java/com/superbiz/agent/service/DiagnosisTraceService.java +++ b/src/main/java/com/superbiz/agent/service/DiagnosisTraceService.java @@ -3,16 +3,20 @@ package com.superbiz.agent.service; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.AgentStep; +import com.superbiz.agent.domain.entity.AgentReasoningAudit; import com.superbiz.agent.domain.entity.ChatSession; import com.superbiz.agent.domain.entity.DiagnosisRun; import com.superbiz.agent.domain.entity.DiagnosisSession; +import com.superbiz.agent.domain.entity.DiagnosisTraceEvent; import com.superbiz.agent.domain.entity.ToolInvocation; import com.superbiz.agent.dto.DiagnosisTraceResponse; import com.superbiz.agent.exception.SessionNotFoundException; import com.superbiz.agent.repository.AgentStepRepository; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import com.superbiz.agent.repository.ChatSessionRepository; import com.superbiz.agent.repository.DiagnosisRunRepository; import com.superbiz.agent.repository.DiagnosisSessionRepository; +import com.superbiz.agent.repository.DiagnosisTraceEventRepository; import com.superbiz.agent.repository.ToolInvocationRepository; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -32,7 +36,9 @@ public class DiagnosisTraceService { private final ChatSessionRepository chatSessionRepository; private final DiagnosisRunRepository diagnosisRunRepository; private final AgentStepRepository agentStepRepository; + private final AgentReasoningAuditRepository agentReasoningAuditRepository; private final ToolInvocationRepository toolInvocationRepository; + private final DiagnosisTraceEventRepository diagnosisTraceEventRepository; private final ObjectMapper objectMapper; public DiagnosisTraceResponse getTrace(String sessionId) { @@ -71,6 +77,8 @@ public class DiagnosisTraceService { private DiagnosisTraceResponse buildRunTraceResponse(String sessionId, DiagnosisRun run) { List steps = orderStepsForTrace(agentStepRepository.findByRunIdOrderByStepIndex(run.getRunId())); List toolInvocations = toolInvocationRepository.findByRunIdOrderByIdAsc(run.getRunId()); + List timeline = + diagnosisTraceEventRepository.findByRunIdOrderBySequenceNoAscIdAsc(run.getRunId()); DiagnosisTraceResponse.RunTrace runTrace = toRunTrace(run); return DiagnosisTraceResponse.builder() @@ -82,7 +90,8 @@ public class DiagnosisTraceService { .run(runTrace) .steps(steps.stream().map(this::toAgentStepTrace).toList()) .toolInvocations(toolInvocations.stream().map(this::toToolInvocationTrace).toList()) - .summary(toSummary(runTrace, steps, toolInvocations)) + .timeline(timeline.stream().map(this::toTraceEventTrace).toList()) + .summary(toSummary(runTrace, steps, toolInvocations, timeline)) .build(); } @@ -101,6 +110,7 @@ public class DiagnosisTraceService { .run(null) .steps(steps.stream().map(this::toAgentStepTrace).toList()) .toolInvocations(toolInvocations.stream().map(this::toToolInvocationTrace).toList()) + .timeline(List.of()) .summary(toSummary(session, steps, toolInvocations)) .build(); } @@ -153,6 +163,8 @@ public class DiagnosisTraceService { .query(run.getQuery()) .status(run.getStatus()) .agentFlow(run.getAgentFlow()) + .intent(run.getIntent()) + .releaseOutcome(run.getReleaseOutcome()) .totalDurationMs(run.getTotalDurationMs()) .totalTokenCount(run.getTotalTokenCount()) .stepCount(run.getStepCount()) @@ -174,6 +186,8 @@ public class DiagnosisTraceService { .query(run.getQuery()) .status(run.getStatus()) .agentFlow(run.getAgentFlow()) + .intent(run.getIntent() == null ? null : run.getIntent().name()) + .releaseOutcome(run.getReleaseOutcome() == null ? null : run.getReleaseOutcome().name()) .totalDurationMs(run.getTotalDurationMs()) .totalTokenCount(run.getTotalTokenCount()) .stepCount(run.getStepCount()) @@ -187,6 +201,49 @@ public class DiagnosisTraceService { .build(); } + public List getReasoning( + String sessionId, String runId) { + if (runId == null || runId.isBlank()) { + throw new IllegalArgumentException("runId is required for reasoning audit"); + } + diagnosisRunRepository.findBySessionIdAndRunId(sessionId, runId) + .orElseThrow(() -> buildRunLookupException(sessionId, runId)); + return agentReasoningAuditRepository.findByRunIdOrderByStepIndex(runId).stream() + .map(this::toReasoningTrace) + .toList(); + } + + private DiagnosisTraceResponse.TraceEventTrace toTraceEventTrace(DiagnosisTraceEvent event) { + return DiagnosisTraceResponse.TraceEventTrace.builder() + .id(event.getId()) + .sessionId(event.getSessionId()) + .runId(event.getRunId()) + .sequenceNo(event.getSequenceNo()) + .phase(event.getPhase()) + .eventType(event.getEventType()) + .status(event.getStatus()) + .attemptNo(event.getAttemptNo()) + .durationMs(event.getDurationMs()) + .detailsRaw(event.getDetails()) + .details(parseJsonObject(event.getDetails())) + .createdAt(event.getCreatedAt()) + .build(); + } + + private DiagnosisTraceResponse.AgentReasoningTrace toReasoningTrace(AgentReasoningAudit audit) { + return DiagnosisTraceResponse.AgentReasoningTrace.builder() + .id(audit.getId()) + .sessionId(audit.getSessionId()) + .runId(audit.getRunId()) + .stepIndex(audit.getStepIndex()) + .agentName(audit.getAgentName()) + .reasoningAvailable(audit.getReasoningAvailable()) + .reasoningContent(audit.getReasoningContent()) + .contentBytes(audit.getContentBytes()) + .createdAt(audit.getCreatedAt()) + .build(); + } + private DiagnosisTraceResponse.AgentStepTrace toAgentStepTrace(AgentStep step) { return DiagnosisTraceResponse.AgentStepTrace.builder() .id(step.getId()) @@ -237,6 +294,8 @@ public class DiagnosisTraceService { .query(run.getQuery()) .status(run.getStatus()) .agentFlow(run.getAgentFlow()) + .intent(run.getIntent() == null ? null : run.getIntent().name()) + .releaseOutcome(run.getReleaseOutcome() == null ? null : run.getReleaseOutcome().name()) .answerPreview(preview(run.getAnswer(), 160)) .stepCount(run.getStepCount()) .toolCallCount(run.getToolCallCount()) @@ -257,6 +316,8 @@ public class DiagnosisTraceService { .returnedStepCount(steps.size()) .persistedToolCallCount(defaultInt(session.getToolCallCount())) .returnedToolCallCount(toolInvocations.size()) + .persistedTraceEventCount(0) + .returnedTraceEventCount(0) .hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation")) .hasFeedback(session.getFeedback() != null && !session.getFeedback().isBlank()) .build(); @@ -265,7 +326,8 @@ public class DiagnosisTraceService { private DiagnosisTraceResponse.TraceSummary toSummary( DiagnosisTraceResponse.RunTrace run, List steps, - List toolInvocations + List toolInvocations, + List timeline ) { Map selfEvaluation = run.getSelfEvaluation(); return DiagnosisTraceResponse.TraceSummary.builder() @@ -274,6 +336,8 @@ public class DiagnosisTraceService { .returnedStepCount(steps.size()) .persistedToolCallCount(defaultInt(run.getToolCallCount())) .returnedToolCallCount(toolInvocations.size()) + .persistedTraceEventCount(timeline.size()) + .returnedTraceEventCount(timeline.size()) .hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation")) .hasFeedback(run.getFeedback() != null && !run.getFeedback().isBlank()) .build(); diff --git a/src/main/resources/db/migration/V014__create_diagnosis_trace_event.sql b/src/main/resources/db/migration/V014__create_diagnosis_trace_event.sql new file mode 100644 index 0000000..efeab11 --- /dev/null +++ b/src/main/resources/db/migration/V014__create_diagnosis_trace_event.sql @@ -0,0 +1,17 @@ +CREATE TABLE diagnosis_trace_event ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + session_id VARCHAR(64) NOT NULL COMMENT 'Owning Chat Session identity', + run_id VARCHAR(64) NOT NULL COMMENT 'Owning Diagnosis Run identity', + sequence_no INT NOT NULL COMMENT 'Monotonic event order within one run', + phase VARCHAR(32) NOT NULL COMMENT 'RUN/ROUTING/AGENT/TOOL/EVIDENCE/SEMANTIC/RELEASE', + event_type VARCHAR(48) NOT NULL COMMENT 'Stable trace event type', + status VARCHAR(32) NOT NULL COMMENT 'Stable event outcome', + attempt_no INT NULL COMMENT 'Retry or model attempt number when applicable', + duration_ms INT NULL COMMENT 'Event duration in milliseconds when available', + details JSON NOT NULL COMMENT 'Bounded safe metadata; no prompt, thought, draft text, or raw tool payload', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_trace_event_run_sequence (run_id, sequence_no, id), + INDEX idx_trace_event_session_created (session_id, created_at, id), + INDEX idx_trace_event_phase (phase, event_type) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Append-only unified diagnosis trace timeline'; diff --git a/src/main/resources/db/migration/V015__create_agent_reasoning_audit.sql b/src/main/resources/db/migration/V015__create_agent_reasoning_audit.sql new file mode 100644 index 0000000..ddfa408 --- /dev/null +++ b/src/main/resources/db/migration/V015__create_agent_reasoning_audit.sql @@ -0,0 +1,14 @@ +CREATE TABLE agent_reasoning_audit ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + session_id VARCHAR(64) NOT NULL COMMENT 'Chat session identity', + run_id VARCHAR(64) NOT NULL COMMENT 'Diagnosis run identity', + step_index INT NOT NULL COMMENT 'Agent model step index', + agent_name VARCHAR(64) NOT NULL COMMENT 'Agent identity', + reasoning_available BOOLEAN NOT NULL COMMENT 'Whether the provider returned reasoning content', + reasoning_content LONGTEXT NULL COMMENT 'Provider reasoning content, bounded by the audit hook', + content_bytes INT NOT NULL COMMENT 'UTF-8 byte length of reasoning content', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + + INDEX idx_reasoning_run_step (run_id, step_index), + INDEX idx_reasoning_session_created (session_id, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='Restricted agent reasoning audit records'; diff --git a/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java b/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java index d6db2cc..3560ab2 100644 --- a/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java +++ b/src/test/java/com/superbiz/agent/config/HarnessChatConfigurationTest.java @@ -2,6 +2,7 @@ package com.superbiz.agent.config; import com.superbiz.agent.harness.core.DiagnosisHarnessCore; import com.superbiz.agent.harness.audit.ToolInvocationAuditSink; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; import com.superbiz.agent.harness.application.persistence.ChatRunStore; import com.superbiz.agent.harness.tool.contract.MysqlToolRequest; import com.superbiz.agent.harness.tool.mysql.MysqlSqlValidator; @@ -13,6 +14,7 @@ import com.superbiz.agent.service.KnowledgeEvidencePostProcessor; import com.superbiz.agent.service.KnowledgeQueryTransformer; import com.superbiz.agent.service.LookupResultAssembler; import com.superbiz.agent.repository.AgentStepRepository; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import org.springframework.ai.chat.model.ChatModel; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.data.redis.core.RedisTemplate; @@ -75,7 +77,10 @@ class HarnessChatConfigurationTest { .withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class)) .withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class)) .withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop) + .withBean(DiagnosisTraceRecorder.class, DiagnosisTraceRecorder::noop) .withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class)) + .withBean(AgentReasoningAuditRepository.class, + () -> mock(AgentReasoningAuditRepository.class)) .withBean(ChatRunStore.class, () -> mock(ChatRunStore.class)) .withBean(MysqlToolProperties.class, MysqlToolProperties::new) .run(context -> { diff --git a/src/test/java/com/superbiz/agent/harness/application/ApplicationExecutorsTest.java b/src/test/java/com/superbiz/agent/harness/application/ApplicationExecutorsTest.java index c2aa2b5..2d05db4 100644 --- a/src/test/java/com/superbiz/agent/harness/application/ApplicationExecutorsTest.java +++ b/src/test/java/com/superbiz/agent/harness/application/ApplicationExecutorsTest.java @@ -7,6 +7,9 @@ import com.superbiz.agent.harness.application.executor.KnowledgeQueryLimits; import com.superbiz.agent.harness.application.executor.SingleTurnExecutorLimits; import com.superbiz.agent.harness.application.executor.SystemChatExecutor; import com.superbiz.agent.harness.application.routing.IntentRouter; +import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceEventType; import com.superbiz.agent.harness.application.routing.IntentRouterInput; import com.superbiz.agent.harness.application.routing.IntentRouterLimits; import com.superbiz.agent.harness.application.routing.IntentRoutingException; @@ -61,7 +64,8 @@ class ApplicationExecutorsTest { ScriptedModel model = new ScriptedModel( "{\"intent\":\"OTHER\"}", "{\"intent\":\"DIAGNOSIS\"}"); List attempts = new ArrayList<>(); - IntentRouter router = router(core, model, attempts::add); + List trace = new ArrayList<>(); + IntentRouter router = router(core, model, attempts::add, trace::add); IntentType intent = router.route(context, new IntentRouterInput("那退款服务呢？", IntentType.DIAGNOSIS, "订单支付为什么超时？")); @@ -72,6 +76,9 @@ class ApplicationExecutorsTest { assertTrue(model.prompts.get(0).contains("那退款服务呢？")); assertFalse(model.prompts.get(0).contains("tool_call_id")); assertEquals(2, attempts.size()); + assertEquals(List.of(TraceEventType.ROUTING_ATTEMPT, TraceEventType.ROUTING_ATTEMPT), + trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList()); + assertEquals("INVALID_OUTPUT", trace.get(0).details().get("failure")); } @Test @@ -118,6 +125,10 @@ class ApplicationExecutorsTest { assertEquals(1, lookupCalls.get()); assertEquals(1, model.calls.get()); + assertTrue(model.prompts.get(0).contains("answer_items")); + assertTrue(model.prompts.get(0).contains("tool_call_id")); + assertTrue(model.prompts.get(0).contains("document_ids")); + assertTrue(model.prompts.get(0).contains("limitations")); assertEquals("doc-1", content.references().get(0).documentId()); assertFalse(objectMapper.writeValueAsString(content).contains("knowledge-1")); assertFalse(objectMapper.writeValueAsString(content).contains("tool_call_id")); @@ -144,12 +155,18 @@ class ApplicationExecutorsTest { } private IntentRouter router(DiagnosisHarnessCore core, ChatModel model, - java.util.function.Consumer recorder) { + java.util.function.Consumer recorder) { + return router(core, model, recorder, DiagnosisTraceRecorder.noop()); + } + + private IntentRouter router(DiagnosisHarnessCore core, ChatModel model, + java.util.function.Consumer recorder, + DiagnosisTraceRecorder traceRecorder) { return new IntentRouter( core, new HarnessRetryExecutor(core), new GuardModelCall(core, model, executor), objectMapper, new IntentRouterLimits(20_000, 2_000, - Duration.ofSeconds(2), Duration.ofSeconds(3)), recorder); + Duration.ofSeconds(2), Duration.ofSeconds(3)), recorder, traceRecorder); } private KnowledgeQueryExecutor knowledge(DiagnosisHarnessCore core, diff --git a/src/test/java/com/superbiz/agent/harness/application/ChatApplicationUseCaseTest.java b/src/test/java/com/superbiz/agent/harness/application/ChatApplicationUseCaseTest.java index 84af087..2071dd5 100644 --- a/src/test/java/com/superbiz/agent/harness/application/ChatApplicationUseCaseTest.java +++ b/src/test/java/com/superbiz/agent/harness/application/ChatApplicationUseCaseTest.java @@ -3,6 +3,9 @@ package com.superbiz.agent.harness.application; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.harness.application.persistence.ChatRunStore; import com.superbiz.agent.harness.application.persistence.RoutingHistory; +import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceEventType; import com.superbiz.agent.harness.application.routing.IntentRoutingException; import com.superbiz.agent.harness.contract.AnalysisKind; import com.superbiz.agent.harness.contract.DiagnosisDraft; @@ -72,6 +75,29 @@ class ChatApplicationUseCaseTest { } } + @Test + void unifiedTraceRecordsRunRoutingAndTerminalEvents() { + FakeStore store = new FakeStore(); + List trace = new ArrayList<>(); + ChatApplicationUseCase useCase = useCase( + store, + (context, input) -> IntentType.SYSTEM_CHAT, + (context, query) -> new SystemChatContent("ok"), + (context, query) -> new KnowledgeContent("unused", List.of(), List.of()), + (context, query, previous, status) -> diagnosisSuccess(query), + trace::add); + + useCase.execute(new ChatApplicationRequest("hello", "session-trace")); + + assertEquals(List.of( + TraceEventType.RUN_STARTED, + TraceEventType.ROUTING_DECISION, + TraceEventType.RUN_FINISHED), + trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList()); + assertEquals("SYSTEM_CHAT", trace.get(1).details().get("intent")); + assertEquals("SUCCESS", trace.get(2).details().get("release_outcome")); + } + @Test void priorSafeTurnIsReadBeforeCurrentRunAndOnlyDiagnosisReceivesIt() { FakeStore store = new FakeStore(); @@ -168,8 +194,18 @@ class ChatApplicationUseCaseTest { private ChatApplicationUseCase useCase(FakeStore store, IntentRouting routing, SystemChatOperation system, - KnowledgeQueryOperation knowledge, - DiagnosisOperation diagnosis) { + KnowledgeQueryOperation knowledge, + DiagnosisOperation diagnosis) { + return useCase(store, routing, system, knowledge, diagnosis, + DiagnosisTraceRecorder.noop()); + } + + private ChatApplicationUseCase useCase(FakeStore store, + IntentRouting routing, + SystemChatOperation system, + KnowledgeQueryOperation knowledge, + DiagnosisOperation diagnosis, + DiagnosisTraceRecorder traceRecorder) { AtomicInteger ids = new AtomicInteger(); DiagnosisHarnessCore core = new DiagnosisHarnessCore( Clock.systemUTC(), () -> "run-app-" + ids.incrementAndGet(), Duration.ofMinutes(5), @@ -177,7 +213,7 @@ class ChatApplicationUseCaseTest { HarnessRetryPolicies.strict()); return new ChatApplicationUseCase( core, () -> "generated-session", store, routing, - system, knowledge, diagnosis, objectMapper); + system, knowledge, diagnosis, objectMapper, traceRecorder); } private DiagnosisExecutionResult diagnosisSuccess(String query) { diff --git a/src/test/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHookTest.java b/src/test/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHookTest.java index 229f9e1..b8a7c0a 100644 --- a/src/test/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHookTest.java +++ b/src/test/java/com/superbiz/agent/harness/audit/HarnessAgentAuditHookTest.java @@ -3,8 +3,10 @@ package com.superbiz.agent.harness.audit; import com.alibaba.cloud.ai.graph.RunnableConfig; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.AgentStep; +import com.superbiz.agent.domain.entity.AgentReasoningAudit; import com.superbiz.agent.harness.agent.DiagnosisAgentFactory; import com.superbiz.agent.repository.AgentStepRepository; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.springframework.ai.chat.messages.AssistantMessage; @@ -12,10 +14,13 @@ import org.springframework.ai.chat.messages.UserMessage; import java.util.List; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -25,13 +30,16 @@ import static org.mockito.Mockito.when; class HarnessAgentAuditHookTest { @Test - void persistsExactIdentityAndMetadataWithoutContentOrArguments() { + void persistsMetadataAndReasoningInSeparateAudit() { AgentStepRepository repository = mock(AgentStepRepository.class); + AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class); AgentStep persisted = AgentStep.builder().id(7L).build(); when(repository.save(any(AgentStep.class))).thenReturn(persisted); when(repository.findById(7L)).thenReturn(Optional.of(persisted)); + AtomicReference trace = new AtomicReference<>(); HarnessAgentAuditHook hook = new HarnessAgentAuditHook( - repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME); + repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME, + trace::set, reasoningRepository); RunnableConfig config = RunnableConfig.builder() .addMetadata("sessionId", "session-audit") .addMetadata("runId", "run-audit") @@ -40,6 +48,8 @@ class HarnessAgentAuditHookTest { hook.beforeModel(List.of(new UserMessage("secret-query")), config); AssistantMessage response = AssistantMessage.builder() .content("secret-model-output") + .properties(java.util.Map.of( + "reasoning_content", "inspect bounded evidence before selecting query_logs")) .toolCalls(List.of(new AssistantMessage.ToolCall( "call-1", "function", "query_logs", "{\"query\":\"secret-argument\"}"))) .build(); @@ -54,8 +64,28 @@ class HarnessAgentAuditHookTest { assertFalse(started.getModelInput().contains("secret-query")); assertFalse(completed.getModelOutput().contains("secret-model-output")); assertFalse(completed.getModelOutput().contains("secret-argument")); - assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"]}", completed.getModelOutput()); + assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"]," + + "\"reasoning_available\":true,\"reasoning_bytes\":52}", + completed.getModelOutput()); assertNull(completed.getThought()); + ArgumentCaptor reasoningCaptor = + ArgumentCaptor.forClass(AgentReasoningAudit.class); + verify(reasoningRepository).save(reasoningCaptor.capture()); + AgentReasoningAudit reasoning = reasoningCaptor.getValue(); + assertEquals("session-audit", reasoning.getSessionId()); + assertEquals("run-audit", reasoning.getRunId()); + assertEquals(0, reasoning.getStepIndex()); + assertTrue(reasoning.getReasoningAvailable()); + assertEquals("inspect bounded evidence before selecting query_logs", + reasoning.getReasoningContent()); + assertNotNull(trace.get()); + assertEquals(TraceEventType.AGENT_MODEL_STEP, trace.get().eventType()); + String traceText = trace.get().details().toString(); + assertFalse(traceText.contains("secret-query")); + assertFalse(traceText.contains("secret-model-output")); + assertFalse(traceText.contains("secret-argument")); + assertFalse(traceText.contains("inspect bounded evidence")); + assertTrue(traceText.contains("reasoning_available=true")); } @Test diff --git a/src/test/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorderTest.java b/src/test/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorderTest.java new file mode 100644 index 0000000..d047e03 --- /dev/null +++ b/src/test/java/com/superbiz/agent/harness/audit/JpaDiagnosisTraceRecorderTest.java @@ -0,0 +1,78 @@ +package com.superbiz.agent.harness.audit; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.domain.entity.DiagnosisTraceEvent; +import com.superbiz.agent.repository.DiagnosisTraceEventRepository; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.core.io.ClassPathResource; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class JpaDiagnosisTraceRecorderTest { + + @Test + void persistsIndependentMonotonicSafeEvents() { + DiagnosisTraceEventRepository repository = mock(DiagnosisTraceEventRepository.class); + when(repository.findMaxSequenceNoByRunId("run-1")).thenReturn(7); + JpaDiagnosisTraceRecorder recorder = new JpaDiagnosisTraceRecorder( + repository, new ObjectMapper()); + + recorder.record(event(TraceEventType.RUN_STARTED, TraceEventStatus.STARTED)); + recorder.record(event(TraceEventType.RELEASE_DECISION, TraceEventStatus.FALLBACK)); + + ArgumentCaptor captor = + ArgumentCaptor.forClass(DiagnosisTraceEvent.class); + verify(repository, times(2)).save(captor.capture()); + assertEquals(8, captor.getAllValues().get(0).getSequenceNo()); + assertEquals(9, captor.getAllValues().get(1).getSequenceNo()); + assertEquals("{\"failure\":\"SCHEMA_INVALID\"}", + captor.getAllValues().get(0).getDetails()); + assertFalse(captor.getAllValues().get(0).getDetails().contains("prompt")); + } + + @Test + void persistenceFailureNeverChangesMainFlow() { + DiagnosisTraceEventRepository repository = mock(DiagnosisTraceEventRepository.class); + when(repository.findMaxSequenceNoByRunId("run-1")).thenReturn(0); + doThrow(new IllegalStateException("database unavailable")) + .when(repository).save(any(DiagnosisTraceEvent.class)); + JpaDiagnosisTraceRecorder recorder = new JpaDiagnosisTraceRecorder( + repository, new ObjectMapper()); + + assertDoesNotThrow(() -> recorder.record( + event(TraceEventType.RUN_STARTED, TraceEventStatus.STARTED))); + } + + @Test + void migrationCreatesIndependentAppendOnlyTraceTable() throws Exception { + String sql; + try (var input = new ClassPathResource( + "db/migration/V014__create_diagnosis_trace_event.sql").getInputStream()) { + sql = new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + + assertTrue(sql.contains("CREATE TABLE diagnosis_trace_event")); + assertTrue(sql.contains("details JSON NOT NULL")); + assertTrue(sql.contains("idx_trace_event_run_sequence")); + assertFalse(sql.toUpperCase().contains("FOREIGN KEY")); + } + + private DiagnosisTraceAuditEvent event(TraceEventType type, TraceEventStatus status) { + return new DiagnosisTraceAuditEvent( + "session-1", "run-1", TracePhase.RUN, type, status, + null, null, Map.of("failure", "SCHEMA_INVALID")); + } +} diff --git a/src/test/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSinkTest.java b/src/test/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSinkTest.java index 4cf351d..c1d0671 100644 --- a/src/test/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSinkTest.java +++ b/src/test/java/com/superbiz/agent/harness/audit/JpaToolInvocationAuditSinkTest.java @@ -18,7 +18,9 @@ class JpaToolInvocationAuditSinkTest { @Test void persistsOnlyBoundedStableMetadata() { ToolInvocationRepository repository = mock(ToolInvocationRepository.class); - JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(repository, new ObjectMapper()); + DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class); + JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink( + repository, new ObjectMapper(), traceRecorder); sink.record(new ToolInvocationAuditEvent( "session-1", "run-1", "call-1", "query_logs", @@ -37,5 +39,11 @@ class JpaToolInvocationAuditSinkTest { String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails(); assertFalse(serialized.contains("query")); assertFalse(serialized.contains("raw_response")); + ArgumentCaptor traceCaptor = + ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class); + verify(traceRecorder).record(traceCaptor.capture()); + assertEquals(TraceEventType.TOOL_INVOCATION, traceCaptor.getValue().eventType()); + assertEquals("call-1", traceCaptor.getValue().details().get("tool_call_id")); + assertFalse(traceCaptor.getValue().details().toString().contains("raw_response")); } } diff --git a/src/test/java/com/superbiz/agent/harness/contract/HarnessContractTest.java b/src/test/java/com/superbiz/agent/harness/contract/HarnessContractTest.java index 9457dde..841527f 100644 --- a/src/test/java/com/superbiz/agent/harness/contract/HarnessContractTest.java +++ b/src/test/java/com/superbiz/agent/harness/contract/HarnessContractTest.java @@ -92,5 +92,7 @@ class HarnessContractTest { previousJson.path("source_documents").get(0).path("document_id").asText()); assertFalse(previousJson.toString().contains("tool_call_id")); assertFalse(previousJson.toString().contains("raw_response")); + assertTrue(fallbackJson.path("observed_facts").isArray()); + assertTrue(fallbackJson.path("validation_issues").isArray()); } } diff --git a/src/test/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCaseTest.java b/src/test/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCaseTest.java index 8d537d1..4147443 100644 --- a/src/test/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCaseTest.java +++ b/src/test/java/com/superbiz/agent/harness/release/DiagnosisReleaseUseCaseTest.java @@ -2,6 +2,9 @@ package com.superbiz.agent.harness.release; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.harness.contract.AnalysisKind; +import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent; +import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder; +import com.superbiz.agent.harness.audit.TraceEventType; import com.superbiz.agent.harness.contract.DiagnosisDraft; import com.superbiz.agent.harness.contract.EvidenceStatus; import com.superbiz.agent.harness.contract.FallbackType; @@ -46,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; class DiagnosisReleaseUseCaseTest { @@ -127,7 +131,21 @@ class DiagnosisReleaseUseCaseTest { fixture.context, "Why did payment fail?", duplicateDraft()); assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0); + assertEquals("EVIDENCE_VALIDATION", result.fallback().failureStage()); + assertFalse(result.fallback().validationIssues().isEmpty()); + assertTrue(result.fallback().validationIssues().stream() + .anyMatch(issue -> "ANALYSIS_ID_DUPLICATE".equals(issue.code()))); assertEquals(1, fixture.model.calls.get()); + assertEquals(List.of( + TraceEventType.EVIDENCE_GUARD_INITIAL, + TraceEventType.EVIDENCE_REPAIR_ATTEMPT, + TraceEventType.EVIDENCE_GUARD_RECHECK, + TraceEventType.RELEASE_DECISION), + fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList()); + String trace = fixture.trace.events.toString(); + assertTrue(trace.contains("ANALYSIS_ID_DUPLICATE")); + assertTrue(trace.contains("call-rag-1")); + assertFalse(trace.contains("Pool reached its limit")); } @Test @@ -140,10 +158,18 @@ class DiagnosisReleaseUseCaseTest { fixture.context, "Why did payment fail?", draft); assertFallback(result, FallbackType.SEMANTIC_UNSUPPORTED, 1); + assertEquals("SEMANTIC_VALIDATION", result.fallback().failureStage()); + assertEquals(1, result.fallback().observedFacts().size()); + assertTrue(result.fallback().observedFacts().get(0).summary().contains("active=50")); String json = objectMapper.writeValueAsString(result); assertFalse(json.contains("Pool exhausted")); assertFalse(json.contains("secret-audit-reason")); assertEquals(1, fixture.model.calls.get()); + String trace = fixture.trace.events.toString(); + assertTrue(trace.contains("SEMANTIC_GUARD_DECISION")); + assertTrue(trace.contains("UNSUPPORTED")); + assertFalse(trace.contains("secret-audit-reason")); + assertFalse(trace.contains("Pool exhausted")); } @Test @@ -167,15 +193,18 @@ class DiagnosisReleaseUseCaseTest { ScriptedChatModel model = new ScriptedChatModel(responses); HarnessRetryExecutor retries = new HarnessRetryExecutor(core); GuardModelCall modelCall = new GuardModelCall(core, model, executor); + RecordingTraceRecorder trace = new RecordingTraceRecorder(); DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase( new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper), new EvidenceRepair(core, retries, modelCall, objectMapper, - new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }), + new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), + ignored -> { }, trace), new SemanticGuard(core, retries, modelCall, objectMapper, new SemanticGuardLimits(100_000, 20_000, - Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }), - new SafeFallbackFactory()); - return new Fixture(context, model, useCase); + Duration.ofSeconds(2), Duration.ofSeconds(3)), + ignored -> { }, trace), + new SafeFallbackFactory(), trace); + return new Fixture(context, model, useCase, trace); } private DiagnosisDraft duplicateDraft() { @@ -309,6 +338,16 @@ class DiagnosisReleaseUseCaseTest { private record Fixture( RunContext context, ScriptedChatModel model, - DiagnosisReleaseUseCase useCase) { + DiagnosisReleaseUseCase useCase, + RecordingTraceRecorder trace) { + } + + private static final class RecordingTraceRecorder implements DiagnosisTraceRecorder { + private final List events = new ArrayList<>(); + + @Override + public void record(DiagnosisTraceAuditEvent event) { + events.add(event); + } } } diff --git a/src/test/java/com/superbiz/agent/service/DiagnosisTraceServiceTest.java b/src/test/java/com/superbiz/agent/service/DiagnosisTraceServiceTest.java index 25529f2..18e0788 100644 --- a/src/test/java/com/superbiz/agent/service/DiagnosisTraceServiceTest.java +++ b/src/test/java/com/superbiz/agent/service/DiagnosisTraceServiceTest.java @@ -2,17 +2,23 @@ package com.superbiz.agent.service; import com.fasterxml.jackson.databind.ObjectMapper; import com.superbiz.agent.domain.entity.AgentStep; +import com.superbiz.agent.domain.entity.AgentReasoningAudit; import com.superbiz.agent.domain.entity.ChatSession; import com.superbiz.agent.domain.entity.DiagnosisRun; import com.superbiz.agent.domain.entity.DiagnosisSession; +import com.superbiz.agent.domain.entity.DiagnosisTraceEvent; import com.superbiz.agent.domain.entity.ToolInvocation; import com.superbiz.agent.dto.DiagnosisTraceResponse; import com.superbiz.agent.exception.SessionNotFoundException; import com.superbiz.agent.repository.AgentStepRepository; +import com.superbiz.agent.repository.AgentReasoningAuditRepository; import com.superbiz.agent.repository.ChatSessionRepository; import com.superbiz.agent.repository.DiagnosisRunRepository; import com.superbiz.agent.repository.DiagnosisSessionRepository; +import com.superbiz.agent.repository.DiagnosisTraceEventRepository; import com.superbiz.agent.repository.ToolInvocationRepository; +import com.superbiz.agent.harness.contract.IntentType; +import com.superbiz.agent.harness.contract.ReleaseOutcome; import org.junit.jupiter.api.Test; import java.time.LocalDateTime; @@ -36,16 +42,50 @@ class DiagnosisTraceServiceTest { private final ChatSessionRepository chatSessionRepository = mock(ChatSessionRepository.class); private final DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class); private final AgentStepRepository agentStepRepository = mock(AgentStepRepository.class); + private final AgentReasoningAuditRepository agentReasoningAuditRepository = + mock(AgentReasoningAuditRepository.class); private final ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class); + private final DiagnosisTraceEventRepository diagnosisTraceEventRepository = + mock(DiagnosisTraceEventRepository.class); private final DiagnosisTraceService service = new DiagnosisTraceService( diagnosisSessionRepository, chatSessionRepository, diagnosisRunRepository, agentStepRepository, + agentReasoningAuditRepository, toolInvocationRepository, + diagnosisTraceEventRepository, new ObjectMapper() ); + @Test + void reasoningAuditRequiresExactRunAndReturnsRestrictedContent() { + String sessionId = "trace-session-reasoning"; + String runId = "run-reasoning"; + LocalDateTime createdAt = LocalDateTime.of(2026, 7, 23, 10, 0); + when(diagnosisRunRepository.findBySessionIdAndRunId(sessionId, runId)) + .thenReturn(Optional.of(run(9L, sessionId, runId, "question", createdAt))); + when(agentReasoningAuditRepository.findByRunIdOrderByStepIndex(runId)) + .thenReturn(List.of(AgentReasoningAudit.builder() + .id(11L) + .sessionId(sessionId) + .runId(runId) + .stepIndex(0) + .agentName("diagnosis_agent") + .reasoningAvailable(true) + .reasoningContent("bounded reasoning") + .contentBytes(17) + .createdAt(createdAt) + .build())); + + List result = + service.getReasoning(sessionId, runId); + + assertEquals(1, result.size()); + assertEquals("bounded reasoning", result.get(0).getReasoningContent()); + assertTrue(result.get(0).getReasoningAvailable()); + } + @Test void getTraceWithoutRunIdResolvesLatestRun() { String sessionId = "trace-session-latest"; @@ -53,12 +93,16 @@ class DiagnosisTraceServiceTest { DiagnosisRun latest = run(2L, sessionId, "run-latest", "second question", base.plusMinutes(1)); AgentStep step = step(20L, sessionId, "run-latest", 0, "composer", base.plusMinutes(1)); ToolInvocation invocation = invocation(30L, sessionId, "run-latest", "query_metrics", base.plusMinutes(1)); + DiagnosisTraceEvent event = traceEvent(40L, sessionId, "run-latest", 1, + "RUN", "RUN_STARTED", "STARTED", base.plusMinutes(1)); when(diagnosisRunRepository.findFirstBySessionIdOrderByCreatedAtDescIdDesc(sessionId)) .thenReturn(Optional.of(latest)); when(chatSessionRepository.findBySessionId(sessionId)).thenReturn(Optional.of(chatSession(sessionId))); when(agentStepRepository.findByRunIdOrderByStepIndex("run-latest")).thenReturn(List.of(step)); when(toolInvocationRepository.findByRunIdOrderByIdAsc("run-latest")).thenReturn(List.of(invocation)); + when(diagnosisTraceEventRepository.findByRunIdOrderBySequenceNoAscIdAsc("run-latest")) + .thenReturn(List.of(event)); DiagnosisTraceResponse response = service.getTrace(sessionId); @@ -69,6 +113,10 @@ class DiagnosisTraceServiceTest { assertEquals("run-latest", response.getSteps().get(0).getRunId()); assertEquals("run-latest", response.getToolInvocations().get(0).getRunId()); assertEquals(1, response.getChatSession().getMessagePairCount()); + assertEquals("DIAGNOSIS", response.getRun().getIntent()); + assertEquals("SUCCESS", response.getRun().getReleaseOutcome()); + assertEquals("RUN_STARTED", response.getTimeline().get(0).getEventType()); + assertEquals(1, response.getSummary().getReturnedTraceEventCount()); } @Test @@ -216,6 +264,8 @@ class DiagnosisTraceServiceTest { .query(query) .status("SUCCESS") .agentFlow("CHAT") + .intent(IntentType.DIAGNOSIS) + .releaseOutcome(ReleaseOutcome.SUCCESS) .answer("answer for " + runId) .selfEvaluation("{\"verifier_evaluation\":{\"verdict\":\"PASS\"}}") .feedback("useful") @@ -226,6 +276,22 @@ class DiagnosisTraceServiceTest { .build(); } + private DiagnosisTraceEvent traceEvent( + Long id, String sessionId, String runId, int sequenceNo, + String phase, String eventType, String status, LocalDateTime createdAt) { + return DiagnosisTraceEvent.builder() + .id(id) + .sessionId(sessionId) + .runId(runId) + .sequenceNo(sequenceNo) + .phase(phase) + .eventType(eventType) + .status(status) + .details("{\"safe\":true}") + .createdAt(createdAt) + .build(); + } + private ChatSession chatSession(String sessionId) { return ChatSession.builder() .id(99L)