diff --git a/mvp/architecture/confidence-feedback.md b/mvp/architecture/confidence-feedback.md new file mode 100644 index 0000000..24d3016 --- /dev/null +++ b/mvp/architecture/confidence-feedback.md @@ -0,0 +1,157 @@ +# 证据评分与用户反馈架构 + +## 一、整体架构 + +``` +用户对话 + ↓ +ChatService.executeChat / executeChatComplex + ↓ SUCCESS 后写入 answer,异步触发 +EvaluationService.evaluate(sessionId, answer) + └─ 读取 tool_invocation 事实 → 规则引擎 → 写 selfEvaluation + +用户提交反馈 + ↓ +POST /api/feedback { sessionId, feedback: "useful" | "not_useful" } + ↓ +FeedbackService.submitFeedback + ├─ 写 DiagnosisSession.feedback + ├─ useful → CaseLibraryService.createFromSession → 写 case_library + └─ not_useful → 仅写 feedback,status 不变 +``` + +--- + +## 二、评分规则(evidence_score) + +### 定位 + +`evidence_score` 衡量的是**证据收集充分度**,不是答案准确性。 + +- 能证明的:Agent 是否有尝试收集证据、检索是否命中 +- 不能证明的:答案是否有幻觉、推理是否正确 + +### 数据来源 + +规则引擎只消费 `tool_invocation` 表的事实记录,不依赖 LLM 判断。 + +### 规则定义 + +| 规则名 | 条件 | delta | +|---|---|---| +| `no_tool_call` | 无任何工具调用 | 直接 0 分,不参与加权 | +| `execution_failed` | status = FAILED | 直接 0 分,不参与加权 | +| `has_successful_tool_call` | 至少 1 次成功调用 | +30 | +| `l0_exact_match` | 任意调用有 L0 精确匹配命中 | +35 | +| `l1_semantic_match` | 无 L0 命中但有 L1 语义匹配 | +20 | +| `retrieval_no_hit` | 有检索调用但无任何命中 | -10 | +| `all_tool_calls_failed` | 全部调用失败 | -20 | + +> L0 和 L1 互斥取高优先级(L0 命中时跳过 L1 分支)。 + +### selfEvaluation 字段格式 + +```json +{ + "evidence_score": 65, + "source": "rule", + "factors": [ + {"name": "has_successful_tool_call", "delta": 30, "description": "有成功的工具调用(20次)"}, + {"name": "l0_exact_match", "delta": 35, "description": "L0 精确匹配命中"} + ] +} +``` + +| 字段 | 说明 | +|---|---| +| `evidence_score` | 0-100 整数 | +| `source` | 当前固定为 `"rule"`;预留 `"llm"` 供后续扩展 | +| `factors` | 命中的规则列表,含 name / delta / description | +| `llm_opinion` | 预留字段(未实现),LLM 观点叠加时在此扩展 | + +### 已知边界 + +- 非检索工具(DateTimeTools、QueryMetricsTools 等)不写 `tool_invocation`,这类 session 的 evidence_score = 0,属于设计边界 +- 评分为异步写入(`@Async`),失败时 `selfEvaluation` 保持 null,前端需处理 null + +--- + +## 三、反馈机制 + +### API + +``` +POST /api/feedback +Content-Type: application/json + +{ + "sessionId": "xxx", + "feedback": "useful" | "not_useful" +} +``` + +**响应** + +```json +{ + "success": true, + "message": "反馈已记录", + "caseId": "uuid 或 null" +} +``` + +### 后端行为 + +| feedback 值 | 操作 | +|---|---| +| `useful` | 写 `DiagnosisSession.feedback = "useful"`,生成 `CaseLibrary` 记录,返回 caseId | +| `not_useful` | 写 `DiagnosisSession.feedback = "not_useful"`,status 不变 | +| 其他值 | 返回 HTTP 400 | + +### 重要设计决策 + +**BAD_CASE 不改 status 字段** + +`status` 表示执行状态(RUNNING/SUCCESS/FAILED),是独立维度,不能被质量标签覆盖。 +查询 BadCase 使用:`WHERE feedback = 'not_useful'` + +**useful 触发案例沉淀规则** + +| CaseLibrary 字段 | 来源 | +|---|---| +| caseId | UUID | +| diagnosisId | DiagnosisSession.sessionId | +| sourceType | AUTO | +| faultCategory | GENERAL(暂时,后续人工补充) | +| title | query 前 100 字符 | +| rootCause / solution | DiagnosisSession.answer(完整答案) | +| createdBy | "system" | + +**幂等性**:同一 sessionId 重复提交 useful,返回已有 caseId,不重复插入 case_library。 + +--- + +## 四、数据库变更 + +### V008(新增) + +```sql +ALTER TABLE diagnosis_session ADD COLUMN answer LONGTEXT COMMENT 'Agent 返回给用户的完整答案'; +``` + +### diagnosis_session 关键字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `answer` | LONGTEXT | Agent 完整回答,useful 案例沉淀的内容来源 | +| `self_evaluation` | JSON | 证据评分结果,格式见上 | +| `feedback` | VARCHAR(16) | useful / not_useful / null | +| `status` | VARCHAR(16) | 执行状态,不受 feedback 影响 | + +--- + +## 五、扩展方向(Phase 2) + +- **LLM 观点层**:在 `selfEvaluation` 的 `llm_opinion` 字段叠加 LLM 结构化观点(has_root_cause、has_solution 等),作为独立 factors,不改变现有规则逻辑 +- **案例结构化字段**:useful 触发时自动提取 faultCategory / errorCode,替代暂时的 GENERAL +- **重复召回问题**:Executor Prompt 约束或工具层 session 维度去重(见 [ISS-001](../issues/ISS-001-duplicate-retrieval.md)) diff --git a/src/main/java/com/superbiz/agent/config/AsyncConfig.java b/src/main/java/com/superbiz/agent/config/AsyncConfig.java new file mode 100644 index 0000000..1646faa --- /dev/null +++ b/src/main/java/com/superbiz/agent/config/AsyncConfig.java @@ -0,0 +1,9 @@ +package com.superbiz.agent.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; + +@Configuration +@EnableAsync +public class AsyncConfig { +} diff --git a/src/main/java/com/superbiz/agent/controller/FeedbackController.java b/src/main/java/com/superbiz/agent/controller/FeedbackController.java new file mode 100644 index 0000000..556443d --- /dev/null +++ b/src/main/java/com/superbiz/agent/controller/FeedbackController.java @@ -0,0 +1,25 @@ +package com.superbiz.agent.controller; + +import com.superbiz.agent.dto.FeedbackRequest; +import com.superbiz.agent.dto.FeedbackResponse; +import com.superbiz.agent.service.FeedbackService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/api") +public class FeedbackController { + + @Autowired + private FeedbackService feedbackService; + + @PostMapping("/feedback") + public ResponseEntity submitFeedback(@RequestBody FeedbackRequest request) { + FeedbackResponse response = feedbackService.submitFeedback(request.getSessionId(), request.getFeedback()); + if (!response.isSuccess()) { + return ResponseEntity.badRequest().body(response); + } + return ResponseEntity.ok(response); + } +} diff --git a/src/main/java/com/superbiz/agent/dto/FeedbackRequest.java b/src/main/java/com/superbiz/agent/dto/FeedbackRequest.java new file mode 100644 index 0000000..8bf8649 --- /dev/null +++ b/src/main/java/com/superbiz/agent/dto/FeedbackRequest.java @@ -0,0 +1,11 @@ +package com.superbiz.agent.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class FeedbackRequest { + private String sessionId; + private String feedback; +} diff --git a/src/main/java/com/superbiz/agent/dto/FeedbackResponse.java b/src/main/java/com/superbiz/agent/dto/FeedbackResponse.java new file mode 100644 index 0000000..226095b --- /dev/null +++ b/src/main/java/com/superbiz/agent/dto/FeedbackResponse.java @@ -0,0 +1,12 @@ +package com.superbiz.agent.dto; + +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +public class FeedbackResponse { + private boolean success; + private String message; + private String caseId; +} diff --git a/src/main/java/com/superbiz/agent/service/CaseLibraryService.java b/src/main/java/com/superbiz/agent/service/CaseLibraryService.java new file mode 100644 index 0000000..09cad07 --- /dev/null +++ b/src/main/java/com/superbiz/agent/service/CaseLibraryService.java @@ -0,0 +1,53 @@ +package com.superbiz.agent.service; + +import com.superbiz.agent.domain.entity.CaseLibrary; +import com.superbiz.agent.domain.entity.DiagnosisSession; +import com.superbiz.agent.domain.enums.FaultCategory; +import com.superbiz.agent.domain.enums.SourceType; +import com.superbiz.agent.repository.CaseLibraryRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.UUID; + +@Service +public class CaseLibraryService { + + private static final Logger logger = LoggerFactory.getLogger(CaseLibraryService.class); + + @Autowired + private CaseLibraryRepository caseLibraryRepository; + + public CaseLibrary createFromSession(DiagnosisSession session) { + return caseLibraryRepository.findByDiagnosisId(session.getSessionId()) + .orElseGet(() -> { + String content = session.getAnswer(); + if (content == null || content.isBlank()) { + content = session.getQuery() + "\n(自动提取失败,请人工补充)"; + } + + String title = session.getQuery(); + if (title.length() > 100) { + title = title.substring(0, 100); + } + + CaseLibrary caseLibrary = CaseLibrary.builder() + .caseId(UUID.randomUUID().toString()) + .diagnosisId(session.getSessionId()) + .sourceType(SourceType.AUTO) + .faultCategory(FaultCategory.GENERAL) + .title(title) + .rootCause(content) + .solution(content) + .createdBy("system") + .referenceCount(0) + .build(); + + CaseLibrary saved = caseLibraryRepository.save(caseLibrary); + logger.info("案例已沉淀: caseId={}, sessionId={}", saved.getCaseId(), session.getSessionId()); + return saved; + }); + } +} diff --git a/src/main/java/com/superbiz/agent/service/EvaluationService.java b/src/main/java/com/superbiz/agent/service/EvaluationService.java new file mode 100644 index 0000000..157f181 --- /dev/null +++ b/src/main/java/com/superbiz/agent/service/EvaluationService.java @@ -0,0 +1,141 @@ +package com.superbiz.agent.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.superbiz.agent.domain.entity.DiagnosisSession; +import com.superbiz.agent.domain.entity.ToolInvocation; +import com.superbiz.agent.repository.DiagnosisSessionRepository; +import com.superbiz.agent.repository.ToolInvocationRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 证据评分服务 + * + * 当前实现:基于 tool_invocation 事实的规则引擎,输出 evidence_score(0-100)。 + * 扩展预留:LLM 观点辅助评估(evaluateWithLlm),未来可叠加到 factors 中作为独立维度。 + */ +@Service +public class EvaluationService { + + private static final Logger logger = LoggerFactory.getLogger(EvaluationService.class); + + @Autowired + private DiagnosisSessionRepository diagnosisSessionRepository; + + @Autowired + private ToolInvocationRepository toolInvocationRepository; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Async + public void evaluate(String sessionId, String answer) { + diagnosisSessionRepository.findBySessionId(sessionId).ifPresent(session -> { + try { + List toolInvocations = toolInvocationRepository.findBySessionId(sessionId); + String selfEvaluation = evaluateWithRules(session, toolInvocations); + session.setSelfEvaluation(selfEvaluation); + diagnosisSessionRepository.save(session); + logger.info("证据评分已写入: sessionId={}, result={}", sessionId, selfEvaluation); + } catch (Exception e) { + logger.error("评分失败: sessionId={}", sessionId, e); + } + }); + } + + // ------------------------------------------------------------------------- + // 规则引擎(事实层) + // ------------------------------------------------------------------------- + + private String evaluateWithRules(DiagnosisSession session, List invocations) { + List> factors = new ArrayList<>(); + + if ("FAILED".equals(session.getStatus())) { + factors.add(factor("execution_failed", -100, "执行失败")); + return buildResult(0, factors); + } + + int total = invocations.size(); + long successCount = invocations.stream().filter(t -> Boolean.TRUE.equals(t.getSuccess())).count(); + boolean hasRetrieval = invocations.stream().anyMatch(t -> t.getRetrievalLayer() != null); + boolean hasL0Hit = invocations.stream() + .anyMatch(t -> t.getL0MatchCount() != null && t.getL0MatchCount() > 0); + boolean hasL1Hit = invocations.stream() + .anyMatch(t -> t.getL1MatchCount() != null && t.getL1MatchCount() > 0); + + if (total == 0) { + factors.add(factor("no_tool_call", 0, "无工具调用,无法评估证据充分度")); + return buildResult(0, factors); + } + + int score = 0; + + // 工具成功调用 + if (successCount > 0) { + int delta = 30; + factors.add(factor("has_successful_tool_call", delta, "有成功的工具调用(" + successCount + "次)")); + score += delta; + } + + // 检索命中(L0 精确匹配,证据最强) + if (hasL0Hit) { + int delta = 35; + factors.add(factor("l0_exact_match", delta, "L0 精确匹配命中")); + score += delta; + } + // 检索命中(L1 语义匹配) + else if (hasL1Hit) { + int delta = 20; + factors.add(factor("l1_semantic_match", delta, "L1 语义匹配命中")); + score += delta; + } + // 有检索但无命中 + else if (hasRetrieval) { + int delta = -10; + factors.add(factor("retrieval_no_hit", delta, "检索工具调用但无匹配结果")); + score += delta; + } + + // 全部工具调用失败 + if (successCount == 0) { + int delta = -20; + factors.add(factor("all_tool_calls_failed", delta, "所有工具调用均失败")); + score += delta; + } + + score = Math.max(0, Math.min(100, score)); + return buildResult(score, factors); + } + + private Map factor(String name, int delta, String description) { + return Map.of("name", name, "delta", delta, "description", description); + } + + private String buildResult(int score, List> factors) { + try { + Map result = Map.of( + "evidence_score", score, + "source", "rule", + "factors", factors + // llm_opinion: null ← 预留字段,LLM 观点叠加时在此处扩展 + ); + return objectMapper.writeValueAsString(result); + } catch (Exception e) { + logger.error("序列化评分结果失败", e); + return "{\"evidence_score\":0,\"source\":\"rule\",\"factors\":[]}"; + } + } + + // ------------------------------------------------------------------------- + // 预留:LLM 观点辅助(Phase 2) + // 实现时在此处添加 evaluateWithLlm(session, answer) 方法, + // 返回结构化观点(如 has_root_cause、has_solution 等), + // 作为独立 factors 叠加到 buildResult 中,不改变现有规则逻辑。 + // ------------------------------------------------------------------------- +} diff --git a/src/main/java/com/superbiz/agent/service/FeedbackService.java b/src/main/java/com/superbiz/agent/service/FeedbackService.java new file mode 100644 index 0000000..a78657e --- /dev/null +++ b/src/main/java/com/superbiz/agent/service/FeedbackService.java @@ -0,0 +1,57 @@ +package com.superbiz.agent.service; + +import com.superbiz.agent.domain.entity.DiagnosisSession; +import com.superbiz.agent.dto.FeedbackResponse; +import com.superbiz.agent.repository.DiagnosisSessionRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class FeedbackService { + + private static final Logger logger = LoggerFactory.getLogger(FeedbackService.class); + + private static final String FEEDBACK_USEFUL = "useful"; + private static final String FEEDBACK_NOT_USEFUL = "not_useful"; + + @Autowired + private DiagnosisSessionRepository diagnosisSessionRepository; + + @Autowired + private CaseLibraryService caseLibraryService; + + public FeedbackResponse submitFeedback(String sessionId, String feedback) { + if (sessionId == null || sessionId.isBlank()) { + return FeedbackResponse.builder().success(false).message("sessionId 不能为空").build(); + } + if (!FEEDBACK_USEFUL.equals(feedback) && !FEEDBACK_NOT_USEFUL.equals(feedback)) { + return FeedbackResponse.builder().success(false) + .message("feedback 只能是 useful 或 not_useful").build(); + } + + DiagnosisSession session = diagnosisSessionRepository.findBySessionId(sessionId) + .orElse(null); + if (session == null) { + return FeedbackResponse.builder().success(false).message("会话不存在").build(); + } + + session.setFeedback(feedback); + + String caseId = null; + if (FEEDBACK_USEFUL.equals(feedback)) { + var caseLibrary = caseLibraryService.createFromSession(session); + caseId = caseLibrary.getCaseId(); + } + + diagnosisSessionRepository.save(session); + logger.info("反馈已记录: sessionId={}, feedback={}, caseId={}", sessionId, feedback, caseId); + + return FeedbackResponse.builder() + .success(true) + .message("反馈已记录") + .caseId(caseId) + .build(); + } +} diff --git a/src/main/resources/db/migration/V008__add_answer_to_diagnosis_session.sql b/src/main/resources/db/migration/V008__add_answer_to_diagnosis_session.sql new file mode 100644 index 0000000..45005d8 --- /dev/null +++ b/src/main/resources/db/migration/V008__add_answer_to_diagnosis_session.sql @@ -0,0 +1 @@ +ALTER TABLE diagnosis_session ADD COLUMN answer LONGTEXT COMMENT 'Agent 返回给用户的完整答案';