feat(rag): chunk evidence identity, dedup, and search port

Preserve same-document multi-chunk evidence with evidenceKey identity, per-document caps, retrieve-k/return-n split, and a dense KnowledgeSearchPort. Archives Delivery 1 OpenSpec change as the foundation for hybrid retrieval.
2026-07-27 18:26:15 +08:00
parent 99d4f6f216
commit ac1f831903
34 changed files with 2880 additions and 298 deletions
@@ -8,17 +8,28 @@ import java.util.List;
 /**
 * 内部证据块（后处理输出，尚未投影到 Agent 契约）。
 *
- * <p>一条 block 通常对应一次向量命中的一个 chunk 内容（截断后）。
- * 注意：当前后处理按 source 去重，同文档多 chunk 可能被合并掉，只保留最高分 content。</p>
+ * <p>一条 block 对应一个 chunk 级证据身份（{@code evidenceKey}）。
+ * 同文档多个相关 chunk 可以同时存在，受 maxChunksPerDocument / return-n 约束。</p>
 *
- * <p>投影到 Agent 时由 {@code RagResultProjector} 转为 {@code RagEvidence}
- * （document_id / source / title / breadcrumb / excerpt）。</p>
+ * <p>投影到 Agent 时由 {@code RagResultProjector} 转为 {@code RagEvidence}，
+ * {@code document_id} 优先使用 evidenceKey（chunk 级）。</p>
 */
@Data
@Builder
 public class EvidenceBlock {

-    /** 来源标识，常见为文件路径、upload:docId 或 docId。 */
+    /** 文档级 id；用于每文档上限统计。 */
+    private String docId;
+
+    /** 切片序号。 */
+    private Integer chunkIndex;
+
+    /**
+     * 片段级身份；去重与投影 document_id 的首选。
+     */
+    private String evidenceKey;
+
+    /** 来源标识，常见为文件路径、upload:docId 或 docId；可重复。 */
    private String source;

    private String title;
@@ -9,8 +9,8 @@ import java.util.Map;
 /**
 * L1 向量命中后、后处理前的统一候选结构。
 *
- * <p>由 {@code KnowledgeDocumentRetriever} 从 {@code VectorSearchService.SearchResult} 映射而来。
- * 后处理会基于它做归一化、规则 boost、去重并生成 {@link EvidenceBlock}。</p>
+ * <p>由检索适配器从 {@code KnowledgeSearchHit} / 向量结果映射而来。
+ * 后处理会基于它做归一化、规则 boost、chunk 级去重并生成 {@link EvidenceBlock}。</p>
 */
@Data
@Builder
@@ -19,9 +19,24 @@ public class RetrievedEvidenceCandidate {
    /** 向量库记录 id。 */
    private String id;

+    /**
+     * 文档级 id（metadata.docId 等）。
+     * 用于每文档 chunk 上限；不等于 evidenceKey。
+     */
+    private String docId;
+
+    /** 文档内切片序号；可能为空（老数据）。 */
+    private Integer chunkIndex;
+
+    /**
+     * 片段级去重/投影主键。
+     * 通常为 docId#chunk-N，fallback 为 vector:{id}。
+     */
+    private String evidenceKey;
+
    /**
     * 来源标识（_source / source / filePath / docId 等）。
-     * 当前后处理去重主要依赖该字段，粒度偏文档级。
+     * 可与同文档其他 chunk 重复；不再作为唯一去重键。
     */
    private String source;

@@ -54,7 +69,7 @@ public class RetrievedEvidenceCandidate {

    /**
     * 扁平化 metadata（string map）。
-     * 可能含 docId、chunkIndex、category、kb_scope 等；chunkIndex 尚未提升为一等字段。
+     * 可能含 docId、chunkIndex、category、kb_scope 等。
     */
    private Map<String, String> metadata;

@@ -18,27 +18,9 @@ import java.util.Set;
 /**
 * 把 legacy {@code LookupResult} JSON 投影成冻结的 Agent 可见 RAG 契约。
 *
- * <h3>为什么需要投影</h3>
- * 内部检索结果字段较多（retrievalTrace、rerankTrace、score、hitReasons、contextPack…），
- * Agent / EvidenceGuard 只应看到受控、有界、可引用的子集。
- *
- * <h3>保留给 Agent 的字段</h3>
- * <ul>
- *   <li>evidenceStatus / toolCallId / query</li>
- *   <li>evidence[]：document_id, source, title, breadcrumb, excerpt</li>
- *   <li>relevanceLevel、truncated、returned_count</li>
- * </ul>
- *
- * <h3>刻意丢弃</h3>
- * score、hitReasons、retrievalTrace、rerankTrace、contextPack 等内部可观测细节。
- *
- * <h3>去重与预算（读代码关键）</h3>
- * <ul>
- *   <li>按 {@code document_id} 去重；若 block 无 document_id，则回退 source/title</li>
- *   <li>因此同 source 的多个 chunk 在此也会被压成 1 条（与后处理文档级去重叠加）</li>
- *   <li>条数上限 {@link ToolProjectionLimits#maxEvidence()}，excerpt 字符上限，
- *       以及总 UTF-8 字节预算（超限从后往前删 evidence）</li>
- * </ul>
+ * <h3>去重</h3>
+ * 按 chunk 级证据身份去重（优先 evidenceKey / document_id），
+ * <b>不再</b>因为 source 相同就丢弃第二条 chunk。
 */
 public final class RagResultProjector {

@@ -50,11 +32,6 @@ public final class RagResultProjector {
        this.limits = limits;
    }

-    /**
-     * @param request     Agent 原始请求（用于回填/截断 query）
-     * @param toolCallId  框架分配的本次工具调用 id，进入结果供证据引用
-     * @param rawResponse legacy LookupResult 的 JSON 字符串
-     */
    public ProjectedToolResult project(RagToolRequest request, String toolCallId,
                                       String rawResponse) throws Exception {
        if (request == null || toolCallId == null || toolCallId.isBlank()) {
@@ -69,8 +46,7 @@ public final class RagResultProjector {
        String query = bounded(request.query(), limits.maxQueryChars());
        truncated = !query.equals(request.query());
        List<RagEvidence> evidence = new ArrayList<>();
-        // 文档级唯一集合：相同 documentId 只保留首次出现
-        Set<String> documentIds = new HashSet<>();
+        Set<String> evidenceIds = new HashSet<>();
        JsonNode blocks = root.has("evidenceBlocks") ? root.get("evidenceBlocks") : root.get("evidence_blocks");
        if (blocks != null && blocks.isArray()) {
            int ordinal = 0;
@@ -89,11 +65,16 @@ public final class RagResultProjector {
                }
                String source = text(block, "source");
                String title = text(block, "title");
-                // legacy EvidenceBlock 通常没有 document_id，实际常退化为 source
-                String documentId = firstNonBlank(text(block, "document_id"), source, title,
-                        "legacy-document-" + ordinal);
-                if (!documentIds.add(documentId)) {
-                    // 同 documentId 重复：丢弃后续条，并标记 truncated
+                // Chunk-scoped identity first; do not collapse on source alone.
+                String documentId = firstPresent(
+                        text(block, "evidenceKey"),
+                        text(block, "evidence_key"),
+                        text(block, "document_id"),
+                        text(block, "documentId"),
+                        composeChunkId(text(block, "docId"), text(block, "doc_id"),
+                                text(block, "chunkIndex"), text(block, "chunk_index")),
+                        "legacy-evidence-" + ordinal);
+                if (!evidenceIds.add(documentId)) {
                    truncated = true;
                    continue;
                }
@@ -118,12 +99,36 @@ public final class RagResultProjector {
                ? relevanceLevel(root) : null;
        RagToolResult result = new RagToolResult(
                status, toolCallId, query, evidence, evidence.size(), relevanceLevel, truncated);
-        // 总字节预算：仍超限则从尾部删 evidence，直到放得下或变 no_evidence
        result = fitBudget(result, truncated);
        return new ProjectedToolResult(objectMapper.writeValueAsString(result), result.evidenceStatus());
    }

-    /** 按 maxAgentUtf8Bytes 从后往前删 evidence，保证 Agent 侧 payload 有界。 */
+    private static String composeChunkId(String docId, String docIdAlt, String chunkIndex, String chunkIndexAlt) {
+        String id = firstPresentOrNull(docId, docIdAlt);
+        String idx = firstPresentOrNull(chunkIndex, chunkIndexAlt);
+        if (id == null || idx == null) {
+            return null;
+        }
+        return id + "#chunk-" + idx;
+    }
+
+    private static String firstPresent(String... values) {
+        String found = firstPresentOrNull(values);
+        return found == null ? "unknown-document" : found;
+    }
+
+    private static String firstPresentOrNull(String... values) {
+        if (values == null) {
+            return null;
+        }
+        for (String value : values) {
+            if (value != null && !value.isBlank()) {
+                return value;
+            }
+        }
+        return null;
+    }
+
    private RagToolResult fitBudget(RagToolResult result, boolean truncated) throws Exception {
        RagToolResult current = result;
        while (bytes(objectMapper.writeValueAsString(current)) > limits.maxAgentUtf8Bytes()
@@ -146,15 +151,6 @@ public final class RagResultProjector {
        return value == null || value.isNull() ? "" : value.asText("");
    }

-    private static String firstNonBlank(String... values) {
-        for (String value : values) {
-            if (value != null && !value.isBlank()) {
-                return value;
-            }
-        }
-        return "unknown-document";
-    }
-
    private static String nullable(String value) {
        return value == null || value.isBlank() ? null : value;
    }
@@ -1,40 +1,30 @@
 package com.superbiz.agent.service;

-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.superbiz.agent.dto.RetrievalTrace;
 import com.superbiz.agent.dto.RetrievedEvidenceCandidate;
+import com.superbiz.agent.service.retrieval.KnowledgeSearchHit;
+import com.superbiz.agent.service.retrieval.KnowledgeSearchPort;
+import com.superbiz.agent.service.retrieval.KnowledgeSearchRequest;
 import org.springframework.stereotype.Service;

 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;

 /**
 * L1 向量检索适配器。
 *
- * <p>职责是把 {@link VectorSearchService} 的原始命中，转成 pipeline 统一使用的
+ * <p>通过 {@link KnowledgeSearchPort} 拉取候选，映射为 pipeline 统一的
 * {@link RetrievedEvidenceCandidate}，并记录单次 attempt 的 trace。</p>
 *
 * <p>本类不做质量判断、不做 rerank、不做上下文打包；那些属于后处理阶段。</p>
- *
- * <h3>字段映射要点</h3>
- * <ul>
- *   <li>{@code source}：优先 metadata._source / source / filePath / docId</li>
- *   <li>{@code title/breadcrumb}：来自 chunk metadata，用于展示与规则 boost</li>
- *   <li>{@code score}：兼容后的距离分（SDK 为 L2；Spring AI 路径会映射成兼容 L2）</li>
- *   <li>metadata 中的 chunkIndex 目前只留在 map 里，未提升为一等字段</li>
- * </ul>
 */
@Service
 public class KnowledgeDocumentRetriever {

-    private final VectorSearchService vectorSearchService;
-    private final ObjectMapper objectMapper;
+    private final KnowledgeSearchPort knowledgeSearchPort;

-    public KnowledgeDocumentRetriever(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
-        this.vectorSearchService = vectorSearchService;
-        this.objectMapper = objectMapper;
+    public KnowledgeDocumentRetriever(KnowledgeSearchPort knowledgeSearchPort) {
+        this.knowledgeSearchPort = knowledgeSearchPort;
    }

    /**
@@ -43,22 +33,21 @@ public class KnowledgeDocumentRetriever {
     * @param attemptName    写入 trace 的 attempt 名（FILTERED_VECTOR / UNFILTERED_VECTOR 等）
     * @param query          检索文本
     * @param categoryFilter 可选 category 元数据过滤；null 表示不过滤
-     * @param topK           召回条数
+     * @param topK           召回条数（retrieve-k）
     * @return attempt 元信息 + 候选列表；异常时 candidates 为空，error 记在 attempt 上
     */
    public RetrievalAttemptResult retrieve(String attemptName, String query, String categoryFilter, int topK) {
        long start = System.currentTimeMillis();
        try {
-            List<VectorSearchService.SearchResult> results =
-                    vectorSearchService.searchSimilarDocuments(query, topK, categoryFilter);
-            List<RetrievedEvidenceCandidate> candidates = toCandidates(attemptName, results);
+            List<KnowledgeSearchHit> hits = knowledgeSearchPort.search(
+                    KnowledgeSearchRequest.dense(query, topK, categoryFilter));
+            List<RetrievedEvidenceCandidate> candidates = toCandidates(attemptName, hits);
            return new RetrievalAttemptResult(
                    attempt(attemptName, query, categoryFilter, candidates.size(), null,
-                            (int) (System.currentTimeMillis() - start), topScore(results)),
+                            (int) (System.currentTimeMillis() - start), topScore(hits)),
                    candidates
            );
        } catch (Exception e) {
-            // 检索失败不向上抛：由上层按“无候选 / 低质量”路径继续（例如 fallback retry）
            return new RetrievalAttemptResult(
                    attempt(attemptName, query, categoryFilter, 0, e.getMessage(),
                            (int) (System.currentTimeMillis() - start), null),
@@ -67,42 +56,29 @@ public class KnowledgeDocumentRetriever {
        }
    }

-    /**
-     * 将向量库原始结果规范化为候选证据。
-     * originalRank 从 1 开始，对应向量召回顺序（尚未规则 rerank）。
-     */
-    private List<RetrievedEvidenceCandidate> toCandidates(String attemptName,
-                                                          List<VectorSearchService.SearchResult> results) {
-        if (results == null || results.isEmpty()) {
+    private List<RetrievedEvidenceCandidate> toCandidates(String attemptName, List<KnowledgeSearchHit> hits) {
+        if (hits == null || hits.isEmpty()) {
            return List.of();
        }
-        List<RetrievedEvidenceCandidate> candidates = new ArrayList<>();
-        for (int i = 0; i < results.size(); i++) {
-            VectorSearchService.SearchResult result = results.get(i);
-            Map<String, String> metadata = parseMetadata(result.getMetadata());
-            // source 是后续去重/展示的主标识；当前实现偏“文档级”，同文档多 chunk 可能共享 source
-            String source = firstNonBlank(
-                    metadata.get("_source"),
-                    metadata.get("source"),
-                    metadata.get("filePath"),
-                    metadata.get("docId"),
-                    result.getMetadata(),
-                    result.getId()
-            );
+        List<RetrievedEvidenceCandidate> candidates = new ArrayList<>(hits.size());
+        for (KnowledgeSearchHit hit : hits) {
            candidates.add(RetrievedEvidenceCandidate.builder()
-                    .id(result.getId())
-                    .source(source)
-                    .title(metadata.get("title"))
-                    .breadcrumb(metadata.get("breadcrumb"))
-                    .content(result.getContent())
+                    .id(hit.id())
+                    .docId(hit.docId())
+                    .chunkIndex(hit.chunkIndex())
+                    .evidenceKey(hit.evidenceKey())
+                    .source(hit.source())
+                    .title(hit.title())
+                    .breadcrumb(hit.breadcrumb())
+                    .content(hit.content())
                    .retrievalLayer("L1")
                    .retrievalAttempt(attemptName)
-                    .score((double) result.getScore())
-                    .rawScore(result.getRawScore())
-                    .scoreLabel(result.getScoreLabel())
-                    .originalRank(i + 1)
-                    .metadata(metadata)
-                    .hitReasons(List.of("semantic_rank:" + (i + 1), "attempt:" + attemptName))
+                    .score(hit.score())
+                    .rawScore(hit.rawScore())
+                    .scoreLabel(hit.scoreLabel())
+                    .originalRank(hit.originalRank())
+                    .metadata(hit.metadata() == null ? java.util.Map.of() : hit.metadata())
+                    .hitReasons(List.of("semantic_rank:" + hit.originalRank(), "attempt:" + attemptName))
                    .build());
        }
        return candidates;
@@ -120,7 +96,6 @@ public class KnowledgeDocumentRetriever {
                .query(query)
                .categoryFilter(categoryFilter)
                .candidateCount(candidateCount)
-                // 此处 usable 只表示“有候选且无错误”；后处理还会用相似度阈值再收紧
                .usable(errorMessage == null && candidateCount > 0)
                .errorMessage(errorMessage)
                .durationMs(durationMs)
@@ -128,46 +103,15 @@ public class KnowledgeDocumentRetriever {
                .build();
    }

-    private Double topScore(List<VectorSearchService.SearchResult> results) {
-        if (results == null || results.isEmpty()) {
+    private Double topScore(List<KnowledgeSearchHit> hits) {
+        if (hits == null || hits.isEmpty() || hits.get(0).score() == null) {
            return null;
        }
-        return (double) results.get(0).getScore();
-    }
-
-    /** metadata 在向量库中多为 JSON 字符串，这里压成 string map 方便后处理读取。 */
-    private Map<String, String> parseMetadata(String metadata) {
-        if (metadata == null || metadata.isBlank()) {
-            return Map.of();
-        }
-        try {
-            Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
-            Map<String, String> parsed = new LinkedHashMap<>();
-            for (Map.Entry<?, ?> entry : raw.entrySet()) {
-                if (entry.getKey() != null && entry.getValue() != null) {
-                    parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
-                }
-            }
-            return parsed;
-        } catch (Exception ignored) {
-            return Map.of();
-        }
-    }
-
-    private String firstNonBlank(String... values) {
-        for (String value : values) {
-            if (value != null && !value.isBlank()) {
-                return value;
-            }
-        }
-        return null;
+        return hits.get(0).score();
    }

    /**
     * 单次检索 attempt 的结果包。
-     *
-     * @param attempt    可观测元数据（耗时、过滤条件、错误等）
-     * @param candidates 规范化后的证据候选
     */
    public record RetrievalAttemptResult(RetrievalTrace.Attempt attempt,
                                         List<RetrievedEvidenceCandidate> candidates) {
@@ -5,11 +5,13 @@ import com.superbiz.agent.dto.EvidencePostprocessResult;
 import com.superbiz.agent.dto.KnowledgeQuery;
 import com.superbiz.agent.dto.RerankTrace;
 import com.superbiz.agent.dto.RetrievedEvidenceCandidate;
+import com.superbiz.agent.service.retrieval.EvidenceIdentity;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;

 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -18,24 +20,18 @@ import java.util.Map;
 import java.util.Set;

 /**
- * 检索后处理：分数归一化、规则 rerank、证据块组装、相关等级判定。
+ * 检索后处理：分数归一化、规则 rerank、chunk 级证据组装、相关等级判定。
 *
 * <h3>处理步骤</h3>
 * <ol>
 *   <li>把候选 L2 距离归一成 0~1 的 baseScore</li>
- *   <li>用 L0 hint（domain/entity/keyword）做规则加分，得到 finalScore</li>
+ *   <li>用 L0 hint 做规则加分，得到 finalScore</li>
 *   <li>按 finalScore 降序排序</li>
- *   <li>按 sourceKey 去重后生成 {@link EvidenceBlock}</li>
+ *   <li>按 evidenceKey 去重</li>
+ *   <li>按 maxChunksPerDocument 截断同文档 chunk</li>
+ *   <li>按 return-n 截断最终 evidence 条数</li>
 *   <li>根据 top baseScore + hint 支撑计算 relevanceLevel</li>
 * </ol>
- *
- * <h3>重要行为（读代码时容易误解）</h3>
- * <ul>
- *   <li>“rerank” 是规则加权，不是 cross-encoder / LLM rerank</li>
- *   <li>去重 key 优先 source/title，粒度偏文档级：同文档多个 chunk 可能被压成一条，
- *       且 merge 时只合并 hitReasons，不拼接 content</li>
- *   <li>content 在此截断到 800 字符；Agent 侧 projector 还可能再截断</li>
- * </ul>
 */
@Service
 public class KnowledgeEvidencePostProcessor {
@@ -48,24 +44,28 @@ public class KnowledgeEvidencePostProcessor {
    private static final String HINT_HIGHLY_RELEVANT = "当前结果已高度相关，继续检索不太可能找到更精准的文档";
    private static final String HINT_REFERENCE = "当前结果为相关参考，如需更精准信息请明确缺少的具体维度";

-    /** L2 距离上限，用于把距离映射到 [0,1] 相似度。 */
    @Value("${retrieval.normalization.max-l2-distance:2.0}")
    private double maxL2Distance = 2.0;

-    /** baseScore &gt;= 该阈值，才可能判 HIGHLY_RELEVANT / PRECISE。 */
    @Value("${retrieval.normalization.highly-relevant-threshold:0.75}")
    private double highlyRelevantThreshold = 0.75;

-    /** baseScore &gt;= 该阈值视为可用参考；低于则 isLowQuality=true，可能触发 unfiltered retry。 */
    @Value("${retrieval.normalization.reference-threshold:0.5}")
    private double referenceThreshold = 0.5;

+    /** 同一 docId 最多保留的 chunk 数。 */
+    @Value("${rag.max-chunks-per-document:2}")
+    private int maxChunksPerDocument = 2;
+
    /**
-     * 对一次 attempt 的候选做后处理，产出可交给打包/组装的证据结果。
+     * 后处理后最多返回的 evidence 条数。
+     * 0 或负数表示不在此层截断（仍可能被 projector 预算截断）。
     */
+    @Value("${rag.return-n:5}")
+    private int returnN = 5;
+
    public EvidencePostprocessResult process(KnowledgeQuery query, List<RetrievedEvidenceCandidate> candidates) {
        List<RetrievedEvidenceCandidate> safeCandidates = candidates == null ? List.of() : candidates;
-        // 先打分排序：baseScore 来自向量距离，finalScore = base + 规则 boost
        List<ScoredCandidate> ranked = safeCandidates.stream()
                .map(candidate -> score(query, candidate))
                .sorted(Comparator.comparingDouble(ScoredCandidate::finalScore).reversed())
@@ -73,37 +73,44 @@ public class KnowledgeEvidencePostProcessor {

        Map<String, EvidenceBlock> deduped = new LinkedHashMap<>();
        List<RerankTrace.Item> traceItems = new ArrayList<>();
+        Map<String, Integer> chunksPerDocument = new HashMap<>();
        int finalRank = 1;
+        int accepted = 0;
+        int effectiveReturnN = returnN > 0 ? returnN : Integer.MAX_VALUE;
+        int effectiveMaxChunks = maxChunksPerDocument > 0 ? maxChunksPerDocument : Integer.MAX_VALUE;
+
        for (ScoredCandidate scored : ranked) {
-            RetrievedEvidenceCandidate candidate = scored.candidate();
-            EvidenceBlock block = EvidenceBlock.builder()
-                    .source(candidate.getSource())
-                    .title(candidate.getTitle())
-                    .breadcrumb(candidate.getBreadcrumb())
-                    .retrievalLayer(candidate.getRetrievalLayer())
-                    .content(truncate(candidate.getContent(), 800))
-                    .score(candidate.getScore())
-                    .hitReasons(mergeReasons(candidate.getHitReasons(), scored.boostReasons()))
-                    .build();
-            // 注意：key 未使用 chunkIndex，同 source 的多个 chunk 会走 merge 分支
-            String key = sourceKey(block, "candidate-" + candidate.getOriginalRank());
-            if (!deduped.containsKey(key)) {
-                deduped.put(key, block);
-                traceItems.add(RerankTrace.Item.builder()
-                        .finalRank(finalRank++)
-                        .source(candidate.getSource())
-                        .baseScore(scored.baseScore())
-                        .finalScore(scored.finalScore())
-                        .boostReasons(scored.boostReasons())
-                        .build());
-            } else {
-                // 重复 key：保留先放入的（更高分）content，只补充 reasons/breadcrumb
-                mergeEvidence(deduped.get(key), block);
+            if (accepted >= effectiveReturnN) {
+                break;
            }
+            RetrievedEvidenceCandidate candidate = scored.candidate();
+            String evidenceKey = resolveEvidenceKey(candidate);
+            String docBucket = resolveDocBucket(candidate, evidenceKey);
+
+            if (deduped.containsKey(evidenceKey)) {
+                mergeEvidence(deduped.get(evidenceKey), toBlock(candidate, evidenceKey, scored));
+                continue;
+            }
+
+            int used = chunksPerDocument.getOrDefault(docBucket, 0);
+            if (used >= effectiveMaxChunks) {
+                continue;
+            }
+
+            EvidenceBlock block = toBlock(candidate, evidenceKey, scored);
+            deduped.put(evidenceKey, block);
+            chunksPerDocument.put(docBucket, used + 1);
+            accepted++;
+            traceItems.add(RerankTrace.Item.builder()
+                    .finalRank(finalRank++)
+                    .source(candidate.getSource())
+                    .baseScore(scored.baseScore())
+                    .finalScore(scored.finalScore())
+                    .boostReasons(scored.boostReasons())
+                    .build());
        }

        List<EvidenceBlock> blocks = new ArrayList<>(deduped.values());
-        // topSimilarity 用排序后第一名的 baseScore（未含 boost），供质量阈值判断
        Double topSimilarity = ranked.isEmpty() ? null : ranked.get(0).baseScore();
        RelevanceAssessment assessment = computeRelevance(query, ranked);
        return EvidencePostprocessResult.builder()
@@ -117,10 +124,6 @@ public class KnowledgeEvidencePostProcessor {
                .build();
    }

-    /**
-     * 是否低质量，用于触发 filtered -> unfiltered 降级。
-     * 无可用证据，或 topSimilarity 低于 referenceThreshold，都视为低质量。
-     */
    public boolean isLowQuality(EvidencePostprocessResult result) {
        if (result == null || !result.hasUsableEvidence()) {
            return true;
@@ -129,10 +132,6 @@ public class KnowledgeEvidencePostProcessor {
        return topSimilarity == null || topSimilarity < referenceThreshold;
    }

-    /**
-     * L2 距离 -> 相似度。
-     * 距离越小越相似：similarity = 1 - min(l2, max) / max。
-     */
    public double normalizeL2(Double l2Score) {
        if (l2Score == null) {
            return 0.0;
@@ -145,10 +144,43 @@ public class KnowledgeEvidencePostProcessor {
        return referenceThreshold;
    }

-    /**
-     * 规则打分：baseScore + domain/entity/keyword/source_type boost。
-     * boost 只影响排序，不改变用于阈值判断的 baseScore。
-     */
+    private EvidenceBlock toBlock(RetrievedEvidenceCandidate candidate,
+                                  String evidenceKey,
+                                  ScoredCandidate scored) {
+        return EvidenceBlock.builder()
+                .docId(candidate.getDocId())
+                .chunkIndex(candidate.getChunkIndex())
+                .evidenceKey(evidenceKey)
+                .source(candidate.getSource())
+                .title(candidate.getTitle())
+                .breadcrumb(candidate.getBreadcrumb())
+                .retrievalLayer(candidate.getRetrievalLayer())
+                .content(truncate(candidate.getContent(), 800))
+                .score(candidate.getScore())
+                .hitReasons(mergeReasons(candidate.getHitReasons(), scored.boostReasons()))
+                .build();
+    }
+
+    private String resolveEvidenceKey(RetrievedEvidenceCandidate candidate) {
+        if (candidate.getEvidenceKey() != null && !candidate.getEvidenceKey().isBlank()) {
+            return candidate.getEvidenceKey();
+        }
+        return EvidenceIdentity.evidenceKey(
+                candidate.getDocId(),
+                candidate.getChunkIndex(),
+                candidate.getId(),
+                candidate.getOriginalRank());
+    }
+
+    private String resolveDocBucket(RetrievedEvidenceCandidate candidate, String evidenceKey) {
+        String docId = EvidenceIdentity.trimToNull(candidate.getDocId());
+        if (docId != null) {
+            return docId;
+        }
+        // No docId: do not collapse unrelated fallback keys under one bucket.
+        return evidenceKey;
+    }
+
    private ScoredCandidate score(KnowledgeQuery query, RetrievedEvidenceCandidate candidate) {
        double baseScore = normalizeL2(candidate.getScore());
        double finalScore = baseScore;
@@ -174,7 +206,6 @@ public class KnowledgeEvidencePostProcessor {
        return new ScoredCandidate(candidate, baseScore, finalScore, boosts);
    }

-    /** 在 source/title/breadcrumb/content/metadata 拼接串上做子串匹配（大小写不敏感）。 */
    private boolean matchesAny(RetrievedEvidenceCandidate candidate, List<String> hints) {
        if (hints == null || hints.isEmpty()) {
            return false;
@@ -194,7 +225,6 @@ public class KnowledgeEvidencePostProcessor {
        return false;
    }

-    /** runbook / guide / case 类来源轻微加分。 */
    private boolean isPreferredSourceType(RetrievedEvidenceCandidate candidate) {
        Map<String, String> metadata = candidate.getMetadata();
        if (metadata == null || metadata.isEmpty()) {
@@ -208,10 +238,6 @@ public class KnowledgeEvidencePostProcessor {
        return normalized.contains("runbook") || normalized.contains("guide") || normalized.contains("case");
    }

-    /**
-     * 相关等级只看 top1 的 baseScore（向量相似度），PRECISE 额外要求 L0 hint 有支撑。
-     * 低于 referenceThreshold 时 level/hint 都为 null，表示不可用参考。
-     */
    private RelevanceAssessment computeRelevance(KnowledgeQuery query, List<ScoredCandidate> ranked) {
        if (ranked.isEmpty()) {
            return new RelevanceAssessment(null, null);
@@ -235,9 +261,6 @@ public class KnowledgeEvidencePostProcessor {
                || matchesAny(top.candidate(), query.getMatchedKeywords());
    }

-    /**
-     * 同 key 合并策略：不覆盖已有 content（保留更高分的那条），只补 reasons 和空 breadcrumb。
-     */
    private void mergeEvidence(EvidenceBlock existing, EvidenceBlock incoming) {
        Set<String> reasons = new LinkedHashSet<>();
        if (existing.getHitReasons() != null) {
@@ -264,14 +287,6 @@ public class KnowledgeEvidencePostProcessor {
        return new ArrayList<>(merged);
    }

-    /**
-     * 当前去重 key：source -> title -> breadcrumb -> fallback。
-     * 因此“同文档不同 chunk”若 source 相同，会被视为重复。
-     */
-    private String sourceKey(EvidenceBlock block, String fallback) {
-        return firstNonBlank(block.getSource(), block.getTitle(), block.getBreadcrumb(), fallback);
-    }
-
    private String truncate(String text, int maxLength) {
        if (text == null || text.length() <= maxLength) {
            return text;
@@ -280,19 +295,13 @@ public class KnowledgeEvidencePostProcessor {
    }

    private String firstNonBlank(String... values) {
-        for (String value : values) {
-            if (value != null && !value.isBlank()) {
-                return value;
-            }
-        }
-        return null;
+        return EvidenceIdentity.firstNonBlank(values);
    }

    private String nullToEmpty(String value) {
        return value == null ? "" : value;
    }

-    /** 内部打分结果：baseScore 用于阈值，finalScore 用于排序。 */
    private record ScoredCandidate(RetrievedEvidenceCandidate candidate,
                                   double baseScore,
                                   double finalScore,
@@ -0,0 +1,76 @@
+package com.superbiz.agent.service.retrieval;
+
+import java.util.Map;
+
+/**
+ * Chunk-level evidence identity helpers shared by retrieval mapping and post-processing.
+ */
+public final class EvidenceIdentity {
+
+    private EvidenceIdentity() {
+    }
+
+    public static String evidenceKey(String docId, Integer chunkIndex, String vectorId, Integer originalRank) {
+        String normalizedDocId = trimToNull(docId);
+        if (normalizedDocId != null && chunkIndex != null) {
+            return normalizedDocId + "#chunk-" + chunkIndex;
+        }
+        String normalizedVectorId = trimToNull(vectorId);
+        if (normalizedVectorId != null) {
+            return "vector:" + normalizedVectorId;
+        }
+        int rank = originalRank == null ? 0 : originalRank;
+        return "rank:" + rank;
+    }
+
+    public static String extractDocId(Map<String, String> metadata, String... fallbacks) {
+        String fromMeta = firstNonBlank(
+                metadataValue(metadata, "docId"),
+                metadataValue(metadata, "doc_id"));
+        if (fromMeta != null) {
+            return fromMeta;
+        }
+        return firstNonBlank(fallbacks);
+    }
+
+    public static Integer extractChunkIndex(Map<String, String> metadata) {
+        String raw = firstNonBlank(
+                metadataValue(metadata, "chunkIndex"),
+                metadataValue(metadata, "chunk_index"));
+        if (raw == null) {
+            return null;
+        }
+        try {
+            return Integer.valueOf(raw.trim());
+        } catch (NumberFormatException ignored) {
+            return null;
+        }
+    }
+
+    public static String metadataValue(Map<String, String> metadata, String key) {
+        if (metadata == null || key == null) {
+            return null;
+        }
+        return trimToNull(metadata.get(key));
+    }
+
+    public static String firstNonBlank(String... values) {
+        if (values == null) {
+            return null;
+        }
+        for (String value : values) {
+            String trimmed = trimToNull(value);
+            if (trimmed != null) {
+                return trimmed;
+            }
+        }
+        return null;
+    }
+
+    public static String trimToNull(String value) {
+        if (value == null || value.isBlank()) {
+            return null;
+        }
+        return value.trim();
+    }
+}
@@ -0,0 +1,24 @@
+package com.superbiz.agent.service.retrieval;
+
+import java.util.Map;
+
+/**
+ * Normalized search hit returned by {@link KnowledgeSearchPort}.
+ */
+public record KnowledgeSearchHit(
+        String id,
+        String content,
+        Double score,
+        Double rawScore,
+        String scoreLabel,
+        String metadataJson,
+        Map<String, String> metadata,
+        String docId,
+        Integer chunkIndex,
+        String evidenceKey,
+        String source,
+        String title,
+        String breadcrumb,
+        int originalRank
+) {
+}
@@ -0,0 +1,10 @@
+package com.superbiz.agent.service.retrieval;
+
+/**
+ * Retrieval mode for {@link KnowledgeSearchPort}.
+ * Delivery 1 only requires {@link #DENSE}; hybrid arrives in a later change.
+ */
+public enum KnowledgeSearchMode {
+    DENSE,
+    HYBRID
+}
@@ -0,0 +1,12 @@
+package com.superbiz.agent.service.retrieval;
+
+import java.util.List;
+
+/**
+ * Application boundary for knowledge semantic search.
+ * Implementations may wrap VectorStore, hybrid engines, etc. without leaking SDK details upward.
+ */
+public interface KnowledgeSearchPort {
+
+    List<KnowledgeSearchHit> search(KnowledgeSearchRequest request);
+}
@@ -0,0 +1,23 @@
+package com.superbiz.agent.service.retrieval;
+
+/**
+ * Portable search request used by the knowledge pipeline.
+ */
+public record KnowledgeSearchRequest(
+        String query,
+        int topK,
+        String categoryFilter,
+        KnowledgeSearchMode mode
+) {
+    public KnowledgeSearchRequest {
+        if (topK <= 0) {
+            throw new IllegalArgumentException("topK must be positive");
+        }
+        mode = mode == null ? KnowledgeSearchMode.DENSE : mode;
+        query = query == null ? "" : query;
+    }
+
+    public static KnowledgeSearchRequest dense(String query, int topK, String categoryFilter) {
+        return new KnowledgeSearchRequest(query, topK, categoryFilter, KnowledgeSearchMode.DENSE);
+    }
+}
@@ -0,0 +1,94 @@
+package com.superbiz.agent.service.retrieval;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.superbiz.agent.service.VectorSearchService;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Dense-only {@link KnowledgeSearchPort} backed by the existing vector search facade.
+ */
+@Component
+public class VectorKnowledgeSearchAdapter implements KnowledgeSearchPort {
+
+    private final VectorSearchService vectorSearchService;
+    private final ObjectMapper objectMapper;
+
+    public VectorKnowledgeSearchAdapter(VectorSearchService vectorSearchService, ObjectMapper objectMapper) {
+        this.vectorSearchService = vectorSearchService;
+        this.objectMapper = objectMapper;
+    }
+
+    @Override
+    public List<KnowledgeSearchHit> search(KnowledgeSearchRequest request) {
+        if (request.mode() == KnowledgeSearchMode.HYBRID) {
+            // Delivery 2 will implement true hybrid. Until then, fall back to dense.
+        }
+        List<VectorSearchService.SearchResult> results = vectorSearchService.searchSimilarDocuments(
+                request.query(),
+                request.topK(),
+                request.categoryFilter());
+        if (results == null || results.isEmpty()) {
+            return List.of();
+        }
+        List<KnowledgeSearchHit> hits = new ArrayList<>(results.size());
+        for (int i = 0; i < results.size(); i++) {
+            hits.add(toHit(results.get(i), i + 1));
+        }
+        return hits;
+    }
+
+    private KnowledgeSearchHit toHit(VectorSearchService.SearchResult result, int originalRank) {
+        Map<String, String> metadata = parseMetadata(result.getMetadata());
+        String docId = EvidenceIdentity.extractDocId(
+                metadata,
+                EvidenceIdentity.metadataValue(metadata, "_source"),
+                EvidenceIdentity.metadataValue(metadata, "source"));
+        Integer chunkIndex = EvidenceIdentity.extractChunkIndex(metadata);
+        String evidenceKey = EvidenceIdentity.evidenceKey(docId, chunkIndex, result.getId(), originalRank);
+        String source = EvidenceIdentity.firstNonBlank(
+                EvidenceIdentity.metadataValue(metadata, "_source"),
+                EvidenceIdentity.metadataValue(metadata, "source"),
+                EvidenceIdentity.metadataValue(metadata, "filePath"),
+                docId,
+                result.getId());
+        return new KnowledgeSearchHit(
+                result.getId(),
+                result.getContent(),
+                (double) result.getScore(),
+                result.getRawScore(),
+                result.getScoreLabel(),
+                result.getMetadata(),
+                metadata,
+                docId,
+                chunkIndex,
+                evidenceKey,
+                source,
+                EvidenceIdentity.metadataValue(metadata, "title"),
+                EvidenceIdentity.metadataValue(metadata, "breadcrumb"),
+                originalRank
+        );
+    }
+
+    private Map<String, String> parseMetadata(String metadata) {
+        if (metadata == null || metadata.isBlank()) {
+            return Map.of();
+        }
+        try {
+            Map<?, ?> raw = objectMapper.readValue(metadata, Map.class);
+            Map<String, String> parsed = new LinkedHashMap<>();
+            for (Map.Entry<?, ?> entry : raw.entrySet()) {
+                if (entry.getKey() != null && entry.getValue() != null) {
+                    parsed.put(String.valueOf(entry.getKey()), String.valueOf(entry.getValue()));
+                }
+            }
+            return parsed;
+        } catch (Exception ignored) {
+            return Map.of();
+        }
+    }
+}
@@ -10,6 +10,7 @@ import com.superbiz.agent.service.KnowledgeDocumentRetriever;
 import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
 import com.superbiz.agent.service.KnowledgeQueryTransformer;
 import com.superbiz.agent.service.LookupResultAssembler;
+import jakarta.annotation.PostConstruct;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -29,39 +30,37 @@ import java.util.Map;
 * <h3>主链路</h3>
 * <pre>
 * query
- *   -> KnowledgeQueryTransformer      // L0：domain/keyword hint，可选 category filter
- *   -> KnowledgeDocumentRetriever     // L1：向量召回 topK
- *   -> KnowledgeEvidencePostProcessor // 归一化、规则 rerank、组装 evidenceBlocks
- *   -> [可选] 去掉 category 后重试      // filtered 结果质量不足时
- *   -> KnowledgeContextPacker         // 按字符预算打包文本
- *   -> LookupResultAssembler          // 统一 LookupResult
+ *   -> KnowledgeQueryTransformer
+ *   -> KnowledgeDocumentRetriever (via KnowledgeSearchPort, retrieve-k)
+ *   -> KnowledgeEvidencePostProcessor (chunk dedup / caps / return-n)
+ *   -> [optional] unfiltered retry
+ *   -> KnowledgeContextPacker
+ *   -> LookupResultAssembler
 * </pre>
- *
- * <h3>降级策略</h3>
- * <ul>
- *   <li>有 categoryFilter：先 FILTERED_VECTOR</li>
- *   <li>若结果无证据或 topSimilarity &lt; referenceThreshold：再 UNFILTERED_VECTOR_RETRY</li>
- *   <li>无 categoryFilter：直接 UNFILTERED_VECTOR</li>
- * </ul>
- *
- * <p>注意：本类返回的是内部 {@link LookupResult}，不是 Agent 最终看到的 JSON。
- * 会话级文档去重目前不在这里做。</p>
 */
@Slf4j
@Component
 public class LookupKnowledgeTool {

-    /** 首次检索：带 L0 推导出的 category 过滤。 */
    private static final String ATTEMPT_FILTERED_VECTOR = "FILTERED_VECTOR";
-    /** 首次检索：L0 未给出唯一 domain，不做 category 过滤。 */
    private static final String ATTEMPT_UNFILTERED_VECTOR = "UNFILTERED_VECTOR";
-    /** 降级重试：去掉 category 过滤，用原始 query 再搜一次。 */
    private static final String ATTEMPT_UNFILTERED_VECTOR_RETRY = "UNFILTERED_VECTOR_RETRY";
    private static final String FALLBACK_NO_EVIDENCE = "filtered_vector_no_evidence";
    private static final String FALLBACK_LOW_QUALITY = "filtered_vector_low_quality";

-    @Value("${rag.top-k:3}")
-    private int topK = 3;
+    /**
+     * Legacy single knob. Used only when retrieve-k / return-n are absent.
+     */
+    @Value("${rag.top-k:0}")
+    private int legacyTopK = 0;
+
+    @Value("${rag.retrieve-k:0}")
+    private int retrieveKConfig = 0;
+
+    @Value("${rag.return-n:0}")
+    private int returnNConfig = 0;
+
+    private int retrieveK = 20;

    @Autowired
    private KnowledgeQueryTransformer queryTransformer;
@@ -78,19 +77,24 @@ public class LookupKnowledgeTool {
    @Autowired
    private LookupResultAssembler resultAssembler;

-    /**
-     * 执行一次知识库检索，返回 evidence-first 的内部结果。
-     *
-     * @param query Agent / Harness 传入的检索语句（不是最终用户原话的完整上下文）
-     * @return 含 evidenceBlocks、contextPack、retrievalTrace 的 LookupResult
-     */
+    @PostConstruct
+    void resolveRetrievalWidths() {
+        int fallback = legacyTopK > 0 ? legacyTopK : 3;
+        this.retrieveK = retrieveKConfig > 0 ? retrieveKConfig : fallback;
+        // return-n is owned by post-processor config; keep field for observability only.
+        if (returnNConfig <= 0 && legacyTopK > 0) {
+            log.info("rag.return-n not set; post-processor will use its own default or rag.return-n binding");
+        }
+        log.info("lookup_knowledge widths: retrieveK={}, legacyTopK={}, returnNConfig={}",
+                retrieveK, legacyTopK, returnNConfig);
+    }
+
    public LookupResult lookupKnowledge(String query) {
        log.info("========================================");
        log.info(">>> [工具调用] lookup_knowledge");
-        log.info(">>> metadata: query_chars={}", query == null ? 0 : query.length());
+        log.info(">>> metadata: query_chars={}, retrieveK={}", query == null ? 0 : query.length(), retrieveK);
        log.info("----------------------------------------");

-        // 1) Query understanding：L0 只产 hint/filter，不直接当事实证据
        KnowledgeQuery knowledgeQuery = queryTransformer.transform(query);
        log.info("[QueryTransformer] categoryFilter={}, domainHintCount={}, keywordCount={}",
                knowledgeQuery.getCategoryFilter(),
@@ -100,7 +104,6 @@ public class LookupKnowledgeTool {
        List<RetrievalTrace.Attempt> attempts = new ArrayList<>();
        String fallbackReason = null;

-        // 2) 首次 L1 向量检索（有唯一 domain 则带 category filter）
        String firstAttemptName = knowledgeQuery.getCategoryFilter() == null
                ? ATTEMPT_UNFILTERED_VECTOR
                : ATTEMPT_FILTERED_VECTOR;
@@ -108,7 +111,7 @@ public class LookupKnowledgeTool {
                documentRetriever.retrieve(firstAttemptName,
                        knowledgeQuery.getRewrittenQuery(),
                        knowledgeQuery.getCategoryFilter(),
-                        topK);
+                        retrieveK);
        EvidencePostprocessResult selectedEvidence = evidencePostProcessor.process(
                knowledgeQuery,
                firstAttempt.candidates());
@@ -116,8 +119,6 @@ public class LookupKnowledgeTool {
        attempts.add(firstAttempt.attempt());
        String selectedAttemptName = firstAttemptName;

-        // 3) filtered 路径质量不足时，去掉 category 用原始 query 重试一次
-        //    重试结果会整体替换首次结果（不是与首次融合）
        if (knowledgeQuery.getCategoryFilter() != null && evidencePostProcessor.isLowQuality(selectedEvidence)) {
            fallbackReason = selectedEvidence.hasUsableEvidence()
                    ? FALLBACK_LOW_QUALITY
@@ -128,7 +129,7 @@ public class LookupKnowledgeTool {
                    documentRetriever.retrieve(ATTEMPT_UNFILTERED_VECTOR_RETRY,
                            knowledgeQuery.getOriginalQuery(),
                            null,
-                            topK);
+                            retrieveK);
            EvidencePostprocessResult retryEvidence = evidencePostProcessor.process(
                    knowledgeQuery,
                    retryAttempt.candidates());
@@ -138,7 +139,6 @@ public class LookupKnowledgeTool {
            selectedAttemptName = ATTEMPT_UNFILTERED_VECTOR_RETRY;
        }

-        // 4) 打包 + 组装最终内部结果（供 projector / 审计消费）
        ContextPack contextPack = contextPacker.pack(selectedEvidence.getEvidenceBlocks());
        RetrievalTrace retrievalTrace = buildRetrievalTrace(knowledgeQuery, attempts, selectedAttemptName,
                fallbackReason, selectedEvidence);
@@ -148,10 +148,6 @@ public class LookupKnowledgeTool {
        return result;
    }

-    /**
-     * 用后处理后的相似度回填 attempt 可观测字段。
-     * usable 要求：有可用证据，且 topSimilarity 达到 reference 阈值。
-     */
    private void enrichAttempt(RetrievalTrace.Attempt attempt, EvidencePostprocessResult evidence) {
        attempt.setTopSimilarity(evidence.getTopSimilarity());
        attempt.setUsable(evidence.hasUsableEvidence()
@@ -159,7 +155,6 @@ public class LookupKnowledgeTool {
                && evidence.getTopSimilarity() >= evidencePostProcessor.getReferenceThreshold());
    }

-    /** 汇总本次检索的 query hint、attempt 列表与最终选用路径，便于 trace 回放。 */
    private RetrievalTrace buildRetrievalTrace(KnowledgeQuery query,
                                               List<RetrievalTrace.Attempt> attempts,
                                               String selectedAttempt,