feat(rag): chunk evidence identity, dedup, and search port

Preserve same-document multi-chunk evidence with evidenceKey identity,
per-document caps, retrieve-k/return-n split, and a dense KnowledgeSearchPort.
Archives Delivery 1 OpenSpec change as the foundation for hybrid retrieval.

src/main/java/com/superbiz/agent/harness/tool/projection/RagResultProjector.java

@@ -18,27 +18,9 @@ import java.util.Set;
 /**
  * 把 legacy {@code LookupResult} JSON 投影成冻结的 Agent 可见 RAG 契约。
  *
- * <h3>为什么需要投影</h3>
- * 内部检索结果字段较多（retrievalTrace、rerankTrace、score、hitReasons、contextPack…），
- * Agent / EvidenceGuard 只应看到受控、有界、可引用的子集。
- *
- * <h3>保留给 Agent 的字段</h3>
- * <ul>
- *   <li>evidenceStatus / toolCallId / query</li>
- *   <li>evidence[]：document_id, source, title, breadcrumb, excerpt</li>
- *   <li>relevanceLevel、truncated、returned_count</li>
- * </ul>
- *
- * <h3>刻意丢弃</h3>
- * score、hitReasons、retrievalTrace、rerankTrace、contextPack 等内部可观测细节。
- *
- * <h3>去重与预算（读代码关键）</h3>
- * <ul>
- *   <li>按 {@code document_id} 去重；若 block 无 document_id，则回退 source/title</li>
- *   <li>因此同 source 的多个 chunk 在此也会被压成 1 条（与后处理文档级去重叠加）</li>
- *   <li>条数上限 {@link ToolProjectionLimits#maxEvidence()}，excerpt 字符上限，
- *       以及总 UTF-8 字节预算（超限从后往前删 evidence）</li>
- * </ul>
+ * <h3>去重</h3>
+ * 按 chunk 级证据身份去重（优先 evidenceKey / document_id），
+ * <b>不再</b>因为 source 相同就丢弃第二条 chunk。
  */
 public final class RagResultProjector {
 
@@ -50,11 +32,6 @@ public final class RagResultProjector {
         this.limits = limits;
     }
 
-    /**
-     * @param request     Agent 原始请求（用于回填/截断 query）
-     * @param toolCallId  框架分配的本次工具调用 id，进入结果供证据引用
-     * @param rawResponse legacy LookupResult 的 JSON 字符串
-     */
     public ProjectedToolResult project(RagToolRequest request, String toolCallId,
                                        String rawResponse) throws Exception {
         if (request == null || toolCallId == null || toolCallId.isBlank()) {
@@ -69,8 +46,7 @@ public final class RagResultProjector {
         String query = bounded(request.query(), limits.maxQueryChars());
         truncated = !query.equals(request.query());
         List<RagEvidence> evidence = new ArrayList<>();
-        // 文档级唯一集合：相同 documentId 只保留首次出现
-        Set<String> documentIds = new HashSet<>();
+        Set<String> evidenceIds = new HashSet<>();
         JsonNode blocks = root.has("evidenceBlocks") ? root.get("evidenceBlocks") : root.get("evidence_blocks");
         if (blocks != null && blocks.isArray()) {
             int ordinal = 0;
@@ -89,11 +65,16 @@ public final class RagResultProjector {
                 }
                 String source = text(block, "source");
                 String title = text(block, "title");
-                // legacy EvidenceBlock 通常没有 document_id，实际常退化为 source
-                String documentId = firstNonBlank(text(block, "document_id"), source, title,
-                        "legacy-document-" + ordinal);
-                if (!documentIds.add(documentId)) {
-                    // 同 documentId 重复：丢弃后续条，并标记 truncated
+                // Chunk-scoped identity first; do not collapse on source alone.
+                String documentId = firstPresent(
+                        text(block, "evidenceKey"),
+                        text(block, "evidence_key"),
+                        text(block, "document_id"),
+                        text(block, "documentId"),
+                        composeChunkId(text(block, "docId"), text(block, "doc_id"),
+                                text(block, "chunkIndex"), text(block, "chunk_index")),
+                        "legacy-evidence-" + ordinal);
+                if (!evidenceIds.add(documentId)) {
                     truncated = true;
                     continue;
                 }
@@ -118,12 +99,36 @@ public final class RagResultProjector {
                 ? relevanceLevel(root) : null;
         RagToolResult result = new RagToolResult(
                 status, toolCallId, query, evidence, evidence.size(), relevanceLevel, truncated);
-        // 总字节预算：仍超限则从尾部删 evidence，直到放得下或变 no_evidence
         result = fitBudget(result, truncated);
         return new ProjectedToolResult(objectMapper.writeValueAsString(result), result.evidenceStatus());
     }
 
-    /** 按 maxAgentUtf8Bytes 从后往前删 evidence，保证 Agent 侧 payload 有界。 */
+    private static String composeChunkId(String docId, String docIdAlt, String chunkIndex, String chunkIndexAlt) {
+        String id = firstPresentOrNull(docId, docIdAlt);
+        String idx = firstPresentOrNull(chunkIndex, chunkIndexAlt);
+        if (id == null || idx == null) {
+            return null;
+        }
+        return id + "#chunk-" + idx;
+    }
+
+    private static String firstPresent(String... values) {
+        String found = firstPresentOrNull(values);
+        return found == null ? "unknown-document" : found;
+    }
+
+    private static String firstPresentOrNull(String... values) {
+        if (values == null) {
+            return null;
+        }
+        for (String value : values) {
+            if (value != null && !value.isBlank()) {
+                return value;
+            }
+        }
+        return null;
+    }
+
     private RagToolResult fitBudget(RagToolResult result, boolean truncated) throws Exception {
         RagToolResult current = result;
         while (bytes(objectMapper.writeValueAsString(current)) > limits.maxAgentUtf8Bytes()
@@ -146,15 +151,6 @@ public final class RagResultProjector {
         return value == null || value.isNull() ? "" : value.asText("");
     }
 
-    private static String firstNonBlank(String... values) {
-        for (String value : values) {
-            if (value != null && !value.isBlank()) {
-                return value;
-            }
-        }
-        return "unknown-document";
-    }
-
     private static String nullable(String value) {
         return value == null || value.isBlank() ? null : value;
     }