osiman/SuperBizAgent-java
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): add executor evidence v2 contract

Browse Source
This commit is contained in:
aruo
2026-07-08 01:37:15 +08:00
parent a6afbfaa9d
commit 050cbc8fee
21 changed files with 2707 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/superbiz/agent/service/ChatService.java
+98 -2
View File
@@ -444,8 +444,12 @@ public class ChatService {
}
if ("PASS".equals(finalDecision.verdict())) {
answer = extractUserFacingAnswer(answer)
.orElse(answer == null || answer.isBlank() ? "抱歉，多 Agent 分析未能生成有效结论。" : answer);
String executorAnswer = answer;
answer = extractUserFacingAnswer(executorAnswer)
.or(() -> renderStructuredExecutorAnswer(executorAnswer))
.orElse(executorAnswer == null || executorAnswer.isBlank()
? "抱歉，多 Agent 分析未能生成有效结论。"
: executorAnswer);
persistVerifierEvaluation(session, finalDecision, round);
break;
}
@@ -816,6 +820,98 @@ public class ChatService {
return Optional.empty();
}
private Optional<String> renderStructuredExecutorAnswer(String executorAnswer) {
if (executorAnswer == null || executorAnswer.isBlank()) {
return Optional.empty();
}
try {
JsonNode root = objectMapper.readTree(sanitizeJsonPayload(executorAnswer));
if (!"executor_evidence_v2".equals(root.path("answer_version").asText(""))) {
return Optional.empty();
}
StringBuilder output = new StringBuilder();
appendTextArraySection(output, "已确认信息", root.path("claims"), "claim_text", "暂无可稳定确认的信息");
appendHypothesesSection(output, root.path("hypotheses"));
appendStringArraySection(output, "当前缺口", root.path("missing_info"), "当前缺少足够的直接证据支撑完整结论");
appendRecommendedActionsSection(output, root.path("recommended_actions"));
String rendered = output.toString().trim();
return rendered.isBlank() ? Optional.empty() : Optional.of(rendered);
} catch (Exception e) {
logger.debug("Failed to render executor_evidence_v2 answer", e);
return Optional.empty();
}
}
private void appendTextArraySection(StringBuilder output, String title, JsonNode items,
String fieldName, String emptyText) {
output.append(title).append("：");
if (!items.isArray() || items.isEmpty()) {
output.append("\n- ").append(emptyText);
return;
}
for (JsonNode item : items) {
String text = item.path(fieldName).asText("");
if (!text.isBlank()) {
output.append("\n- ").append(text);
}
}
if (output.charAt(output.length() - 1) == '：') {
output.append("\n- ").append(emptyText);
}
}
private void appendHypothesesSection(StringBuilder output, JsonNode hypotheses) {
if (!hypotheses.isArray() || hypotheses.isEmpty()) {
return;
}
output.append("\n\n可能方向：");
for (JsonNode hypothesis : hypotheses) {
String text = hypothesis.path("hypothesis_text").asText("");
if (text.isBlank()) {
continue;
}
String basis = hypothesis.path("basis").asText("");
output.append("\n- ").append(text);
if (!basis.isBlank()) {
output.append("（").append(basis).append("）");
}
}
}
private void appendStringArraySection(StringBuilder output, String title, JsonNode items, String emptyText) {
output.append("\n\n").append(title).append("：");
if (!items.isArray() || items.isEmpty()) {
output.append("\n- ").append(emptyText);
return;
}
for (JsonNode item : items) {
String text = item.asText("");
if (!text.isBlank()) {
output.append("\n- ").append(text);
}
}
}
private void appendRecommendedActionsSection(StringBuilder output, JsonNode actions) {
output.append("\n\n建议下一步：");
if (!actions.isArray() || actions.isEmpty()) {
output.append("\n- 围绕上述证据缺口补充只读查询，再由人工复核最终结论");
return;
}
for (JsonNode action : actions) {
String text = action.path("action_text").asText("");
if (text.isBlank()) {
continue;
}
String reason = action.path("reason").asText("");
output.append("\n- ").append(text);
if (!reason.isBlank()) {
output.append("：").append(reason);
}
}
}
private String buildDegradedOutput(VerifierDecision decision) {
StringBuilder output = new StringBuilder(DEGRADED_PREFIX);
src/main/resources/prompts/chat-executor-prompt.md
+5 -6
View File
@@ -51,7 +51,6 @@
支持等级：
- `direct`：工具返回中有直接事实。
- `indirect`：工具返回可支撑方向，但没有直接陈述完整结论。
- `none`：不能放入 `claims`，应放入 `hypotheses`、`recommended_actions` 或 `missing_info`。
### hypotheses
`hypotheses` 用来放合理怀疑但未被工具证实的方向。
@@ -71,8 +70,7 @@
```json
{
"answer_version": "executor_evidence_v1",
"diagnosis_summary": "1-2句话总结，仅包含有证据支撑的事实和证据边界",
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
@@ -106,15 +104,16 @@
],
"missing_info": [
"导致无法确认完整根因的证据缺口"
],
"user_facing_answer": "面向用户的中文回答。必须与 claims/hypotheses/recommended_actions/missing_info 一致，不得额外加入未绑定证据的确认式事实。"
]
}
```
## 输出校验
- `answer_version` 必须是 `executor_evidence_v2`。
- 不得输出 `diagnosis_summary`。
- 不得输出 `user_facing_answer`。
- `claims[*].support_level` 只能是 `direct` 或 `indirect`。
- `claims[*].evidence_bindings` 不能为空。
- `evidence_excerpt` 必须来自工具返回，不允许编造。
- 如果没有任何可确认事实，`claims` 返回空数组，并在 `missing_info` 说明缺少什么。
- `user_facing_answer` 不得出现 `claims` 中没有、且又被写成确认结论的事实。
- 不要把其它服务、其它历史案例、其它会话的事实迁移为当前会话事实。
src/main/resources/prompts/chat-verifier-prompt.md
+2 -2
View File
@@ -10,7 +10,7 @@
- `original_query`：用户原始问题
- `executor_final_answer`：本轮 Executor 最终答案
- `executor_structured_output`：如果 Executor 输出了合法证据归因 JSON，这里会提供解析后的对象。结构包含 `claims`、`hypotheses`、`recommended_actions`、`missing_info`、`user_facing_answer`
- `executor_structured_output`：如果 Executor 输出了合法证据归因 JSON，这里会提供解析后的对象。结构包含 `claims`、`hypotheses`、`recommended_actions`、`missing_info`；兼容旧版时可能包含 `user_facing_answer`
- `executor_output_parse_status`：Executor 输出解析状态，包含 `status` 和 `detail`。`status` 可能是 `valid` / `missing` / `malformed`
- `tool_trace_summary`：基于真实工具调用整理出的证据索引。每一项都带有：
- `trace_ref`
@@ -31,7 +31,7 @@
- 必须检查 claim 的 `evidence_bindings` 是否能对应到 `tool_trace_summary` 中真实存在的 trace、tool 或 source_invocation_ids
- 如果 claim 声称 direct/indirect 支撑，但 evidence binding 不存在、无法定位、或 excerpt 与工具摘要不匹配，不得判为 `direct_evidence`
然后必须扫描 `executor_structured_output.user_facing_answer`：
如果 `executor_structured_output.user_facing_answer` 存在，则必须扫描它：
- 如果其中出现 confirmed-sounding facts（确认式事实、根因、指标值、错误码、服务名、修复结论）
- 且这些事实没有出现在 `executor_structured_output.claims`
- 必须额外加入 `facts_checked` 并按工具证据校验
src/test/java/com/superbiz/agent/hook/VerifierInputHookTest.java
+49
View File
@@ -84,6 +84,55 @@ class VerifierInputHookTest {
assertEquals("valid", VerifierContextHolder.getExecutorOutputParseStatus().get("status"));
}
@Test
void beforeModelAddsStructuredExecutorOutputWhenV2ContractHasNoUserFacingAnswer() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
when(traceSummaryService.buildVerifierTraceSummary(anyString(), anyString())).thenReturn(List.of(
Map.of("trace_ref", "trace-1", "tool_name", "query_metrics")
));
VerifierInputHook hook = new VerifierInputHook(traceSummaryService);
VerifierContextHolder.setOriginalQuery("分析 MySQL 连接池耗尽");
String executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [],
"recommended_actions": [],
"missing_info": []
}
""";
AgentCommand command = hook.beforeModel(
List.of(new AssistantMessage(executorOutput)),
RunnableConfig.builder().addMetadata("sessionId", "structured-v2-session").build()
);
JsonNode payload = readPayload(command);
assertEquals("valid", payload.path("executor_output_parse_status").path("status").asText());
assertEquals("executor_evidence_v2",
payload.path("executor_structured_output").path("answer_version").asText());
assertFalse(payload.path("executor_structured_output").has("user_facing_answer"));
assertEquals("连接池 active 达到上限",
payload.path("executor_structured_output").path("claims").get(0).path("claim_text").asText());
}
@Test
void beforeModelExtractsStructuredOutputFromPrefixedJsonFence() throws Exception {
ToolTraceSummaryService traceSummaryService = mock(ToolTraceSummaryService.class);
src/test/java/com/superbiz/agent/service/ChatServiceSequentialAgentTest.java
+57
View File
@@ -273,6 +273,63 @@ class ChatServiceSequentialAgentTest {
assertTrue(chatModel.verifierPromptText.contains("连接池 active 达到上限"));
}
@Test
void executeChatComplexRendersExecutorEvidenceV2InsteadOfRawJsonOnPass() throws Exception {
ChatService chatService = createChatService();
ScriptedChatModel chatModel = new ScriptedChatModel();
chatModel.executorOutput = """
{
"answer_version": "executor_evidence_v2",
"claims": [
{
"claim_id": "claim-1",
"claim_type": "symptom",
"claim_text": "连接池 active 达到上限",
"support_level": "direct",
"evidence_bindings": [
{
"source_type": "tool_trace",
"source_id": "trace-1",
"tool_name": "query_metrics",
"source_invocation_ids": [101],
"evidence_excerpt": "active=50 max=50"
}
]
}
],
"hypotheses": [
{
"hypothesis_text": "连接泄漏可能参与了连接池耗尽",
"basis": "已有连接池满载证据，但缺少泄漏检测日志",
"needed_evidence": ["连接泄漏检测日志"]
}
],
"recommended_actions": [
{
"action_text": "补充查询连接池泄漏检测日志",
"reason": "用于确认是否存在连接未释放"
}
],
"missing_info": ["缺少连接泄漏检测日志"]
}
""";
ChatService.ChatResult result = chatService.executeChatComplex(
chatModel,
new ToolCallback[0],
"请分析 MySQL 连接池耗尽",
List.of(),
"sequential-v2-render-session"
);
assertTrue(result.answer().contains("已确认信息"));
assertTrue(result.answer().contains("连接池 active 达到上限"));
assertTrue(result.answer().contains("可能方向"));
assertTrue(result.answer().contains("建议下一步"));
assertFalse(result.answer().contains("\"answer_version\""));
assertFalse(result.answer().contains("executor_evidence_v2"));
}
@Test
void buildMethodToolsArrayIncludesLogsAndMetricsWhenAvailable() {
ChatService chatService = new ChatService();