feat: add aiops lightweight verifier

2026-07-05 13:44:30 +08:00
parent 2658742119
commit ed267d753d
15 changed files with 417 additions and 5 deletions
@@ -248,7 +248,7 @@ public class ChatController {
                if (finalReportOptional.isPresent()) {
                    String finalReportText = finalReportOptional.get();
                    logger.info("提取到 Planner 最终报告，长度: {}", finalReportText.length());
-                    aiOpsService.persistFinalReport(sessionId, finalReportText);
+                    aiOpsService.persistFinalReport(sessionId, finalReportText, request);
                    
                    // 发送分隔线
                    emitter.send(SseEmitter.event().name("message")
@@ -97,6 +97,7 @@ public class DiagnosisTraceResponse {
        private int persistedToolCallCount;
        private int returnedToolCallCount;
        private boolean hasVerifierEvaluation;
+        private boolean hasAiOpsRuleEvaluation;
        private boolean hasFeedback;
    }
 }
@@ -0,0 +1,146 @@
+package com.superbiz.agent.service;
+
+import com.superbiz.agent.domain.entity.ToolInvocation;
+import com.superbiz.agent.dto.AIOpsRequest;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+@Service
+public class AiOpsRuleEvaluationService {
+
+    public static final String PASS = "PASS";
+    public static final String WARN = "WARN";
+    public static final String FAIL = "FAIL";
+
+    public Map<String, Object> evaluate(AIOpsRequest request,
+                                        String finalReport,
+                                        List<ToolInvocation> toolInvocations) {
+        List<Map<String, Object>> checks = new ArrayList<>();
+        checks.add(checkReportPresent(finalReport));
+        checks.add(checkPayloadFocus(request, finalReport));
+        checks.add(checkEvidenceCoverage(toolInvocations));
+
+        String verdict = aggregateVerdict(checks);
+        Map<String, Object> evaluation = new LinkedHashMap<>();
+        evaluation.put("verdict", verdict);
+        evaluation.put("checks", checks);
+        evaluation.put("rationale", buildRationale(verdict, checks));
+        evaluation.put("traceability_version", "aiops-rule-v1");
+        return evaluation;
+    }
+
+    private Map<String, Object> checkReportPresent(String finalReport) {
+        boolean passed = finalReport != null && finalReport.trim().length() >= 40;
+        return check(
+                "final_report_present",
+                passed ? PASS : FAIL,
+                passed ? "Final report is present." : "Final report is missing or too short."
+        );
+    }
+
+    private Map<String, Object> checkPayloadFocus(AIOpsRequest request, String finalReport) {
+        if (!hasAlertPayload(request)) {
+            return check("payload_focus", PASS, "No alert payload was supplied; payload focus is not required.");
+        }
+
+        String report = lower(finalReport);
+        List<String> missing = new ArrayList<>();
+        if (!contains(report, request.getAlertName())) {
+            missing.add("alertName");
+        }
+        if (!contains(report, request.getService())) {
+            missing.add("service");
+        }
+
+        if (missing.isEmpty()) {
+            return check("payload_focus", PASS, "Final report mentions the supplied alert and service.");
+        }
+        return check(
+                "payload_focus",
+                WARN,
+                "Final report is missing payload focus terms: " + String.join(", ", missing)
+        );
+    }
+
+    private Map<String, Object> checkEvidenceCoverage(List<ToolInvocation> toolInvocations) {
+        List<String> evidenceTools = safeTools(toolInvocations).stream()
+                .filter(tool -> tool.equals("lookup_knowledge")
+                        || tool.equals("query_metrics")
+                        || tool.equals("query_logs"))
+                .distinct()
+                .toList();
+
+        if (evidenceTools.isEmpty()) {
+            return check("evidence_tool_coverage", WARN, "No persisted AIOps evidence tool calls were found.");
+        }
+        return check(
+                "evidence_tool_coverage",
+                PASS,
+                "Persisted evidence tools: " + String.join(", ", evidenceTools)
+        );
+    }
+
+    private List<String> safeTools(List<ToolInvocation> toolInvocations) {
+        if (toolInvocations == null) {
+            return List.of();
+        }
+        return toolInvocations.stream()
+                .map(ToolInvocation::getToolName)
+                .filter(name -> name != null && !name.isBlank())
+                .map(name -> name.trim().toLowerCase(Locale.ROOT))
+                .toList();
+    }
+
+    private String aggregateVerdict(List<Map<String, Object>> checks) {
+        boolean hasFail = checks.stream().anyMatch(check -> FAIL.equals(check.get("verdict")));
+        if (hasFail) {
+            return FAIL;
+        }
+        boolean hasWarn = checks.stream().anyMatch(check -> WARN.equals(check.get("verdict")));
+        return hasWarn ? WARN : PASS;
+    }
+
+    private String buildRationale(String verdict, List<Map<String, Object>> checks) {
+        long passCount = checks.stream().filter(check -> PASS.equals(check.get("verdict"))).count();
+        long warnCount = checks.stream().filter(check -> WARN.equals(check.get("verdict"))).count();
+        long failCount = checks.stream().filter(check -> FAIL.equals(check.get("verdict"))).count();
+        return "AIOps rule evaluation %s: pass=%d, warn=%d, fail=%d"
+                .formatted(verdict, passCount, warnCount, failCount);
+    }
+
+    private Map<String, Object> check(String name, String verdict, String detail) {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("name", name);
+        result.put("verdict", verdict);
+        result.put("detail", detail);
+        return result;
+    }
+
+    private boolean hasAlertPayload(AIOpsRequest request) {
+        if (request == null) {
+            return false;
+        }
+        return !isBlank(request.getAlertName())
+                || !isBlank(request.getService())
+                || !isBlank(request.getSeverity())
+                || !isBlank(request.getDescription())
+                || !isBlank(request.getTimeRange());
+    }
+
+    private boolean contains(String lowerText, String value) {
+        return isBlank(value) || lowerText.contains(value.trim().toLowerCase(Locale.ROOT));
+    }
+
+    private String lower(String value) {
+        return value == null ? "" : value.toLowerCase(Locale.ROOT);
+    }
+
+    private boolean isBlank(String value) {
+        return value == null || value.trim().isEmpty();
+    }
+}
@@ -27,6 +27,7 @@ import com.superbiz.agent.config.AiOpsPromptProperties;
 import com.superbiz.agent.tool.LookupKnowledgeTool;

 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;

@@ -66,6 +67,12 @@ public class AiOpsService {
    @Autowired
    private ToolInvocationRepository toolInvocationRepository;

+    @Autowired
+    private AiOpsRuleEvaluationService aiOpsRuleEvaluationService;
+
+    @Autowired
+    private SelfEvaluationMergeService selfEvaluationMergeService;
+
    /**
     * 执行 AI Ops 告警分析流程
     *
@@ -170,11 +177,20 @@ public class AiOpsService {
    }

    public void persistFinalReport(String sessionId, String finalReport) {
+        persistFinalReport(sessionId, finalReport, null);
+    }
+
+    public void persistFinalReport(String sessionId, String finalReport, AIOpsRequest request) {
        if (isBlank(sessionId) || isBlank(finalReport)) {
            return;
        }
        diagnosisSessionRepository.findBySessionId(sessionId.trim()).ifPresent(session -> {
            session.setAnswer(finalReport);
+            List<com.superbiz.agent.domain.entity.ToolInvocation> invocations =
+                    toolInvocationRepository.findBySessionIdOrderByIdAsc(session.getSessionId());
+            Map<String, Object> evaluation = aiOpsRuleEvaluationService.evaluate(request, finalReport, invocations);
+            session.setSelfEvaluation(selfEvaluationMergeService.mergeAiOpsRuleEvaluation(
+                    session.getSelfEvaluation(), evaluation));
            diagnosisSessionRepository.save(session);
        });
    }
@@ -115,6 +115,7 @@ public class DiagnosisTraceService {
                .persistedToolCallCount(defaultInt(session.getToolCallCount()))
                .returnedToolCallCount(toolInvocations.size())
                .hasVerifierEvaluation(selfEvaluation != null && selfEvaluation.containsKey("verifier_evaluation"))
+                .hasAiOpsRuleEvaluation(selfEvaluation != null && selfEvaluation.containsKey("aiops_rule_evaluation"))
                .hasFeedback(session.getFeedback() != null && !session.getFeedback().isBlank())
                .build();
    }
@@ -27,6 +27,10 @@ public class SelfEvaluationMergeService {
        return merge(existingJson, "verifier_evaluation", verifierEvaluation);
    }

+    public String mergeAiOpsRuleEvaluation(String existingJson, Map<String, Object> aiOpsRuleEvaluation) {
+        return merge(existingJson, "aiops_rule_evaluation", aiOpsRuleEvaluation);
+    }
+
    private String merge(String existingJson, String key, Map<String, Object> value) {
        try {
            Map<String, Object> root = parseRoot(existingJson);
@@ -44,7 +48,9 @@ public class SelfEvaluationMergeService {
        }

        Map<String, Object> parsed = objectMapper.readValue(existingJson, MAP_TYPE);
-        if (parsed.containsKey("rule_evaluation") || parsed.containsKey("verifier_evaluation")) {
+        if (parsed.containsKey("rule_evaluation")
+                || parsed.containsKey("verifier_evaluation")
+                || parsed.containsKey("aiops_rule_evaluation")) {
            return new LinkedHashMap<>(parsed);
        }