feat(harness): improve trace fallback and reasoning audit

2026-07-23 17:52:01 +08:00
parent 8fbc443f76
commit e20249c5d9
38 changed files with 1355 additions and 75 deletions
@@ -3,6 +3,8 @@ package com.superbiz.agent.harness.application;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.superbiz.agent.harness.application.persistence.ChatRunStore;
+import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
+import com.superbiz.agent.harness.audit.TraceAuditEvents;
 import com.superbiz.agent.harness.application.persistence.RoutingHistory;
 import com.superbiz.agent.harness.application.routing.IntentRouterInput;
 import com.superbiz.agent.harness.application.routing.IntentRoutingException;
@@ -33,6 +35,7 @@ public final class ChatApplicationUseCase {
    private final KnowledgeQueryOperation knowledgeQuery;
    private final DiagnosisOperation diagnosis;
    private final ObjectMapper objectMapper;
+    private final DiagnosisTraceRecorder traceRecorder;

    public ChatApplicationUseCase(DiagnosisHarnessCore core,
                                  Supplier<String> sessionIdSupplier,
@@ -42,6 +45,19 @@ public final class ChatApplicationUseCase {
                                  KnowledgeQueryOperation knowledgeQuery,
                                  DiagnosisOperation diagnosis,
                                  ObjectMapper objectMapper) {
+        this(core, sessionIdSupplier, runStore, router, systemChat, knowledgeQuery,
+                diagnosis, objectMapper, DiagnosisTraceRecorder.noop());
+    }
+
+    public ChatApplicationUseCase(DiagnosisHarnessCore core,
+                                  Supplier<String> sessionIdSupplier,
+                                  ChatRunStore runStore,
+                                  IntentRouting router,
+                                  SystemChatOperation systemChat,
+                                  KnowledgeQueryOperation knowledgeQuery,
+                                  DiagnosisOperation diagnosis,
+                                  ObjectMapper objectMapper,
+                                  DiagnosisTraceRecorder traceRecorder) {
        this.core = Objects.requireNonNull(core, "core must not be null");
        this.sessionIdSupplier = Objects.requireNonNull(
                sessionIdSupplier, "sessionIdSupplier must not be null");
@@ -51,6 +67,7 @@ public final class ChatApplicationUseCase {
        this.knowledgeQuery = Objects.requireNonNull(knowledgeQuery, "knowledgeQuery must not be null");
        this.diagnosis = Objects.requireNonNull(diagnosis, "diagnosis must not be null");
        this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
+        this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
    }

    public ChatApplicationResult execute(ChatApplicationRequest request) {
@@ -78,12 +95,14 @@ public final class ChatApplicationUseCase {
        IntentType intent = null;
        try {
            persistStart(context, request.query());
+            traceRecorder.record(TraceAuditEvents.runStarted(context));
            observer.onStarted(new CoreRunControl(core, context));
            observer.onStatus(ChatApplicationStatus.ROUTING);
            intent = router.route(context, new IntentRouterInput(
                    request.query(),
                    history.map(RoutingHistory::intent).orElse(null),
                    history.map(RoutingHistory::userQuery).orElse(null)));
+            traceRecorder.record(TraceAuditEvents.routingDecision(context, intent));
            persistIntent(context.runId(), intent);

            PathResult path = executePath(
@@ -93,6 +112,8 @@ public final class ChatApplicationUseCase {
            String safeJson = write(path.content());
            persistFinish(context, intent, path.outcome(), safeJson,
                    path.publishedResult(), durationMillis(startedNanos));
+            traceRecorder.record(TraceAuditEvents.runFinished(
+                    context, intent, path.outcome(), durationMillis(startedNanos)));
            return new ChatApplicationResult(
                    context.sessionId(), context.runId(), intent, path.outcome(),
                    path.content().contentType(), path.content());
@@ -104,6 +125,8 @@ public final class ChatApplicationUseCase {
            } catch (RuntimeException persistenceFailure) {
                exception.addSuppressed(persistenceFailure);
            }
+            traceRecorder.record(TraceAuditEvents.runFinished(
+                    context, intent, terminal, durationMillis(startedNanos)));
            throw safeFailure(exception, terminal);
        }
    }
@@ -24,6 +24,7 @@ import com.superbiz.agent.harness.tool.contract.RagToolResult;
 import org.springframework.ai.chat.messages.SystemMessage;
 import org.springframework.ai.chat.messages.UserMessage;
 import org.springframework.ai.chat.prompt.Prompt;
+import org.springframework.ai.converter.BeanOutputConverter;

 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -39,7 +40,8 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {

    private static final String SYSTEM_PROMPT = """
            你只根据提供的有界知识库证据回答原始问题。每个 answer item 必须引用 exact tool_call_id 和实际 document_ids。
-            不要使用外部知识，不要改写问题，不要输出 markdown fence。返回严格 KnowledgeAnswerDraft JSON。
+            不要使用外部知识，不要改写问题。answer_items 必须是非空数组；每个 item 必须包含非空 text、
+            exact tool_call_id 和至少一个 actual document_id。limitations 必须是字符串数组。
            """;

    private final DiagnosisHarnessCore core;
@@ -48,6 +50,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
    private final ObjectMapper objectMapper;
    private final ObjectReader ragReader;
    private final ObjectReader answerReader;
+    private final String prompt;
    private final Supplier<String> callIdSupplier;
    private final KnowledgeQueryLimits limits;

@@ -67,6 +70,8 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
        this.answerReader = objectMapper.readerFor(KnowledgeAnswerDraft.class)
                .with(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
                .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
+        this.prompt = SYSTEM_PROMPT + System.lineSeparator()
+                + new BeanOutputConverter<>(KnowledgeAnswerDraft.class, objectMapper).getFormat();
        this.callIdSupplier = Objects.requireNonNull(callIdSupplier, "callIdSupplier must not be null");
        this.limits = Objects.requireNonNull(limits, "limits must not be null");
    }
@@ -102,7 +107,7 @@ public final class KnowledgeQueryExecutor implements KnowledgeQueryOperation {
        core.reserveRunBytes(context, bytes);
        String output = modelCall.call(
                context,
-                new Prompt(List.of(new SystemMessage(SYSTEM_PROMPT), new UserMessage(modelInput))),
+                new Prompt(List.of(new SystemMessage(prompt), new UserMessage(modelInput))),
                limits.modelTimeout(),
                limits.maxModelOutputBytes());
        KnowledgeAnswerDraft draft = readAnswer(output);
@@ -4,6 +4,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.superbiz.agent.harness.contract.IntentType;
+import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
+import com.superbiz.agent.harness.audit.TraceAuditEvents;
 import com.superbiz.agent.harness.application.IntentRouting;
 import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
 import com.superbiz.agent.harness.core.RunContext;
@@ -36,6 +38,7 @@ public final class IntentRouter implements IntentRouting {
    private final ObjectMapper objectMapper;
    private final IntentRouterLimits limits;
    private final Consumer<RetryAttempt> attemptRecorder;
+    private final DiagnosisTraceRecorder traceRecorder;
    private final String systemPrompt;

    public IntentRouter(DiagnosisHarnessCore core,
@@ -44,12 +47,24 @@ public final class IntentRouter implements IntentRouting {
                        ObjectMapper objectMapper,
                        IntentRouterLimits limits,
                        Consumer<RetryAttempt> attemptRecorder) {
+        this(core, retryExecutor, modelCall, objectMapper, limits,
+                attemptRecorder, DiagnosisTraceRecorder.noop());
+    }
+
+    public IntentRouter(DiagnosisHarnessCore core,
+                        HarnessRetryExecutor retryExecutor,
+                        GuardModelCall modelCall,
+                        ObjectMapper objectMapper,
+                        IntentRouterLimits limits,
+                        Consumer<RetryAttempt> attemptRecorder,
+                        DiagnosisTraceRecorder traceRecorder) {
        this.core = Objects.requireNonNull(core, "core must not be null");
        this.retryExecutor = Objects.requireNonNull(retryExecutor, "retryExecutor must not be null");
        this.modelCall = Objects.requireNonNull(modelCall, "modelCall must not be null");
        this.objectMapper = Objects.requireNonNull(objectMapper, "objectMapper must not be null");
        this.limits = Objects.requireNonNull(limits, "limits must not be null");
        this.attemptRecorder = Objects.requireNonNull(attemptRecorder, "attemptRecorder must not be null");
+        this.traceRecorder = Objects.requireNonNull(traceRecorder, "traceRecorder must not be null");
        this.systemPrompt = IntentRouterPrompt.load();
    }

@@ -73,7 +88,10 @@ public final class IntentRouter implements IntentRouting {
                    () -> parse(modelCall.call(
                            context, prompt, remaining(started), limits.maxOutputBytes())),
                    this::classify,
-                    attemptRecorder);
+                    attempt -> {
+                        attemptRecorder.accept(attempt);
+                        traceRecorder.record(TraceAuditEvents.routingAttempt(context, attempt));
+                    });
        } catch (RetryExecutionException exception) {
            if (exception.failure() == RetryFailure.CANCELLED
                    || exception.failure() == RetryFailure.BUDGET_EXHAUSTED) {