feat(harness): improve trace fallback and reasoning audit
This commit is contained in:
@@ -2,6 +2,7 @@ package com.superbiz.agent.config;
|
||||
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.audit.ToolInvocationAuditSink;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.application.persistence.ChatRunStore;
|
||||
import com.superbiz.agent.harness.tool.contract.MysqlToolRequest;
|
||||
import com.superbiz.agent.harness.tool.mysql.MysqlSqlValidator;
|
||||
@@ -13,6 +14,7 @@ import com.superbiz.agent.service.KnowledgeEvidencePostProcessor;
|
||||
import com.superbiz.agent.service.KnowledgeQueryTransformer;
|
||||
import com.superbiz.agent.service.LookupResultAssembler;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.data.redis.core.RedisTemplate;
|
||||
@@ -75,7 +77,10 @@ class HarnessChatConfigurationTest {
|
||||
.withBean(KnowledgeContextPacker.class, () -> mock(KnowledgeContextPacker.class))
|
||||
.withBean(LookupResultAssembler.class, () -> mock(LookupResultAssembler.class))
|
||||
.withBean(ToolInvocationAuditSink.class, ToolInvocationAuditSink::noop)
|
||||
.withBean(DiagnosisTraceRecorder.class, DiagnosisTraceRecorder::noop)
|
||||
.withBean(AgentStepRepository.class, () -> mock(AgentStepRepository.class))
|
||||
.withBean(AgentReasoningAuditRepository.class,
|
||||
() -> mock(AgentReasoningAuditRepository.class))
|
||||
.withBean(ChatRunStore.class, () -> mock(ChatRunStore.class))
|
||||
.withBean(MysqlToolProperties.class, MysqlToolProperties::new)
|
||||
.run(context -> {
|
||||
|
||||
@@ -7,6 +7,9 @@ import com.superbiz.agent.harness.application.executor.KnowledgeQueryLimits;
|
||||
import com.superbiz.agent.harness.application.executor.SingleTurnExecutorLimits;
|
||||
import com.superbiz.agent.harness.application.executor.SystemChatExecutor;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouter;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouterInput;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouterLimits;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
@@ -61,7 +64,8 @@ class ApplicationExecutorsTest {
|
||||
ScriptedModel model = new ScriptedModel(
|
||||
"{\"intent\":\"OTHER\"}", "{\"intent\":\"DIAGNOSIS\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
IntentRouter router = router(core, model, attempts::add);
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
IntentRouter router = router(core, model, attempts::add, trace::add);
|
||||
|
||||
IntentType intent = router.route(context,
|
||||
new IntentRouterInput("那退款服务呢?", IntentType.DIAGNOSIS, "订单支付为什么超时?"));
|
||||
@@ -72,6 +76,9 @@ class ApplicationExecutorsTest {
|
||||
assertTrue(model.prompts.get(0).contains("那退款服务呢?"));
|
||||
assertFalse(model.prompts.get(0).contains("tool_call_id"));
|
||||
assertEquals(2, attempts.size());
|
||||
assertEquals(List.of(TraceEventType.ROUTING_ATTEMPT, TraceEventType.ROUTING_ATTEMPT),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertEquals("INVALID_OUTPUT", trace.get(0).details().get("failure"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -118,6 +125,10 @@ class ApplicationExecutorsTest {
|
||||
|
||||
assertEquals(1, lookupCalls.get());
|
||||
assertEquals(1, model.calls.get());
|
||||
assertTrue(model.prompts.get(0).contains("answer_items"));
|
||||
assertTrue(model.prompts.get(0).contains("tool_call_id"));
|
||||
assertTrue(model.prompts.get(0).contains("document_ids"));
|
||||
assertTrue(model.prompts.get(0).contains("limitations"));
|
||||
assertEquals("doc-1", content.references().get(0).documentId());
|
||||
assertFalse(objectMapper.writeValueAsString(content).contains("knowledge-1"));
|
||||
assertFalse(objectMapper.writeValueAsString(content).contains("tool_call_id"));
|
||||
@@ -144,12 +155,18 @@ class ApplicationExecutorsTest {
|
||||
}
|
||||
|
||||
private IntentRouter router(DiagnosisHarnessCore core, ChatModel model,
|
||||
java.util.function.Consumer<RetryAttempt> recorder) {
|
||||
java.util.function.Consumer<RetryAttempt> recorder) {
|
||||
return router(core, model, recorder, DiagnosisTraceRecorder.noop());
|
||||
}
|
||||
|
||||
private IntentRouter router(DiagnosisHarnessCore core, ChatModel model,
|
||||
java.util.function.Consumer<RetryAttempt> recorder,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
return new IntentRouter(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new IntentRouterLimits(20_000, 2_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), recorder);
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), recorder, traceRecorder);
|
||||
}
|
||||
|
||||
private KnowledgeQueryExecutor knowledge(DiagnosisHarnessCore core,
|
||||
|
||||
+39
-3
@@ -3,6 +3,9 @@ package com.superbiz.agent.harness.application;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.application.persistence.ChatRunStore;
|
||||
import com.superbiz.agent.harness.application.persistence.RoutingHistory;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
@@ -72,6 +75,29 @@ class ChatApplicationUseCaseTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void unifiedTraceRecordsRunRoutingAndTerminalEvents() {
|
||||
FakeStore store = new FakeStore();
|
||||
List<DiagnosisTraceAuditEvent> trace = new ArrayList<>();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> IntentType.SYSTEM_CHAT,
|
||||
(context, query) -> new SystemChatContent("ok"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> diagnosisSuccess(query),
|
||||
trace::add);
|
||||
|
||||
useCase.execute(new ChatApplicationRequest("hello", "session-trace"));
|
||||
|
||||
assertEquals(List.of(
|
||||
TraceEventType.RUN_STARTED,
|
||||
TraceEventType.ROUTING_DECISION,
|
||||
TraceEventType.RUN_FINISHED),
|
||||
trace.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
assertEquals("SYSTEM_CHAT", trace.get(1).details().get("intent"));
|
||||
assertEquals("SUCCESS", trace.get(2).details().get("release_outcome"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void priorSafeTurnIsReadBeforeCurrentRunAndOnlyDiagnosisReceivesIt() {
|
||||
FakeStore store = new FakeStore();
|
||||
@@ -168,8 +194,18 @@ class ChatApplicationUseCaseTest {
|
||||
private ChatApplicationUseCase useCase(FakeStore store,
|
||||
IntentRouting routing,
|
||||
SystemChatOperation system,
|
||||
KnowledgeQueryOperation knowledge,
|
||||
DiagnosisOperation diagnosis) {
|
||||
KnowledgeQueryOperation knowledge,
|
||||
DiagnosisOperation diagnosis) {
|
||||
return useCase(store, routing, system, knowledge, diagnosis,
|
||||
DiagnosisTraceRecorder.noop());
|
||||
}
|
||||
|
||||
private ChatApplicationUseCase useCase(FakeStore store,
|
||||
IntentRouting routing,
|
||||
SystemChatOperation system,
|
||||
KnowledgeQueryOperation knowledge,
|
||||
DiagnosisOperation diagnosis,
|
||||
DiagnosisTraceRecorder traceRecorder) {
|
||||
AtomicInteger ids = new AtomicInteger();
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "run-app-" + ids.incrementAndGet(), Duration.ofMinutes(5),
|
||||
@@ -177,7 +213,7 @@ class ChatApplicationUseCaseTest {
|
||||
HarnessRetryPolicies.strict());
|
||||
return new ChatApplicationUseCase(
|
||||
core, () -> "generated-session", store, routing,
|
||||
system, knowledge, diagnosis, objectMapper);
|
||||
system, knowledge, diagnosis, objectMapper, traceRecorder);
|
||||
}
|
||||
|
||||
private DiagnosisExecutionResult diagnosisSuccess(String query) {
|
||||
|
||||
@@ -3,8 +3,10 @@ package com.superbiz.agent.harness.audit;
|
||||
import com.alibaba.cloud.ai.graph.RunnableConfig;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.AgentReasoningAudit;
|
||||
import com.superbiz.agent.harness.agent.DiagnosisAgentFactory;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
@@ -12,10 +14,13 @@ import org.springframework.ai.chat.messages.UserMessage;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
@@ -25,13 +30,16 @@ import static org.mockito.Mockito.when;
|
||||
class HarnessAgentAuditHookTest {
|
||||
|
||||
@Test
|
||||
void persistsExactIdentityAndMetadataWithoutContentOrArguments() {
|
||||
void persistsMetadataAndReasoningInSeparateAudit() {
|
||||
AgentStepRepository repository = mock(AgentStepRepository.class);
|
||||
AgentReasoningAuditRepository reasoningRepository = mock(AgentReasoningAuditRepository.class);
|
||||
AgentStep persisted = AgentStep.builder().id(7L).build();
|
||||
when(repository.save(any(AgentStep.class))).thenReturn(persisted);
|
||||
when(repository.findById(7L)).thenReturn(Optional.of(persisted));
|
||||
AtomicReference<DiagnosisTraceAuditEvent> trace = new AtomicReference<>();
|
||||
HarnessAgentAuditHook hook = new HarnessAgentAuditHook(
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME);
|
||||
repository, new ObjectMapper(), DiagnosisAgentFactory.AGENT_NAME,
|
||||
trace::set, reasoningRepository);
|
||||
RunnableConfig config = RunnableConfig.builder()
|
||||
.addMetadata("sessionId", "session-audit")
|
||||
.addMetadata("runId", "run-audit")
|
||||
@@ -40,6 +48,8 @@ class HarnessAgentAuditHookTest {
|
||||
hook.beforeModel(List.of(new UserMessage("secret-query")), config);
|
||||
AssistantMessage response = AssistantMessage.builder()
|
||||
.content("secret-model-output")
|
||||
.properties(java.util.Map.of(
|
||||
"reasoning_content", "inspect bounded evidence before selecting query_logs"))
|
||||
.toolCalls(List.of(new AssistantMessage.ToolCall(
|
||||
"call-1", "function", "query_logs", "{\"query\":\"secret-argument\"}")))
|
||||
.build();
|
||||
@@ -54,8 +64,28 @@ class HarnessAgentAuditHookTest {
|
||||
assertFalse(started.getModelInput().contains("secret-query"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-model-output"));
|
||||
assertFalse(completed.getModelOutput().contains("secret-argument"));
|
||||
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"]}", completed.getModelOutput());
|
||||
assertEquals("{\"has_text\":true,\"tool_names\":[\"query_logs\"],"
|
||||
+ "\"reasoning_available\":true,\"reasoning_bytes\":52}",
|
||||
completed.getModelOutput());
|
||||
assertNull(completed.getThought());
|
||||
ArgumentCaptor<AgentReasoningAudit> reasoningCaptor =
|
||||
ArgumentCaptor.forClass(AgentReasoningAudit.class);
|
||||
verify(reasoningRepository).save(reasoningCaptor.capture());
|
||||
AgentReasoningAudit reasoning = reasoningCaptor.getValue();
|
||||
assertEquals("session-audit", reasoning.getSessionId());
|
||||
assertEquals("run-audit", reasoning.getRunId());
|
||||
assertEquals(0, reasoning.getStepIndex());
|
||||
assertTrue(reasoning.getReasoningAvailable());
|
||||
assertEquals("inspect bounded evidence before selecting query_logs",
|
||||
reasoning.getReasoningContent());
|
||||
assertNotNull(trace.get());
|
||||
assertEquals(TraceEventType.AGENT_MODEL_STEP, trace.get().eventType());
|
||||
String traceText = trace.get().details().toString();
|
||||
assertFalse(traceText.contains("secret-query"));
|
||||
assertFalse(traceText.contains("secret-model-output"));
|
||||
assertFalse(traceText.contains("secret-argument"));
|
||||
assertFalse(traceText.contains("inspect bounded evidence"));
|
||||
assertTrue(traceText.contains("reasoning_available=true"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package com.superbiz.agent.harness.audit;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisTraceEvent;
|
||||
import com.superbiz.agent.repository.DiagnosisTraceEventRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class JpaDiagnosisTraceRecorderTest {
|
||||
|
||||
@Test
|
||||
void persistsIndependentMonotonicSafeEvents() {
|
||||
DiagnosisTraceEventRepository repository = mock(DiagnosisTraceEventRepository.class);
|
||||
when(repository.findMaxSequenceNoByRunId("run-1")).thenReturn(7);
|
||||
JpaDiagnosisTraceRecorder recorder = new JpaDiagnosisTraceRecorder(
|
||||
repository, new ObjectMapper());
|
||||
|
||||
recorder.record(event(TraceEventType.RUN_STARTED, TraceEventStatus.STARTED));
|
||||
recorder.record(event(TraceEventType.RELEASE_DECISION, TraceEventStatus.FALLBACK));
|
||||
|
||||
ArgumentCaptor<DiagnosisTraceEvent> captor =
|
||||
ArgumentCaptor.forClass(DiagnosisTraceEvent.class);
|
||||
verify(repository, times(2)).save(captor.capture());
|
||||
assertEquals(8, captor.getAllValues().get(0).getSequenceNo());
|
||||
assertEquals(9, captor.getAllValues().get(1).getSequenceNo());
|
||||
assertEquals("{\"failure\":\"SCHEMA_INVALID\"}",
|
||||
captor.getAllValues().get(0).getDetails());
|
||||
assertFalse(captor.getAllValues().get(0).getDetails().contains("prompt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void persistenceFailureNeverChangesMainFlow() {
|
||||
DiagnosisTraceEventRepository repository = mock(DiagnosisTraceEventRepository.class);
|
||||
when(repository.findMaxSequenceNoByRunId("run-1")).thenReturn(0);
|
||||
doThrow(new IllegalStateException("database unavailable"))
|
||||
.when(repository).save(any(DiagnosisTraceEvent.class));
|
||||
JpaDiagnosisTraceRecorder recorder = new JpaDiagnosisTraceRecorder(
|
||||
repository, new ObjectMapper());
|
||||
|
||||
assertDoesNotThrow(() -> recorder.record(
|
||||
event(TraceEventType.RUN_STARTED, TraceEventStatus.STARTED)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void migrationCreatesIndependentAppendOnlyTraceTable() throws Exception {
|
||||
String sql;
|
||||
try (var input = new ClassPathResource(
|
||||
"db/migration/V014__create_diagnosis_trace_event.sql").getInputStream()) {
|
||||
sql = new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
assertTrue(sql.contains("CREATE TABLE diagnosis_trace_event"));
|
||||
assertTrue(sql.contains("details JSON NOT NULL"));
|
||||
assertTrue(sql.contains("idx_trace_event_run_sequence"));
|
||||
assertFalse(sql.toUpperCase().contains("FOREIGN KEY"));
|
||||
}
|
||||
|
||||
private DiagnosisTraceAuditEvent event(TraceEventType type, TraceEventStatus status) {
|
||||
return new DiagnosisTraceAuditEvent(
|
||||
"session-1", "run-1", TracePhase.RUN, type, status,
|
||||
null, null, Map.of("failure", "SCHEMA_INVALID"));
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,9 @@ class JpaToolInvocationAuditSinkTest {
|
||||
@Test
|
||||
void persistsOnlyBoundedStableMetadata() {
|
||||
ToolInvocationRepository repository = mock(ToolInvocationRepository.class);
|
||||
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(repository, new ObjectMapper());
|
||||
DiagnosisTraceRecorder traceRecorder = mock(DiagnosisTraceRecorder.class);
|
||||
JpaToolInvocationAuditSink sink = new JpaToolInvocationAuditSink(
|
||||
repository, new ObjectMapper(), traceRecorder);
|
||||
|
||||
sink.record(new ToolInvocationAuditEvent(
|
||||
"session-1", "run-1", "call-1", "query_logs",
|
||||
@@ -37,5 +39,11 @@ class JpaToolInvocationAuditSinkTest {
|
||||
String serialized = saved.getInputParams() + saved.getOutputPreview() + saved.getRetrievalDetails();
|
||||
assertFalse(serialized.contains("query"));
|
||||
assertFalse(serialized.contains("raw_response"));
|
||||
ArgumentCaptor<DiagnosisTraceAuditEvent> traceCaptor =
|
||||
ArgumentCaptor.forClass(DiagnosisTraceAuditEvent.class);
|
||||
verify(traceRecorder).record(traceCaptor.capture());
|
||||
assertEquals(TraceEventType.TOOL_INVOCATION, traceCaptor.getValue().eventType());
|
||||
assertEquals("call-1", traceCaptor.getValue().details().get("tool_call_id"));
|
||||
assertFalse(traceCaptor.getValue().details().toString().contains("raw_response"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,5 +92,7 @@ class HarnessContractTest {
|
||||
previousJson.path("source_documents").get(0).path("document_id").asText());
|
||||
assertFalse(previousJson.toString().contains("tool_call_id"));
|
||||
assertFalse(previousJson.toString().contains("raw_response"));
|
||||
assertTrue(fallbackJson.path("observed_facts").isArray());
|
||||
assertTrue(fallbackJson.path("validation_issues").isArray());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,9 @@ package com.superbiz.agent.harness.release;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceAuditEvent;
|
||||
import com.superbiz.agent.harness.audit.DiagnosisTraceRecorder;
|
||||
import com.superbiz.agent.harness.audit.TraceEventType;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.FallbackType;
|
||||
@@ -46,6 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class DiagnosisReleaseUseCaseTest {
|
||||
|
||||
@@ -127,7 +131,21 @@ class DiagnosisReleaseUseCaseTest {
|
||||
fixture.context, "Why did payment fail?", duplicateDraft());
|
||||
|
||||
assertFallback(result, FallbackType.EVIDENCE_VALIDATION_FAILED, 0);
|
||||
assertEquals("EVIDENCE_VALIDATION", result.fallback().failureStage());
|
||||
assertFalse(result.fallback().validationIssues().isEmpty());
|
||||
assertTrue(result.fallback().validationIssues().stream()
|
||||
.anyMatch(issue -> "ANALYSIS_ID_DUPLICATE".equals(issue.code())));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
assertEquals(List.of(
|
||||
TraceEventType.EVIDENCE_GUARD_INITIAL,
|
||||
TraceEventType.EVIDENCE_REPAIR_ATTEMPT,
|
||||
TraceEventType.EVIDENCE_GUARD_RECHECK,
|
||||
TraceEventType.RELEASE_DECISION),
|
||||
fixture.trace.events.stream().map(DiagnosisTraceAuditEvent::eventType).toList());
|
||||
String trace = fixture.trace.events.toString();
|
||||
assertTrue(trace.contains("ANALYSIS_ID_DUPLICATE"));
|
||||
assertTrue(trace.contains("call-rag-1"));
|
||||
assertFalse(trace.contains("Pool reached its limit"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -140,10 +158,18 @@ class DiagnosisReleaseUseCaseTest {
|
||||
fixture.context, "Why did payment fail?", draft);
|
||||
|
||||
assertFallback(result, FallbackType.SEMANTIC_UNSUPPORTED, 1);
|
||||
assertEquals("SEMANTIC_VALIDATION", result.fallback().failureStage());
|
||||
assertEquals(1, result.fallback().observedFacts().size());
|
||||
assertTrue(result.fallback().observedFacts().get(0).summary().contains("active=50"));
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("Pool exhausted"));
|
||||
assertFalse(json.contains("secret-audit-reason"));
|
||||
assertEquals(1, fixture.model.calls.get());
|
||||
String trace = fixture.trace.events.toString();
|
||||
assertTrue(trace.contains("SEMANTIC_GUARD_DECISION"));
|
||||
assertTrue(trace.contains("UNSUPPORTED"));
|
||||
assertFalse(trace.contains("secret-audit-reason"));
|
||||
assertFalse(trace.contains("Pool exhausted"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -167,15 +193,18 @@ class DiagnosisReleaseUseCaseTest {
|
||||
ScriptedChatModel model = new ScriptedChatModel(responses);
|
||||
HarnessRetryExecutor retries = new HarnessRetryExecutor(core);
|
||||
GuardModelCall modelCall = new GuardModelCall(core, model, executor);
|
||||
RecordingTraceRecorder trace = new RecordingTraceRecorder();
|
||||
DiagnosisReleaseUseCase useCase = new DiagnosisReleaseUseCase(
|
||||
new EvidenceGuard(store, new ToolCallKeyFactory(PREFIX), objectMapper),
|
||||
new EvidenceRepair(core, retries, modelCall, objectMapper,
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)), ignored -> { }),
|
||||
new EvidenceRepairLimits(100_000, 20_000, Duration.ofSeconds(2)),
|
||||
ignored -> { }, trace),
|
||||
new SemanticGuard(core, retries, modelCall, objectMapper,
|
||||
new SemanticGuardLimits(100_000, 20_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), ignored -> { }),
|
||||
new SafeFallbackFactory());
|
||||
return new Fixture(context, model, useCase);
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)),
|
||||
ignored -> { }, trace),
|
||||
new SafeFallbackFactory(), trace);
|
||||
return new Fixture(context, model, useCase, trace);
|
||||
}
|
||||
|
||||
private DiagnosisDraft duplicateDraft() {
|
||||
@@ -309,6 +338,16 @@ class DiagnosisReleaseUseCaseTest {
|
||||
private record Fixture(
|
||||
RunContext context,
|
||||
ScriptedChatModel model,
|
||||
DiagnosisReleaseUseCase useCase) {
|
||||
DiagnosisReleaseUseCase useCase,
|
||||
RecordingTraceRecorder trace) {
|
||||
}
|
||||
|
||||
private static final class RecordingTraceRecorder implements DiagnosisTraceRecorder {
|
||||
private final List<DiagnosisTraceAuditEvent> events = new ArrayList<>();
|
||||
|
||||
@Override
|
||||
public void record(DiagnosisTraceAuditEvent event) {
|
||||
events.add(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,17 +2,23 @@ package com.superbiz.agent.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.AgentStep;
|
||||
import com.superbiz.agent.domain.entity.AgentReasoningAudit;
|
||||
import com.superbiz.agent.domain.entity.ChatSession;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisRun;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisSession;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisTraceEvent;
|
||||
import com.superbiz.agent.domain.entity.ToolInvocation;
|
||||
import com.superbiz.agent.dto.DiagnosisTraceResponse;
|
||||
import com.superbiz.agent.exception.SessionNotFoundException;
|
||||
import com.superbiz.agent.repository.AgentStepRepository;
|
||||
import com.superbiz.agent.repository.AgentReasoningAuditRepository;
|
||||
import com.superbiz.agent.repository.ChatSessionRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisSessionRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisTraceEventRepository;
|
||||
import com.superbiz.agent.repository.ToolInvocationRepository;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
@@ -36,16 +42,50 @@ class DiagnosisTraceServiceTest {
|
||||
private final ChatSessionRepository chatSessionRepository = mock(ChatSessionRepository.class);
|
||||
private final DiagnosisRunRepository diagnosisRunRepository = mock(DiagnosisRunRepository.class);
|
||||
private final AgentStepRepository agentStepRepository = mock(AgentStepRepository.class);
|
||||
private final AgentReasoningAuditRepository agentReasoningAuditRepository =
|
||||
mock(AgentReasoningAuditRepository.class);
|
||||
private final ToolInvocationRepository toolInvocationRepository = mock(ToolInvocationRepository.class);
|
||||
private final DiagnosisTraceEventRepository diagnosisTraceEventRepository =
|
||||
mock(DiagnosisTraceEventRepository.class);
|
||||
private final DiagnosisTraceService service = new DiagnosisTraceService(
|
||||
diagnosisSessionRepository,
|
||||
chatSessionRepository,
|
||||
diagnosisRunRepository,
|
||||
agentStepRepository,
|
||||
agentReasoningAuditRepository,
|
||||
toolInvocationRepository,
|
||||
diagnosisTraceEventRepository,
|
||||
new ObjectMapper()
|
||||
);
|
||||
|
||||
@Test
|
||||
void reasoningAuditRequiresExactRunAndReturnsRestrictedContent() {
|
||||
String sessionId = "trace-session-reasoning";
|
||||
String runId = "run-reasoning";
|
||||
LocalDateTime createdAt = LocalDateTime.of(2026, 7, 23, 10, 0);
|
||||
when(diagnosisRunRepository.findBySessionIdAndRunId(sessionId, runId))
|
||||
.thenReturn(Optional.of(run(9L, sessionId, runId, "question", createdAt)));
|
||||
when(agentReasoningAuditRepository.findByRunIdOrderByStepIndex(runId))
|
||||
.thenReturn(List.of(AgentReasoningAudit.builder()
|
||||
.id(11L)
|
||||
.sessionId(sessionId)
|
||||
.runId(runId)
|
||||
.stepIndex(0)
|
||||
.agentName("diagnosis_agent")
|
||||
.reasoningAvailable(true)
|
||||
.reasoningContent("bounded reasoning")
|
||||
.contentBytes(17)
|
||||
.createdAt(createdAt)
|
||||
.build()));
|
||||
|
||||
List<DiagnosisTraceResponse.AgentReasoningTrace> result =
|
||||
service.getReasoning(sessionId, runId);
|
||||
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("bounded reasoning", result.get(0).getReasoningContent());
|
||||
assertTrue(result.get(0).getReasoningAvailable());
|
||||
}
|
||||
|
||||
@Test
|
||||
void getTraceWithoutRunIdResolvesLatestRun() {
|
||||
String sessionId = "trace-session-latest";
|
||||
@@ -53,12 +93,16 @@ class DiagnosisTraceServiceTest {
|
||||
DiagnosisRun latest = run(2L, sessionId, "run-latest", "second question", base.plusMinutes(1));
|
||||
AgentStep step = step(20L, sessionId, "run-latest", 0, "composer", base.plusMinutes(1));
|
||||
ToolInvocation invocation = invocation(30L, sessionId, "run-latest", "query_metrics", base.plusMinutes(1));
|
||||
DiagnosisTraceEvent event = traceEvent(40L, sessionId, "run-latest", 1,
|
||||
"RUN", "RUN_STARTED", "STARTED", base.plusMinutes(1));
|
||||
|
||||
when(diagnosisRunRepository.findFirstBySessionIdOrderByCreatedAtDescIdDesc(sessionId))
|
||||
.thenReturn(Optional.of(latest));
|
||||
when(chatSessionRepository.findBySessionId(sessionId)).thenReturn(Optional.of(chatSession(sessionId)));
|
||||
when(agentStepRepository.findByRunIdOrderByStepIndex("run-latest")).thenReturn(List.of(step));
|
||||
when(toolInvocationRepository.findByRunIdOrderByIdAsc("run-latest")).thenReturn(List.of(invocation));
|
||||
when(diagnosisTraceEventRepository.findByRunIdOrderBySequenceNoAscIdAsc("run-latest"))
|
||||
.thenReturn(List.of(event));
|
||||
|
||||
DiagnosisTraceResponse response = service.getTrace(sessionId);
|
||||
|
||||
@@ -69,6 +113,10 @@ class DiagnosisTraceServiceTest {
|
||||
assertEquals("run-latest", response.getSteps().get(0).getRunId());
|
||||
assertEquals("run-latest", response.getToolInvocations().get(0).getRunId());
|
||||
assertEquals(1, response.getChatSession().getMessagePairCount());
|
||||
assertEquals("DIAGNOSIS", response.getRun().getIntent());
|
||||
assertEquals("SUCCESS", response.getRun().getReleaseOutcome());
|
||||
assertEquals("RUN_STARTED", response.getTimeline().get(0).getEventType());
|
||||
assertEquals(1, response.getSummary().getReturnedTraceEventCount());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -216,6 +264,8 @@ class DiagnosisTraceServiceTest {
|
||||
.query(query)
|
||||
.status("SUCCESS")
|
||||
.agentFlow("CHAT")
|
||||
.intent(IntentType.DIAGNOSIS)
|
||||
.releaseOutcome(ReleaseOutcome.SUCCESS)
|
||||
.answer("answer for " + runId)
|
||||
.selfEvaluation("{\"verifier_evaluation\":{\"verdict\":\"PASS\"}}")
|
||||
.feedback("useful")
|
||||
@@ -226,6 +276,22 @@ class DiagnosisTraceServiceTest {
|
||||
.build();
|
||||
}
|
||||
|
||||
private DiagnosisTraceEvent traceEvent(
|
||||
Long id, String sessionId, String runId, int sequenceNo,
|
||||
String phase, String eventType, String status, LocalDateTime createdAt) {
|
||||
return DiagnosisTraceEvent.builder()
|
||||
.id(id)
|
||||
.sessionId(sessionId)
|
||||
.runId(runId)
|
||||
.sequenceNo(sequenceNo)
|
||||
.phase(phase)
|
||||
.eventType(eventType)
|
||||
.status(status)
|
||||
.details("{\"safe\":true}")
|
||||
.createdAt(createdAt)
|
||||
.build();
|
||||
}
|
||||
|
||||
private ChatSession chatSession(String sessionId) {
|
||||
return ChatSession.builder()
|
||||
.id(99L)
|
||||
|
||||
Reference in New Issue
Block a user