feat(harness): add chat application use case
This commit is contained in:
zhuyongxin
@@ -0,0 +1,215 @@
|
||||
package com.superbiz.agent.harness.application;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.agent.HarnessEvidenceTools;
|
||||
import com.superbiz.agent.harness.application.executor.KnowledgeQueryExecutor;
|
||||
import com.superbiz.agent.harness.application.executor.KnowledgeQueryLimits;
|
||||
import com.superbiz.agent.harness.application.executor.SingleTurnExecutorLimits;
|
||||
import com.superbiz.agent.harness.application.executor.SystemChatExecutor;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouter;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouterInput;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRouterLimits;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
import com.superbiz.agent.harness.contract.EvidenceStatus;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.semantic.GuardModelCall;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryExecutor;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.harness.retry.RetryAttempt;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryErrorCode;
|
||||
import com.superbiz.agent.harness.tool.boundary.ToolBoundaryResult;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatResponseMetadata;
|
||||
import org.springframework.ai.chat.metadata.DefaultUsage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ApplicationExecutorsTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
private final ExecutorService executor = Executors.newCachedThreadPool();
|
||||
|
||||
@AfterEach
|
||||
void shutdown() {
|
||||
executor.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
void routerRetriesInvalidOutputWithSameMinimalInput() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-router", "run-router");
|
||||
ScriptedModel model = new ScriptedModel(
|
||||
"{\"intent\":\"OTHER\"}", "{\"intent\":\"DIAGNOSIS\"}");
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
IntentRouter router = router(core, model, attempts::add);
|
||||
|
||||
IntentType intent = router.route(context,
|
||||
new IntentRouterInput("那退款服务呢？", IntentType.DIAGNOSIS, "订单支付为什么超时？"));
|
||||
|
||||
assertEquals(IntentType.DIAGNOSIS, intent);
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(model.prompts.get(0), model.prompts.get(1));
|
||||
assertTrue(model.prompts.get(0).contains("那退款服务呢？"));
|
||||
assertFalse(model.prompts.get(0).contains("tool_call_id"));
|
||||
assertEquals(2, attempts.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void routerNeverDefaultsToDiagnosisAfterTwoInvalidOutputs() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
ScriptedModel model = new ScriptedModel("bad", "still-bad");
|
||||
IntentRouter router = router(core, model, ignored -> { });
|
||||
|
||||
assertThrows(IntentRoutingException.class, () -> router.route(
|
||||
core.startRun("session-fail", "run-fail"),
|
||||
new IntentRouterInput("排查故障", null, null)));
|
||||
|
||||
assertEquals(2, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void systemChatUsesOneDirectModelCall() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
ScriptedModel model = new ScriptedModel("你好，我可以帮助查询知识和诊断只读数据。");
|
||||
SystemChatExecutor system = new SystemChatExecutor(
|
||||
core, new GuardModelCall(core, model, executor),
|
||||
new SingleTurnExecutorLimits(10_000, 10_000, Duration.ofSeconds(2)));
|
||||
|
||||
SystemChatContent content = system.execute(
|
||||
core.startRun("session-system", "run-system"), "你能做什么？");
|
||||
|
||||
assertTrue(content.answer().contains("知识"));
|
||||
assertEquals(1, model.calls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void knowledgeAnswerValidatesExactReferencesAndRemovesToolId() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
AtomicInteger lookupCalls = new AtomicInteger();
|
||||
HarnessEvidenceTools tools = tools(lookupCalls, EvidenceStatus.EVIDENCE_FOUND);
|
||||
ScriptedModel model = new ScriptedModel("""
|
||||
{"answer_items":[{"text":"先检查连接池等待线程。","tool_call_id":"knowledge-1",
|
||||
"document_ids":["doc-1"]}],"limitations":["未查询实时日志"]}
|
||||
""");
|
||||
KnowledgeQueryExecutor knowledge = knowledge(core, tools, model);
|
||||
|
||||
KnowledgeContent content = knowledge.execute(
|
||||
core.startRun("session-knowledge", "run-knowledge"), "连接池超时怎么排查？");
|
||||
|
||||
assertEquals(1, lookupCalls.get());
|
||||
assertEquals(1, model.calls.get());
|
||||
assertEquals("doc-1", content.references().get(0).documentId());
|
||||
assertFalse(objectMapper.writeValueAsString(content).contains("knowledge-1"));
|
||||
assertFalse(objectMapper.writeValueAsString(content).contains("tool_call_id"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void knowledgeNoEvidenceSkipsModelAndFabricatedDocumentFailsClosed() {
|
||||
DiagnosisHarnessCore core = core();
|
||||
AtomicInteger emptyCalls = new AtomicInteger();
|
||||
ScriptedModel unused = new ScriptedModel();
|
||||
KnowledgeContent empty = knowledge(core, tools(emptyCalls, EvidenceStatus.NO_EVIDENCE), unused)
|
||||
.execute(core.startRun("session-empty", "run-empty"), "不存在的手册");
|
||||
assertEquals(1, emptyCalls.get());
|
||||
assertEquals(0, unused.calls.get());
|
||||
assertTrue(empty.references().isEmpty());
|
||||
|
||||
ScriptedModel fabricated = new ScriptedModel("""
|
||||
{"answer_items":[{"text":"fabricated","tool_call_id":"knowledge-1",
|
||||
"document_ids":["unknown-doc"]}],"limitations":[]}
|
||||
""");
|
||||
assertThrows(ChatApplicationException.class, () ->
|
||||
knowledge(core, tools(new AtomicInteger(), EvidenceStatus.EVIDENCE_FOUND), fabricated)
|
||||
.execute(core.startRun("session-fake", "run-fake"), "排查手册"));
|
||||
}
|
||||
|
||||
private IntentRouter router(DiagnosisHarnessCore core, ChatModel model,
|
||||
java.util.function.Consumer<RetryAttempt> recorder) {
|
||||
return new IntentRouter(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
new GuardModelCall(core, model, executor), objectMapper,
|
||||
new IntentRouterLimits(20_000, 2_000,
|
||||
Duration.ofSeconds(2), Duration.ofSeconds(3)), recorder);
|
||||
}
|
||||
|
||||
private KnowledgeQueryExecutor knowledge(DiagnosisHarnessCore core,
|
||||
HarnessEvidenceTools tools,
|
||||
ChatModel model) {
|
||||
return new KnowledgeQueryExecutor(
|
||||
core, tools, new GuardModelCall(core, model, executor), objectMapper,
|
||||
() -> "knowledge-1",
|
||||
new KnowledgeQueryLimits(50_000, 20_000, Duration.ofSeconds(2)));
|
||||
}
|
||||
|
||||
private HarnessEvidenceTools tools(AtomicInteger calls, EvidenceStatus status) {
|
||||
var rag = (com.superbiz.agent.harness.agent.EvidenceToolInvoker)
|
||||
(context, callId, arguments) -> {
|
||||
calls.incrementAndGet();
|
||||
if (status == EvidenceStatus.NO_EVIDENCE) {
|
||||
return ToolBoundaryResult.ready(callId, """
|
||||
{"evidence_status":"NO_EVIDENCE","tool_call_id":"knowledge-1",
|
||||
"query":"missing","evidence":[],"returned_count":0,"truncated":false}
|
||||
""", status);
|
||||
}
|
||||
return ToolBoundaryResult.ready(callId, """
|
||||
{"evidence_status":"EVIDENCE_FOUND","tool_call_id":"knowledge-1",
|
||||
"query":"pool timeout","evidence":[{"document_id":"doc-1",
|
||||
"source":"runbook.md","title":"连接池手册","breadcrumb":"DB > Pool",
|
||||
"excerpt":"active=50 max=50"}],"returned_count":1,"truncated":false}
|
||||
""", status);
|
||||
};
|
||||
var unavailable = (com.superbiz.agent.harness.agent.EvidenceToolInvoker)
|
||||
(context, callId, arguments) ->
|
||||
ToolBoundaryResult.error(callId, ToolBoundaryErrorCode.INVALID_REQUEST);
|
||||
return new HarnessEvidenceTools(rag, unavailable, unavailable);
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(20, 10, 10, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
|
||||
private static final class ScriptedModel implements ChatModel {
|
||||
private final List<String> responses;
|
||||
private final List<String> prompts = new ArrayList<>();
|
||||
private final AtomicInteger calls = new AtomicInteger();
|
||||
|
||||
private ScriptedModel(String... responses) {
|
||||
this.responses = List.of(responses);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt.getContents());
|
||||
int index = calls.getAndIncrement();
|
||||
if (index >= responses.size()) {
|
||||
throw new AssertionError("unexpected model call");
|
||||
}
|
||||
return new ChatResponse(
|
||||
List.of(new Generation(new AssistantMessage(responses.get(index)))),
|
||||
ChatResponseMetadata.builder().usage(new DefaultUsage(3, 2)).build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package com.superbiz.agent.harness.application;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.harness.application.persistence.ChatRunStore;
|
||||
import com.superbiz.agent.harness.application.persistence.RoutingHistory;
|
||||
import com.superbiz.agent.harness.application.routing.IntentRoutingException;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.PreviousTurn;
|
||||
import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.semantic.SemanticDraftView;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class ChatApplicationUseCaseTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void dispatchesAllThreeIntentsToOnlyTheirFixedExecutor() {
|
||||
for (IntentType selected : IntentType.values()) {
|
||||
FakeStore store = new FakeStore();
|
||||
AtomicInteger systemCalls = new AtomicInteger();
|
||||
AtomicInteger knowledgeCalls = new AtomicInteger();
|
||||
AtomicInteger diagnosisCalls = new AtomicInteger();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> selected,
|
||||
(context, query) -> {
|
||||
systemCalls.incrementAndGet();
|
||||
return new SystemChatContent("system:" + query);
|
||||
},
|
||||
(context, query) -> {
|
||||
knowledgeCalls.incrementAndGet();
|
||||
return new KnowledgeContent("knowledge:" + query, List.of(), List.of());
|
||||
},
|
||||
(context, query, previous, status) -> {
|
||||
diagnosisCalls.incrementAndGet();
|
||||
return diagnosisSuccess(query);
|
||||
});
|
||||
|
||||
ChatApplicationResult result = useCase.execute(
|
||||
new ChatApplicationRequest("original query", "session-fixed"));
|
||||
|
||||
assertEquals(selected, result.intent());
|
||||
assertEquals("session-fixed", result.sessionId());
|
||||
assertEquals(selected == IntentType.SYSTEM_CHAT ? 1 : 0, systemCalls.get());
|
||||
assertEquals(selected == IntentType.KNOWLEDGE_QUERY ? 1 : 0, knowledgeCalls.get());
|
||||
assertEquals(selected == IntentType.DIAGNOSIS ? 1 : 0, diagnosisCalls.get());
|
||||
assertEquals(ReleaseOutcome.SUCCESS, store.finishedOutcome);
|
||||
assertEquals(result.runId(), store.startedContext.runId());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void priorSafeTurnIsReadBeforeCurrentRunAndOnlyDiagnosisReceivesIt() {
|
||||
FakeStore store = new FakeStore();
|
||||
store.history = new RoutingHistory(IntentType.DIAGNOSIS, "old query");
|
||||
store.previous = new PreviousTurn(
|
||||
"old query", "old conclusion", "old scope", List.of("old limitation"),
|
||||
List.of(new SourceDocument("doc-1", "guide")));
|
||||
AtomicReference<PreviousTurn> received = new AtomicReference<>();
|
||||
AtomicReference<String> routedQuery = new AtomicReference<>();
|
||||
List<String> observerEvents = new ArrayList<>();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> {
|
||||
routedQuery.set(input.query());
|
||||
assertEquals(IntentType.DIAGNOSIS, input.lastIntent());
|
||||
assertEquals("old query", input.lastUserQuery());
|
||||
return IntentType.DIAGNOSIS;
|
||||
},
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> {
|
||||
received.set(previous);
|
||||
return diagnosisSuccess(query);
|
||||
});
|
||||
|
||||
ChatApplicationResult result = useCase.execute(
|
||||
new ChatApplicationRequest("那退款服务呢？", "session-history"),
|
||||
observer(observerEvents));
|
||||
|
||||
assertEquals("那退款服务呢？", routedQuery.get());
|
||||
assertEquals(store.previous, received.get());
|
||||
assertEquals(List.of("history", "previous", "start"), store.events.subList(0, 3));
|
||||
assertEquals("started:" + result.runId(), observerEvents.get(0));
|
||||
assertEquals("status:ROUTING", observerEvents.get(1));
|
||||
assertEquals(IntentType.DIAGNOSIS, store.finishedIntent);
|
||||
assertNotNull(store.finishedPublishedResult);
|
||||
}
|
||||
|
||||
@Test
|
||||
void routingFailurePersistsFailedAndNeverCallsAnyExecutor() {
|
||||
FakeStore store = new FakeStore();
|
||||
AtomicInteger executors = new AtomicInteger();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> { throw new IntentRoutingException(new RuntimeException("invalid")); },
|
||||
(context, query) -> { executors.incrementAndGet(); return new SystemChatContent("x"); },
|
||||
(context, query) -> { executors.incrementAndGet(); return new KnowledgeContent("x", List.of(), List.of()); },
|
||||
(context, query, previous, status) -> { executors.incrementAndGet(); return diagnosisSuccess(query); });
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest("route me", "session-route-fail")));
|
||||
|
||||
assertEquals(ChatFailureCode.ROUTING_UNAVAILABLE, failure.code());
|
||||
assertEquals(0, executors.get());
|
||||
assertEquals(ReleaseOutcome.FAILED, store.finishedOutcome);
|
||||
assertEquals(com.superbiz.agent.harness.core.RunState.FAILED,
|
||||
store.startedContext.lifecycle().state());
|
||||
}
|
||||
|
||||
@Test
|
||||
void clientDisconnectControlCancelsSameRunAndPersistsCancelled() {
|
||||
FakeStore store = new FakeStore();
|
||||
ChatApplicationUseCase useCase = useCase(
|
||||
store,
|
||||
(context, input) -> {
|
||||
assertTrue(context.cancellation().isCancelled());
|
||||
throw new com.superbiz.agent.harness.core.RunAbortedException(
|
||||
context.lifecycle().termination().orElseThrow());
|
||||
},
|
||||
(context, query) -> new SystemChatContent("unused"),
|
||||
(context, query) -> new KnowledgeContent("unused", List.of(), List.of()),
|
||||
(context, query, previous, status) -> diagnosisSuccess(query));
|
||||
ChatApplicationObserver cancelOnStart = new ChatApplicationObserver() {
|
||||
@Override
|
||||
public void onStarted(ChatRunControl runControl) {
|
||||
assertEquals(store.startedContext.runId(), runControl.runId());
|
||||
assertTrue(runControl.cancelClientDisconnect());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStatus(ChatApplicationStatus status) {
|
||||
}
|
||||
};
|
||||
|
||||
ChatApplicationException failure = assertThrows(ChatApplicationException.class, () ->
|
||||
useCase.execute(new ChatApplicationRequest("cancel", "session-cancel"), cancelOnStart));
|
||||
|
||||
assertEquals(ChatFailureCode.RUN_CANCELLED, failure.code());
|
||||
assertEquals(ReleaseOutcome.CANCELLED, store.finishedOutcome);
|
||||
assertEquals(com.superbiz.agent.harness.core.RunState.CANCELLED,
|
||||
store.startedContext.lifecycle().state());
|
||||
}
|
||||
|
||||
private ChatApplicationUseCase useCase(FakeStore store,
|
||||
IntentRouting routing,
|
||||
SystemChatOperation system,
|
||||
KnowledgeQueryOperation knowledge,
|
||||
DiagnosisOperation diagnosis) {
|
||||
AtomicInteger ids = new AtomicInteger();
|
||||
DiagnosisHarnessCore core = new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "run-app-" + ids.incrementAndGet(), Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(20, 20, 20, 100_000, 100_000, 200_000, 1_000_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
return new ChatApplicationUseCase(
|
||||
core, () -> "generated-session", store, routing,
|
||||
system, knowledge, diagnosis, objectMapper);
|
||||
}
|
||||
|
||||
private DiagnosisExecutionResult diagnosisSuccess(String query) {
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("safe conclusion", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "supported analysis", List.of("internal-call"))),
|
||||
List.of(), List.of(), new DiagnosisDraft.Limitations("scope", List.of()));
|
||||
PublishedResult published = new PublishedResult(
|
||||
query, "safe conclusion", "scope", List.of(), List.of());
|
||||
return new DiagnosisExecutionResult(
|
||||
ReleaseOutcome.SUCCESS,
|
||||
new DiagnosisContent(SemanticDraftView.from(draft), List.of()),
|
||||
published);
|
||||
}
|
||||
|
||||
private ChatApplicationObserver observer(List<String> events) {
|
||||
return new ChatApplicationObserver() {
|
||||
@Override
|
||||
public void onStarted(ChatRunControl runControl) {
|
||||
events.add("started:" + runControl.runId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onStatus(ChatApplicationStatus status) {
|
||||
events.add("status:" + status.name());
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static final class FakeStore implements ChatRunStore {
|
||||
private final List<String> events = new ArrayList<>();
|
||||
private RoutingHistory history;
|
||||
private PreviousTurn previous;
|
||||
private RunContext startedContext;
|
||||
private IntentType finishedIntent;
|
||||
private ReleaseOutcome finishedOutcome;
|
||||
private PublishedResult finishedPublishedResult;
|
||||
|
||||
@Override
|
||||
public Optional<RoutingHistory> findLatestRoutingHistory(String sessionId) {
|
||||
events.add("history");
|
||||
return Optional.ofNullable(history);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<PreviousTurn> findPreviousTurn(String sessionId) {
|
||||
events.add("previous");
|
||||
return Optional.ofNullable(previous);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start(RunContext context, String query) {
|
||||
events.add("start");
|
||||
startedContext = context;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void markIntent(String runId, IntentType intent) {
|
||||
events.add("intent:" + intent);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish(RunContext context, IntentType intent, ReleaseOutcome outcome,
|
||||
String safeContentJson, PublishedResult publishedResult, int durationMs) {
|
||||
events.add("finish:" + outcome);
|
||||
finishedIntent = intent;
|
||||
finishedOutcome = outcome;
|
||||
finishedPublishedResult = publishedResult;
|
||||
}
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package com.superbiz.agent.harness.application;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.superbiz.agent.domain.entity.DiagnosisRun;
|
||||
import com.superbiz.agent.harness.application.persistence.JpaChatRunStore;
|
||||
import com.superbiz.agent.harness.application.persistence.PreviousTurnLimits;
|
||||
import com.superbiz.agent.harness.application.persistence.PublishedResultPolicy;
|
||||
import com.superbiz.agent.harness.contract.AnalysisKind;
|
||||
import com.superbiz.agent.harness.contract.DiagnosisDraft;
|
||||
import com.superbiz.agent.harness.contract.IntentType;
|
||||
import com.superbiz.agent.harness.contract.PublishedResult;
|
||||
import com.superbiz.agent.harness.contract.ReleaseOutcome;
|
||||
import com.superbiz.agent.harness.contract.SourceDocument;
|
||||
import com.superbiz.agent.harness.core.DiagnosisHarnessCore;
|
||||
import com.superbiz.agent.harness.core.RunBudgetLimits;
|
||||
import com.superbiz.agent.harness.core.RunContext;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedAnalysisEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidence;
|
||||
import com.superbiz.agent.harness.guard.evidence.VerifiedEvidenceSnapshot;
|
||||
import com.superbiz.agent.harness.retry.HarnessRetryPolicies;
|
||||
import com.superbiz.agent.repository.ChatSessionRepository;
|
||||
import com.superbiz.agent.repository.DiagnosisRunRepository;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.core.io.ClassPathResource;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class PublishedResultPersistenceTest {
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void policyBuildsBoundedRagOnlyPublishedResultWithoutInternalEvidence() throws Exception {
|
||||
PublishedResultPolicy policy = new PublishedResultPolicy(
|
||||
new PreviousTurnLimits(8, 10, 12, 1, 6, 1, 10, 8));
|
||||
DiagnosisDraft draft = new DiagnosisDraft(
|
||||
new DiagnosisDraft.Conclusion("connection pool exhausted", List.of("a-1")),
|
||||
List.of(new DiagnosisDraft.AnalysisItem(
|
||||
"a-1", AnalysisKind.NORMAL, "active=50", List.of("call-1"))),
|
||||
List.of(), List.of(),
|
||||
new DiagnosisDraft.Limitations("order-service last 30 minutes", List.of("missing slow SQL")));
|
||||
VerifiedEvidenceSnapshot snapshot = new VerifiedEvidenceSnapshot(List.of(
|
||||
new VerifiedAnalysisEvidence("a-1", "active=50", AnalysisKind.NORMAL, List.of(
|
||||
new VerifiedEvidence("RAG", "guide.md", "pool", null, "excerpt",
|
||||
Map.of("document_id", "document-123", "title", "Long guide title")),
|
||||
new VerifiedEvidence("LOG", "APPLICATION", "30m", null, "raw log", Map.of())))));
|
||||
|
||||
PublishedResult result = policy.create("why pool exhausted", draft, snapshot).orElseThrow();
|
||||
|
||||
assertEquals(8, result.userQuery().length());
|
||||
assertEquals(10, result.publishedConclusion().length());
|
||||
assertEquals(1, result.sourceDocuments().size());
|
||||
assertEquals(10, result.sourceDocuments().get(0).documentId().length());
|
||||
String json = objectMapper.writeValueAsString(result);
|
||||
assertFalse(json.contains("tool_call_id"));
|
||||
assertFalse(json.contains("raw log"));
|
||||
assertFalse(json.contains("excerpt"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void storeUsesExactSafePreviousTurnQueryAndCorruptJsonFailsClosed() {
|
||||
DiagnosisRunRepository runs = mock(DiagnosisRunRepository.class);
|
||||
ChatSessionRepository sessions = mock(ChatSessionRepository.class);
|
||||
DiagnosisRun corrupt = DiagnosisRun.builder()
|
||||
.runId("run-old").sessionId("session-1").query("q")
|
||||
.intent(IntentType.DIAGNOSIS).releaseOutcome(ReleaseOutcome.SUCCESS)
|
||||
.publishedResult("not-json").build();
|
||||
when(runs.findFirstBySessionIdAndIntentAndReleaseOutcomeAndPublishedResultIsNotNullOrderByCreatedAtDescIdDesc(
|
||||
"session-1", IntentType.DIAGNOSIS, ReleaseOutcome.SUCCESS))
|
||||
.thenReturn(Optional.of(corrupt));
|
||||
JpaChatRunStore store = new JpaChatRunStore(
|
||||
sessions, runs, objectMapper,
|
||||
new PublishedResultPolicy(PreviousTurnLimits.defaults()));
|
||||
|
||||
assertTrue(store.findPreviousTurn("session-1").isEmpty());
|
||||
verify(runs).findFirstBySessionIdAndIntentAndReleaseOutcomeAndPublishedResultIsNotNullOrderByCreatedAtDescIdDesc(
|
||||
"session-1", IntentType.DIAGNOSIS, ReleaseOutcome.SUCCESS);
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallbackCompletionStoresSafeAnswerButNeverPublishedResult() {
|
||||
DiagnosisRunRepository runs = mock(DiagnosisRunRepository.class);
|
||||
ChatSessionRepository sessions = mock(ChatSessionRepository.class);
|
||||
DiagnosisRun entity = DiagnosisRun.builder()
|
||||
.runId("run-fallback").sessionId("session-1").query("q").status("RUNNING").build();
|
||||
when(runs.findByRunId("run-fallback")).thenReturn(Optional.of(entity));
|
||||
when(runs.save(any(DiagnosisRun.class))).thenAnswer(invocation -> invocation.getArgument(0));
|
||||
JpaChatRunStore store = new JpaChatRunStore(sessions, runs, objectMapper);
|
||||
RunContext context = core().startRun("session-1", "run-fallback");
|
||||
|
||||
store.finish(context, IntentType.DIAGNOSIS, ReleaseOutcome.FALLBACK,
|
||||
"{\"type\":\"SEMANTIC_UNAVAILABLE\"}",
|
||||
new PublishedResult("q", "c", "scope", List.of(),
|
||||
List.of(new SourceDocument("doc", "title"))), 12);
|
||||
|
||||
assertEquals("SUCCESS", entity.getStatus());
|
||||
assertEquals(ReleaseOutcome.FALLBACK, entity.getReleaseOutcome());
|
||||
assertNull(entity.getPublishedResult());
|
||||
assertEquals(12, entity.getTotalDurationMs());
|
||||
}
|
||||
|
||||
@Test
|
||||
void migrationContainsEntityColumnsAndPreviousTurnIndex() throws Exception {
|
||||
String sql;
|
||||
try (var input = new ClassPathResource(
|
||||
"db/migration/V012__add_chat_release_contract.sql").getInputStream()) {
|
||||
sql = new String(input.readAllBytes(), StandardCharsets.UTF_8);
|
||||
}
|
||||
assertTrue(sql.contains("ADD COLUMN intent VARCHAR(32)"));
|
||||
assertTrue(sql.contains("ADD COLUMN release_outcome VARCHAR(16)"));
|
||||
assertTrue(sql.contains("ADD COLUMN published_result JSON"));
|
||||
assertTrue(sql.contains("idx_diagnosis_run_previous_turn"));
|
||||
}
|
||||
|
||||
private DiagnosisHarnessCore core() {
|
||||
return new DiagnosisHarnessCore(
|
||||
Clock.systemUTC(), () -> "unused", Duration.ofMinutes(5),
|
||||
new RunBudgetLimits(10, 10, 10, 10_000, 10_000, 20_000, 100_000),
|
||||
HarnessRetryPolicies.strict());
|
||||
}
|
||||
}
|
||||
@@ -109,10 +109,10 @@ class SemanticGuardTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void attemptTimeoutCancelsBothPermittedModelCalls() throws Exception {
|
||||
void attemptTimeoutCancelsEveryPermittedAttempt() throws Exception {
|
||||
DiagnosisHarnessCore core = core();
|
||||
RunContext context = core.startRun("session-timeout", "run-timeout");
|
||||
BlockingChatModel model = new BlockingChatModel(2);
|
||||
BlockingChatModel model = new BlockingChatModel(1);
|
||||
List<RetryAttempt> attempts = new ArrayList<>();
|
||||
SemanticGuard guard = new SemanticGuard(
|
||||
core, new HarnessRetryExecutor(core),
|
||||
@@ -128,8 +128,10 @@ class SemanticGuardTest {
|
||||
assertEquals(com.superbiz.agent.harness.retry.RetryFailure.TIMEOUT, failure.failure());
|
||||
assertEquals(2, failure.attempts());
|
||||
assertTrue(model.interrupted.await(2, TimeUnit.SECONDS));
|
||||
assertEquals(2, model.calls.get());
|
||||
assertEquals(2, attempts.size());
|
||||
assertTrue(attempts.stream().allMatch(attempt ->
|
||||
attempt.failure() == com.superbiz.agent.harness.retry.RetryFailure.TIMEOUT));
|
||||
assertEquals(2, context.budget().snapshot().modelCalls());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Reference in New Issue
Block a user