feat(agent): add executor evidence output contract

This commit is contained in:
zhuyongxin
2026-07-07 19:02:02 +08:00
parent 7f2e47ca38
commit 04eb50e2b4
19 changed files with 986 additions and 25 deletions
@@ -101,6 +101,8 @@ public class DiagnosisTraceEvaluator {
failedChecks.add("reject output does not use degraded template");
}
failedChecks.addAll(validateExecutorStructuredOutput(trace));
Integer toolCallCount = trace.getToolInvocations() == null ? 0 : trace.getToolInvocations().size();
Integer durationMs = trace.getSession() == null ? null : trace.getSession().getTotalDurationMs();
@@ -174,6 +176,32 @@ public class DiagnosisTraceEvaluator {
return value == null ? null : String.valueOf(value);
}
private List<String> validateExecutorStructuredOutput(DiagnosisTraceResponse trace) {
Object structuredOutput = nestedValue(trace, "verifier_evaluation", "executor_structured_output");
if (!(structuredOutput instanceof Map<?, ?> output)) {
return List.of();
}
Object claims = output.get("claims");
if (!(claims instanceof List<?> claimList)) {
return List.of("executor structured output missing claims array");
}
List<String> failedChecks = new ArrayList<>();
for (Object item : claimList) {
if (!(item instanceof Map<?, ?> claim)) {
failedChecks.add("executor structured claim is not an object");
continue;
}
Object claimIdValue = claim.get("claim_id");
String claimId = claimIdValue == null ? "unknown" : String.valueOf(claimIdValue);
Object bindings = claim.get("evidence_bindings");
if (!(bindings instanceof List<?> bindingList) || bindingList.isEmpty()) {
failedChecks.add("executor confirmed claim missing evidence bindings: " + claimId);
}
}
return failedChecks;
}
private Object nestedValue(DiagnosisTraceResponse trace, String firstKey, String secondKey) {
if (trace.getSession() == null || trace.getSession().getSelfEvaluation() == null) {
return null;