234 lines
7.7 KiB
PowerShell
234 lines
7.7 KiB
PowerShell
param(
|
|
[string]$BaseUrl = "http://localhost:9900",
|
|
[string]$SessionId = "mvp-demo-interview-payment-timeout-001",
|
|
[string]$RequestFile = "$PSScriptRoot/../requests/payment-timeout-chat.json",
|
|
[string]$OutputDir = "$PSScriptRoot/../output"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Test-ServiceReachable {
|
|
param([string]$Url)
|
|
|
|
try {
|
|
$request = [System.Net.WebRequest]::Create($Url)
|
|
$request.Method = "GET"
|
|
$request.Timeout = 5000
|
|
$response = $request.GetResponse()
|
|
$response.Close()
|
|
return $true
|
|
} catch [System.Net.WebException] {
|
|
if ($_.Exception.Response -ne $null) {
|
|
$_.Exception.Response.Close()
|
|
return $true
|
|
}
|
|
return $false
|
|
}
|
|
}
|
|
|
|
function Get-TraceData {
|
|
param($TraceResponse)
|
|
|
|
if ($TraceResponse.PSObject.Properties.Name -contains "data") {
|
|
return $TraceResponse.data
|
|
}
|
|
return $TraceResponse
|
|
}
|
|
|
|
function Get-SelfEvaluation {
|
|
param($TraceData)
|
|
|
|
if ($null -eq $TraceData -or $null -eq $TraceData.session) {
|
|
return $null
|
|
}
|
|
return $TraceData.session.selfEvaluation
|
|
}
|
|
|
|
function Get-ToolNames {
|
|
param($TraceData)
|
|
|
|
if ($null -eq $TraceData -or $null -eq $TraceData.toolInvocations) {
|
|
return @()
|
|
}
|
|
return @($TraceData.toolInvocations | ForEach-Object { $_.toolName } | Where-Object { $_ } | Sort-Object -Unique)
|
|
}
|
|
|
|
New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null
|
|
|
|
Write-Host "Running interview demo preflight..."
|
|
Write-Host "BaseUrl: $BaseUrl"
|
|
Write-Host "SessionId: $SessionId"
|
|
|
|
if (-not (Test-ServiceReachable -Url $BaseUrl)) {
|
|
throw "Service is not reachable: $BaseUrl. Start the app with mvp-demo profile first: mvn spring-boot:run -Dspring-boot.run.profiles=mvp-demo"
|
|
}
|
|
|
|
$request = Get-Content -Raw -Encoding UTF8 -Path $RequestFile | ConvertFrom-Json
|
|
$request.Id = $SessionId
|
|
$body = $request | ConvertTo-Json -Depth 8
|
|
|
|
$chatRequest = @{
|
|
Method = "Post"
|
|
Uri = "$BaseUrl/api/chat"
|
|
ContentType = "application/json; charset=utf-8"
|
|
Body = $body
|
|
}
|
|
$chat = Invoke-RestMethod @chatRequest
|
|
|
|
$chatPath = Join-Path $OutputDir "chat-response.json"
|
|
$chat | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $chatPath
|
|
|
|
$runId = $chat.data.runId
|
|
if ($chat.data.success -ne $true) {
|
|
throw "Chat response was not successful."
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace([string]$chat.data.answer)) {
|
|
throw "Chat response did not include a non-empty answer."
|
|
}
|
|
if ($chat.data.sessionId -ne $SessionId) {
|
|
throw "Chat response sessionId '$($chat.data.sessionId)' did not match requested sessionId '$SessionId'."
|
|
}
|
|
if (-not $runId) {
|
|
throw "Chat response did not include runId; exact trace verification cannot continue."
|
|
}
|
|
|
|
$traceRequest = @{
|
|
Method = "Get"
|
|
Uri = "$BaseUrl/api/diagnosis/$SessionId/trace?runId=$([System.Uri]::EscapeDataString($runId))"
|
|
}
|
|
$trace = Invoke-RestMethod @traceRequest
|
|
|
|
$tracePath = Join-Path $OutputDir "trace-response.json"
|
|
$trace | ConvertTo-Json -Depth 80 | Set-Content -Encoding UTF8 -Path $tracePath
|
|
|
|
$traceData = Get-TraceData -TraceResponse $trace
|
|
if ($null -eq $traceData -or $null -eq $traceData.run) {
|
|
throw "Exact Trace response did not include data.run."
|
|
}
|
|
if ($traceData.runId -ne $runId -or $traceData.run.runId -ne $runId) {
|
|
throw "Exact Trace runId did not match Chat runId '$runId'."
|
|
}
|
|
if ($traceData.run.sessionId -ne $SessionId) {
|
|
throw "Exact Trace run did not belong to requested sessionId '$SessionId'."
|
|
}
|
|
|
|
$orchestrationTrace = $traceData.run.orchestrationTrace
|
|
if ($null -eq $orchestrationTrace) {
|
|
throw "Exact Trace data.run.orchestrationTrace is missing."
|
|
}
|
|
foreach ($field in @("version", "final_node", "termination_reason")) {
|
|
if (-not ($orchestrationTrace.PSObject.Properties.Name -contains $field) -or
|
|
[string]::IsNullOrWhiteSpace([string]$orchestrationTrace.$field)) {
|
|
throw "Exact Trace data.run.orchestrationTrace.$field is missing."
|
|
}
|
|
}
|
|
foreach ($field in @("transitions", "degraded", "evidence_retry_count")) {
|
|
if (-not ($orchestrationTrace.PSObject.Properties.Name -contains $field)) {
|
|
throw "Exact Trace data.run.orchestrationTrace.$field is missing."
|
|
}
|
|
}
|
|
if ($null -eq $orchestrationTrace.transitions) {
|
|
throw "Exact Trace data.run.orchestrationTrace.transitions must be an array."
|
|
}
|
|
if ([int]$orchestrationTrace.evidence_retry_count -lt 0) {
|
|
throw "Exact Trace data.run.orchestrationTrace.evidence_retry_count must not be negative."
|
|
}
|
|
if ($traceData.run.status -ne "SUCCESS" -or $traceData.run.agentFlow -ne "CHAT") {
|
|
throw "Exact Trace run must be CHAT/SUCCESS."
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace([string]$traceData.run.answer)) {
|
|
throw "Exact Trace run did not include a non-empty answer."
|
|
}
|
|
if (@($traceData.steps).Count -eq 0 -or @($traceData.toolInvocations).Count -eq 0) {
|
|
throw "Exact Trace did not include both Agent steps and tool invocation evidence."
|
|
}
|
|
if ($null -eq $traceData.run.selfEvaluation) {
|
|
throw "Exact Trace run did not include selfEvaluation."
|
|
}
|
|
|
|
$feedbackBody = @{
|
|
sessionId = $SessionId
|
|
runId = $runId
|
|
feedback = "useful"
|
|
} | ConvertTo-Json
|
|
|
|
$feedbackRequest = @{
|
|
Method = "Post"
|
|
Uri = "$BaseUrl/api/feedback"
|
|
ContentType = "application/json; charset=utf-8"
|
|
Body = $feedbackBody
|
|
}
|
|
$feedback = Invoke-RestMethod @feedbackRequest
|
|
if ($feedback.success -ne $true) {
|
|
throw "Feedback request was not successful for runId '$runId'."
|
|
}
|
|
|
|
$feedbackPath = Join-Path $OutputDir "feedback-response.json"
|
|
$feedback | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $feedbackPath
|
|
|
|
$selfEvaluation = Get-SelfEvaluation -TraceData $traceData
|
|
$verifierEvaluation = $null
|
|
if ($null -ne $selfEvaluation) {
|
|
$verifierEvaluation = $selfEvaluation.verifier_evaluation
|
|
}
|
|
|
|
$gatekeeperResult = $null
|
|
$promptAudit = $null
|
|
if ($null -ne $verifierEvaluation) {
|
|
$gatekeeperResult = $verifierEvaluation.gatekeeper_result
|
|
$promptAudit = $verifierEvaluation.prompt_audit
|
|
}
|
|
|
|
$verdict = $null
|
|
$gatekeeperStatus = $null
|
|
$gatekeeperRuleSetVersion = $null
|
|
$promptAuditVersion = $null
|
|
if ($null -ne $verifierEvaluation) {
|
|
$verdict = $verifierEvaluation.verdict
|
|
}
|
|
if ($null -ne $gatekeeperResult) {
|
|
$gatekeeperStatus = $gatekeeperResult.status
|
|
$gatekeeperRuleSetVersion = $gatekeeperResult.rule_set_version
|
|
}
|
|
if ($null -ne $promptAudit) {
|
|
$promptAuditVersion = $promptAudit.version
|
|
}
|
|
$toolNames = Get-ToolNames -TraceData $traceData
|
|
$transitionCount = @($orchestrationTrace.transitions).Count
|
|
$summaryPath = Join-Path $OutputDir "interview-demo-summary.json"
|
|
|
|
$summary = [ordered]@{
|
|
sessionId = $SessionId
|
|
runId = $runId
|
|
baseUrl = $BaseUrl
|
|
chatSuccess = $chat.data.success
|
|
verdict = $verdict
|
|
gatekeeperStatus = $gatekeeperStatus
|
|
gatekeeperRuleSetVersion = $gatekeeperRuleSetVersion
|
|
promptAuditVersion = $promptAuditVersion
|
|
orchestrationVersion = $orchestrationTrace.version
|
|
finalNode = $orchestrationTrace.final_node
|
|
terminationReason = $orchestrationTrace.termination_reason
|
|
degraded = [bool]$orchestrationTrace.degraded
|
|
transitionCount = $transitionCount
|
|
evidenceRetryCount = [int]$orchestrationTrace.evidence_retry_count
|
|
toolNames = $toolNames
|
|
paths = [ordered]@{
|
|
chat = $chatPath
|
|
trace = $tracePath
|
|
feedback = $feedbackPath
|
|
summary = $summaryPath
|
|
}
|
|
}
|
|
|
|
$summary | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 -Path $summaryPath
|
|
|
|
Write-Host ""
|
|
Write-Host "Interview demo preflight completed."
|
|
Write-Host "Verdict: $($summary.verdict)"
|
|
Write-Host "Gatekeeper rules: $($summary.gatekeeperRuleSetVersion)"
|
|
Write-Host "Prompt audit: $($summary.promptAuditVersion)"
|
|
Write-Host "Graph final node: $($summary.finalNode)"
|
|
Write-Host "Graph termination: $($summary.terminationReason)"
|
|
Write-Host "Summary: $summaryPath"
|