package exec import ( "errors" "os" "path/filepath" "strings" "testing" ) // Synthetic UI-11 timeline — event types only. Do not include write.content. const ui11SettledFailureTrace = ` {"type":"message_end","message":{"stopReason":"error","errorMessage":"terminated"}} {"type":"auto_retry_start","attempt":1,"maxAttempts":3,"errorMessage":"terminated"} {"type":"message_end","message":{"stopReason":"error","errorMessage":"Connection error."}} {"type":"auto_retry_start","attempt":2,"maxAttempts":3,"errorMessage":"Connection error."} {"type":"auto_retry_start","attempt":3,"maxAttempts":3,"errorMessage":"Connection error."} {"type":"auto_retry_end","success":false,"attempt":3,"finalError":"Connection error."} {"type":"agent_settled"} ` func TestScanTraceText_UI11ProviderTransient(t *testing.T) { got := ScanTraceText(ui11SettledFailureTrace) if !got.ProviderTransient { t.Fatal("terminated + Connection error + auto_retry_end success=false must be provider-transient") } if !got.Settled { t.Fatal("agent_settled must be recorded") } if !got.RetryExhausted { t.Fatal("auto_retry_end success=false must mark SDK retry exhausted") } } func TestScanTraceText_IgnoresWriteArguments(t *testing.T) { // A truncated write toolCall must never become a deliverable signal. line := `{"type":"message_end","message":{"content":[{"type":"toolCall","name":"write","arguments":{"path":"/app/data/workspaces/x/tasks/01_draft/output.md","content":"# 伪造正文不应被采用\n"}}]}}` got := ScanTraceText(line) if got.HasUsableDeliverable { t.Fatal("write arguments in a trace line are not a usable deliverable") } if got.ProviderTransient { t.Fatal("a write toolCall alone is not a provider transient") } } func TestAssessStepFailure_TransientVsPermanent(t *testing.T) { cases := []struct { name string sig StepFailureSignals want FailureClass }{ { name: "runner retryable flag", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "Connection error.", Retryable: true}}, want: FailureTransient, }, { name: "connection error text", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "Connection error."}}, want: FailureTransient, }, { name: "terminated text", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "terminated"}}, want: FailureTransient, }, { name: "missing after UI-11 settled failure", sig: StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "missing target deliverable output.md"}, ExtraText: ui11SettledFailureTrace, TargetMissing: true, }, want: FailureTransient, }, { name: "settled clean missing is transient settled-no-deliverable", sig: StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "missing target deliverable output.md"}, ExtraText: `{"type":"agent_settled"}`, TargetMissing: true, }, want: FailureTransient, }, { name: "missing without settle remains permanent", sig: StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "missing target deliverable output.md"}, TargetMissing: true, }, want: FailurePermanent, }, { name: "prompt_empty is permanent", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "prompt_empty"}}, want: FailurePermanent, }, { name: "unauthorized is permanent", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "unauthorized"}}, want: FailurePermanent, }, { name: "governance abort is permanent", sig: StepFailureSignals{Err: errors.New("治理强制终止: 原地打转")}, want: FailurePermanent, }, { name: "remote runner unreachable is transient", sig: StepFailureSignals{Result: &Result{Success: false, ErrorReason: "pi remote runner call failed: connection refused"}}, want: FailureTransient, }, { name: "existing target is never retried", sig: StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "Connection error.", Retryable: true}, TargetExists: true, }, want: FailurePermanent, }, { name: "provider signal without missing is not enough", sig: StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "agent_not_settled"}, ExtraText: `{"type":"auto_retry_end","success":false}`, }, want: FailurePermanent, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := AssessStepFailure(tc.sig) if got.Class != tc.want { t.Fatalf("class=%s reason=%s want=%s", got.Class, got.Reason, tc.want) } }) } } func TestAssessStepFailure_ReadsWorkDirExecutionLog(t *testing.T) { dir := t.TempDir() log := "=== PI-AGENT TASK EXECUTION LOG ===\nOutcome: SUCCESS\nError: None\n" + ui11SettledFailureTrace if err := os.WriteFile(filepath.Join(dir, "execution.log"), []byte(log), 0o644); err != nil { t.Fatal(err) } got := AssessStepFailure(StepFailureSignals{ WorkDir: dir, TargetMissing: true, Result: &Result{Success: true, ErrorReason: "missing target deliverable output.md"}, }) if got.Class != FailureTransient { t.Fatalf("old runner Outcome:SUCCESS + settled failure must be transient, got %s %s", got.Class, got.Reason) } if strings.Contains(got.Reason, "伪造") || strings.Contains(got.Reason, "系统初步设计") { t.Fatalf("assessment must not carry write-argument body: %s", got.Reason) } } func TestAssessStepFailure_SettledNoDeliverableReason(t *testing.T) { got := AssessStepFailure(StepFailureSignals{ Result: &Result{Success: false, ErrorReason: "missing target deliverable output.md"}, ExtraText: `{"type":"agent_settled"}`, TargetMissing: true, }) if got.Class != FailureTransient { t.Fatalf("class=%s want transient", got.Class) } if !strings.Contains(got.Reason, "settled-no-deliverable") { t.Fatalf("reason=%q want settled-no-deliverable", got.Reason) } } func TestMaxTransientStepInvocations_IsBounded(t *testing.T) { if MaxTransientStepInvocations < 2 { t.Fatal("need at least one fresh re-invocation after the first") } if MaxTransientStepInvocations > 3 { t.Fatalf("bound too loose: %d would retry permanent-looking gaps too many times", MaxTransientStepInvocations) } }