Agent-Native/exec/step_retry_test.go

package exec

import (
	"errors"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// Synthetic UI-11 timeline — event types only. Do not include write.content.
const ui11SettledFailureTrace = `
{"type":"message_end","message":{"stopReason":"error","errorMessage":"terminated"}}
{"type":"auto_retry_start","attempt":1,"maxAttempts":3,"errorMessage":"terminated"}
{"type":"message_end","message":{"stopReason":"error","errorMessage":"Connection error."}}
{"type":"auto_retry_start","attempt":2,"maxAttempts":3,"errorMessage":"Connection error."}
{"type":"auto_retry_start","attempt":3,"maxAttempts":3,"errorMessage":"Connection error."}
{"type":"auto_retry_end","success":false,"attempt":3,"finalError":"Connection error."}
{"type":"agent_settled"}
`

func TestScanTraceText_UI11ProviderTransient(t *testing.T) {
	got := ScanTraceText(ui11SettledFailureTrace)
	if !got.ProviderTransient {
		t.Fatal("terminated + Connection error + auto_retry_end success=false must be provider-transient")
	}
	if !got.Settled {
		t.Fatal("agent_settled must be recorded")
	}
	if !got.RetryExhausted {
		t.Fatal("auto_retry_end success=false must mark SDK retry exhausted")
	}
}

func TestScanTraceText_IgnoresWriteArguments(t *testing.T) {
	// A truncated write toolCall must never become a deliverable signal.
	line := `{"type":"message_end","message":{"content":[{"type":"toolCall","name":"write","arguments":{"path":"/app/data/workspaces/x/tasks/01_draft/output.md","content":"# 伪造正文不应被采用\n"}}]}}`
	got := ScanTraceText(line)
	if got.HasUsableDeliverable {
		t.Fatal("write arguments in a trace line are not a usable deliverable")
	}
	if got.ProviderTransient {
		t.Fatal("a write toolCall alone is not a provider transient")
	}
}

func TestAssessStepFailure_TransientVsPermanent(t *testing.T) {
	cases := []struct {
		name string
		sig  StepFailureSignals
		want FailureClass
	}{
		{
			name: "runner retryable flag",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "Connection error.", Retryable: true}},
			want: FailureTransient,
		},
		{
			name: "connection error text",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "Connection error."}},
			want: FailureTransient,
		},
		{
			name: "terminated text",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "terminated"}},
			want: FailureTransient,
		},
		{
			name: "missing after UI-11 settled failure",
			sig: StepFailureSignals{
				Result:        &Result{Success: false, ErrorReason: "missing target deliverable output.md"},
				ExtraText:     ui11SettledFailureTrace,
				TargetMissing: true,
			},
			want: FailureTransient,
		},
		{
			name: "settled clean missing is transient settled-no-deliverable",
			sig: StepFailureSignals{
				Result:        &Result{Success: false, ErrorReason: "missing target deliverable output.md"},
				ExtraText:     `{"type":"agent_settled"}`,
				TargetMissing: true,
			},
			want: FailureTransient,
		},
		{
			name: "missing without settle remains permanent",
			sig: StepFailureSignals{
				Result:        &Result{Success: false, ErrorReason: "missing target deliverable output.md"},
				TargetMissing: true,
			},
			want: FailurePermanent,
		},
		{
			name: "prompt_empty is permanent",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "prompt_empty"}},
			want: FailurePermanent,
		},
		{
			name: "unauthorized is permanent",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "unauthorized"}},
			want: FailurePermanent,
		},
		{
			name: "governance abort is permanent",
			sig:  StepFailureSignals{Err: errors.New("治理强制终止: 原地打转")},
			want: FailurePermanent,
		},
		{
			name: "remote runner unreachable is transient",
			sig:  StepFailureSignals{Result: &Result{Success: false, ErrorReason: "pi remote runner call failed: connection refused"}},
			want: FailureTransient,
		},
		{
			name: "existing target is never retried",
			sig: StepFailureSignals{
				Result:       &Result{Success: false, ErrorReason: "Connection error.", Retryable: true},
				TargetExists: true,
			},
			want: FailurePermanent,
		},
		{
			name: "provider signal without missing is not enough",
			sig: StepFailureSignals{
				Result:    &Result{Success: false, ErrorReason: "agent_not_settled"},
				ExtraText: `{"type":"auto_retry_end","success":false}`,
			},
			want: FailurePermanent,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got := AssessStepFailure(tc.sig)
			if got.Class != tc.want {
				t.Fatalf("class=%s reason=%s want=%s", got.Class, got.Reason, tc.want)
			}
		})
	}
}

func TestAssessStepFailure_ReadsWorkDirExecutionLog(t *testing.T) {
	dir := t.TempDir()
	log := "=== PI-AGENT TASK EXECUTION LOG ===\nOutcome: SUCCESS\nError: None\n" + ui11SettledFailureTrace
	if err := os.WriteFile(filepath.Join(dir, "execution.log"), []byte(log), 0o644); err != nil {
		t.Fatal(err)
	}
	got := AssessStepFailure(StepFailureSignals{
		WorkDir:       dir,
		TargetMissing: true,
		Result:        &Result{Success: true, ErrorReason: "missing target deliverable output.md"},
	})
	if got.Class != FailureTransient {
		t.Fatalf("old runner Outcome:SUCCESS + settled failure must be transient, got %s %s", got.Class, got.Reason)
	}
	if strings.Contains(got.Reason, "伪造") || strings.Contains(got.Reason, "系统初步设计") {
		t.Fatalf("assessment must not carry write-argument body: %s", got.Reason)
	}
}

func TestAssessStepFailure_SettledNoDeliverableReason(t *testing.T) {
	got := AssessStepFailure(StepFailureSignals{
		Result:        &Result{Success: false, ErrorReason: "missing target deliverable output.md"},
		ExtraText:     `{"type":"agent_settled"}`,
		TargetMissing: true,
	})
	if got.Class != FailureTransient {
		t.Fatalf("class=%s want transient", got.Class)
	}
	if !strings.Contains(got.Reason, "settled-no-deliverable") {
		t.Fatalf("reason=%q want settled-no-deliverable", got.Reason)
	}
}

func TestMaxTransientStepInvocations_IsBounded(t *testing.T) {
	if MaxTransientStepInvocations < 2 {
		t.Fatal("need at least one fresh re-invocation after the first")
	}
	if MaxTransientStepInvocations > 3 {
		t.Fatalf("bound too loose: %d would retry permanent-looking gaps too many times", MaxTransientStepInvocations)
	}
}