Agent-Native/adapters/native/result_fidelity_test.go

package native

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"agent-native/exec"
	"agent-native/runtime/llm"
)

// TestExecuteCanceledIsNotRetryable: aborted/canceled runs carry Canceled
// semantics with Retryable=false and SideEffects=possible.
func TestExecuteCanceledIsNotRetryable(t *testing.T) {
	dir := t.TempDir()
	b := New(Options{
		ModelOptions: ModelOptions{
			Model: "fake",
			Stream: func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
				ch := make(chan llm.Event, 1)
				go func() {
					<-ctx.Done()
					ch <- llm.Event{
						Kind:      llm.KindError,
						Err:       ctx.Err(),
						Retryable: true, // provider may mark abort retryable; adapter must not.
						Message:   llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopAborted},
					}
					close(ch)
				}()
				return ch
			},
		},
	})
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
	defer cancel()
	res, err := b.Execute(ctx, exec.Request{
		WorkDir: dir, Prompt: "cancel me", Targets: []string{"output.md"},
	})
	if err != nil {
		t.Fatalf("transport error: %v", err)
	}
	if res.Success {
		t.Fatalf("canceled run must not succeed: %+v", res)
	}
	if !res.Canceled {
		t.Fatalf("want Canceled=true, got %+v", res)
	}
	if res.Retryable {
		t.Fatalf("canceled must not be Retryable: %+v", res)
	}
	if res.EffectiveSideEffects() != exec.SideEffectPossible {
		t.Fatalf("canceled SideEffects = %q, want possible", res.EffectiveSideEffects())
	}
}

// TestExecuteToolFailureSideEffectsPossible: failure after tools may have run
// reports SideEffects=possible.
func TestExecuteToolFailureSideEffectsPossible(t *testing.T) {
	dir := t.TempDir()
	var call int
	b := New(Options{
		ModelOptions: ModelOptions{
			Model: "fake",
			Stream: func(ctx context.Context, model string, messages []llm.Message, decls []llm.ToolDecl, opts llm.StreamOptions) <-chan llm.Event {
				ch := make(chan llm.Event, 2)
				go func() {
					defer close(ch)
					call++
					if call == 1 {
						msg := llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
							ID: "c1", Name: "write_file", Arguments: `{"path":"scratch.md","content":"side effect"}`,
						}}}
						ch <- llm.Event{Kind: llm.KindMessageEnd, Message: msg}
						return
					}
					ch <- llm.Event{
						Kind:    llm.KindError,
						Err:     errString("provider failed after tools"),
						Message: llm.Message{Role: llm.RoleAssistant, Content: "boom", StopReason: llm.StopError},
					}
				}()
				return ch
			},
		},
	})
	res, err := b.Execute(context.Background(), exec.Request{
		WorkDir: dir, Prompt: "write then fail", Targets: []string{"output.md"},
	})
	if err != nil {
		t.Fatalf("transport error: %v", err)
	}
	if res.Success || res.Canceled {
		t.Fatalf("want definitive failure after tools, got %+v", res)
	}
	if res.SideEffects != exec.SideEffectPossible {
		t.Fatalf("SideEffects = %q, want possible", res.SideEffects)
	}
	if _, statErr := os.Stat(filepath.Join(dir, "scratch.md")); statErr != nil {
		t.Fatalf("tool side effect missing: %v", statErr)
	}
}

// TestExecutePreflightSideEffectsNone: validation rejection before tools
// reports SideEffects=none.
func TestExecutePreflightSideEffectsNone(t *testing.T) {
	cases := []struct {
		name string
		opts Options
		req  exec.Request
		want string
	}{
		{
			name: "missing-config",
			opts: Options{},
			req:  exec.Request{WorkDir: t.TempDir(), Targets: []string{"x.md"}},
			want: "requires",
		},
		{
			name: "bad-profile",
			opts: Options{ModelOptions: ModelOptions{Stream: script(), Model: "fake"}},
			req:  exec.Request{WorkDir: t.TempDir(), Targets: []string{"x.md"}, ToolProfile: "bogus"},
			want: "tool profile",
		},
		{
			name: "missing-model",
			opts: Options{ModelOptions: ModelOptions{APIBase: "http://x", APIKey: "k"}},
			req:  exec.Request{WorkDir: t.TempDir(), Targets: []string{"x.md"}},
			want: "model",
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			res, err := New(tc.opts).Execute(context.Background(), tc.req)
			if err != nil {
				t.Fatalf("transport error: %v", err)
			}
			if res.Success || res.Canceled || res.Unknown || res.Retryable {
				t.Fatalf("preflight must be definitive non-retry failure: %+v", res)
			}
			if res.SideEffects != exec.SideEffectNone {
				t.Fatalf("SideEffects = %q, want none", res.SideEffects)
			}
			if !strings.Contains(res.ErrorReason, tc.want) {
				t.Fatalf("ErrorReason = %q, want substring %q", res.ErrorReason, tc.want)
			}
		})
	}
}

// TestExecuteSuccessSideEffectsReflectTools: success after tool dispatch
// reports SideEffects=possible; patches also imply possible.
func TestExecuteSuccessSideEffectsReflectTools(t *testing.T) {
	dir := t.TempDir()
	b := New(Options{
		ModelOptions: ModelOptions{
			Model: "fake",
			Stream: script(
				llm.Message{Role: llm.RoleAssistant, StopReason: llm.StopToolCalls, ToolCalls: []llm.ToolCall{{
					ID: "c1", Name: "write_file", Arguments: `{"path":"output.md","content":"ok"}`,
				}}},
				llm.Message{Role: llm.RoleAssistant, Content: "done", StopReason: llm.StopEnd},
			),
		},
	})
	res, err := b.Execute(context.Background(), exec.Request{
		WorkDir: dir, Prompt: "write", Targets: []string{"output.md"},
	})
	if err != nil || !res.Success {
		t.Fatalf("want success, got err=%v res=%+v", err, res)
	}
	if res.SideEffects != exec.SideEffectPossible {
		t.Fatalf("SideEffects = %q, want possible", res.SideEffects)
	}
}