Agent-System-Backend/internal/server/work_checkpoint_eval_route_contract_test.go

//go:build live

package server

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"

	agentllm "agent-native/runtime/llm"
	"agent-system-backend/internal/settings"
	"agent-system-backend/internal/store"
)

const (
	evalRouteContractModel = "fixture-model"
	evalRouteContractBase  = "https://api.example.com/v1"
)

func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) {
	cases := []struct {
		basis, check         string
		wantBasis, wantCheck string
	}{
		{"done_marker", "", "done_marker", ""},
		{"route_contract_eof", "", "route_contract_eof", ""},
		{"", "C1", "", "C1"},
		{"", "C9", "", "C9"},
		{"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"},
		{"Done_Marker", "C10", "unknown", "unknown"},
		{"", "c3", "", "unknown"},
		{"", "", "", ""},
	}
	for _, tc := range cases {
		got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check})
		if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck {
			t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck)
		}
	}
}

func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) {
	for _, tc := range []struct{ basis, want string }{
		{"done_marker", "done_marker"},
		{"route_contract_eof", "route_contract_eof"},
		{"PRIVATE_BASIS_I2B", "unknown"},
		{"", ""},
	} {
		dir, _ := newEvaluationEventStore(t)
		provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
			out := make(chan agentllm.Event, 1)
			out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
				Message:         agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
				Usage:           agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
				FinishReason:    "stop",
				StreamComplete:  true,
				CompletionBasis: tc.basis,
				TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true,
					CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"},
			}
			close(out)
			return out
		}
		for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
		}
		raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
		if err != nil {
			t.Fatal(err)
		}
		if strings.Contains(string(raw), "PRIVATE_") {
			t.Fatalf("provider-controlled completion label escaped into record: %s", raw)
		}
		var record struct {
			Calls []struct {
				CompletionBasis *string                       `json:"completionBasis"`
				Observation     *agentllm.TerminalObservation `json:"terminalObservation"`
			} `json:"calls"`
		}
		if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil {
			t.Fatalf("unexpected record: %s err=%v", raw, err)
		}
		call := record.Calls[0]
		if tc.want == "" {
			if call.CompletionBasis != nil {
				t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis)
			}
		} else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want {
			t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want)
		}
		if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" {
			t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want)
		}
	}
}

func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) {
	t.Helper()
	route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"}
	if contract != nil {
		route["routeContractEOF"] = contract
	}
	models, ok := evalModelSettings(route)
	if !ok {
		t.Fatal("route file rejected")
	}
	if _, leaked := route["routeContractEOF"]; leaked {
		t.Fatal("routeContractEOF left in the chat route")
	}
	config := evalFixtureSettings()
	config["models"] = models
	st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { st.Close() })
	svc := settings.NewService(st)
	svc.SetSystemOverride(config)
	return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route
}

func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) {
	digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase)
	matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest}
	client, _ := evalRouteContractClient(t, matching)
	if client.RouteContract == nil {
		t.Fatal("matching effective settings must attach the route contract")
	}
	if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) {
		t.Fatalf("unexpected route contract %+v", *client.RouteContract)
	}
	if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" {
		t.Fatalf("client route changed: base=%q", client.BaseURL)
	}

	if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil {
		t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract)
	}
	for _, key := range []string{"model", "baseUrl", "identitySha256"} {
		partial := map[string]any{}
		for k, v := range matching {
			if k != key {
				partial[k] = v
			}
		}
		if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil {
			t.Fatalf("missing %s must leave RouteContract nil", key)
		}
	}
	for name, contract := range map[string]map[string]any{
		"other model":     {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)},
		"other base":      {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")},
		"digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)},
	} {
		if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil {
			t.Fatalf("%s must leave RouteContract nil", name)
		}
	}
}

func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) {
	for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} {
		route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract}
		if _, ok := evalModelSettings(route); ok {
			t.Fatalf("malformed routeContractEOF %v accepted", contract)
		}
	}
}