217 lines
9.0 KiB
Go
217 lines
9.0 KiB
Go
//go:build live
|
|
|
|
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
agentllm "agent-native/runtime/llm"
|
|
"agent-system-backend/internal/settings"
|
|
"agent-system-backend/internal/store"
|
|
)
|
|
|
|
const (
|
|
evalRouteContractModel = "fixture-model"
|
|
evalRouteContractBase = "https://api.example.com/v1"
|
|
)
|
|
|
|
func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) {
|
|
cases := []struct {
|
|
basis, check string
|
|
wantBasis, wantCheck string
|
|
}{
|
|
{"done_marker", "", "done_marker", ""},
|
|
{"route_contract_eof", "", "route_contract_eof", ""},
|
|
{"", "C1", "", "C1"},
|
|
{"", "C9", "", "C9"},
|
|
{"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"},
|
|
{"Done_Marker", "C10", "unknown", "unknown"},
|
|
{"", "c3", "", "unknown"},
|
|
{"", "", "", ""},
|
|
}
|
|
for _, tc := range cases {
|
|
got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check})
|
|
if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck {
|
|
t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) {
|
|
for _, tc := range []struct{ basis, want string }{
|
|
{"done_marker", "done_marker"},
|
|
{"route_contract_eof", "route_contract_eof"},
|
|
{"PRIVATE_BASIS_I2B", "unknown"},
|
|
{"", ""},
|
|
} {
|
|
dir, _ := newEvaluationEventStore(t)
|
|
provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
|
|
out := make(chan agentllm.Event, 1)
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
|
|
Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
|
|
Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
|
|
FinishReason: "stop",
|
|
StreamComplete: true,
|
|
CompletionBasis: tc.basis,
|
|
TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true,
|
|
CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"},
|
|
}
|
|
close(out)
|
|
return out
|
|
}
|
|
for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(raw), "PRIVATE_") {
|
|
t.Fatalf("provider-controlled completion label escaped into record: %s", raw)
|
|
}
|
|
var record struct {
|
|
Calls []struct {
|
|
CompletionBasis *string `json:"completionBasis"`
|
|
Observation *agentllm.TerminalObservation `json:"terminalObservation"`
|
|
} `json:"calls"`
|
|
}
|
|
if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil {
|
|
t.Fatalf("unexpected record: %s err=%v", raw, err)
|
|
}
|
|
call := record.Calls[0]
|
|
if tc.want == "" {
|
|
if call.CompletionBasis != nil {
|
|
t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis)
|
|
}
|
|
} else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want {
|
|
t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want)
|
|
}
|
|
if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" {
|
|
t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) {
|
|
t.Helper()
|
|
route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"}
|
|
if contract != nil {
|
|
route["routeContractEOF"] = contract
|
|
}
|
|
models, ok := evalModelSettings(route)
|
|
if !ok {
|
|
t.Fatal("route file rejected")
|
|
}
|
|
if _, leaked := route["routeContractEOF"]; leaked {
|
|
t.Fatal("routeContractEOF left in the chat route")
|
|
}
|
|
config := evalFixtureSettings()
|
|
config["models"] = models
|
|
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
svc := settings.NewService(st)
|
|
svc.SetSystemOverride(config)
|
|
return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route
|
|
}
|
|
|
|
func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) {
|
|
digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase)
|
|
matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest}
|
|
client, _ := evalRouteContractClient(t, matching)
|
|
if client.RouteContract == nil {
|
|
t.Fatal("matching effective settings must attach the route contract")
|
|
}
|
|
if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) {
|
|
t.Fatalf("unexpected route contract %+v", *client.RouteContract)
|
|
}
|
|
if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" {
|
|
t.Fatalf("client route changed: base=%q", client.BaseURL)
|
|
}
|
|
|
|
if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil {
|
|
t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract)
|
|
}
|
|
for _, key := range []string{"model", "baseUrl", "identitySha256"} {
|
|
partial := map[string]any{}
|
|
for k, v := range matching {
|
|
if k != key {
|
|
partial[k] = v
|
|
}
|
|
}
|
|
if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil {
|
|
t.Fatalf("missing %s must leave RouteContract nil", key)
|
|
}
|
|
}
|
|
for name, contract := range map[string]map[string]any{
|
|
"other model": {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)},
|
|
"other base": {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")},
|
|
"digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)},
|
|
} {
|
|
if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil {
|
|
t.Fatalf("%s must leave RouteContract nil", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func evalRouteContractModels(t *testing.T, contract any) map[string]any {
|
|
t.Helper()
|
|
route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"}
|
|
if contract != nil {
|
|
route["routeContractEOF"] = contract
|
|
}
|
|
models, ok := evalModelSettings(route)
|
|
if !ok {
|
|
t.Fatal("route file rejected")
|
|
}
|
|
return models
|
|
}
|
|
|
|
func TestEvalRouteContractFailFastWhenConfiguredButNotAttached(t *testing.T) {
|
|
digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase)
|
|
matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest}
|
|
client, _ := evalRouteContractClient(t, matching)
|
|
if !evalRouteContractAttached(evalRouteContractModels(t, matching), client) {
|
|
t.Fatal("matching route contract must be reported attached")
|
|
}
|
|
|
|
for name, contract := range map[string]map[string]any{
|
|
"digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)},
|
|
"other model": {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)},
|
|
"other base": {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")},
|
|
"missing digest": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase},
|
|
"empty object": {},
|
|
} {
|
|
client, _ := evalRouteContractClient(t, contract)
|
|
if evalRouteContractAttached(evalRouteContractModels(t, contract), client) {
|
|
t.Fatalf("%s: configured but unattached route contract must fail fast", name)
|
|
}
|
|
}
|
|
|
|
client, _ = evalRouteContractClient(t, nil)
|
|
if !evalRouteContractAttached(evalRouteContractModels(t, nil), client) {
|
|
t.Fatal("route without routeContractEOF must keep running without a contract")
|
|
}
|
|
if !evalRouteContractAttached(map[string]any{"chat": map[string]any{}}, nil) {
|
|
t.Fatal("absent routeContractEOF must not depend on the client")
|
|
}
|
|
if evalRouteContractAttached(map[string]any{"routeContractEOF": map[string]any{}}, nil) {
|
|
t.Fatal("configured routeContractEOF without a client must fail fast")
|
|
}
|
|
}
|
|
|
|
func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) {
|
|
for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} {
|
|
route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract}
|
|
if _, ok := evalModelSettings(route); ok {
|
|
t.Fatalf("malformed routeContractEOF %v accepted", contract)
|
|
}
|
|
}
|
|
}
|