agent-exec transport 4792e7917ab99a9f3fd0e21823af20c0/c578cddeabbd

internal/runtime/llm.go

@@ -627,6 +627,11 @@ func routeContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm
 	return &agentllm.RouteContractEOF{Model: model, BaseURL: base, IdentitySHA256: digest}
 }
 
+// RouteContractFor 供评测等组合根按与生产完全相同的规则判定 RouteContract。
+func RouteContractFor(eff core.EffectiveSettings, route ResolvedModel) *agentllm.RouteContractEOF {
+	return routeContractFor(eff, route)
+}
+
 // SubLLMRunner 子代理执行器：按内置 profile 跑一轮带工具的内核循环。
 type SubLLMRunner struct {
 	// Prompts 提示词库（nil = 使用内联回退文案与只读回退 spec）。

internal/server/work_checkpoint_eval_route_contract_test.go

@@ -0,0 +1,170 @@
+//go:build live
+
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	agentllm "agent-native/runtime/llm"
+	"agent-system-backend/internal/settings"
+	"agent-system-backend/internal/store"
+)
+
+const (
+	evalRouteContractModel = "fixture-model"
+	evalRouteContractBase  = "https://api.example.com/v1"
+)
+
+func TestEvalRouteContractObservationLabelsWhitelisted(t *testing.T) {
+	cases := []struct {
+		basis, check         string
+		wantBasis, wantCheck string
+	}{
+		{"done_marker", "", "done_marker", ""},
+		{"route_contract_eof", "", "route_contract_eof", ""},
+		{"", "C1", "", "C1"},
+		{"", "C9", "", "C9"},
+		{"PRIVATE_BASIS_I2B", "PRIVATE_CHECK_I2B", "unknown", "unknown"},
+		{"Done_Marker", "C10", "unknown", "unknown"},
+		{"", "c3", "", "unknown"},
+		{"", "", "", ""},
+	}
+	for _, tc := range cases {
+		got := evalTerminalObservation(&agentllm.TerminalObservation{ReadOutcome: "done_marker", CompletionBasis: tc.basis, ContractCheck: tc.check})
+		if got.CompletionBasis != tc.wantBasis || got.ContractCheck != tc.wantCheck {
+			t.Fatalf("basis=%q check=%q => basis=%q check=%q, want %q/%q", tc.basis, tc.check, got.CompletionBasis, got.ContractCheck, tc.wantBasis, tc.wantCheck)
+		}
+	}
+}
+
+func TestEvalRouteContractRecordCompletionBasisWhitelisted(t *testing.T) {
+	for _, tc := range []struct{ basis, want string }{
+		{"done_marker", "done_marker"},
+		{"route_contract_eof", "route_contract_eof"},
+		{"PRIVATE_BASIS_I2B", "unknown"},
+		{"", ""},
+	} {
+		dir, _ := newEvaluationEventStore(t)
+		provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
+			out := make(chan agentllm.Event, 1)
+			out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
+				Message:         agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
+				Usage:           agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
+				FinishReason:    "stop",
+				StreamComplete:  true,
+				CompletionBasis: tc.basis,
+				TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "done_marker", UsagePresent: true,
+					CompletionBasis: tc.basis, ContractCheck: "PRIVATE_CHECK_I2B"},
+			}
+			close(out)
+			return out
+		}
+		for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
+		}
+		raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Contains(string(raw), "PRIVATE_") {
+			t.Fatalf("provider-controlled completion label escaped into record: %s", raw)
+		}
+		var record struct {
+			Calls []struct {
+				CompletionBasis *string                       `json:"completionBasis"`
+				Observation     *agentllm.TerminalObservation `json:"terminalObservation"`
+			} `json:"calls"`
+		}
+		if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 || record.Calls[0].Observation == nil {
+			t.Fatalf("unexpected record: %s err=%v", raw, err)
+		}
+		call := record.Calls[0]
+		if tc.want == "" {
+			if call.CompletionBasis != nil {
+				t.Fatalf("empty completion basis must be omitted, got %q", *call.CompletionBasis)
+			}
+		} else if call.CompletionBasis == nil || *call.CompletionBasis != tc.want {
+			t.Fatalf("basis %q recorded as %v, want %q", tc.basis, call.CompletionBasis, tc.want)
+		}
+		if call.Observation.CompletionBasis != tc.want || call.Observation.ContractCheck != "unknown" {
+			t.Fatalf("observation labels = %q/%q, want %q/unknown", call.Observation.CompletionBasis, call.Observation.ContractCheck, tc.want)
+		}
+	}
+}
+
+func evalRouteContractClient(t *testing.T, contract any) (*agentllm.Client, map[string]any) {
+	t.Helper()
+	route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "apiKey": "fixture-key"}
+	if contract != nil {
+		route["routeContractEOF"] = contract
+	}
+	models, ok := evalModelSettings(route)
+	if !ok {
+		t.Fatal("route file rejected")
+	}
+	if _, leaked := route["routeContractEOF"]; leaked {
+		t.Fatal("routeContractEOF left in the chat route")
+	}
+	config := evalFixtureSettings()
+	config["models"] = models
+	st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "agent-system.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	svc := settings.NewService(st)
+	svc.SetSystemOverride(config)
+	return evalProviderClient(route, svc.Effective(store.OwnerDefault, "")), route
+}
+
+func TestEvalRouteContractAttachedOnlyWhenProductionRuleAdmits(t *testing.T) {
+	digest := agentllm.RouteContractIdentity(evalRouteContractModel, evalRouteContractBase)
+	matching := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": digest}
+	client, _ := evalRouteContractClient(t, matching)
+	if client.RouteContract == nil {
+		t.Fatal("matching effective settings must attach the route contract")
+	}
+	if *client.RouteContract != (agentllm.RouteContractEOF{Model: evalRouteContractModel, BaseURL: evalRouteContractBase, IdentitySHA256: digest}) {
+		t.Fatalf("unexpected route contract %+v", *client.RouteContract)
+	}
+	if client.BaseURL != evalRouteContractBase || client.APIKey != "fixture-key" {
+		t.Fatalf("client route changed: base=%q", client.BaseURL)
+	}
+
+	if client, _ = evalRouteContractClient(t, nil); client.RouteContract != nil {
+		t.Fatalf("absent settings must leave RouteContract nil, got %+v", client.RouteContract)
+	}
+	for _, key := range []string{"model", "baseUrl", "identitySha256"} {
+		partial := map[string]any{}
+		for k, v := range matching {
+			if k != key {
+				partial[k] = v
+			}
+		}
+		if client, _ = evalRouteContractClient(t, partial); client.RouteContract != nil {
+			t.Fatalf("missing %s must leave RouteContract nil", key)
+		}
+	}
+	for name, contract := range map[string]map[string]any{
+		"other model":     {"model": "other-model", "baseUrl": evalRouteContractBase, "identitySha256": agentllm.RouteContractIdentity("other-model", evalRouteContractBase)},
+		"other base":      {"model": evalRouteContractModel, "baseUrl": "https://other.example.com/v1", "identitySha256": agentllm.RouteContractIdentity(evalRouteContractModel, "https://other.example.com/v1")},
+		"digest mismatch": {"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "identitySha256": strings.Repeat("0", 64)},
+	} {
+		if client, _ = evalRouteContractClient(t, contract); client.RouteContract != nil {
+			t.Fatalf("%s must leave RouteContract nil", name)
+		}
+	}
+}
+
+func TestEvalRouteContractRejectsMalformedRouteFileContract(t *testing.T) {
+	for _, contract := range []any{"text", map[string]any{"model": 1, "baseUrl": evalRouteContractBase, "identitySha256": "x"}} {
+		route := map[string]any{"model": evalRouteContractModel, "baseUrl": evalRouteContractBase, "routeContractEOF": contract}
+		if _, ok := evalModelSettings(route); ok {
+			t.Fatalf("malformed routeContractEOF %v accepted", contract)
+		}
+	}
+}

internal/server/work_checkpoint_eval_test.go

@@ -204,9 +204,13 @@ func TestServeWorkCheckpointEval(t *testing.T) {
 		if json.Unmarshal(raw, &route) != nil {
 			t.Fatal("bad route file")
 		}
+		models, ok := evalModelSettings(route)
+		if !ok {
+			t.Fatal("bad route file")
+		}
 		route["maxOutputTokens"] = float64(4000)
 		route["fallbacks"] = ""
-		config["models"] = map[string]any{"chat": route}
+		config["models"] = models
 		modelRoute = route
 	}
 	// The isolated fixture owns its startup configuration; managed model/mock
@@ -241,7 +245,7 @@ func TestServeWorkCheckpointEval(t *testing.T) {
 	front := &runtime.LLMRunner{Prompts: promptSvc}
 	sub := &runtime.SubLLMRunner{Prompts: promptSvc}
 	if mode == "model" {
-		stream := evalProviderStream(modelRoute, dir)
+		stream := evalProviderStreamWithSettings(modelRoute, settingSvc.Effective(store.OwnerDefault, ""), dir)
 		if guardedA01 {
 			stream = limitExtraA01Runs(st, stream)
 		}
@@ -425,12 +429,50 @@ func evalRejectedCheckpointPersisted(dir string) (bool, error) {
 	return false, nil
 }
 
-func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
+// evalModelSettings moves the protected route file's optional routeContractEOF
+// object out of the chat route and into the system-layer key production reads.
+func evalModelSettings(route map[string]any) (map[string]any, bool) {
+	models := map[string]any{"chat": route}
+	raw, present := route["routeContractEOF"]
+	if !present {
+		return models, true
+	}
+	delete(route, "routeContractEOF")
+	contract, ok := raw.(map[string]any)
+	if !ok {
+		return nil, false
+	}
+	clean := map[string]any{}
+	for _, key := range []string{"model", "baseUrl", "identitySha256"} {
+		value, present := contract[key]
+		if !present {
+			continue
+		}
+		text, ok := value.(string)
+		if !ok {
+			return nil, false
+		}
+		clean[key] = text
+	}
+	models["routeContractEOF"] = clean
+	return models, true
+}
+
+func evalProviderClient(route map[string]any, eff core.EffectiveSettings) *agentllm.Client {
 	base, _ := route["baseUrl"].(string)
 	key, _ := route["apiKey"].(string)
 	model, _ := route["model"].(string)
-	provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream
-	return evalProviderStreamWithProvider(provider, dir)
+	client := &agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}
+	client.RouteContract = runtime.RouteContractFor(eff, runtime.ResolvedModel{Model: model, Base: base, Key: key})
+	return client
+}
+
+func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
+	return evalProviderStreamWithSettings(route, core.EffectiveSettings{}, dir)
+}
+
+func evalProviderStreamWithSettings(route map[string]any, eff core.EffectiveSettings, dir string) agentllm.StreamFunc {
+	return evalProviderStreamWithProvider(evalProviderClient(route, eff).Stream, dir)
 }
 
 // evalCallRecord contains only accounting and terminal facts. Historical
@@ -447,6 +489,7 @@ type evalCallRecord struct {
 	HTTPStatus          int                           `json:"httpStatus"`
 	FinishReason        string                        `json:"finishReason"`
 	StreamComplete      bool                          `json:"streamComplete"`
+	CompletionBasis     string                        `json:"completionBasis,omitempty"`
 	TerminalObservation *agentllm.TerminalObservation `json:"terminalObservation,omitempty"`
 	ToolArgumentBytes   int                           `json:"toolArgumentBytes"`
 	ToolArgumentsValid  bool                          `json:"toolArgumentsValid"`
@@ -479,6 +522,21 @@ func evalLabel(value, fallback string, allowed ...string) string {
 	return fallback
 }
 
+// Absent completion facts stay absent; never record free text.
+func evalCompletionBasis(value string) string {
+	if value == "" {
+		return ""
+	}
+	return evalLabel(value, "unknown", "done_marker", "route_contract_eof")
+}
+
+func evalContractCheck(value string) string {
+	if value == "" {
+		return ""
+	}
+	return evalLabel(value, "unknown", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9")
+}
+
 func clampCount(value, limit int64) int64 {
 	if value < 0 {
 		return 0
@@ -503,7 +561,9 @@ func evalTerminalObservation(observation *agentllm.TerminalObservation) *agentll
 		ToolArgumentsValid:    observation.ToolArgumentsValid,
 		TailEventKind: evalLabel(observation.TailEventKind, "unknown",
 			"done_literal", "json_frame", "comment", "empty_data", "partial_frame", "unknown_non_json", "unknown"),
-		TailEventCount: int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)),
+		TailEventCount:  int(clampCount(int64(observation.TailEventCount), evalMaxTailEvents)),
+		CompletionBasis: evalCompletionBasis(observation.CompletionBasis),
+		ContractCheck:   evalContractCheck(observation.ContractCheck),
 	}
 	if body := observation.BodyBoundary; body != nil {
 		clean.BodyBoundary = &agentllm.BodyBoundaryObservation{
@@ -667,12 +727,13 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag
 			if terminal != nil && fault {
 				terminal.Message.ToolCalls = nil
 			}
-			finishReason, streamComplete := "", false
+			finishReason, streamComplete, completionBasis := "", false, ""
 			var returnedUsage agentllm.Usage
 			var terminalObservation *agentllm.TerminalObservation
 			httpStatus := 0
 			if terminal != nil {
 				finishReason, streamComplete = evalFinishReasonClass(terminal.FinishReason), terminal.StreamComplete
+				completionBasis = evalCompletionBasis(terminal.CompletionBasis)
 				returnedUsage = terminal.Usage
 				terminalObservation = evalTerminalObservation(terminal.TerminalObservation)
 				httpStatus = terminal.HTTPStatus
@@ -702,7 +763,7 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag
 			record := evalCallRecord{RunID: runID, RunKey: runKey, Call: a.Calls, Model: m,
 				OutputCap: opts.MaxTokens, ReportedTokens: usage, UsagePresent: usage > 0,
 				Usage: returnedUsage, HTTPStatus: httpStatus, FinishReason: finishReason,
-				StreamComplete: streamComplete, TerminalObservation: terminalObservation,
+				StreamComplete: streamComplete, CompletionBasis: completionBasis, TerminalObservation: terminalObservation,
 				ToolArgumentBytes: argumentBytes, ToolArgumentsValid: argumentsValid,
 				FailureClass: failureClass}
 			records = append(records, record)