Bound provider terminal labels in evaluation records
ci / vet-and-test (pull_request) Successful in 53s

Why: A provider-controlled finish reason could leak arbitrary content into the supposed content-free audit file.

Evidence: Red then green terminal-label regression and full Backend vet/Go test in Dev receipts.

Impact: Only evaluation records map unknown terminal labels to a finite class; tool execution rules are unchanged.

Refs: U3 r0.5 independent review finding
This commit is contained in:
2026-10-14 09:07:07 +00:00
parent bd41302310
commit 8f5e32dca1
2 changed files with 60 additions and 3 deletions
@@ -74,6 +74,41 @@ func TestEvaluationRecordsContentFreeTerminal(t *testing.T) {
}
}
func TestEvaluationRedactsProviderControlledTerminalLabels(t *testing.T) {
dir, _ := newEvaluationEventStore(t)
provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
out := make(chan agentllm.Event, 1)
out <- agentllm.Event{Kind: agentllm.KindMessageEnd,
Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: "safe", StopReason: agentllm.StopEnd},
Usage: agentllm.Usage{PromptTokens: 12, CompletionTokens: 5, TotalTokens: 17},
FinishReason: "PRIVATE_REASON_R05", StreamComplete: true,
TerminalObservation: &agentllm.TerminalObservation{ReadOutcome: "PRIVATE_OUTCOME_R05", UsagePresent: true},
}
close(out)
return out
}
for range evalProviderStreamWithProvider(provider, dir)(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
}
raw, err := os.ReadFile(filepath.Join(dir, "provider-records.json"))
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "PRIVATE_REASON_R05") || strings.Contains(string(raw), "PRIVATE_OUTCOME_R05") {
t.Fatal("provider-controlled terminal label escaped into a content-free record")
}
var record struct {
Calls []struct {
FinishReason string `json:"finishReason"`
Observation *agentllm.TerminalObservation `json:"terminalObservation"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &record); err != nil || len(record.Calls) != 1 ||
record.Calls[0].FinishReason != "unknown" || record.Calls[0].Observation == nil ||
record.Calls[0].Observation.ReadOutcome != "unknown" {
t.Fatalf("unexpected sanitized terminal labels: %+v err=%v", record, err)
}
}
func TestEvaluationRedactsErrorTerminalAndBlocksTool(t *testing.T) {
dir, _ := newEvaluationEventStore(t)
var calls atomic.Int32
+25 -3
View File
@@ -413,6 +413,28 @@ type evalCallRecord struct {
FailureClass string `json:"failureClass,omitempty"`
}
func evalFinishReasonClass(reason string) string {
switch reason {
case "", "stop", "length", "tool_calls", "function_call", "content_filter":
return reason
default:
return "unknown"
}
}
func evalTerminalObservation(observation *agentllm.TerminalObservation) *agentllm.TerminalObservation {
if observation == nil {
return nil
}
copy := *observation
switch copy.ReadOutcome {
case "done_marker", "clean_eof", "read_error", "cancelled", "invalid_frame", "provider_error", "transport_error":
default:
copy.ReadOutcome = "unknown"
}
return &copy
}
func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) agentllm.StreamFunc {
batchLimit := 120000
if raw := os.Getenv("WORK_CHECKPOINT_REMAINING_TOKENS"); raw != "" {
@@ -565,12 +587,12 @@ func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) ag
var terminalObservation *agentllm.TerminalObservation
httpStatus := 0
if terminal != nil {
finishReason, streamComplete = terminal.FinishReason, terminal.StreamComplete
finishReason, streamComplete = evalFinishReasonClass(terminal.FinishReason), terminal.StreamComplete
returnedUsage = terminal.Usage
terminalObservation = terminal.TerminalObservation
terminalObservation = evalTerminalObservation(terminal.TerminalObservation)
httpStatus = terminal.HTTPStatus
} else if streamError != nil {
terminalObservation = streamError.TerminalObservation
terminalObservation = evalTerminalObservation(streamError.TerminalObservation)
httpStatus = streamError.HTTPStatus
}
failureClass := ""