Close U3 review gaps before provider continuation
ci / vet-and-test (pull_request) Successful in 50s

Why: Missing Usage must stop a paid sequence before a tool terminal is executable; exact attachment names and host-owned choices must also remain reliable.

Evidence: Each reported behavior had a failing regression before its fix. Full Backend vet/Go tests, live-tag budget tests, ten race replays, and six task-owned HTTP/Playwright scenarios passed with the aligned Agent-Native and Agent-Squad pins.

Impact: Evaluation terminal accounting becomes durable before delivery; material fallback matches bounded filenames; ready choices use host options without redundant model fields. No new model calls or budget authorization.

Refs: U3 r0.4 independent review Critical 1 and Important 3-4
This commit is contained in:
2026-10-14 03:57:33 +00:00
parent 726e4a6d2e
commit 583c074aa4
8 changed files with 193 additions and 18 deletions
+42 -2
View File
@@ -4,6 +4,8 @@ import (
"crypto/sha256"
"encoding/hex"
"strings"
"unicode"
"unicode/utf8"
core "agent-system-core"
)
@@ -34,7 +36,7 @@ func (c *runCtx) materialAttachmentFallback(q core.RetrievalQuery) ([]core.Retri
var matched *core.ParsedDocument
for i := range c.attachments {
name := strings.TrimSpace(c.attachments[i].Meta.Filename)
if name == "" || !strings.Contains(q.Text, name) {
if exactFilenameIndex(q.Text, name) < 0 {
continue
}
if matched != nil {
@@ -109,11 +111,49 @@ func (c *runCtx) materialAttachmentFallback(q core.RetrievalQuery) ([]core.Retri
return []core.RetrievalHit{{Doc: doc, Score: 1, Kind: "fulltext"}}, nil
}
// A filename must stand as a whole query item, including when it has spaces.
// Nearby letters, digits, dots and hyphens may belong to another filename.
func exactFilenameIndex(query, name string) int {
if name == "" {
return -1
}
boundary := func(r rune) bool {
return unicode.IsSpace(r) || strings.ContainsRune(`"'“”‘’「」『』()()[]【】,,;;::`, r)
}
for from := 0; from < len(query); {
at := strings.Index(query[from:], name)
if at < 0 {
return -1
}
at += from
before := at == 0
if !before {
r, _ := utf8.DecodeLastRuneInString(query[:at])
before = boundary(r)
}
end := at + len(name)
after := end == len(query)
if !after {
r, _ := utf8.DecodeRuneInString(query[end:])
after = boundary(r)
}
if before && after {
return at
}
from = at + 1
}
return -1
}
func materialExcerpt(full, query, filename string, docs []core.ParsedDocument, id core.ID, eff core.EffectiveSettings, limit int) string {
runes := []rune(full)
start := 0
matchedTerm := false
terms := strings.Fields(strings.TrimSpace(strings.Replace(query, filename, "", 1)))
withoutName := query
if at := exactFilenameIndex(query, filename); at >= 0 {
withoutName = query[:at] + query[at+len(filename):]
}
terms := strings.Fields(strings.TrimSpace(withoutName))
for _, term := range terms {
if at := strings.Index(full, term); at >= 0 {
matchedTerm = true
+19
View File
@@ -38,6 +38,25 @@ func TestMaterialAttachmentFallbackQuery(t *testing.T) {
}
}
func TestMaterialAttachmentFallbackNeedsExactFilename(t *testing.T) {
c := materialRetrievalContext(t)
for _, tc := range []struct {
query string
want int
}{
{"决定.txt 上线 日期", 1},
{`"决定.txt" 上线`, 1},
{"非决定.txt 上线", 0},
{"决定.txt备份 上线", 0},
{"前缀-决定.txt 上线", 0},
} {
hits, err := c.materialAttachmentFallback(core.RetrievalQuery{ProjectID: c.run.ProjectID, Text: tc.query})
if err != nil || len(hits) != tc.want {
t.Fatalf("query %q matched %d current attachments; want %d (err=%v)", tc.query, len(hits), tc.want, err)
}
}
}
func TestMaterialAttachmentFallbackScope(t *testing.T) {
for _, tc := range []struct {
name string
+3 -1
View File
@@ -882,7 +882,9 @@ func (st *directRun) runToolCtx(ctx context.Context, name, args string) agenttoo
FreeInput *bool `json:"free_input"`
TimeoutSec int `json:"timeout_sec"`
}
if uerr := json.Unmarshal([]byte(call.Arguments), &a); uerr != nil || strings.TrimSpace(a.Question) == "" || len(a.Options) < 2 {
if uerr := json.Unmarshal([]byte(call.Arguments), &a); uerr != nil ||
(!st.materialNotice && (strings.TrimSpace(a.Question) == "" || len(a.Options) < 2)) ||
(st.materialNotice && len(st.materialChoiceOptions) < 3) {
out = "confirm_choice 参数无效:需要 question 与 2-4 个选项(id/label/description/impact)"
break
}
+16
View File
@@ -251,6 +251,22 @@ func TestMaterialNoticeChoiceUsesHostOptions(t *testing.T) {
}
}
func TestMaterialNoticeChoiceNeedsNoModelOptions(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlErr: errors.New("no answer")}
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态:completed\n" +
"[material-choice-options-v1]:[{\"label\":\"会议摘要\",\"basis\":\"会议决定分两期。\"},{\"label\":\"验收清单\",\"basis\":\"阈值待确认。\"},{\"label\":\"联调计划\",\"basis\":\"负责人待确认。\"}]\n" +
"[material-choice-ready]\n</system-reminder>"}}}}
runMaterial(t, [][]agentllm.Event{
hitlTurn(toolCall("confirm_choice", `{}`)),
textDeltaTurn("请说明下一步目标。"),
}, rc)
if len(rc.hitls) != 1 || len(rc.hitls[0].Choices) != 3 || rc.hitls[0].Choices[0].Label != "会议摘要" ||
rc.hitls[0].Question != "基于整理结果,接下来生成什么?" {
t.Fatalf("host-owned choice rejected because model omitted redundant options: %+v", rc.hitls)
}
}
func TestMaterialNoticeWithoutSuggestionsRejectsDelegation(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
@@ -17,6 +17,7 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
agentllm "agent-native/runtime/llm"
)
@@ -169,6 +170,44 @@ func TestEvaluationStopsWhenUsageMissing(t *testing.T) {
}
}
func TestEvaluationDoesNotForwardToolTerminalBeforeUsageRecorded(t *testing.T) {
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
dir, _ := newEvaluationEventStore(t)
var calls atomic.Int32
release := make(chan struct{})
provider := func(context.Context, string, []agentllm.Message, []agentllm.ToolDecl, agentllm.StreamOptions) <-chan agentllm.Event {
calls.Add(1)
out := make(chan agentllm.Event)
go func() {
defer close(out)
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant,
StopReason: agentllm.StopToolCalls, ToolCalls: []agentllm.ToolCall{{ID: "call-1", Name: "publish_work_checkpoint", Arguments: `{}`}}},
FinishReason: "tool_calls", StreamComplete: true}
<-release // keep the provider stream open after its terminal
}()
return out
}
stream := evalProviderStreamWithProvider(provider, dir)
time.AfterFunc(100*time.Millisecond, func() { close(release) })
var terminal agentllm.Event
first := stream(evalTestContext(t), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000})
for ev := range first {
if ev.Kind == agentllm.KindMessageEnd {
terminal = ev
break
}
}
blocked := false
for ev := range stream(bruntime.WithProviderRunID(context.Background(), t.Name()+"-next"), "fixture", nil, nil, agentllm.StreamOptions{MaxTokens: 4000}) {
blocked = blocked || ev.Kind == agentllm.KindError
}
for range first {
}
if terminal.Err == nil || len(terminal.Message.ToolCalls) != 0 || !blocked || calls.Load() != 1 {
t.Fatalf("missing Usage tool terminal escaped stop gate: terminal=%#v blocked=%v providerCalls=%d", terminal, blocked, calls.Load())
}
}
func TestEvaluationStopsWhenProviderRecordUnavailable(t *testing.T) {
t.Setenv("WORK_CHECKPOINT_REMAINING_TOKENS", "60000")
var providerCalls atomic.Int32
+72 -13
View File
@@ -101,6 +101,7 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
if len(calls) > 0 {
reason = agentllm.StopToolCalls
}
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: content}
out <- agentllm.Event{Kind: agentllm.KindMessageEnd, Message: agentllm.Message{Role: agentllm.RoleAssistant, Content: content, ToolCalls: calls, StopReason: reason}, Usage: agentllm.Usage{PromptTokens: 10, CompletionTokens: 10, TotalTokens: 20}}
return out
}
@@ -147,6 +148,9 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
if toolReplies >= 2 && strings.Contains(sourceText, "[fixture:choice]") {
content = "整理结束,日期与验收条件需要核对;完整范围以原件为准。\n\n## 建议后续处理\n1. **会议摘要**\n - 依据:材料记载上线六月。\n2. **验收清单**\n - 依据:验收条件需要补齐。\n3. **联调计划**\n - 依据:三项需求依赖外部接口。"
}
if !checkpoint && toolReplies >= 2 && strings.Contains(sourceText, "[fixture:slow-stages]") {
time.Sleep(1500 * time.Millisecond)
}
if len(calls) == 0 {
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: content}
}
@@ -382,6 +386,14 @@ func evalRejectedCheckpointPersisted(dir string) (bool, error) {
}
func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
base, _ := route["baseUrl"].(string)
key, _ := route["apiKey"].(string)
model, _ := route["model"].(string)
provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream
return evalProviderStreamWithProvider(provider, dir)
}
func evalProviderStreamWithProvider(provider agentllm.StreamFunc, dir string) agentllm.StreamFunc {
batchLimit := 120000
if raw := os.Getenv("WORK_CHECKPOINT_REMAINING_TOKENS"); raw != "" {
limit, err := strconv.Atoi(raw)
@@ -390,10 +402,6 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
}
batchLimit = limit
}
base, _ := route["baseUrl"].(string)
key, _ := route["apiKey"].(string)
model, _ := route["model"].(string)
provider := (&agentllm.Client{BaseURL: base, APIKey: key, HTTPClient: &http.Client{}, ReasoningSplit: strings.HasPrefix(model, "MiniMax-M")}).Stream
type allowance struct {
Calls int
Started time.Time
@@ -468,24 +476,45 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
defer cancel()
usage := 0
var terminal *agentllm.Event
var streamError *agentllm.Event
arguments := map[int]string{}
argumentBytes := 0
for event := range in {
// A terminal is executable to the runner. Hold it until the
// provider stream has closed and its Usage/stop record is durable.
if event.Kind == agentllm.KindMessageEnd {
copy := event
if terminal != nil {
streamError = &agentllm.Event{Kind: agentllm.KindError, Err: fmt.Errorf("evaluation budget: duplicate provider terminal")}
} else {
terminal = &copy
usage = event.Usage.TotalTokens
if usage == 0 {
usage = event.Usage.PromptTokens + event.Usage.CompletionTokens
}
}
continue
}
if event.Kind == agentllm.KindError {
copy := event
streamError = &copy
continue
}
if terminal != nil {
streamError = &agentllm.Event{Kind: agentllm.KindError, Err: fmt.Errorf("evaluation budget: provider emitted data after terminal")}
continue
}
if event.Kind == agentllm.KindToolCallDelta {
arguments[event.ToolIndex] += event.ArgumentsDelta
argumentBytes += len(event.ArgumentsDelta)
}
if event.Kind == agentllm.KindMessageEnd {
copy := event
terminal = &copy
usage = event.Usage.TotalTokens
if usage == 0 {
usage = event.Usage.PromptTokens + event.Usage.CompletionTokens
}
}
select {
case out <- event:
case <-ctx.Done():
mu.Lock()
terminalFault = true
unknown = true
mu.Unlock()
return
}
}
@@ -497,7 +526,17 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
break
}
}
fault := terminal == nil || terminal.Err != nil || terminal.Message.StopReason == agentllm.StopError
fault := terminal == nil || terminal.Err != nil || terminal.Message.StopReason == agentllm.StopError || streamError != nil
if terminal != nil && (streamError != nil || usage == 0) {
if streamError != nil {
terminal.Err = fmt.Errorf("evaluation budget: provider stream failed after terminal: %v", streamError.Err)
} else {
terminal.Err = fmt.Errorf("evaluation budget: provider usage unavailable")
}
terminal.Message.StopReason = agentllm.StopError
terminal.Message.ToolCalls = nil
fault = true
}
finishReason, streamComplete := "", false
if terminal != nil {
finishReason, streamComplete = terminal.FinishReason, terminal.StreamComplete
@@ -520,8 +559,28 @@ func evalProviderStream(route map[string]any, dir string) agentllm.StreamFunc {
"usageUnknown": unknown, "terminalFault": terminalFault, "modelRuns": len(groups)}, "", " ")
if err := os.WriteFile(filepath.Join(dir, "provider-records.json"), raw, 0600); err != nil {
terminalFault = true // no durable accounting: never issue another paid request
if terminal != nil {
terminal.Err = fmt.Errorf("evaluation budget: provider record unavailable: %w", err)
terminal.Message.StopReason = agentllm.StopError
terminal.Message.ToolCalls = nil
}
}
mu.Unlock()
if terminal != nil {
select {
case out <- *terminal:
case <-ctx.Done():
}
} else {
failure := agentllm.Event{Kind: agentllm.KindError, Err: fmt.Errorf("evaluation budget: provider terminal unavailable")}
if streamError != nil {
failure = *streamError
}
select {
case out <- failure:
case <-ctx.Done():
}
}
}()
return out
}