Announce final-only on every material tool result #235

Merged
weichen merged 1 commits from feature/fast-slow-o2-final-only-notice-all-tools into main 2026-10-21 04:40:07 +00:00
3 changed files with 199 additions and 13 deletions
@@ -1,5 +1,7 @@
package runtime
import agenttools "agent-native/runtime/tools"
// materialBudgetState describes only usage returned by the provider for a
// material subagent. A response with no reported usage cannot authorize a
// further request.
@@ -42,6 +44,24 @@ func materialOutputCap(rc RunCtx) int {
return outputCap
}
// withFinalOnlyNotice announces the final-only transition on a tool result the
// model actually reads. The reserve can be reached by a round of ordinary tool
// calls, so the notice belongs on every result handed back after the decision
// flips — not only on checkpoint results. IsError is never changed.
func (st *subRun) withFinalOnlyNotice(res agenttools.Result) agenttools.Result {
if st.materialLimit <= 0 {
return res
}
if finalOnly, stop := st.materialDecision(); !finalOnly && !stop {
return res
}
if res.Content != "" {
res.Content += "\n\n"
}
res.Content += materialFinalOnlyNotice
return res
}
// materialDecision is the single source of the final-only/stop decision for
// both the next request and the checkpoint tool result. Only meaningful when
// materialLimit > 0.
+5 -13
View File
@@ -395,14 +395,7 @@ func (st *subRun) runTool(name, args string) agenttools.Result {
}
st.savedCheckpointIDs[submitted.CheckpointID] = struct{}{}
}
if st.materialLimit > 0 {
if finalOnly, stop := st.materialDecision(); finalOnly || stop {
if res.Content != "" {
res.Content += "\n\n"
}
res.Content += materialFinalOnlyNotice
}
}
res = st.withFinalOnlyNotice(res)
if st.inflight != nil {
st.inflight.summary = res.Content
st.inflight.status = "ok"
@@ -426,15 +419,14 @@ func (st *subRun) runTool(name, args string) agenttools.Result {
if res.Artifact != nil {
ref = &core.ArtifactRef{ArtifactID: res.Artifact.ID, Name: res.Artifact.Name, Kind: res.Artifact.Kind}
}
out := agenttools.Result{Content: summary, IsError: status != "ok"}
out = st.withFinalOnlyNotice(out)
if st.inflight != nil {
st.inflight.summary = summary
st.inflight.summary = out.Content
st.inflight.status = status
st.inflight.artifact = ref
}
if status != "ok" {
return agenttools.Result{Content: summary, IsError: true}
}
return agenttools.Result{Content: summary}
return out
}
func subagentBudgetError(reason string) <-chan agentllm.Event {
+174
View File
@@ -303,6 +303,180 @@ func TestMaterialCheckpointResultAnnouncesFinalOnly(t *testing.T) {
})
}
// TestMaterialToolResultAnnouncesFinalOnly: the budget reserve can be reached
// by a round of ordinary tool calls, so every tool result the model reads
// after the transition must carry the notice — not only checkpoint results.
func TestMaterialToolResultAnnouncesFinalOnly(t *testing.T) {
toolMsgs := func(msgs []agentllm.Message) []agentllm.Message {
var out []agentllm.Message
for _, msg := range msgs {
if msg.Role == agentllm.RoleTool {
out = append(out, msg)
}
}
return out
}
partSummaries := func(parts []core.Part) []string {
var out []string
for _, p := range parts {
if p.Type == core.PartToolResult && p.ToolResult != nil {
out = append(out, p.ToolResult.Summary)
}
}
return out
}
declares := func(decls []agentllm.ToolDecl, name string) bool {
for _, d := range decls {
if d.Name == name {
return true
}
}
return false
}
turn := func(usage agentllm.Usage, body string, call ...agentllm.ToolCall) []agentllm.Event {
events := textDeltaTurn(body, call...)
events[len(events)-1].Usage = usage
return events
}
// ample keeps the reserve out of reach for one request; narrow is the
// second request that trips it.
ample := agentllm.Usage{PromptTokens: 4000, CompletionTokens: 1000, TotalTokens: 5000}
narrow := agentllm.Usage{PromptTokens: 8000, CompletionTokens: 2000, TotalTokens: 10000}
searchArgs := `{"query":"上线时间"}`
// materialRC exposes one non-checkpoint tool to the material subagent and
// fixes the outcome its handler reports back.
materialRC := func(limit int, execStatus, execOutput string) *cappedCheckpointRC {
prof := subProfile()
if limit > 0 {
prof.Budget.MaxTokens = &limit
}
return &cappedCheckpointRC{&checkpointRC{subRC: subRC{
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: prof, execStatus: execStatus, execOutput: execOutput},
id: materialSubagentID,
tools: []core.ToolDesc{{Name: "knowledge_search", Description: "检索材料", Parameters: []byte(`{"type":"object"}`)}},
}}}
}
t.Run("budget reserve announces on the next tool result", func(t *testing.T) {
rc := materialRC(30000, "", "命中 2 条材料")
base := scriptStream([][]agentllm.Event{
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
textDeltaTurn("整理完成"),
}, nil)
calls := 0
var seen [][]agentllm.Message
var secondDecls, thirdDecls []agentllm.ToolDecl
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
seen = append(seen, toolMsgs(msgs))
switch calls {
case 2:
secondDecls = decls
case 3:
thirdDecls = decls
}
return base(ctx, model, msgs, decls, opts)
}}
if err := r.Execute(rc); err != nil {
t.Fatal(err)
}
if calls != 3 || len(rc.checkpoints) != 1 || rc.toolRuns != 1 {
t.Fatalf("unexpected flow: calls=%d saved=%d toolRuns=%d", calls, len(rc.checkpoints), rc.toolRuns)
}
if len(seen[1]) != 1 || strings.Contains(seen[1][0].Content, materialFinalOnlyNotice) {
t.Fatalf("first checkpoint result announced final-only early: %+v", seen[1])
}
if !declares(secondDecls, workCheckpointTool) || !declares(secondDecls, "knowledge_search") || declares(thirdDecls, workCheckpointTool) {
t.Fatalf("checkpoint declaration not withdrawn exactly at the transition: second=%+v third=%+v", secondDecls, thirdDecls)
}
if len(seen[2]) != 2 || strings.Contains(seen[2][0].Content, materialFinalOnlyNotice) {
t.Fatalf("unexpected results before the final request: %+v", seen[2])
}
got := seen[2][1]
if !strings.HasPrefix(got.Content, "命中 2 条材料\n\n"+materialFinalOnlyNotice) || got.IsError {
t.Fatalf("generic tool result lacks the notice or changed IsError: %+v", got)
}
summaries := partSummaries(rc.assistant)
if len(summaries) != 2 || strings.Contains(summaries[0], materialFinalOnlyNotice) || summaries[1] != got.Content {
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
}
})
t.Run("failed tool result keeps IsError while announcing final-only", func(t *testing.T) {
rc := materialRC(30000, "error", "检索超时")
base := scriptStream([][]agentllm.Event{
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
textDeltaTurn("整理完成"),
}, nil)
calls := 0
var third []agentllm.Message
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
if calls == 3 {
third = toolMsgs(msgs)
}
return base(ctx, model, msgs, decls, opts)
}}
if err := r.Execute(rc); err != nil {
t.Fatal(err)
}
if calls != 3 || len(third) != 2 || !third[1].IsError || third[1].Content != "检索超时\n\n"+materialFinalOnlyNotice {
t.Fatalf("failed result lost IsError or the notice: calls=%d results=%+v", calls, third)
}
summaries := partSummaries(rc.assistant)
if len(summaries) != 2 || summaries[1] != third[1].Content {
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
}
})
t.Run("ample budget leaves a generic result unchanged", func(t *testing.T) {
rc := materialRC(100000, "", "命中 2 条材料")
base := scriptStream([][]agentllm.Event{
turn(ample, "", toolCall("knowledge_search", searchArgs)),
textDeltaTurn("整理完成"),
}, nil)
calls := 0
var last []agentllm.Message
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
last = toolMsgs(msgs)
return base(ctx, model, msgs, decls, opts)
}}
if err := r.Execute(rc); err != nil {
t.Fatal(err)
}
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
t.Fatalf("ample budget changed the generic result: calls=%d results=%+v", calls, last)
}
if summaries := partSummaries(rc.assistant); len(summaries) != 1 || summaries[0] != "命中 2 条材料" {
t.Fatalf("persisted summary does not match the model-visible content: %q", summaries)
}
})
t.Run("no material budget leaves a generic result unchanged", func(t *testing.T) {
rc := materialRC(0, "", "命中 2 条材料")
base := scriptStream([][]agentllm.Event{
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
textDeltaTurn("整理完成"),
}, nil)
calls := 0
var last []agentllm.Message
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
calls++
last = toolMsgs(msgs)
return base(ctx, model, msgs, decls, opts)
}}
if err := r.Execute(rc); err != nil {
t.Fatal(err)
}
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
t.Fatalf("notice added without material budget: calls=%d results=%+v", calls, last)
}
})
}
func TestMaterialFinalReserveStopsBeforeUnaffordableRequest(t *testing.T) {
for _, tc := range []struct {
name string