Announce final-only on every material tool result #235
@@ -1,5 +1,7 @@
|
|||||||
package runtime
|
package runtime
|
||||||
|
|
||||||
|
import agenttools "agent-native/runtime/tools"
|
||||||
|
|
||||||
// materialBudgetState describes only usage returned by the provider for a
|
// materialBudgetState describes only usage returned by the provider for a
|
||||||
// material subagent. A response with no reported usage cannot authorize a
|
// material subagent. A response with no reported usage cannot authorize a
|
||||||
// further request.
|
// further request.
|
||||||
@@ -42,6 +44,24 @@ func materialOutputCap(rc RunCtx) int {
|
|||||||
return outputCap
|
return outputCap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// withFinalOnlyNotice announces the final-only transition on a tool result the
|
||||||
|
// model actually reads. The reserve can be reached by a round of ordinary tool
|
||||||
|
// calls, so the notice belongs on every result handed back after the decision
|
||||||
|
// flips — not only on checkpoint results. IsError is never changed.
|
||||||
|
func (st *subRun) withFinalOnlyNotice(res agenttools.Result) agenttools.Result {
|
||||||
|
if st.materialLimit <= 0 {
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
if finalOnly, stop := st.materialDecision(); !finalOnly && !stop {
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
if res.Content != "" {
|
||||||
|
res.Content += "\n\n"
|
||||||
|
}
|
||||||
|
res.Content += materialFinalOnlyNotice
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
// materialDecision is the single source of the final-only/stop decision for
|
// materialDecision is the single source of the final-only/stop decision for
|
||||||
// both the next request and the checkpoint tool result. Only meaningful when
|
// both the next request and the checkpoint tool result. Only meaningful when
|
||||||
// materialLimit > 0.
|
// materialLimit > 0.
|
||||||
|
|||||||
@@ -395,14 +395,7 @@ func (st *subRun) runTool(name, args string) agenttools.Result {
|
|||||||
}
|
}
|
||||||
st.savedCheckpointIDs[submitted.CheckpointID] = struct{}{}
|
st.savedCheckpointIDs[submitted.CheckpointID] = struct{}{}
|
||||||
}
|
}
|
||||||
if st.materialLimit > 0 {
|
res = st.withFinalOnlyNotice(res)
|
||||||
if finalOnly, stop := st.materialDecision(); finalOnly || stop {
|
|
||||||
if res.Content != "" {
|
|
||||||
res.Content += "\n\n"
|
|
||||||
}
|
|
||||||
res.Content += materialFinalOnlyNotice
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if st.inflight != nil {
|
if st.inflight != nil {
|
||||||
st.inflight.summary = res.Content
|
st.inflight.summary = res.Content
|
||||||
st.inflight.status = "ok"
|
st.inflight.status = "ok"
|
||||||
@@ -426,15 +419,14 @@ func (st *subRun) runTool(name, args string) agenttools.Result {
|
|||||||
if res.Artifact != nil {
|
if res.Artifact != nil {
|
||||||
ref = &core.ArtifactRef{ArtifactID: res.Artifact.ID, Name: res.Artifact.Name, Kind: res.Artifact.Kind}
|
ref = &core.ArtifactRef{ArtifactID: res.Artifact.ID, Name: res.Artifact.Name, Kind: res.Artifact.Kind}
|
||||||
}
|
}
|
||||||
|
out := agenttools.Result{Content: summary, IsError: status != "ok"}
|
||||||
|
out = st.withFinalOnlyNotice(out)
|
||||||
if st.inflight != nil {
|
if st.inflight != nil {
|
||||||
st.inflight.summary = summary
|
st.inflight.summary = out.Content
|
||||||
st.inflight.status = status
|
st.inflight.status = status
|
||||||
st.inflight.artifact = ref
|
st.inflight.artifact = ref
|
||||||
}
|
}
|
||||||
if status != "ok" {
|
return out
|
||||||
return agenttools.Result{Content: summary, IsError: true}
|
|
||||||
}
|
|
||||||
return agenttools.Result{Content: summary}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func subagentBudgetError(reason string) <-chan agentllm.Event {
|
func subagentBudgetError(reason string) <-chan agentllm.Event {
|
||||||
|
|||||||
@@ -303,6 +303,180 @@ func TestMaterialCheckpointResultAnnouncesFinalOnly(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestMaterialToolResultAnnouncesFinalOnly: the budget reserve can be reached
|
||||||
|
// by a round of ordinary tool calls, so every tool result the model reads
|
||||||
|
// after the transition must carry the notice — not only checkpoint results.
|
||||||
|
func TestMaterialToolResultAnnouncesFinalOnly(t *testing.T) {
|
||||||
|
toolMsgs := func(msgs []agentllm.Message) []agentllm.Message {
|
||||||
|
var out []agentllm.Message
|
||||||
|
for _, msg := range msgs {
|
||||||
|
if msg.Role == agentllm.RoleTool {
|
||||||
|
out = append(out, msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
partSummaries := func(parts []core.Part) []string {
|
||||||
|
var out []string
|
||||||
|
for _, p := range parts {
|
||||||
|
if p.Type == core.PartToolResult && p.ToolResult != nil {
|
||||||
|
out = append(out, p.ToolResult.Summary)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
declares := func(decls []agentllm.ToolDecl, name string) bool {
|
||||||
|
for _, d := range decls {
|
||||||
|
if d.Name == name {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
turn := func(usage agentllm.Usage, body string, call ...agentllm.ToolCall) []agentllm.Event {
|
||||||
|
events := textDeltaTurn(body, call...)
|
||||||
|
events[len(events)-1].Usage = usage
|
||||||
|
return events
|
||||||
|
}
|
||||||
|
// ample keeps the reserve out of reach for one request; narrow is the
|
||||||
|
// second request that trips it.
|
||||||
|
ample := agentllm.Usage{PromptTokens: 4000, CompletionTokens: 1000, TotalTokens: 5000}
|
||||||
|
narrow := agentllm.Usage{PromptTokens: 8000, CompletionTokens: 2000, TotalTokens: 10000}
|
||||||
|
searchArgs := `{"query":"上线时间"}`
|
||||||
|
// materialRC exposes one non-checkpoint tool to the material subagent and
|
||||||
|
// fixes the outcome its handler reports back.
|
||||||
|
materialRC := func(limit int, execStatus, execOutput string) *cappedCheckpointRC {
|
||||||
|
prof := subProfile()
|
||||||
|
if limit > 0 {
|
||||||
|
prof.Budget.MaxTokens = &limit
|
||||||
|
}
|
||||||
|
return &cappedCheckpointRC{&checkpointRC{subRC: subRC{
|
||||||
|
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: prof, execStatus: execStatus, execOutput: execOutput},
|
||||||
|
id: materialSubagentID,
|
||||||
|
tools: []core.ToolDesc{{Name: "knowledge_search", Description: "检索材料", Parameters: []byte(`{"type":"object"}`)}},
|
||||||
|
}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("budget reserve announces on the next tool result", func(t *testing.T) {
|
||||||
|
rc := materialRC(30000, "", "命中 2 条材料")
|
||||||
|
base := scriptStream([][]agentllm.Event{
|
||||||
|
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
|
||||||
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
||||||
|
textDeltaTurn("整理完成"),
|
||||||
|
}, nil)
|
||||||
|
calls := 0
|
||||||
|
var seen [][]agentllm.Message
|
||||||
|
var secondDecls, thirdDecls []agentllm.ToolDecl
|
||||||
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||||
|
calls++
|
||||||
|
seen = append(seen, toolMsgs(msgs))
|
||||||
|
switch calls {
|
||||||
|
case 2:
|
||||||
|
secondDecls = decls
|
||||||
|
case 3:
|
||||||
|
thirdDecls = decls
|
||||||
|
}
|
||||||
|
return base(ctx, model, msgs, decls, opts)
|
||||||
|
}}
|
||||||
|
if err := r.Execute(rc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 3 || len(rc.checkpoints) != 1 || rc.toolRuns != 1 {
|
||||||
|
t.Fatalf("unexpected flow: calls=%d saved=%d toolRuns=%d", calls, len(rc.checkpoints), rc.toolRuns)
|
||||||
|
}
|
||||||
|
if len(seen[1]) != 1 || strings.Contains(seen[1][0].Content, materialFinalOnlyNotice) {
|
||||||
|
t.Fatalf("first checkpoint result announced final-only early: %+v", seen[1])
|
||||||
|
}
|
||||||
|
if !declares(secondDecls, workCheckpointTool) || !declares(secondDecls, "knowledge_search") || declares(thirdDecls, workCheckpointTool) {
|
||||||
|
t.Fatalf("checkpoint declaration not withdrawn exactly at the transition: second=%+v third=%+v", secondDecls, thirdDecls)
|
||||||
|
}
|
||||||
|
if len(seen[2]) != 2 || strings.Contains(seen[2][0].Content, materialFinalOnlyNotice) {
|
||||||
|
t.Fatalf("unexpected results before the final request: %+v", seen[2])
|
||||||
|
}
|
||||||
|
got := seen[2][1]
|
||||||
|
if !strings.HasPrefix(got.Content, "命中 2 条材料\n\n"+materialFinalOnlyNotice) || got.IsError {
|
||||||
|
t.Fatalf("generic tool result lacks the notice or changed IsError: %+v", got)
|
||||||
|
}
|
||||||
|
summaries := partSummaries(rc.assistant)
|
||||||
|
if len(summaries) != 2 || strings.Contains(summaries[0], materialFinalOnlyNotice) || summaries[1] != got.Content {
|
||||||
|
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("failed tool result keeps IsError while announcing final-only", func(t *testing.T) {
|
||||||
|
rc := materialRC(30000, "error", "检索超时")
|
||||||
|
base := scriptStream([][]agentllm.Event{
|
||||||
|
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
|
||||||
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
||||||
|
textDeltaTurn("整理完成"),
|
||||||
|
}, nil)
|
||||||
|
calls := 0
|
||||||
|
var third []agentllm.Message
|
||||||
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||||
|
calls++
|
||||||
|
if calls == 3 {
|
||||||
|
third = toolMsgs(msgs)
|
||||||
|
}
|
||||||
|
return base(ctx, model, msgs, decls, opts)
|
||||||
|
}}
|
||||||
|
if err := r.Execute(rc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 3 || len(third) != 2 || !third[1].IsError || third[1].Content != "检索超时\n\n"+materialFinalOnlyNotice {
|
||||||
|
t.Fatalf("failed result lost IsError or the notice: calls=%d results=%+v", calls, third)
|
||||||
|
}
|
||||||
|
summaries := partSummaries(rc.assistant)
|
||||||
|
if len(summaries) != 2 || summaries[1] != third[1].Content {
|
||||||
|
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("ample budget leaves a generic result unchanged", func(t *testing.T) {
|
||||||
|
rc := materialRC(100000, "", "命中 2 条材料")
|
||||||
|
base := scriptStream([][]agentllm.Event{
|
||||||
|
turn(ample, "", toolCall("knowledge_search", searchArgs)),
|
||||||
|
textDeltaTurn("整理完成"),
|
||||||
|
}, nil)
|
||||||
|
calls := 0
|
||||||
|
var last []agentllm.Message
|
||||||
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||||
|
calls++
|
||||||
|
last = toolMsgs(msgs)
|
||||||
|
return base(ctx, model, msgs, decls, opts)
|
||||||
|
}}
|
||||||
|
if err := r.Execute(rc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
|
||||||
|
t.Fatalf("ample budget changed the generic result: calls=%d results=%+v", calls, last)
|
||||||
|
}
|
||||||
|
if summaries := partSummaries(rc.assistant); len(summaries) != 1 || summaries[0] != "命中 2 条材料" {
|
||||||
|
t.Fatalf("persisted summary does not match the model-visible content: %q", summaries)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no material budget leaves a generic result unchanged", func(t *testing.T) {
|
||||||
|
rc := materialRC(0, "", "命中 2 条材料")
|
||||||
|
base := scriptStream([][]agentllm.Event{
|
||||||
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
||||||
|
textDeltaTurn("整理完成"),
|
||||||
|
}, nil)
|
||||||
|
calls := 0
|
||||||
|
var last []agentllm.Message
|
||||||
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||||||
|
calls++
|
||||||
|
last = toolMsgs(msgs)
|
||||||
|
return base(ctx, model, msgs, decls, opts)
|
||||||
|
}}
|
||||||
|
if err := r.Execute(rc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
|
||||||
|
t.Fatalf("notice added without material budget: calls=%d results=%+v", calls, last)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestMaterialFinalReserveStopsBeforeUnaffordableRequest(t *testing.T) {
|
func TestMaterialFinalReserveStopsBeforeUnaffordableRequest(t *testing.T) {
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
Reference in New Issue
Block a user