|
|
|
@@ -303,6 +303,180 @@ func TestMaterialCheckpointResultAnnouncesFinalOnly(t *testing.T) {
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestMaterialToolResultAnnouncesFinalOnly: the budget reserve can be reached
|
|
|
|
|
// by a round of ordinary tool calls, so every tool result the model reads
|
|
|
|
|
// after the transition must carry the notice — not only checkpoint results.
|
|
|
|
|
func TestMaterialToolResultAnnouncesFinalOnly(t *testing.T) {
|
|
|
|
|
toolMsgs := func(msgs []agentllm.Message) []agentllm.Message {
|
|
|
|
|
var out []agentllm.Message
|
|
|
|
|
for _, msg := range msgs {
|
|
|
|
|
if msg.Role == agentllm.RoleTool {
|
|
|
|
|
out = append(out, msg)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
partSummaries := func(parts []core.Part) []string {
|
|
|
|
|
var out []string
|
|
|
|
|
for _, p := range parts {
|
|
|
|
|
if p.Type == core.PartToolResult && p.ToolResult != nil {
|
|
|
|
|
out = append(out, p.ToolResult.Summary)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
declares := func(decls []agentllm.ToolDecl, name string) bool {
|
|
|
|
|
for _, d := range decls {
|
|
|
|
|
if d.Name == name {
|
|
|
|
|
return true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
turn := func(usage agentllm.Usage, body string, call ...agentllm.ToolCall) []agentllm.Event {
|
|
|
|
|
events := textDeltaTurn(body, call...)
|
|
|
|
|
events[len(events)-1].Usage = usage
|
|
|
|
|
return events
|
|
|
|
|
}
|
|
|
|
|
// ample keeps the reserve out of reach for one request; narrow is the
|
|
|
|
|
// second request that trips it.
|
|
|
|
|
ample := agentllm.Usage{PromptTokens: 4000, CompletionTokens: 1000, TotalTokens: 5000}
|
|
|
|
|
narrow := agentllm.Usage{PromptTokens: 8000, CompletionTokens: 2000, TotalTokens: 10000}
|
|
|
|
|
searchArgs := `{"query":"上线时间"}`
|
|
|
|
|
// materialRC exposes one non-checkpoint tool to the material subagent and
|
|
|
|
|
// fixes the outcome its handler reports back.
|
|
|
|
|
materialRC := func(limit int, execStatus, execOutput string) *cappedCheckpointRC {
|
|
|
|
|
prof := subProfile()
|
|
|
|
|
if limit > 0 {
|
|
|
|
|
prof.Budget.MaxTokens = &limit
|
|
|
|
|
}
|
|
|
|
|
return &cappedCheckpointRC{&checkpointRC{subRC: subRC{
|
|
|
|
|
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: prof, execStatus: execStatus, execOutput: execOutput},
|
|
|
|
|
id: materialSubagentID,
|
|
|
|
|
tools: []core.ToolDesc{{Name: "knowledge_search", Description: "检索材料", Parameters: []byte(`{"type":"object"}`)}},
|
|
|
|
|
}}}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
t.Run("budget reserve announces on the next tool result", func(t *testing.T) {
|
|
|
|
|
rc := materialRC(30000, "", "命中 2 条材料")
|
|
|
|
|
base := scriptStream([][]agentllm.Event{
|
|
|
|
|
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
|
|
|
|
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
|
|
|
|
textDeltaTurn("整理完成"),
|
|
|
|
|
}, nil)
|
|
|
|
|
calls := 0
|
|
|
|
|
var seen [][]agentllm.Message
|
|
|
|
|
var secondDecls, thirdDecls []agentllm.ToolDecl
|
|
|
|
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
|
|
|
calls++
|
|
|
|
|
seen = append(seen, toolMsgs(msgs))
|
|
|
|
|
switch calls {
|
|
|
|
|
case 2:
|
|
|
|
|
secondDecls = decls
|
|
|
|
|
case 3:
|
|
|
|
|
thirdDecls = decls
|
|
|
|
|
}
|
|
|
|
|
return base(ctx, model, msgs, decls, opts)
|
|
|
|
|
}}
|
|
|
|
|
if err := r.Execute(rc); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if calls != 3 || len(rc.checkpoints) != 1 || rc.toolRuns != 1 {
|
|
|
|
|
t.Fatalf("unexpected flow: calls=%d saved=%d toolRuns=%d", calls, len(rc.checkpoints), rc.toolRuns)
|
|
|
|
|
}
|
|
|
|
|
if len(seen[1]) != 1 || strings.Contains(seen[1][0].Content, materialFinalOnlyNotice) {
|
|
|
|
|
t.Fatalf("first checkpoint result announced final-only early: %+v", seen[1])
|
|
|
|
|
}
|
|
|
|
|
if !declares(secondDecls, workCheckpointTool) || !declares(secondDecls, "knowledge_search") || declares(thirdDecls, workCheckpointTool) {
|
|
|
|
|
t.Fatalf("checkpoint declaration not withdrawn exactly at the transition: second=%+v third=%+v", secondDecls, thirdDecls)
|
|
|
|
|
}
|
|
|
|
|
if len(seen[2]) != 2 || strings.Contains(seen[2][0].Content, materialFinalOnlyNotice) {
|
|
|
|
|
t.Fatalf("unexpected results before the final request: %+v", seen[2])
|
|
|
|
|
}
|
|
|
|
|
got := seen[2][1]
|
|
|
|
|
if !strings.HasPrefix(got.Content, "命中 2 条材料\n\n"+materialFinalOnlyNotice) || got.IsError {
|
|
|
|
|
t.Fatalf("generic tool result lacks the notice or changed IsError: %+v", got)
|
|
|
|
|
}
|
|
|
|
|
summaries := partSummaries(rc.assistant)
|
|
|
|
|
if len(summaries) != 2 || strings.Contains(summaries[0], materialFinalOnlyNotice) || summaries[1] != got.Content {
|
|
|
|
|
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("failed tool result keeps IsError while announcing final-only", func(t *testing.T) {
|
|
|
|
|
rc := materialRC(30000, "error", "检索超时")
|
|
|
|
|
base := scriptStream([][]agentllm.Event{
|
|
|
|
|
turn(ample, "", toolCall(workCheckpointTool, checkpointJSON())),
|
|
|
|
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
|
|
|
|
textDeltaTurn("整理完成"),
|
|
|
|
|
}, nil)
|
|
|
|
|
calls := 0
|
|
|
|
|
var third []agentllm.Message
|
|
|
|
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
|
|
|
calls++
|
|
|
|
|
if calls == 3 {
|
|
|
|
|
third = toolMsgs(msgs)
|
|
|
|
|
}
|
|
|
|
|
return base(ctx, model, msgs, decls, opts)
|
|
|
|
|
}}
|
|
|
|
|
if err := r.Execute(rc); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if calls != 3 || len(third) != 2 || !third[1].IsError || third[1].Content != "检索超时\n\n"+materialFinalOnlyNotice {
|
|
|
|
|
t.Fatalf("failed result lost IsError or the notice: calls=%d results=%+v", calls, third)
|
|
|
|
|
}
|
|
|
|
|
summaries := partSummaries(rc.assistant)
|
|
|
|
|
if len(summaries) != 2 || summaries[1] != third[1].Content {
|
|
|
|
|
t.Fatalf("persisted tool result summary does not match the model-visible content: %q", summaries)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("ample budget leaves a generic result unchanged", func(t *testing.T) {
|
|
|
|
|
rc := materialRC(100000, "", "命中 2 条材料")
|
|
|
|
|
base := scriptStream([][]agentllm.Event{
|
|
|
|
|
turn(ample, "", toolCall("knowledge_search", searchArgs)),
|
|
|
|
|
textDeltaTurn("整理完成"),
|
|
|
|
|
}, nil)
|
|
|
|
|
calls := 0
|
|
|
|
|
var last []agentllm.Message
|
|
|
|
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
|
|
|
calls++
|
|
|
|
|
last = toolMsgs(msgs)
|
|
|
|
|
return base(ctx, model, msgs, decls, opts)
|
|
|
|
|
}}
|
|
|
|
|
if err := r.Execute(rc); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
|
|
|
|
|
t.Fatalf("ample budget changed the generic result: calls=%d results=%+v", calls, last)
|
|
|
|
|
}
|
|
|
|
|
if summaries := partSummaries(rc.assistant); len(summaries) != 1 || summaries[0] != "命中 2 条材料" {
|
|
|
|
|
t.Fatalf("persisted summary does not match the model-visible content: %q", summaries)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("no material budget leaves a generic result unchanged", func(t *testing.T) {
|
|
|
|
|
rc := materialRC(0, "", "命中 2 条材料")
|
|
|
|
|
base := scriptStream([][]agentllm.Event{
|
|
|
|
|
turn(narrow, "", toolCall("knowledge_search", searchArgs)),
|
|
|
|
|
textDeltaTurn("整理完成"),
|
|
|
|
|
}, nil)
|
|
|
|
|
calls := 0
|
|
|
|
|
var last []agentllm.Message
|
|
|
|
|
r := &SubLLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, decls []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
|
|
|
calls++
|
|
|
|
|
last = toolMsgs(msgs)
|
|
|
|
|
return base(ctx, model, msgs, decls, opts)
|
|
|
|
|
}}
|
|
|
|
|
if err := r.Execute(rc); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
if calls != 2 || len(last) != 1 || last[0].Content != "命中 2 条材料" || last[0].IsError {
|
|
|
|
|
t.Fatalf("notice added without material budget: calls=%d results=%+v", calls, last)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestMaterialFinalReserveStopsBeforeUnaffordableRequest(t *testing.T) {
|
|
|
|
|
for _, tc := range []struct {
|
|
|
|
|
name string
|
|
|
|
|