Files
Agent-System-Backend/internal/runtime/material_flow_test.go
T
weichen 42867aabf4 Add deliverable-choice chain: confirm_choice tool, notice contract, backstops
Why: after material organization completes, weak models report and end the turn without offering the user a choice of formal deliverables, and even after a user picks one the model may close the turn without starting the background task.
Evidence: new tests (notice-choice chain, deliverable backstop, free-input path, invalid args, notice contract) pass; full go test ./... exit 0 (local, SELF).
Impact: new confirm_choice tool issues a HITL choice card and routes the answer into a hard delegate instruction; material notices instruct the model to report then offer choices; stop hook adds two one-shot backstops (choice not offered in material notice runs; choice made but no background delegation).
Refs: Dev task .task/feature-material-intake-flow.md I2.
2026-10-11 16:53:53 +00:00

223 lines
9.8 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package runtime
import (
"context"
"strings"
"testing"
agentllm "agent-native/runtime/llm"
core "agent-system-core"
)
// matCtx 在 fakeRunCtx 之上实现材料整理的两个可选接口(附件即保证),
// 记录 EnsureMaterial 调用次数以断言 stop 兜底的幂等。
type matCtx struct {
*fakeRunCtx
note string
ensureN int
lastNote string
// confirm_choice 断言(I2)
hitls []core.HITLCard
hitlFree string
hitlErr error
}
func (m *matCtx) MaterialNote() string { return m.note }
func (m *matCtx) EnsureMaterial() string {
m.ensureN++
if m.ensureN > 1 {
return ""
}
return m.lastNote
}
// AskHITL 覆盖:预置选择默认取第一个选项;hitlFree 模拟自由输入。
func (m *matCtx) AskHITL(card core.HITLCard) (core.HITLResponse, error) {
m.hitls = append(m.hitls, card)
if m.hitlErr != nil {
return core.HITLResponse{}, m.hitlErr
}
if m.hitlFree != "" {
return core.HITLResponse{FreeText: m.hitlFree}, nil
}
if len(card.Choices) == 0 {
return core.HITLResponse{}, nil
}
return core.HITLResponse{ChoiceID: card.Choices[0].ID}, nil
}
func matAttachments() []core.ParsedDocument {
return []core.ParsedDocument{
{Meta: core.DocMeta{Filename: "纪要.docx"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "决定 11 月上线"}}},
{Meta: core.DocMeta{Filename: "意见.pdf"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "预算存疑"}}},
{Meta: core.DocMeta{Filename: "清单.md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "12 项需求"}}},
}
}
func runMaterial(t *testing.T, turns [][]agentllm.Event, rc *matCtx) *dualSpeedRecorder {
t.Helper()
rec := &dualSpeedRecorder{clock: &stepClock{}, turns: turns}
r := &LLMRunner{Stream: rec.stream(scriptStream(turns, nil)), Now: rec.clock.now}
if err := r.Execute(rc); err != nil {
t.Fatalf("Execute: %v", err)
}
return rec
}
func requestsContaining(rec *dualSpeedRecorder, needle string) int {
n := 0
for _, req := range rec.request {
if strings.Contains(req, needle) {
n++
}
}
return n
}
// 预启动说明替换建议式提示:第 0 回合注入事实说明而非「可能属于重任务」。
func TestTurn0MaterialNoteReplacesAdvisory(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, note: "系统已自动启动材料整理后台任务(收件箱进度卡「材料整理」已创建)。"}
rc.attachments = matAttachments()
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("收到,先给每份材料一句话概览。")}, rc)
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
t.Fatal("material note not injected at turn 0")
}
if requestsContaining(rec, "可能属于重任务") != 0 {
t.Fatal("advisory text should be replaced when material note present")
}
}
// 未预启动(文字+小材料):保留建议式入口提示。
func TestTurn0AdvisoryWhenNoNote(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
rc.attachments = matAttachments()
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("概览如下。")}, rc)
if requestsContaining(rec, "可能属于重任务") == 0 {
t.Fatal("advisory hint missing when no material note")
}
}
// stop 兜底:模型问一句澄清就收场(用户实测现象)→ 系统续跑一轮要求告知;
// ensureMaterial 幂等保证只续跑一次。
func TestStopMaterialBackstopContinuesOnce(t *testing.T) {
rc := &matCtx{
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
lastNote: "系统已自动启动材料整理后台任务",
}
rc.attachments = matAttachments()
turns := [][]agentllm.Event{
textDeltaTurn("请问你需要哪种产出?"),
textDeltaTurn("材料正在后台整理,完成后自动汇报;先给概览。"),
}
rec := runMaterial(t, turns, rc)
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
t.Fatal("backstop reason not injected as continuation")
}
if rc.ensureN != 1 {
t.Fatalf("EnsureMaterial calls = %d, want exactly 1", rc.ensureN)
}
}
// 模型已自行委派后台(delegatedBackground)→ 尊重模型判断,不触发兜底。
func TestStopMaterialBackstopSkippedWhenDelegated(t *testing.T) {
rc := &matCtx{
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_1"},
}
rc.attachments = matAttachments()
delegate := toolCall("delegate_task", `{"name":"深度解析","task":"解析材料","background":true}`)
turns := [][]agentllm.Event{
textDeltaTurn("", delegate),
textDeltaTurn("已在后台解析,先给即时反馈。"),
}
rec := runMaterial(t, turns, rc)
if rc.ensureN != 0 {
t.Fatalf("EnsureMaterial calls = %d, want 0 (model delegated)", rc.ensureN)
}
if requestsContaining(rec, "自动启动材料整理后台任务") != 0 {
t.Fatal("backstop should not fire when model delegated")
}
}
func hitlTurn(call agentllm.ToolCall) []agentllm.Event { return textDeltaTurn("", call) }
// 完整确认链:材料通知回合 → 汇报但没发选择卡(应急兜底)→ 发起 confirm_choice
// → 用户选择 → 模型委派后台 → 通过(无进一步拦截)。
func TestMaterialNoticeChoiceChain(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_d"}}
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText, Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态:completed\n整理完成\n</system-reminder>"}}}}
turns := [][]agentllm.Event{
textDeltaTurn("整理完成:三份材料要点如下……"), // 未发选择卡 → 应急兜底
hitlTurn(toolCall("confirm_choice", `{"question":"接下来生成什么?","options":[{"id":"a","label":"需求文档","description":"会议含明确需求项","impact":"产出需求文档"},{"id":"b","label":"会议摘要","description":"快速回顾","impact":"产出摘要"}]}`)), // AskHITL → 用户选第一个
textDeltaTurn("好的,启动需求文档任务……", toolCall("delegate_task", `{"name":"生成需求文档","task":"基于整理结果生成需求文档","background":true}`)),
textDeltaTurn("已在后台启动需求文档,完成后自动汇报。"),
}
rec := runMaterial(t, turns, rc)
if requestsContaining(rec, "还没有让用户选择正式交付物") == 0 {
t.Fatal("notice backstop (confirm_choice nudge) missing")
}
if len(rc.hitls) != 1 || rc.hitls[0].Kind != core.HITLKindChoice {
t.Fatalf("hitl cards = %d", len(rc.hitls))
}
if requestsContaining(rec, "用户已选择:需求文档") == 0 {
t.Fatal("choice answer not surfaced to model")
}
if requestsContaining(rec, "还没有启动") != 0 {
t.Fatal("deliverable backstop should not fire after model delegated")
}
if rc.ensureN != 0 {
t.Fatal("material backstop must not run in notice turns (no attachments)")
}
}
// 用户已选定但模型未委派 → 硬性续跑一轮要求启动;第二轮委派后放行。
func TestDeliverableBackstopForcesDelegation(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_e"}}
turns := [][]agentllm.Event{
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么?","options":[{"id":"a","label":"会议摘要","description":"快","impact":"摘要"},{"id":"b","label":"评审报告","description":"全","impact":"报告"}]}`)),
textDeltaTurn("好的,马上生成会议摘要。"), // 未委派 → 兜底
textDeltaTurn("这就启动。", toolCall("delegate_task", `{"name":"生成会议摘要","task":"生成摘要","background":true}`)),
textDeltaTurn("已在后台生成会议摘要,稍后回投。"),
}
rec := runMaterial(t, turns, rc)
// 只触发一次由脚本回合数锁死:再续跑一次会多一次模型调用,脚本耗尽报错。
if requestsContaining(rec, "还没有启动") == 0 {
t.Fatal("deliverable backstop missing")
}
if requestsContaining(rec, "用户已选择:会议摘要") == 0 {
t.Fatal("choice answer not surfaced to model")
}
}
// 自由输入也应回到同一回合并进入待交付状态。
func TestConfirmChoiceFreeTextSetsPending(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlFree: "做成对比表格"}
turns := [][]agentllm.Event{
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么?","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
textDeltaTurn("收到,将输出对比表格。", toolCall("delegate_task", `{"name":"对比表格","task":"输出对比表格","background":true}`)),
textDeltaTurn("已在后台生成对比表格。"),
}
rec := runMaterial(t, turns, rc)
if requestsContaining(rec, "用户自由输入:做成对比表格") == 0 {
t.Fatal("free input not surfaced to model")
}
if requestsContaining(rec, "还没有启动") != 0 {
t.Fatal("no backstop expected when model delegated after free input")
}
}
// 参数无效(选项不足)不建卡,直接给模型纠错说明。
func TestConfirmChoiceRejectsInvalidArgs(t *testing.T) {
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
turns := [][]agentllm.Event{
hitlTurn(toolCall("confirm_choice", `{"question":"选什么?"}`)),
textDeltaTurn("改用两个选项重试。", toolCall("confirm_choice", `{"question":"选什么?","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
textDeltaTurn("这就启动摘要。", toolCall("delegate_task", `{"name":"生成摘要","task":"生成摘要","background":true}`)),
textDeltaTurn("已在后台生成摘要。"),
}
runMaterial(t, turns, rc)
if len(rc.hitls) != 1 {
t.Fatalf("invalid args should not create card; hitls = %d", len(rc.hitls))
}
}