Why: after material organization completes, weak models report and end the turn without offering the user a choice of formal deliverables, and even after a user picks one the model may close the turn without starting the background task. Evidence: new tests (notice-choice chain, deliverable backstop, free-input path, invalid args, notice contract) pass; full go test ./... exit 0 (local, SELF). Impact: new confirm_choice tool issues a HITL choice card and routes the answer into a hard delegate instruction; material notices instruct the model to report then offer choices; stop hook adds two one-shot backstops (choice not offered in material notice runs; choice made but no background delegation). Refs: Dev task .task/feature-material-intake-flow.md I2.
223 lines
9.8 KiB
Go
223 lines
9.8 KiB
Go
package runtime
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"testing"
|
||
|
||
agentllm "agent-native/runtime/llm"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
// matCtx 在 fakeRunCtx 之上实现材料整理的两个可选接口(附件即保证),
|
||
// 记录 EnsureMaterial 调用次数以断言 stop 兜底的幂等。
|
||
type matCtx struct {
|
||
*fakeRunCtx
|
||
note string
|
||
ensureN int
|
||
lastNote string
|
||
// confirm_choice 断言(I2)
|
||
hitls []core.HITLCard
|
||
hitlFree string
|
||
hitlErr error
|
||
}
|
||
|
||
func (m *matCtx) MaterialNote() string { return m.note }
|
||
|
||
func (m *matCtx) EnsureMaterial() string {
|
||
m.ensureN++
|
||
if m.ensureN > 1 {
|
||
return ""
|
||
}
|
||
return m.lastNote
|
||
}
|
||
|
||
// AskHITL 覆盖:预置选择默认取第一个选项;hitlFree 模拟自由输入。
|
||
func (m *matCtx) AskHITL(card core.HITLCard) (core.HITLResponse, error) {
|
||
m.hitls = append(m.hitls, card)
|
||
if m.hitlErr != nil {
|
||
return core.HITLResponse{}, m.hitlErr
|
||
}
|
||
if m.hitlFree != "" {
|
||
return core.HITLResponse{FreeText: m.hitlFree}, nil
|
||
}
|
||
if len(card.Choices) == 0 {
|
||
return core.HITLResponse{}, nil
|
||
}
|
||
return core.HITLResponse{ChoiceID: card.Choices[0].ID}, nil
|
||
}
|
||
|
||
func matAttachments() []core.ParsedDocument {
|
||
return []core.ParsedDocument{
|
||
{Meta: core.DocMeta{Filename: "纪要.docx"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "决定 11 月上线"}}},
|
||
{Meta: core.DocMeta{Filename: "意见.pdf"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "预算存疑"}}},
|
||
{Meta: core.DocMeta{Filename: "清单.md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "12 项需求"}}},
|
||
}
|
||
}
|
||
|
||
func runMaterial(t *testing.T, turns [][]agentllm.Event, rc *matCtx) *dualSpeedRecorder {
|
||
t.Helper()
|
||
rec := &dualSpeedRecorder{clock: &stepClock{}, turns: turns}
|
||
r := &LLMRunner{Stream: rec.stream(scriptStream(turns, nil)), Now: rec.clock.now}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatalf("Execute: %v", err)
|
||
}
|
||
return rec
|
||
}
|
||
|
||
func requestsContaining(rec *dualSpeedRecorder, needle string) int {
|
||
n := 0
|
||
for _, req := range rec.request {
|
||
if strings.Contains(req, needle) {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
// 预启动说明替换建议式提示:第 0 回合注入事实说明而非「可能属于重任务」。
|
||
func TestTurn0MaterialNoteReplacesAdvisory(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, note: "系统已自动启动材料整理后台任务(收件箱进度卡「材料整理」已创建)。"}
|
||
rc.attachments = matAttachments()
|
||
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("收到,先给每份材料一句话概览。")}, rc)
|
||
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
|
||
t.Fatal("material note not injected at turn 0")
|
||
}
|
||
if requestsContaining(rec, "可能属于重任务") != 0 {
|
||
t.Fatal("advisory text should be replaced when material note present")
|
||
}
|
||
}
|
||
|
||
// 未预启动(文字+小材料):保留建议式入口提示。
|
||
func TestTurn0AdvisoryWhenNoNote(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
rc.attachments = matAttachments()
|
||
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("概览如下。")}, rc)
|
||
if requestsContaining(rec, "可能属于重任务") == 0 {
|
||
t.Fatal("advisory hint missing when no material note")
|
||
}
|
||
}
|
||
|
||
// stop 兜底:模型问一句澄清就收场(用户实测现象)→ 系统续跑一轮要求告知;
|
||
// ensureMaterial 幂等保证只续跑一次。
|
||
func TestStopMaterialBackstopContinuesOnce(t *testing.T) {
|
||
rc := &matCtx{
|
||
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
||
lastNote: "系统已自动启动材料整理后台任务",
|
||
}
|
||
rc.attachments = matAttachments()
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("请问你需要哪种产出?"),
|
||
textDeltaTurn("材料正在后台整理,完成后自动汇报;先给概览。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
|
||
t.Fatal("backstop reason not injected as continuation")
|
||
}
|
||
if rc.ensureN != 1 {
|
||
t.Fatalf("EnsureMaterial calls = %d, want exactly 1", rc.ensureN)
|
||
}
|
||
}
|
||
|
||
// 模型已自行委派后台(delegatedBackground)→ 尊重模型判断,不触发兜底。
|
||
func TestStopMaterialBackstopSkippedWhenDelegated(t *testing.T) {
|
||
rc := &matCtx{
|
||
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_1"},
|
||
}
|
||
rc.attachments = matAttachments()
|
||
delegate := toolCall("delegate_task", `{"name":"深度解析","task":"解析材料","background":true}`)
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("", delegate),
|
||
textDeltaTurn("已在后台解析,先给即时反馈。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if rc.ensureN != 0 {
|
||
t.Fatalf("EnsureMaterial calls = %d, want 0 (model delegated)", rc.ensureN)
|
||
}
|
||
if requestsContaining(rec, "自动启动材料整理后台任务") != 0 {
|
||
t.Fatal("backstop should not fire when model delegated")
|
||
}
|
||
}
|
||
|
||
func hitlTurn(call agentllm.ToolCall) []agentllm.Event { return textDeltaTurn("", call) }
|
||
|
||
// 完整确认链:材料通知回合 → 汇报但没发选择卡(应急兜底)→ 发起 confirm_choice
|
||
// → 用户选择 → 模型委派后台 → 通过(无进一步拦截)。
|
||
func TestMaterialNoticeChoiceChain(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_d"}}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText, Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态:completed\n整理完成\n</system-reminder>"}}}}
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("整理完成:三份材料要点如下……"), // 未发选择卡 → 应急兜底
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"接下来生成什么?","options":[{"id":"a","label":"需求文档","description":"会议含明确需求项","impact":"产出需求文档"},{"id":"b","label":"会议摘要","description":"快速回顾","impact":"产出摘要"}]}`)), // AskHITL → 用户选第一个
|
||
textDeltaTurn("好的,启动需求文档任务……", toolCall("delegate_task", `{"name":"生成需求文档","task":"基于整理结果生成需求文档","background":true}`)),
|
||
textDeltaTurn("已在后台启动需求文档,完成后自动汇报。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "还没有让用户选择正式交付物") == 0 {
|
||
t.Fatal("notice backstop (confirm_choice nudge) missing")
|
||
}
|
||
if len(rc.hitls) != 1 || rc.hitls[0].Kind != core.HITLKindChoice {
|
||
t.Fatalf("hitl cards = %d", len(rc.hitls))
|
||
}
|
||
if requestsContaining(rec, "用户已选择:需求文档") == 0 {
|
||
t.Fatal("choice answer not surfaced to model")
|
||
}
|
||
if requestsContaining(rec, "还没有启动") != 0 {
|
||
t.Fatal("deliverable backstop should not fire after model delegated")
|
||
}
|
||
if rc.ensureN != 0 {
|
||
t.Fatal("material backstop must not run in notice turns (no attachments)")
|
||
}
|
||
}
|
||
|
||
// 用户已选定但模型未委派 → 硬性续跑一轮要求启动;第二轮委派后放行。
|
||
func TestDeliverableBackstopForcesDelegation(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_e"}}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么?","options":[{"id":"a","label":"会议摘要","description":"快","impact":"摘要"},{"id":"b","label":"评审报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("好的,马上生成会议摘要。"), // 未委派 → 兜底
|
||
textDeltaTurn("这就启动。", toolCall("delegate_task", `{"name":"生成会议摘要","task":"生成摘要","background":true}`)),
|
||
textDeltaTurn("已在后台生成会议摘要,稍后回投。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
// 只触发一次由脚本回合数锁死:再续跑一次会多一次模型调用,脚本耗尽报错。
|
||
if requestsContaining(rec, "还没有启动") == 0 {
|
||
t.Fatal("deliverable backstop missing")
|
||
}
|
||
if requestsContaining(rec, "用户已选择:会议摘要") == 0 {
|
||
t.Fatal("choice answer not surfaced to model")
|
||
}
|
||
}
|
||
|
||
// 自由输入也应回到同一回合并进入待交付状态。
|
||
func TestConfirmChoiceFreeTextSetsPending(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlFree: "做成对比表格"}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么?","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("收到,将输出对比表格。", toolCall("delegate_task", `{"name":"对比表格","task":"输出对比表格","background":true}`)),
|
||
textDeltaTurn("已在后台生成对比表格。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "用户自由输入:做成对比表格") == 0 {
|
||
t.Fatal("free input not surfaced to model")
|
||
}
|
||
if requestsContaining(rec, "还没有启动") != 0 {
|
||
t.Fatal("no backstop expected when model delegated after free input")
|
||
}
|
||
}
|
||
|
||
// 参数无效(选项不足)不建卡,直接给模型纠错说明。
|
||
func TestConfirmChoiceRejectsInvalidArgs(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"选什么?"}`)),
|
||
textDeltaTurn("改用两个选项重试。", toolCall("confirm_choice", `{"question":"选什么?","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("这就启动摘要。", toolCall("delegate_task", `{"name":"生成摘要","task":"生成摘要","background":true}`)),
|
||
textDeltaTurn("已在后台生成摘要。"),
|
||
}
|
||
runMaterial(t, turns, rc)
|
||
if len(rc.hitls) != 1 {
|
||
t.Fatalf("invalid args should not create card; hitls = %d", len(rc.hitls))
|
||
}
|
||
}
|