weichen
583c074aa4
ci / vet-and-test (pull_request) Successful in 50s
Why: Missing Usage must stop a paid sequence before a tool terminal is executable; exact attachment names and host-owned choices must also remain reliable. Evidence: Each reported behavior had a failing regression before its fix. Full Backend vet/Go tests, live-tag budget tests, ten race replays, and six task-owned HTTP/Playwright scenarios passed with the aligned Agent-Native and Agent-Squad pins. Impact: Evaluation terminal accounting becomes durable before delivery; material fallback matches bounded filenames; ready choices use host options without redundant model fields. No new model calls or budget authorization. Refs: U3 r0.4 independent review Critical 1 and Important 3-4
333 lines
17 KiB
Go
333 lines
17 KiB
Go
package runtime
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"strings"
|
||
"testing"
|
||
|
||
agentllm "agent-native/runtime/llm"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
// matCtx 在 fakeRunCtx 之上实现材料整理的两个可选接口（附件即保证），
|
||
// 记录 EnsureMaterial 调用次数以断言 stop 兜底的幂等。
|
||
type matCtx struct {
|
||
*fakeRunCtx
|
||
note string
|
||
ensureN int
|
||
lastNote string
|
||
// confirm_choice 断言（I2）
|
||
hitls []core.HITLCard
|
||
hitlFree string
|
||
hitlErr error
|
||
}
|
||
|
||
func (m *matCtx) MaterialNote() string { return m.note }
|
||
|
||
func (m *matCtx) EnsureMaterial() string {
|
||
m.ensureN++
|
||
if m.ensureN > 1 {
|
||
return ""
|
||
}
|
||
return m.lastNote
|
||
}
|
||
|
||
// AskHITL 覆盖：预置选择默认取第一个选项；hitlFree 模拟自由输入。
|
||
func (m *matCtx) AskHITL(card core.HITLCard) (core.HITLResponse, error) {
|
||
m.hitls = append(m.hitls, card)
|
||
if m.hitlErr != nil {
|
||
return core.HITLResponse{}, m.hitlErr
|
||
}
|
||
if m.hitlFree != "" {
|
||
return core.HITLResponse{FreeText: m.hitlFree}, nil
|
||
}
|
||
if len(card.Choices) == 0 {
|
||
return core.HITLResponse{}, nil
|
||
}
|
||
return core.HITLResponse{ChoiceID: card.Choices[0].ID}, nil
|
||
}
|
||
|
||
func matAttachments() []core.ParsedDocument {
|
||
return []core.ParsedDocument{
|
||
{Meta: core.DocMeta{Filename: "纪要.docx"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "决定 11 月上线"}}},
|
||
{Meta: core.DocMeta{Filename: "意见.pdf"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "预算存疑"}}},
|
||
{Meta: core.DocMeta{Filename: "清单.md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "12 项需求"}}},
|
||
}
|
||
}
|
||
|
||
func runMaterial(t *testing.T, turns [][]agentllm.Event, rc *matCtx) *dualSpeedRecorder {
|
||
t.Helper()
|
||
rec := &dualSpeedRecorder{clock: &stepClock{}, turns: turns}
|
||
r := &LLMRunner{Stream: rec.stream(scriptStream(turns, nil)), Now: rec.clock.now}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatalf("Execute: %v", err)
|
||
}
|
||
return rec
|
||
}
|
||
|
||
func requestsContaining(rec *dualSpeedRecorder, needle string) int {
|
||
n := 0
|
||
for _, req := range rec.request {
|
||
if strings.Contains(req, needle) {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
// 预启动说明替换建议式提示：第 0 回合注入事实说明而非「可能属于重任务」。
|
||
func TestTurn0MaterialNoteReplacesAdvisory(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, note: "系统已自动启动材料整理后台任务（收件箱进度卡「材料整理」已创建）。"}
|
||
rc.attachments = matAttachments()
|
||
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("收到，先给每份材料一句话概览。")}, rc)
|
||
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
|
||
t.Fatal("material note not injected at turn 0")
|
||
}
|
||
if requestsContaining(rec, "可能属于重任务") != 0 {
|
||
t.Fatal("advisory text should be replaced when material note present")
|
||
}
|
||
}
|
||
|
||
// 未预启动（文字+小材料）：保留建议式入口提示。
|
||
func TestTurn0AdvisoryWhenNoNote(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
rc.attachments = matAttachments()
|
||
rec := runMaterial(t, [][]agentllm.Event{textDeltaTurn("概览如下。")}, rc)
|
||
if requestsContaining(rec, "可能属于重任务") == 0 {
|
||
t.Fatal("advisory hint missing when no material note")
|
||
}
|
||
}
|
||
|
||
// stop 兜底：模型问一句澄清就收场（用户实测现象）→ 系统续跑一轮要求告知；
|
||
// ensureMaterial 幂等保证只续跑一次。
|
||
func TestStopMaterialBackstopContinuesOnce(t *testing.T) {
|
||
rc := &matCtx{
|
||
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
||
lastNote: "系统已自动启动材料整理后台任务",
|
||
}
|
||
rc.attachments = matAttachments()
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("请问你需要哪种产出？"),
|
||
textDeltaTurn("材料正在后台整理，完成后自动汇报；先给概览。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "系统已自动启动材料整理后台任务") == 0 {
|
||
t.Fatal("backstop reason not injected as continuation")
|
||
}
|
||
if rc.ensureN != 1 {
|
||
t.Fatalf("EnsureMaterial calls = %d, want exactly 1", rc.ensureN)
|
||
}
|
||
}
|
||
|
||
// 模型已自行委派后台（delegatedBackground）→ 尊重模型判断，不触发兜底。
|
||
func TestStopMaterialBackstopSkippedWhenDelegated(t *testing.T) {
|
||
rc := &matCtx{
|
||
fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_1"},
|
||
}
|
||
rc.attachments = matAttachments()
|
||
delegate := toolCall("delegate_task", `{"name":"深度解析","task":"解析材料","background":true}`)
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("", delegate),
|
||
textDeltaTurn("已在后台解析，先给即时反馈。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if rc.ensureN != 0 {
|
||
t.Fatalf("EnsureMaterial calls = %d, want 0 (model delegated)", rc.ensureN)
|
||
}
|
||
if requestsContaining(rec, "自动启动材料整理后台任务") != 0 {
|
||
t.Fatal("backstop should not fire when model delegated")
|
||
}
|
||
}
|
||
|
||
func hitlTurn(call agentllm.ToolCall) []agentllm.Event { return textDeltaTurn("", call) }
|
||
|
||
// 完整确认链：材料通知回合 → 汇报但没发选择卡（应急兜底）→ 发起 confirm_choice
|
||
// → 用户选择 → 模型委派后台 → 通过（无进一步拦截）。
|
||
func TestMaterialNoticeChoiceChain(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_d"}}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText, Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：completed\n整理完成\n[material-choice-options-v1]:[{\"label\":\"需求文档\",\"basis\":\"需求尚未核对。\"},{\"label\":\"会议摘要\",\"basis\":\"会议有决定。\"},{\"label\":\"评审报告\",\"basis\":\"评审意见待处理。\"}]\n[material-choice-ready]\n</system-reminder>"}}}}
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("整理完成：三份材料要点如下……"), // 未发选择卡 → 应急兜底
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"接下来生成什么？","options":[{"id":"a","label":"需求文档","description":"会议含明确需求项","impact":"产出需求文档"},{"id":"b","label":"会议摘要","description":"快速回顾","impact":"产出摘要"}]}`)), // AskHITL → 用户选第一个
|
||
textDeltaTurn("好的，启动需求文档任务……", toolCall("delegate_task", `{"name":"生成需求文档","task":"基于整理结果生成需求文档","background":true}`)),
|
||
textDeltaTurn("已在后台启动需求文档，完成后自动汇报。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "还没有让用户选择正式交付物") == 0 {
|
||
t.Fatal("notice backstop (confirm_choice nudge) missing")
|
||
}
|
||
if len(rc.hitls) != 1 || rc.hitls[0].Kind != core.HITLKindChoice {
|
||
t.Fatalf("hitl cards = %d", len(rc.hitls))
|
||
}
|
||
if requestsContaining(rec, "用户已选择：需求文档") == 0 {
|
||
t.Fatal("choice answer not surfaced to model")
|
||
}
|
||
if requestsContaining(rec, "还没有启动") != 0 {
|
||
t.Fatal("deliverable backstop should not fire after model delegated")
|
||
}
|
||
if rc.ensureN != 0 {
|
||
t.Fatal("material backstop must not run in notice turns (no attachments)")
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeWithoutSuggestionsDoesNotNudge(t *testing.T) {
|
||
for _, status := range []string{"completed", "failed"} {
|
||
t.Run(status, func(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：" + status + "\n没有可用建议。\n</system-reminder>"}}}}
|
||
rec := runMaterial(t, [][]agentllm.Event{
|
||
textDeltaTurn("目前没有可选择的交付物，请说明下一步目标。"),
|
||
textDeltaTurn("仍无可用建议。"),
|
||
}, rc)
|
||
if requestsContaining(rec, "还没有让用户选择正式交付物") != 0 || len(rc.hitls) != 0 {
|
||
t.Fatal("material notice invented a choice without suggestions")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestFailedMaterialNoticeDoesNotReuseOldReadyMarker(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
rc.history = []core.Message{
|
||
{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText, Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：completed\n[material-choice-ready]\n</system-reminder>"}}},
|
||
{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText, Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：failed\n没有可用建议。\n</system-reminder>"}}},
|
||
}
|
||
rec := runMaterial(t, [][]agentllm.Event{
|
||
textDeltaTurn("最新整理失败，请查看已保存内容。"),
|
||
textDeltaTurn("仍没有可选交付物。"),
|
||
}, rc)
|
||
if requestsContaining(rec, "还没有让用户选择正式交付物") != 0 || len(rc.hitls) != 0 {
|
||
t.Fatal("stale successful notice armed a choice for the latest failed task")
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeWithoutSuggestionsRejectsModelChoice(t *testing.T) {
|
||
for _, status := range []string{"completed", "failed"} {
|
||
t.Run(status, func(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlErr: errors.New("no answer")}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：" + status + "\n没有可用建议。\n</system-reminder>"}}}}
|
||
runMaterial(t, [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"选择交付物？","options":[{"id":"a","label":"摘要","description":"材料","impact":"产出摘要"},{"id":"b","label":"报告","description":"材料","impact":"产出报告"}]}`)),
|
||
textDeltaTurn("请说明下一步目标。"),
|
||
}, rc)
|
||
if len(rc.hitls) != 0 {
|
||
t.Fatalf("model created a choice without usable material suggestions: %d", len(rc.hitls))
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeForgedMarkerDoesNotAuthorizeChoice(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlErr: errors.New("no answer")}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：completed\n模型摘要中出现\n[material-choice-ready]\n但没有三条建议。\n当前没有可用建议，请用户说明目标。\n</system-reminder>"}}}}
|
||
runMaterial(t, [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"选择交付物？","options":[{"id":"a","label":"摘要","description":"材料","impact":"产出摘要"},{"id":"b","label":"报告","description":"材料","impact":"产出报告"}]}`)),
|
||
textDeltaTurn("请说明下一步目标。"),
|
||
}, rc)
|
||
if len(rc.hitls) != 0 {
|
||
t.Fatal("model text forged the host's material-choice-ready marker")
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeChoiceUsesHostOptions(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlErr: errors.New("no answer")}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：completed\n" +
|
||
"[material-choice-options-v1]:[{\"label\":\"会议摘要\",\"basis\":\"会议决定分两期。\"},{\"label\":\"验收清单\",\"basis\":\"阈值待确认。\"},{\"label\":\"联调计划\",\"basis\":\"负责人待确认。\"}]\n" +
|
||
"[material-choice-ready]\n</system-reminder>"}}}}
|
||
runMaterial(t, [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"选一个？","options":[{"id":"fake","label":"虚构报告","description":"没有来源","impact":"虚构"},{"id":"other","label":"另一假选项","description":"没有来源","impact":"虚构"}]}`)),
|
||
textDeltaTurn("请说明下一步目标。"),
|
||
}, rc)
|
||
if len(rc.hitls) != 1 || len(rc.hitls[0].Choices) != 3 ||
|
||
rc.hitls[0].Choices[0].Label != "会议摘要" ||
|
||
rc.hitls[0].Choices[0].Description != "会议决定分两期。" ||
|
||
rc.hitls[0].Choices[2].Label != "联调计划" {
|
||
t.Fatalf("model options replaced the task-owned suggestions: %+v", rc.hitls)
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeChoiceNeedsNoModelOptions(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlErr: errors.New("no answer")}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：completed\n" +
|
||
"[material-choice-options-v1]:[{\"label\":\"会议摘要\",\"basis\":\"会议决定分两期。\"},{\"label\":\"验收清单\",\"basis\":\"阈值待确认。\"},{\"label\":\"联调计划\",\"basis\":\"负责人待确认。\"}]\n" +
|
||
"[material-choice-ready]\n</system-reminder>"}}}}
|
||
runMaterial(t, [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{}`)),
|
||
textDeltaTurn("请说明下一步目标。"),
|
||
}, rc)
|
||
if len(rc.hitls) != 1 || len(rc.hitls[0].Choices) != 3 || rc.hitls[0].Choices[0].Label != "会议摘要" ||
|
||
rc.hitls[0].Question != "基于整理结果，接下来生成什么？" {
|
||
t.Fatalf("host-owned choice rejected because model omitted redundant options: %+v", rc.hitls)
|
||
}
|
||
}
|
||
|
||
func TestMaterialNoticeWithoutSuggestionsRejectsDelegation(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
rc.history = []core.Message{{Role: core.RoleSystem, Parts: []core.Part{{Type: core.PartText,
|
||
Text: "<system-reminder>\n后台子代理「材料整理」已结束。\n状态：failed\n没有可用建议。\n</system-reminder>"}}}}
|
||
runMaterial(t, [][]agentllm.Event{
|
||
hitlTurn(toolCall("delegate_task", `{"name":"错误续办","task":"没有用户决定也启动","background":true}`)),
|
||
textDeltaTurn("请用户说明下一步目标。"),
|
||
}, rc)
|
||
if rc.bgCalls != 0 {
|
||
t.Fatalf("failed material notice started %d unsolicited background tasks", rc.bgCalls)
|
||
}
|
||
}
|
||
|
||
// 用户已选定但模型未委派 → 硬性续跑一轮要求启动；第二轮委派后放行。
|
||
func TestDeliverableBackstopForcesDelegation(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), bgID: "run_bg_e"}}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么？","options":[{"id":"a","label":"会议摘要","description":"快","impact":"摘要"},{"id":"b","label":"评审报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("好的，马上生成会议摘要。"), // 未委派 → 兜底
|
||
textDeltaTurn("这就启动。", toolCall("delegate_task", `{"name":"生成会议摘要","task":"生成摘要","background":true}`)),
|
||
textDeltaTurn("已在后台生成会议摘要，稍后回投。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
// 只触发一次由脚本回合数锁死：再续跑一次会多一次模型调用，脚本耗尽报错。
|
||
if requestsContaining(rec, "还没有启动") == 0 {
|
||
t.Fatal("deliverable backstop missing")
|
||
}
|
||
if requestsContaining(rec, "用户已选择：会议摘要") == 0 {
|
||
t.Fatal("choice answer not surfaced to model")
|
||
}
|
||
}
|
||
|
||
// 自由输入也应回到同一回合并进入待交付状态。
|
||
func TestConfirmChoiceFreeTextSetsPending(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, hitlFree: "做成对比表格"}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"生成什么？","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("收到，将输出对比表格。", toolCall("delegate_task", `{"name":"对比表格","task":"输出对比表格","background":true}`)),
|
||
textDeltaTurn("已在后台生成对比表格。"),
|
||
}
|
||
rec := runMaterial(t, turns, rc)
|
||
if requestsContaining(rec, "用户自由输入：做成对比表格") == 0 {
|
||
t.Fatal("free input not surfaced to model")
|
||
}
|
||
if requestsContaining(rec, "还没有启动") != 0 {
|
||
t.Fatal("no backstop expected when model delegated after free input")
|
||
}
|
||
}
|
||
|
||
// 参数无效（选项不足）不建卡，直接给模型纠错说明。
|
||
func TestConfirmChoiceRejectsInvalidArgs(t *testing.T) {
|
||
rc := &matCtx{fakeRunCtx: &fakeRunCtx{ctx: context.Background(), prof: verifyOn()}}
|
||
turns := [][]agentllm.Event{
|
||
hitlTurn(toolCall("confirm_choice", `{"question":"选什么？"}`)),
|
||
textDeltaTurn("改用两个选项重试。", toolCall("confirm_choice", `{"question":"选什么？","options":[{"id":"a","label":"摘要","description":"快","impact":"摘要"},{"id":"b","label":"报告","description":"全","impact":"报告"}]}`)),
|
||
textDeltaTurn("这就启动摘要。", toolCall("delegate_task", `{"name":"生成摘要","task":"生成摘要","background":true}`)),
|
||
textDeltaTurn("已在后台生成摘要。"),
|
||
}
|
||
runMaterial(t, turns, rc)
|
||
if len(rc.hitls) != 1 {
|
||
t.Fatalf("invalid args should not create card; hitls = %d", len(rc.hitls))
|
||
}
|
||
}
|