weichenandCursor
46d5f7b668
Why: The composed reply copied the registered artifact id and file size out of the tool summary. Evidence: go test ./internal/runtime/ -run 'TestComposeUserPromptOmitsArtifactIDs|TestComposeReplyDropsInternalRefs' Impact: Those tokens are removed from the compose input and from the reply before it is shown. Refs: feature/final-reply-compose Co-authored-by: Cursor <cursoragent@cursor.com>
335 lines
11 KiB
Go
335 lines
11 KiB
Go
package runtime
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"testing"
|
||
|
||
agentllm "agent-native/runtime/llm"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
func TestEvidencePromptLastCheckWins(t *testing.T) {
|
||
paper := `[{"severity":"error","rule":"paper-size","message":"纸张不是 A4（21.59 × 27.94 cm）"}]`
|
||
empty := `[{"severity":"error","rule":"empty-section","message":"标题下没有内容"}]`
|
||
warn := `[{"severity":"warning","rule":"visual-summary","message":"建议添加视觉摘要"}]`
|
||
evidence := []toolEvidence{
|
||
{Name: "run_shell", Status: "ok", Summary: summarize(paper, nil), VerifyText: tailRunes(paper, 600)},
|
||
{Name: "run_shell", Status: "ok", Summary: summarize(empty, nil), VerifyText: tailRunes(empty, 600)},
|
||
{Name: "run_shell", Status: "ok", Summary: summarize(warn, nil), VerifyText: tailRunes(warn, 600)},
|
||
}
|
||
got := evidencePrompt("给我总结一份会议纪要，以 Word 文档给我", "lint 无 error", evidence, []*core.ArtifactRef{
|
||
{ArtifactID: "art_abc79b0e4486", Name: "纪要.docx"},
|
||
})
|
||
for _, want := range []string{"#1 ", "#2 ", "#3 ", "以最后一次为准", "visual-summary", "这是强证据", "纸张不是 A4"} {
|
||
if !strings.Contains(got, want) {
|
||
t.Fatalf("prompt missing %q:\n%s", want, got)
|
||
}
|
||
}
|
||
if strings.Index(got, "paper-size") > strings.Index(got, "visual-summary") {
|
||
t.Fatal("later check should follow the earlier one")
|
||
}
|
||
long := strings.Repeat("纸张不是 A4。", 200) + "TAIL_MARKER"
|
||
tail := tailRunes(long, 600)
|
||
if !strings.HasPrefix(tail, "…") || !strings.HasSuffix(tail, "TAIL_MARKER") || strings.HasPrefix(tail, "纸张") {
|
||
t.Fatalf("tail did not keep the end: %q", tail[:40])
|
||
}
|
||
}
|
||
|
||
func TestVerifyRejectionIsNotUserSpeech(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("答案是 42。"),
|
||
textDeltaTurn("答案仍是 42。"),
|
||
}
|
||
var seen []string
|
||
base := scriptStream(turns, nil)
|
||
r := &LLMRunner{
|
||
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||
seen = append(seen, joinContents(msgs))
|
||
return base(ctx, model, msgs, tools, opts)
|
||
},
|
||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||
body := `{"passed":false,"reason":"纸张不是 A4","nextAction":"改成 A4"}`
|
||
if len(seen) > 1 {
|
||
body = `{"passed":true,"reason":"完成","nextAction":""}`
|
||
}
|
||
return passedStream(body)(ctx, model, msgs, tools, opts)
|
||
},
|
||
}
|
||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(seen) < 2 {
|
||
t.Fatalf("calls = %d", len(seen))
|
||
}
|
||
second := seen[1]
|
||
if !strings.Contains(second, "<system-reminder>") || !strings.Contains(second, "不是用户发言") || !strings.Contains(second, "改成 A4") {
|
||
t.Fatalf("rejection was not wrapped: %s", second)
|
||
}
|
||
if strings.Contains(second, "您提到") {
|
||
t.Fatalf("rejection speaks as the user: %s", second)
|
||
}
|
||
}
|
||
|
||
func TestComposeReplacesDraftAndPinsArtifact(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("我来写文档。", toolCall("make_report", "{}")),
|
||
textDeltaTurn("草稿：已完成，lint 无 error。"),
|
||
textDeltaTurn("会议纪要已写好，文件在下面。"),
|
||
}
|
||
var toolCounts []int
|
||
base := scriptStream(turns, nil)
|
||
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||
toolCounts = append(toolCounts, len(tools))
|
||
return base(ctx, model, msgs, tools, opts)
|
||
}}
|
||
rc := &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
compose: true,
|
||
execArtifact: &core.Artifact{ID: "art_abc79b0e4486", Name: "纪要.docx"},
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(toolCounts) != 3 || toolCounts[2] != 0 {
|
||
t.Fatalf("tool decl counts = %v, want the compose call to have no tools", toolCounts)
|
||
}
|
||
sawFinalizing := false
|
||
deltaAfter := false
|
||
for _, e := range rc.events {
|
||
if e.typ == core.EvRunPhase {
|
||
p, ok := e.payload.(core.RunPhasePayload)
|
||
if ok && p.Phase == core.PhaseFinalizing {
|
||
sawFinalizing = true
|
||
}
|
||
}
|
||
if sawFinalizing && e.typ == core.EvMessageDelta {
|
||
p, ok := e.payload.(core.MessageDeltaPayload)
|
||
if ok && strings.Contains(p.Text, "会议纪要已写好") {
|
||
deltaAfter = true
|
||
}
|
||
}
|
||
}
|
||
if !sawFinalizing || !deltaAfter {
|
||
t.Fatal("finalizing phase did not precede the compose deltas")
|
||
}
|
||
parts := rc.assistant
|
||
if parts[len(parts)-1].Type != core.PartArtifactRef || parts[len(parts)-1].Artifact.Name != "纪要.docx" {
|
||
t.Fatalf("last part = %+v", parts[len(parts)-1])
|
||
}
|
||
var draft, reply *core.Part
|
||
for i := range parts {
|
||
p := &parts[i]
|
||
if p.Type != core.PartText {
|
||
continue
|
||
}
|
||
if strings.Contains(p.Text, "草稿") {
|
||
draft = p
|
||
}
|
||
if strings.Contains(p.Text, "会议纪要已写好") {
|
||
reply = p
|
||
}
|
||
}
|
||
if draft == nil || !draft.Process || reply == nil || reply.Process {
|
||
t.Fatalf("draft=%+v reply=%+v", draft, reply)
|
||
}
|
||
}
|
||
|
||
func TestComposeStripsThinkBlock(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("我来写。", toolCall("make_report", "{}")),
|
||
textDeltaTurn("草稿：写好了。"),
|
||
textDeltaTurn("<think>\n根据规则，所以我应该解释评分。\n</think>\n\n纪要已写好，文件在下面。"),
|
||
}
|
||
r := &LLMRunner{Stream: scriptStream(turns, nil)}
|
||
rc := &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
compose: true,
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var reply string
|
||
for _, p := range rc.assistant {
|
||
if p.Type == core.PartText && !p.Process {
|
||
reply = p.Text
|
||
}
|
||
if p.Type == core.PartText && strings.Contains(p.Text, "所以我应该") {
|
||
t.Fatalf("think leaked into parts: %+v", p)
|
||
}
|
||
}
|
||
if reply != "纪要已写好，文件在下面。" {
|
||
t.Fatalf("reply = %q", reply)
|
||
}
|
||
}
|
||
|
||
func TestComposeReplyDropsInternalRefs(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("我来写。", toolCall("make_report", "{}")),
|
||
textDeltaTurn("草稿。"),
|
||
textDeltaTurn("纪要已写好。登记信息：art_986876932a39（38,505 字节）。"),
|
||
}
|
||
r := &LLMRunner{Stream: scriptStream(turns, nil)}
|
||
rc := &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
compose: true,
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var reply string
|
||
for _, p := range rc.assistant {
|
||
if p.Type == core.PartText && !p.Process {
|
||
reply = p.Text
|
||
}
|
||
}
|
||
if strings.Contains(reply, "art_") || strings.Contains(reply, "字节") || !strings.Contains(reply, "纪要已写好") {
|
||
t.Fatalf("reply = %q", reply)
|
||
}
|
||
}
|
||
|
||
func TestComposeFailureRestoresDraft(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("", toolCall("make_report", "{}")),
|
||
textDeltaTurn("草稿还在。"),
|
||
}
|
||
r := &LLMRunner{Stream: scriptStream(turns, nil)}
|
||
rc := &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
compose: true,
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var draft *core.Part
|
||
for i := range rc.assistant {
|
||
p := &rc.assistant[i]
|
||
if p.Type == core.PartText && strings.Contains(p.Text, "草稿还在") {
|
||
draft = p
|
||
}
|
||
if p.Type == core.PartText && strings.Contains(p.Text, "整理") {
|
||
t.Fatalf("unexpected compose text: %+v", p)
|
||
}
|
||
}
|
||
if draft == nil || draft.Process {
|
||
t.Fatalf("draft was not restored: %+v", rc.assistant)
|
||
}
|
||
}
|
||
|
||
func TestComposeSkipsPlainChatAndCanBeDisabled(t *testing.T) {
|
||
plain := [][]agentllm.Event{textDeltaTurn("你好。")}
|
||
r := &LLMRunner{Stream: scriptStream(plain, nil)}
|
||
rc := &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
compose: true,
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if hasPhase(rc, core.PhaseFinalizing) {
|
||
t.Fatal("plain chat should not finalize")
|
||
}
|
||
|
||
worked := [][]agentllm.Event{
|
||
textDeltaTurn("我来写。", toolCall("make_report", "{}")),
|
||
textDeltaTurn("写好了。"),
|
||
}
|
||
r = &LLMRunner{Stream: scriptStream(worked, nil)}
|
||
rc = &fakeRunCtx{
|
||
ctx: context.Background(),
|
||
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
||
}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if hasPhase(rc, core.PhaseFinalizing) {
|
||
t.Fatal("compose off should not finalize")
|
||
}
|
||
found := false
|
||
for _, p := range rc.assistant {
|
||
if p.Type == core.PartText && p.Text == "写好了。" && !p.Process {
|
||
found = true
|
||
}
|
||
}
|
||
if !found {
|
||
t.Fatalf("draft missing: %+v", rc.assistant)
|
||
}
|
||
}
|
||
|
||
func TestComposeAbsorbsVerifierNotice(t *testing.T) {
|
||
turns := [][]agentllm.Event{
|
||
textDeltaTurn("我先生成。", toolCall("make_report", "{}")),
|
||
textDeltaTurn("写好了。"),
|
||
textDeltaTurn("文件已生成，请查看。"),
|
||
}
|
||
r := &LLMRunner{
|
||
Stream: scriptStream(turns, nil),
|
||
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
||
out := make(chan agentllm.Event, 1)
|
||
out <- agentllm.Event{Kind: agentllm.KindError, ErrorMessage: "upstream"}
|
||
close(out)
|
||
return out
|
||
},
|
||
}
|
||
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true, compose: true}
|
||
if err := r.Execute(rc); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, p := range rc.assistant {
|
||
if p.Type == core.PartText && strings.Contains(p.Text, "验证器不可用") {
|
||
t.Fatalf("notice was appended beside the compose reply: %+v", rc.assistant)
|
||
}
|
||
}
|
||
if !hasProcessNote(rc.assistant, "调用失败") {
|
||
t.Fatalf("check missing: %+v", rc.assistant)
|
||
}
|
||
found := false
|
||
for _, p := range rc.assistant {
|
||
if p.Type == core.PartText && p.Text == "文件已生成，请查看。" && !p.Process {
|
||
found = true
|
||
}
|
||
}
|
||
if !found {
|
||
t.Fatalf("compose reply missing: %+v", rc.assistant)
|
||
}
|
||
}
|
||
|
||
func TestComposeUserPromptOmitsArtifactIDs(t *testing.T) {
|
||
st := &directRun{
|
||
rc: &fakeRunCtx{ctx: context.Background(), history: []core.Message{{
|
||
Role: core.RoleUser, Parts: []core.Part{{Type: core.PartText, Text: "给我一份纪要"}},
|
||
}}},
|
||
evidence: []toolEvidence{{Name: "register_artifact", Status: "ok", Summary: "已登记 art_abc79b0e4486，38505 字节"}},
|
||
deliverables: []*core.ArtifactRef{{ArtifactID: "art_abc79b0e4486", Name: "纪要.docx"}},
|
||
checks: []core.CheckPart{{Passed: false, Reason: "纸张尺寸"}},
|
||
}
|
||
got := st.composeUserPrompt("草稿里写了 art_abc79b0e4486（38,505 字节）", "本轮回复未通过终轮检查。")
|
||
if strings.Contains(got, "art_") || strings.Contains(got, "字节") {
|
||
t.Fatalf("prompt leaked an internal ref: %s", got)
|
||
}
|
||
for _, want := range []string{"给我一份纪要", "纪要.docx", "纸张尺寸", "仍未完成", "本轮回复未通过终轮检查。"} {
|
||
if !strings.Contains(got, want) {
|
||
t.Fatalf("prompt missing %q:\n%s", want, got)
|
||
}
|
||
}
|
||
}
|
||
|
||
func hasPhase(rc *fakeRunCtx, phase core.RunPhase) bool {
|
||
for _, e := range rc.events {
|
||
if e.typ != core.EvRunPhase {
|
||
continue
|
||
}
|
||
p, ok := e.payload.(core.RunPhasePayload)
|
||
if ok && p.Phase == phase {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|