package runtime import ( "context" "encoding/json" "os" "os/exec" "path/filepath" "strings" "testing" "git-ssh.u1680832.nyat.app/weichen/Agent-Native.git/runtime/gates" agentllm "git-ssh.u1680832.nyat.app/weichen/Agent-Native.git/runtime/llm" core "git-ssh.u1680832.nyat.app/weichen/Agent-System-Core.git" ) func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event { evs := []agentllm.Event{} if body != "" { evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}) } end := agentllm.Event{Kind: agentllm.KindMessageEnd} if len(calls) > 0 { end.Message.ToolCalls = calls } return append(evs, end) } func toolCall(name, args string) agentllm.ToolCall { return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args} } func passedStream(payloads ...string) agentllm.StreamFunc { n := 0 return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { body := `{"passed":true,"reason":"完成","nextAction":""}` if n < len(payloads) { body = payloads[n] n++ } out := make(chan agentllm.Event, 2) out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body} out <- agentllm.Event{Kind: agentllm.KindMessageEnd} close(out) return out } } func verifyOn() core.AgentProfile { return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}} } func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) { for _, tc := range []struct { name string miss bool want string }{ {name: "missing", miss: true, want: "不存在"}, {name: "empty", miss: false, want: "内容为空"}, } { t.Run(tc.name, func(t *testing.T) { turns := padTurns( textDeltaTurn("", toolCall("make_report", "{}")), textDeltaTurn("写好了。"), ) var calls int base := scriptStream(turns, nil) r := &LLMRunner{ Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { calls++ return base(ctx, model, msgs, tools, opts) }, VerifyStream: passedStream(), } rc := &fakeRunCtx{ ctx: context.Background(), prof: verifyOn(), verify: true, execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"}, missingArtifact: tc.miss, emptyArtifact: !tc.miss, } if err := r.Execute(rc); err != nil { t.Fatalf("Execute: %v", err) } if calls < 2 { t.Fatalf("stream calls = %d, want a continuation", calls) } if !hasProcessNote(rc.assistant, tc.want) { t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant)) } }) } } func TestLintErrorContinues(t *testing.T) { style := filepath.Join("..", "skills", "recipes", "house-style") abs, err := filepath.Abs(style) if err != nil { t.Fatal(err) } t.Setenv("PYTHONPATH", abs) docx := badDocx(t) turns := padTurns( textDeltaTurn("", toolCall("make_report", "{}")), textDeltaTurn("写好了。"), ) var verifyCalls int base := scriptStream(turns, nil) r := &LLMRunner{ Stream: base, VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { verifyCalls++ return passedStream()(ctx, model, msgs, tools, opts) }, } rc := &fakeRunCtx{ ctx: context.Background(), prof: verifyOn(), verify: true, execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"}, execArtifactContent: docx, } if err := r.Execute(rc); err != nil { t.Fatalf("Execute: %v", err) } if verifyCalls != 0 { t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls) } if !hasProcessNote(rc.assistant, "版式") { t.Fatalf("lint reason missing: %+v", textParts(rc.assistant)) } } func TestVerifierRejectsThenPasses(t *testing.T) { turns := [][]agentllm.Event{ textDeltaTurn("答案是 42。"), textDeltaTurn("答案仍是 42。"), } var mainCalls int var models []string var toolArgs [][]agentllm.ToolDecl base := scriptStream(turns, nil) r := &LLMRunner{ Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { mainCalls++ return base(ctx, model, msgs, tools, opts) }, VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { models = append(models, model) toolArgs = append(toolArgs, tools) body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}` if len(models) > 1 { body = `{"passed":true,"reason":"完成","nextAction":""}` } return passedStream(body)(ctx, model, msgs, tools, opts) }, } rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} if err := r.Execute(rc); err != nil { t.Fatalf("Execute: %v", err) } if mainCalls != 2 { t.Fatalf("main calls = %d, want 2", mainCalls) } if len(models) != 2 || models[0] != "lite-model" { t.Fatalf("verifier models = %v, want lite-model twice", models) } for i, tools := range toolArgs { if tools != nil { t.Fatalf("verifier call %d tools = %#v, want nil", i, tools) } } texts := textParts(rc.assistant) if len(texts) < 2 || !texts[0].Process || texts[len(texts)-1].Process { t.Fatalf("parts = %+v", texts) } if !hasProcessNote(rc.assistant, "还没回答完") || !hasProcessNote(rc.assistant, "补上结论") { t.Fatalf("check missing: %+v", rc.assistant) } } func TestVerifierGarbageFailsClosed(t *testing.T) { for _, payload := range []agentllm.Event{ {Kind: agentllm.KindTextDelta, Text: "当然通过"}, {Kind: agentllm.KindError, ErrorMessage: "upstream"}, } { t.Run(string(payload.Kind), func(t *testing.T) { turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")} var mainCalls int base := scriptStream(turns, nil) r := &LLMRunner{ Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { mainCalls++ return base(ctx, model, msgs, tools, opts) }, VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { if tools != nil { t.Errorf("verifier tools = %#v, want nil", tools) } out := make(chan agentllm.Event, 2) out <- payload out <- agentllm.Event{Kind: agentllm.KindMessageEnd} close(out) return out }, } rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} if err := r.Execute(rc); err != nil { t.Fatalf("Execute: %v", err) } if mainCalls != 1 { t.Fatalf("main calls = %d, want 1", mainCalls) } texts := textParts(rc.assistant) var kept bool notice := false for _, p := range texts { if p.Text == "答案是 42。" && !p.Process { kept = true } if !p.Process && strings.Contains(p.Text, "验证器") { notice = true } } // 2026-10-10 契约更新:验证器不可用不再作为正文 Notice,落成过程检查行。 if !kept || notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") { t.Fatalf("parts = %+v", texts) } }) } } func TestParseVerdictToleratesFence(t *testing.T) { got, err := gates.ParseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```") if err != nil || got.Passed || got.Reason != "不够" { t.Fatalf("verdict = %+v %v", got, err) } if _, err := gates.ParseVerdict(`{"reason":"no passed"}`); err == nil { t.Fatal("missing passed should fail") } } type checklistFake struct { fakeRunCtx list core.TodoList } func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true } func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event { for len(turns) < 5 { turns = append(turns, textDeltaTurn("仍未修好。")) } return turns } func TestTodoReminderFiresOnceAndResets(t *testing.T) { quiet := textDeltaTurn("", toolCall("make_report", "{}")) turns := [][]agentllm.Event{ textDeltaTurn("", toolCall("make_report", "{}")), textDeltaTurn("", toolCall("make_report", "{}")), textDeltaTurn("", toolCall("make_report", "{}")), textDeltaTurn("完成了。"), } requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}}) if len(requests) != 4 { t.Fatalf("requests = %d, want 4", len(requests)) } for i, req := range requests[:3] { if strings.Contains(req, "system-reminder") { t.Fatalf("request %d reminded too early: %s", i, req) } } if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") { t.Fatalf("final request missing reminder: %s", requests[3]) } update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}` resetTurns := [][]agentllm.Event{ textDeltaTurn("", toolCall("update_todos", update)), quiet, quiet, quiet, textDeltaTurn("完成了。"), } requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}}) if len(requests) != 5 { t.Fatalf("reset requests = %d, want 5", len(requests)) } for i, req := range requests[:4] { if strings.Contains(req, "system-reminder") { t.Fatalf("request %d reminded before three quiet turns: %s", i, req) } } if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") { t.Fatalf("reminder did not refresh todos: %s", requests[4]) } } func TestTodoReminderCapsAtThree(t *testing.T) { var turns [][]agentllm.Event for i := 0; i < 12; i++ { turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}"))) } turns = append(turns, textDeltaTurn("完成了。")) requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}}) n := 0 for _, req := range requests { if strings.Contains(req, "system-reminder") { n++ } } if n != 3 { t.Fatalf("reminders = %d, want 3", n) } } func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string { t.Helper() var requests []string base := scriptStream(turns, nil) r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event { requests = append(requests, joinContents(msgs)) return base(ctx, model, msgs, tools, opts) }} rc := &checklistFake{ fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()}, list: list, } if err := r.Execute(rc); err != nil { t.Fatal(err) } for _, p := range rc.assistant { if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") { t.Fatalf("reminder was persisted: %+v", p) } } return requests } func joinContents(msgs []agentllm.Message) string { var b strings.Builder for _, m := range msgs { b.WriteString(m.Content) b.WriteString("\n") } return b.String() } func hasProcessNote(parts []core.Part, fragment string) bool { for _, p := range parts { if p.Type == core.PartCheck && p.Check != nil && !p.Check.Passed && strings.Contains(p.Check.Reason, fragment) { return true } } return false } func badDocx(t *testing.T) []byte { t.Helper() dir := t.TempDir() path := filepath.Join(dir, "bad.docx") cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")") if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("docx: %v %s", err, out) } data, err := os.ReadFile(path) if err != nil { t.Fatal(err) } return data } func jsonString(s string) string { b, _ := json.Marshal(s) return string(b) } func TestFalseDeliverableFixtures(t *testing.T) { raw, err := os.ReadFile("testdata/false_deliverables.json") if err != nil { t.Fatal(err) } var doc struct { Cases []struct { Name string `json:"name"` Body string `json:"body"` } `json:"cases"` } if err := json.Unmarshal(raw, &doc); err != nil { t.Fatal(err) } if len(doc.Cases) < 5 { t.Fatalf("cases = %d, want at least 5", len(doc.Cases)) } for _, c := range doc.Cases { t.Run(c.Name, func(t *testing.T) { block, reason := deliverableGateOf(c.Body, nil, &fakeRunCtx{ctx: context.Background()}) if !block || strings.TrimSpace(reason) == "" { t.Fatalf("gate = %v %q", block, reason) } turns := [][]agentllm.Event{ textDeltaTurn(c.Body), textDeltaTurn("这一轮没有生成文件。"), } r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()} rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true} if err := r.Execute(rc); err != nil { t.Fatal(err) } if !hasProcessNote(rc.assistant, reason) { t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant)) } }) } }