weichen
18aa99cb95
ci / vet-and-test (pull_request) Successful in 1m1s
Why: real run run_b520bf2e3037 showed the user-facing reply being displaced by internal bookkeeping - the fact-check correction round's closing text plus the same-source verifier skip notice became the visible answer while the actual summary was folded into the process timeline (frontend splits at the last process part). Same tail pollution reproduced on every same-model run since. Evidence: go test ./... -count=1 green (32 pkgs); new regression tests cover the demotion of the pre-check summary and the skip notice becoming a check part; three old-contract tests updated to assert notices land as process checks instead of final text. Impact: stop() fact-check branch now demotes the standing summary (aligned with the gate and cap branches) and asks the model for a fresh full reply; verifier errors record as process checks (pass-state for same-source skip, fail-state for real errors) instead of StopDecision.Notice text; the cap notice likewise becomes a failed check line. Refs: Dev task feature/final-reply-pollution-fix; diagnosis in .task/feature-dual-speed-tasks.md lineage and session analysis 2026-10-10.
417 lines
13 KiB
Go
417 lines
13 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"agent-native/runtime/gates"
|
|
agentllm "agent-native/runtime/llm"
|
|
core "agent-system-core"
|
|
)
|
|
|
|
func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event {
|
|
evs := []agentllm.Event{}
|
|
if body != "" {
|
|
evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body})
|
|
}
|
|
end := agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
if len(calls) > 0 {
|
|
end.Message.ToolCalls = calls
|
|
}
|
|
return append(evs, end)
|
|
}
|
|
|
|
func toolCall(name, args string) agentllm.ToolCall {
|
|
return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args}
|
|
}
|
|
|
|
func passedStream(payloads ...string) agentllm.StreamFunc {
|
|
n := 0
|
|
return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
body := `{"passed":true,"reason":"完成","nextAction":""}`
|
|
if n < len(payloads) {
|
|
body = payloads[n]
|
|
n++
|
|
}
|
|
out := make(chan agentllm.Event, 2)
|
|
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
close(out)
|
|
return out
|
|
}
|
|
}
|
|
|
|
func verifyOn() core.AgentProfile {
|
|
return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}
|
|
}
|
|
|
|
func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
miss bool
|
|
want string
|
|
}{
|
|
{name: "missing", miss: true, want: "不存在"},
|
|
{name: "empty", miss: false, want: "内容为空"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
turns := padTurns(
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("写好了。"),
|
|
)
|
|
var calls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
calls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: passedStream(),
|
|
}
|
|
rc := &fakeRunCtx{
|
|
ctx: context.Background(),
|
|
prof: verifyOn(),
|
|
verify: true,
|
|
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"},
|
|
missingArtifact: tc.miss,
|
|
emptyArtifact: !tc.miss,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if calls < 2 {
|
|
t.Fatalf("stream calls = %d, want a continuation", calls)
|
|
}
|
|
if !hasProcessNote(rc.assistant, tc.want) {
|
|
t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLintErrorContinues(t *testing.T) {
|
|
style := filepath.Join("..", "skills", "recipes", "house-style")
|
|
abs, err := filepath.Abs(style)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PYTHONPATH", abs)
|
|
docx := badDocx(t)
|
|
turns := padTurns(
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("写好了。"),
|
|
)
|
|
var verifyCalls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: base,
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
verifyCalls++
|
|
return passedStream()(ctx, model, msgs, tools, opts)
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{
|
|
ctx: context.Background(),
|
|
prof: verifyOn(),
|
|
verify: true,
|
|
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"},
|
|
execArtifactContent: docx,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if verifyCalls != 0 {
|
|
t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls)
|
|
}
|
|
if !hasProcessNote(rc.assistant, "版式") {
|
|
t.Fatalf("lint reason missing: %+v", textParts(rc.assistant))
|
|
}
|
|
}
|
|
|
|
func TestVerifierRejectsThenPasses(t *testing.T) {
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn("答案是 42。"),
|
|
textDeltaTurn("答案仍是 42。"),
|
|
}
|
|
var mainCalls int
|
|
var models []string
|
|
var toolArgs [][]agentllm.ToolDecl
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
mainCalls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
models = append(models, model)
|
|
toolArgs = append(toolArgs, tools)
|
|
body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}`
|
|
if len(models) > 1 {
|
|
body = `{"passed":true,"reason":"完成","nextAction":""}`
|
|
}
|
|
return passedStream(body)(ctx, model, msgs, tools, opts)
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if mainCalls != 2 {
|
|
t.Fatalf("main calls = %d, want 2", mainCalls)
|
|
}
|
|
if len(models) != 2 || models[0] != "lite-model" {
|
|
t.Fatalf("verifier models = %v, want lite-model twice", models)
|
|
}
|
|
for i, tools := range toolArgs {
|
|
if tools != nil {
|
|
t.Fatalf("verifier call %d tools = %#v, want nil", i, tools)
|
|
}
|
|
}
|
|
texts := textParts(rc.assistant)
|
|
if len(texts) < 2 || !texts[0].Process || texts[len(texts)-1].Process {
|
|
t.Fatalf("parts = %+v", texts)
|
|
}
|
|
if !hasProcessNote(rc.assistant, "还没回答完") || !hasProcessNote(rc.assistant, "补上结论") {
|
|
t.Fatalf("check missing: %+v", rc.assistant)
|
|
}
|
|
}
|
|
|
|
func TestVerifierGarbageFailsClosed(t *testing.T) {
|
|
for _, payload := range []agentllm.Event{
|
|
{Kind: agentllm.KindTextDelta, Text: "当然通过"},
|
|
{Kind: agentllm.KindError, ErrorMessage: "upstream"},
|
|
} {
|
|
t.Run(string(payload.Kind), func(t *testing.T) {
|
|
turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")}
|
|
var mainCalls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
mainCalls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
if tools != nil {
|
|
t.Errorf("verifier tools = %#v, want nil", tools)
|
|
}
|
|
out := make(chan agentllm.Event, 2)
|
|
out <- payload
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
close(out)
|
|
return out
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if mainCalls != 1 {
|
|
t.Fatalf("main calls = %d, want 1", mainCalls)
|
|
}
|
|
texts := textParts(rc.assistant)
|
|
var kept bool
|
|
notice := false
|
|
for _, p := range texts {
|
|
if p.Text == "答案是 42。" && !p.Process {
|
|
kept = true
|
|
}
|
|
if !p.Process && strings.Contains(p.Text, "验证器") {
|
|
notice = true
|
|
}
|
|
}
|
|
// 2026-10-10 契约更新：验证器不可用不再作为正文 Notice，落成过程检查行。
|
|
if !kept || notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") {
|
|
t.Fatalf("parts = %+v", texts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseVerdictToleratesFence(t *testing.T) {
|
|
got, err := gates.ParseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```")
|
|
if err != nil || got.Passed || got.Reason != "不够" {
|
|
t.Fatalf("verdict = %+v %v", got, err)
|
|
}
|
|
if _, err := gates.ParseVerdict(`{"reason":"no passed"}`); err == nil {
|
|
t.Fatal("missing passed should fail")
|
|
}
|
|
}
|
|
|
|
type checklistFake struct {
|
|
fakeRunCtx
|
|
list core.TodoList
|
|
}
|
|
|
|
func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true }
|
|
|
|
func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event {
|
|
for len(turns) < 5 {
|
|
turns = append(turns, textDeltaTurn("仍未修好。"))
|
|
}
|
|
return turns
|
|
}
|
|
|
|
func TestTodoReminderFiresOnceAndResets(t *testing.T) {
|
|
quiet := textDeltaTurn("", toolCall("make_report", "{}"))
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("完成了。"),
|
|
}
|
|
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
|
if len(requests) != 4 {
|
|
t.Fatalf("requests = %d, want 4", len(requests))
|
|
}
|
|
for i, req := range requests[:3] {
|
|
if strings.Contains(req, "system-reminder") {
|
|
t.Fatalf("request %d reminded too early: %s", i, req)
|
|
}
|
|
}
|
|
if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") {
|
|
t.Fatalf("final request missing reminder: %s", requests[3])
|
|
}
|
|
|
|
update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}`
|
|
resetTurns := [][]agentllm.Event{
|
|
textDeltaTurn("", toolCall("update_todos", update)),
|
|
quiet, quiet, quiet,
|
|
textDeltaTurn("完成了。"),
|
|
}
|
|
requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}})
|
|
if len(requests) != 5 {
|
|
t.Fatalf("reset requests = %d, want 5", len(requests))
|
|
}
|
|
for i, req := range requests[:4] {
|
|
if strings.Contains(req, "system-reminder") {
|
|
t.Fatalf("request %d reminded before three quiet turns: %s", i, req)
|
|
}
|
|
}
|
|
if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") {
|
|
t.Fatalf("reminder did not refresh todos: %s", requests[4])
|
|
}
|
|
}
|
|
|
|
func TestTodoReminderCapsAtThree(t *testing.T) {
|
|
var turns [][]agentllm.Event
|
|
for i := 0; i < 12; i++ {
|
|
turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}")))
|
|
}
|
|
turns = append(turns, textDeltaTurn("完成了。"))
|
|
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
|
n := 0
|
|
for _, req := range requests {
|
|
if strings.Contains(req, "system-reminder") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 3 {
|
|
t.Fatalf("reminders = %d, want 3", n)
|
|
}
|
|
}
|
|
|
|
func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string {
|
|
t.Helper()
|
|
var requests []string
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
requests = append(requests, joinContents(msgs))
|
|
return base(ctx, model, msgs, tools, opts)
|
|
}}
|
|
rc := &checklistFake{
|
|
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
|
list: list,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, p := range rc.assistant {
|
|
if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") {
|
|
t.Fatalf("reminder was persisted: %+v", p)
|
|
}
|
|
}
|
|
return requests
|
|
}
|
|
|
|
func joinContents(msgs []agentllm.Message) string {
|
|
var b strings.Builder
|
|
for _, m := range msgs {
|
|
b.WriteString(m.Content)
|
|
b.WriteString("\n")
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func hasProcessNote(parts []core.Part, fragment string) bool {
|
|
for _, p := range parts {
|
|
if p.Type == core.PartCheck && p.Check != nil && !p.Check.Passed && strings.Contains(p.Check.Reason, fragment) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func badDocx(t *testing.T) []byte {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "bad.docx")
|
|
cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")")
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("docx: %v %s", err, out)
|
|
}
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func jsonString(s string) string {
|
|
b, _ := json.Marshal(s)
|
|
return string(b)
|
|
}
|
|
|
|
func TestFalseDeliverableFixtures(t *testing.T) {
|
|
raw, err := os.ReadFile("testdata/false_deliverables.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var doc struct {
|
|
Cases []struct {
|
|
Name string `json:"name"`
|
|
Body string `json:"body"`
|
|
} `json:"cases"`
|
|
}
|
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(doc.Cases) < 5 {
|
|
t.Fatalf("cases = %d, want at least 5", len(doc.Cases))
|
|
}
|
|
for _, c := range doc.Cases {
|
|
t.Run(c.Name, func(t *testing.T) {
|
|
block, reason := deliverableGateOf(c.Body, nil, &fakeRunCtx{ctx: context.Background()})
|
|
if !block || strings.TrimSpace(reason) == "" {
|
|
t.Fatalf("gate = %v %q", block, reason)
|
|
}
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn(c.Body),
|
|
textDeltaTurn("这一轮没有生成文件。"),
|
|
}
|
|
r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !hasProcessNote(rc.assistant, reason) {
|
|
t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant))
|
|
}
|
|
})
|
|
}
|
|
}
|