Agent-System-Backend/internal/runtime/llm_verify_test.go

package runtime

import (
	"context"
	"encoding/json"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"

	agentllm "agent-native/runtime/llm"
	core "agent-system-core"
)

func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event {
	evs := []agentllm.Event{}
	if body != "" {
		evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body})
	}
	end := agentllm.Event{Kind: agentllm.KindMessageEnd}
	if len(calls) > 0 {
		end.Message.ToolCalls = calls
	}
	return append(evs, end)
}

func toolCall(name, args string) agentllm.ToolCall {
	return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args}
}

func passedStream(payloads ...string) agentllm.StreamFunc {
	n := 0
	return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
		body := `{"passed":true,"reason":"完成","nextAction":""}`
		if n < len(payloads) {
			body = payloads[n]
			n++
		}
		out := make(chan agentllm.Event, 2)
		out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}
		out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
		close(out)
		return out
	}
}

func verifyOn() core.AgentProfile {
	return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}
}

func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) {
	for _, tc := range []struct {
		name string
		miss bool
		want string
	}{
		{name: "missing", miss: true, want: "不存在"},
		{name: "empty", miss: false, want: "内容为空"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			turns := padTurns(
				textDeltaTurn("", toolCall("make_report", "{}")),
				textDeltaTurn("写好了。"),
			)
			var calls int
			base := scriptStream(turns, nil)
			r := &LLMRunner{
				Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
					calls++
					return base(ctx, model, msgs, tools, opts)
				},
				VerifyStream: passedStream(),
			}
			rc := &fakeRunCtx{
				ctx:             context.Background(),
				prof:            verifyOn(),
				verify:          true,
				execArtifact:    &core.Artifact{ID: "art_0123456789ab", Name: "note.md"},
				missingArtifact: tc.miss,
				emptyArtifact:   !tc.miss,
			}
			if err := r.Execute(rc); err != nil {
				t.Fatalf("Execute: %v", err)
			}
			if calls < 2 {
				t.Fatalf("stream calls = %d, want a continuation", calls)
			}
			if !hasProcessNote(rc.assistant, tc.want) {
				t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant))
			}
		})
	}
}

func TestLintErrorContinues(t *testing.T) {
	style := filepath.Join("..", "skills", "recipes", "house-style")
	abs, err := filepath.Abs(style)
	if err != nil {
		t.Fatal(err)
	}
	t.Setenv("PYTHONPATH", abs)
	docx := badDocx(t)
	turns := padTurns(
		textDeltaTurn("", toolCall("make_report", "{}")),
		textDeltaTurn("写好了。"),
	)
	var verifyCalls int
	base := scriptStream(turns, nil)
	r := &LLMRunner{
		Stream: base,
		VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
			verifyCalls++
			return passedStream()(ctx, model, msgs, tools, opts)
		},
	}
	rc := &fakeRunCtx{
		ctx:                 context.Background(),
		prof:                verifyOn(),
		verify:              true,
		execArtifact:        &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"},
		execArtifactContent: docx,
	}
	if err := r.Execute(rc); err != nil {
		t.Fatalf("Execute: %v", err)
	}
	if verifyCalls != 0 {
		t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls)
	}
	if !hasProcessNote(rc.assistant, "版式") {
		t.Fatalf("lint reason missing: %+v", textParts(rc.assistant))
	}
}

func TestVerifierRejectsThenPasses(t *testing.T) {
	turns := [][]agentllm.Event{
		textDeltaTurn("答案是 42。"),
		textDeltaTurn("答案仍是 42。"),
	}
	var mainCalls int
	var models []string
	var toolArgs [][]agentllm.ToolDecl
	base := scriptStream(turns, nil)
	r := &LLMRunner{
		Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
			mainCalls++
			return base(ctx, model, msgs, tools, opts)
		},
		VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
			models = append(models, model)
			toolArgs = append(toolArgs, tools)
			body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}`
			if len(models) > 1 {
				body = `{"passed":true,"reason":"完成","nextAction":""}`
			}
			return passedStream(body)(ctx, model, msgs, tools, opts)
		},
	}
	rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
	if err := r.Execute(rc); err != nil {
		t.Fatalf("Execute: %v", err)
	}
	if mainCalls != 2 {
		t.Fatalf("main calls = %d, want 2", mainCalls)
	}
	if len(models) != 2 || models[0] != "lite-model" {
		t.Fatalf("verifier models = %v, want lite-model twice", models)
	}
	for i, tools := range toolArgs {
		if tools != nil {
			t.Fatalf("verifier call %d tools = %#v, want nil", i, tools)
		}
	}
	texts := textParts(rc.assistant)
	if len(texts) < 3 || !texts[0].Process || !strings.Contains(texts[1].Text, "验证未通过") || texts[len(texts)-1].Process {
		t.Fatalf("parts = %+v", texts)
	}
	if !strings.Contains(texts[1].Text, "补上结论") {
		t.Fatalf("nextAction missing: %+v", texts)
	}
}

func TestVerifierGarbageFailsClosed(t *testing.T) {
	for _, payload := range []agentllm.Event{
		{Kind: agentllm.KindTextDelta, Text: "当然通过"},
		{Kind: agentllm.KindError, ErrorMessage: "upstream"},
	} {
		t.Run(string(payload.Kind), func(t *testing.T) {
			turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")}
			var mainCalls int
			base := scriptStream(turns, nil)
			r := &LLMRunner{
				Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
					mainCalls++
					return base(ctx, model, msgs, tools, opts)
				},
				VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
					if tools != nil {
						t.Errorf("verifier tools = %#v, want nil", tools)
					}
					out := make(chan agentllm.Event, 2)
					out <- payload
					out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
					close(out)
					return out
				},
			}
			rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
			if err := r.Execute(rc); err != nil {
				t.Fatalf("Execute: %v", err)
			}
			if mainCalls != 1 {
				t.Fatalf("main calls = %d, want 1", mainCalls)
			}
			texts := textParts(rc.assistant)
			var kept, notice bool
			for _, p := range texts {
				if p.Text == "答案是 42。" && !p.Process {
					kept = true
				}
				if !p.Process && strings.Contains(p.Text, "验证器不可用") {
					notice = true
				}
			}
			if !kept || !notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") {
				t.Fatalf("parts = %+v", texts)
			}
		})
	}
}

func TestParseVerdictToleratesFence(t *testing.T) {
	got, err := parseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```")
	if err != nil || got.Passed || got.Reason != "不够" {
		t.Fatalf("verdict = %+v %v", got, err)
	}
	if _, err := parseVerdict(`{"reason":"no passed"}`); err == nil {
		t.Fatal("missing passed should fail")
	}
}

type checklistFake struct {
	fakeRunCtx
	list core.TodoList
}

func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true }

func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event {
	for len(turns) < 5 {
		turns = append(turns, textDeltaTurn("仍未修好。"))
	}
	return turns
}

func TestTodoReminderFiresOnceAndResets(t *testing.T) {
	quiet := textDeltaTurn("", toolCall("make_report", "{}"))
	turns := [][]agentllm.Event{
		textDeltaTurn("", toolCall("make_report", "{}")),
		textDeltaTurn("", toolCall("make_report", "{}")),
		textDeltaTurn("", toolCall("make_report", "{}")),
		textDeltaTurn("完成了。"),
	}
	requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
	if len(requests) != 4 {
		t.Fatalf("requests = %d, want 4", len(requests))
	}
	for i, req := range requests[:3] {
		if strings.Contains(req, "system-reminder") {
			t.Fatalf("request %d reminded too early: %s", i, req)
		}
	}
	if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") {
		t.Fatalf("final request missing reminder: %s", requests[3])
	}

	update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}`
	resetTurns := [][]agentllm.Event{
		textDeltaTurn("", toolCall("update_todos", update)),
		quiet, quiet, quiet,
		textDeltaTurn("完成了。"),
	}
	requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}})
	if len(requests) != 5 {
		t.Fatalf("reset requests = %d, want 5", len(requests))
	}
	for i, req := range requests[:4] {
		if strings.Contains(req, "system-reminder") {
			t.Fatalf("request %d reminded before three quiet turns: %s", i, req)
		}
	}
	if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") {
		t.Fatalf("reminder did not refresh todos: %s", requests[4])
	}
}

func TestTodoReminderCapsAtThree(t *testing.T) {
	var turns [][]agentllm.Event
	for i := 0; i < 12; i++ {
		turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}")))
	}
	turns = append(turns, textDeltaTurn("完成了。"))
	requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
	n := 0
	for _, req := range requests {
		if strings.Contains(req, "system-reminder") {
			n++
		}
	}
	if n != 3 {
		t.Fatalf("reminders = %d, want 3", n)
	}
}

func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string {
	t.Helper()
	var requests []string
	base := scriptStream(turns, nil)
	r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
		requests = append(requests, joinContents(msgs))
		return base(ctx, model, msgs, tools, opts)
	}}
	rc := &checklistFake{
		fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
		list:       list,
	}
	if err := r.Execute(rc); err != nil {
		t.Fatal(err)
	}
	for _, p := range rc.assistant {
		if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") {
			t.Fatalf("reminder was persisted: %+v", p)
		}
	}
	return requests
}

func joinContents(msgs []agentllm.Message) string {
	var b strings.Builder
	for _, m := range msgs {
		b.WriteString(m.Content)
		b.WriteString("\n")
	}
	return b.String()
}

func hasProcessNote(parts []core.Part, fragment string) bool {
	for _, p := range parts {
		if p.Type == core.PartText && p.Process && strings.Contains(p.Text, "验证未通过") && strings.Contains(p.Text, fragment) {
			return true
		}
	}
	return false
}

func badDocx(t *testing.T) []byte {
	t.Helper()
	dir := t.TempDir()
	path := filepath.Join(dir, "bad.docx")
	cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")")
	if out, err := cmd.CombinedOutput(); err != nil {
		t.Fatalf("docx: %v %s", err, out)
	}
	data, err := os.ReadFile(path)
	if err != nil {
		t.Fatal(err)
	}
	return data
}

func jsonString(s string) string {
	b, _ := json.Marshal(s)
	return string(b)
}

func TestFalseDeliverableFixtures(t *testing.T) {
	raw, err := os.ReadFile("testdata/false_deliverables.json")
	if err != nil {
		t.Fatal(err)
	}
	var doc struct {
		Cases []struct {
			Name string `json:"name"`
			Body string `json:"body"`
		} `json:"cases"`
	}
	if err := json.Unmarshal(raw, &doc); err != nil {
		t.Fatal(err)
	}
	if len(doc.Cases) < 5 {
		t.Fatalf("cases = %d, want at least 5", len(doc.Cases))
	}
	for _, c := range doc.Cases {
		t.Run(c.Name, func(t *testing.T) {
			block, reason := deliverableGate(c.Body, nil, &fakeRunCtx{ctx: context.Background()})
			if !block || strings.TrimSpace(reason) == "" {
				t.Fatalf("gate = %v %q", block, reason)
			}
			turns := [][]agentllm.Event{
				textDeltaTurn(c.Body),
				textDeltaTurn("这一轮没有生成文件。"),
			}
			r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()}
			rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
			if err := r.Execute(rc); err != nil {
				t.Fatal(err)
			}
			if !hasProcessNote(rc.assistant, reason) {
				t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant))
			}
		})
	}
}