weichen
f1302ed5b3
Why: Allow consumers to download an immutable module revision without nested source checkouts or local path replacements. Evidence: SELF package tests and the prior canonical-path rejection are recorded in the Dev dependency-workspace task card; remote consumption is replayed against the committed candidate. Impact: Change the module identity and its Go imports while preserving API behavior. Existing historical submodule revisions remain buildable. Refs: Versioned dependencies U1/U3; feature/dependency-workspace
417 lines
13 KiB
Go
417 lines
13 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git-ssh.u1680832.nyat.app/weichen/Agent-Native.git/runtime/gates"
|
|
agentllm "git-ssh.u1680832.nyat.app/weichen/Agent-Native.git/runtime/llm"
|
|
core "git-ssh.u1680832.nyat.app/weichen/Agent-System-Core.git"
|
|
)
|
|
|
|
func textDeltaTurn(body string, calls ...agentllm.ToolCall) []agentllm.Event {
|
|
evs := []agentllm.Event{}
|
|
if body != "" {
|
|
evs = append(evs, agentllm.Event{Kind: agentllm.KindTextDelta, Text: body})
|
|
}
|
|
end := agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
if len(calls) > 0 {
|
|
end.Message.ToolCalls = calls
|
|
}
|
|
return append(evs, end)
|
|
}
|
|
|
|
func toolCall(name, args string) agentllm.ToolCall {
|
|
return agentllm.ToolCall{ID: "c-" + name, Name: name, Arguments: args}
|
|
}
|
|
|
|
func passedStream(payloads ...string) agentllm.StreamFunc {
|
|
n := 0
|
|
return func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
body := `{"passed":true,"reason":"完成","nextAction":""}`
|
|
if n < len(payloads) {
|
|
body = payloads[n]
|
|
n++
|
|
}
|
|
out := make(chan agentllm.Event, 2)
|
|
out <- agentllm.Event{Kind: agentllm.KindTextDelta, Text: body}
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
close(out)
|
|
return out
|
|
}
|
|
}
|
|
|
|
func verifyOn() core.AgentProfile {
|
|
return core.AgentProfile{Model: core.ModelRoute{Chat: "test-model", Lightweight: "lite-model"}}
|
|
}
|
|
|
|
func TestCheckMissingOrEmptyArtifactContinues(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
miss bool
|
|
want string
|
|
}{
|
|
{name: "missing", miss: true, want: "不存在"},
|
|
{name: "empty", miss: false, want: "内容为空"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
turns := padTurns(
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("写好了。"),
|
|
)
|
|
var calls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
calls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: passedStream(),
|
|
}
|
|
rc := &fakeRunCtx{
|
|
ctx: context.Background(),
|
|
prof: verifyOn(),
|
|
verify: true,
|
|
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.md"},
|
|
missingArtifact: tc.miss,
|
|
emptyArtifact: !tc.miss,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if calls < 2 {
|
|
t.Fatalf("stream calls = %d, want a continuation", calls)
|
|
}
|
|
if !hasProcessNote(rc.assistant, tc.want) {
|
|
t.Fatalf("missing reason part %q in %+v", tc.want, textParts(rc.assistant))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestLintErrorContinues(t *testing.T) {
|
|
style := filepath.Join("..", "skills", "recipes", "house-style")
|
|
abs, err := filepath.Abs(style)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Setenv("PYTHONPATH", abs)
|
|
docx := badDocx(t)
|
|
turns := padTurns(
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("写好了。"),
|
|
)
|
|
var verifyCalls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: base,
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
verifyCalls++
|
|
return passedStream()(ctx, model, msgs, tools, opts)
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{
|
|
ctx: context.Background(),
|
|
prof: verifyOn(),
|
|
verify: true,
|
|
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "note.docx"},
|
|
execArtifactContent: docx,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if verifyCalls != 0 {
|
|
t.Fatalf("verifier calls = %d, want 0 when lint fails first", verifyCalls)
|
|
}
|
|
if !hasProcessNote(rc.assistant, "版式") {
|
|
t.Fatalf("lint reason missing: %+v", textParts(rc.assistant))
|
|
}
|
|
}
|
|
|
|
func TestVerifierRejectsThenPasses(t *testing.T) {
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn("答案是 42。"),
|
|
textDeltaTurn("答案仍是 42。"),
|
|
}
|
|
var mainCalls int
|
|
var models []string
|
|
var toolArgs [][]agentllm.ToolDecl
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
mainCalls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
models = append(models, model)
|
|
toolArgs = append(toolArgs, tools)
|
|
body := `{"passed":false,"reason":"还没回答完","nextAction":"补上结论"}`
|
|
if len(models) > 1 {
|
|
body = `{"passed":true,"reason":"完成","nextAction":""}`
|
|
}
|
|
return passedStream(body)(ctx, model, msgs, tools, opts)
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if mainCalls != 2 {
|
|
t.Fatalf("main calls = %d, want 2", mainCalls)
|
|
}
|
|
if len(models) != 2 || models[0] != "lite-model" {
|
|
t.Fatalf("verifier models = %v, want lite-model twice", models)
|
|
}
|
|
for i, tools := range toolArgs {
|
|
if tools != nil {
|
|
t.Fatalf("verifier call %d tools = %#v, want nil", i, tools)
|
|
}
|
|
}
|
|
texts := textParts(rc.assistant)
|
|
if len(texts) < 2 || !texts[0].Process || texts[len(texts)-1].Process {
|
|
t.Fatalf("parts = %+v", texts)
|
|
}
|
|
if !hasProcessNote(rc.assistant, "还没回答完") || !hasProcessNote(rc.assistant, "补上结论") {
|
|
t.Fatalf("check missing: %+v", rc.assistant)
|
|
}
|
|
}
|
|
|
|
func TestVerifierGarbageFailsClosed(t *testing.T) {
|
|
for _, payload := range []agentllm.Event{
|
|
{Kind: agentllm.KindTextDelta, Text: "当然通过"},
|
|
{Kind: agentllm.KindError, ErrorMessage: "upstream"},
|
|
} {
|
|
t.Run(string(payload.Kind), func(t *testing.T) {
|
|
turns := [][]agentllm.Event{textDeltaTurn("答案是 42。")}
|
|
var mainCalls int
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{
|
|
Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
mainCalls++
|
|
return base(ctx, model, msgs, tools, opts)
|
|
},
|
|
VerifyStream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
if tools != nil {
|
|
t.Errorf("verifier tools = %#v, want nil", tools)
|
|
}
|
|
out := make(chan agentllm.Event, 2)
|
|
out <- payload
|
|
out <- agentllm.Event{Kind: agentllm.KindMessageEnd}
|
|
close(out)
|
|
return out
|
|
},
|
|
}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if mainCalls != 1 {
|
|
t.Fatalf("main calls = %d, want 1", mainCalls)
|
|
}
|
|
texts := textParts(rc.assistant)
|
|
var kept bool
|
|
notice := false
|
|
for _, p := range texts {
|
|
if p.Text == "答案是 42。" && !p.Process {
|
|
kept = true
|
|
}
|
|
if !p.Process && strings.Contains(p.Text, "验证器") {
|
|
notice = true
|
|
}
|
|
}
|
|
// 2026-10-10 契约更新：验证器不可用不再作为正文 Notice，落成过程检查行。
|
|
if !kept || notice || !hasProcessNote(rc.assistant, "无法解析") && !hasProcessNote(rc.assistant, "调用失败") {
|
|
t.Fatalf("parts = %+v", texts)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestParseVerdictToleratesFence(t *testing.T) {
|
|
got, err := gates.ParseVerdict("```json\n{\"passed\":false,\"reason\":\"不够\",\"nextAction\":\"重写\"}\n```")
|
|
if err != nil || got.Passed || got.Reason != "不够" {
|
|
t.Fatalf("verdict = %+v %v", got, err)
|
|
}
|
|
if _, err := gates.ParseVerdict(`{"reason":"no passed"}`); err == nil {
|
|
t.Fatal("missing passed should fail")
|
|
}
|
|
}
|
|
|
|
type checklistFake struct {
|
|
fakeRunCtx
|
|
list core.TodoList
|
|
}
|
|
|
|
func (c *checklistFake) Todos() (core.TodoList, bool) { return c.list, true }
|
|
|
|
func padTurns(turns ...[]agentllm.Event) [][]agentllm.Event {
|
|
for len(turns) < 5 {
|
|
turns = append(turns, textDeltaTurn("仍未修好。"))
|
|
}
|
|
return turns
|
|
}
|
|
|
|
func TestTodoReminderFiresOnceAndResets(t *testing.T) {
|
|
quiet := textDeltaTurn("", toolCall("make_report", "{}"))
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("", toolCall("make_report", "{}")),
|
|
textDeltaTurn("完成了。"),
|
|
}
|
|
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
|
if len(requests) != 4 {
|
|
t.Fatalf("requests = %d, want 4", len(requests))
|
|
}
|
|
for i, req := range requests[:3] {
|
|
if strings.Contains(req, "system-reminder") {
|
|
t.Fatalf("request %d reminded too early: %s", i, req)
|
|
}
|
|
}
|
|
if !strings.Contains(requests[3], "system-reminder") || !strings.Contains(requests[3], "写纪要") {
|
|
t.Fatalf("final request missing reminder: %s", requests[3])
|
|
}
|
|
|
|
update := `{"items":[{"id":"1","text":"改后的待办","status":"pending"}]}`
|
|
resetTurns := [][]agentllm.Event{
|
|
textDeltaTurn("", toolCall("update_todos", update)),
|
|
quiet, quiet, quiet,
|
|
textDeltaTurn("完成了。"),
|
|
}
|
|
requests = runChecklist(t, resetTurns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "旧待办", Status: "pending"}}})
|
|
if len(requests) != 5 {
|
|
t.Fatalf("reset requests = %d, want 5", len(requests))
|
|
}
|
|
for i, req := range requests[:4] {
|
|
if strings.Contains(req, "system-reminder") {
|
|
t.Fatalf("request %d reminded before three quiet turns: %s", i, req)
|
|
}
|
|
}
|
|
if !strings.Contains(requests[4], "改后的待办") || strings.Contains(requests[4], "旧待办") {
|
|
t.Fatalf("reminder did not refresh todos: %s", requests[4])
|
|
}
|
|
}
|
|
|
|
func TestTodoReminderCapsAtThree(t *testing.T) {
|
|
var turns [][]agentllm.Event
|
|
for i := 0; i < 12; i++ {
|
|
turns = append(turns, textDeltaTurn("", toolCall("make_report", "{}")))
|
|
}
|
|
turns = append(turns, textDeltaTurn("完成了。"))
|
|
requests := runChecklist(t, turns, core.TodoList{Items: []core.TodoItem{{ID: "1", Text: "写纪要", Status: "pending"}}})
|
|
n := 0
|
|
for _, req := range requests {
|
|
if strings.Contains(req, "system-reminder") {
|
|
n++
|
|
}
|
|
}
|
|
if n != 3 {
|
|
t.Fatalf("reminders = %d, want 3", n)
|
|
}
|
|
}
|
|
|
|
func runChecklist(t *testing.T, turns [][]agentllm.Event, list core.TodoList) []string {
|
|
t.Helper()
|
|
var requests []string
|
|
base := scriptStream(turns, nil)
|
|
r := &LLMRunner{Stream: func(ctx context.Context, model string, msgs []agentllm.Message, tools []agentllm.ToolDecl, opts agentllm.StreamOptions) <-chan agentllm.Event {
|
|
requests = append(requests, joinContents(msgs))
|
|
return base(ctx, model, msgs, tools, opts)
|
|
}}
|
|
rc := &checklistFake{
|
|
fakeRunCtx: fakeRunCtx{ctx: context.Background(), prof: verifyOn()},
|
|
list: list,
|
|
}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, p := range rc.assistant {
|
|
if strings.Contains(p.Text, "system-reminder") || strings.Contains(p.Text, "连续 3 个工具轮") {
|
|
t.Fatalf("reminder was persisted: %+v", p)
|
|
}
|
|
}
|
|
return requests
|
|
}
|
|
|
|
func joinContents(msgs []agentllm.Message) string {
|
|
var b strings.Builder
|
|
for _, m := range msgs {
|
|
b.WriteString(m.Content)
|
|
b.WriteString("\n")
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func hasProcessNote(parts []core.Part, fragment string) bool {
|
|
for _, p := range parts {
|
|
if p.Type == core.PartCheck && p.Check != nil && !p.Check.Passed && strings.Contains(p.Check.Reason, fragment) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func badDocx(t *testing.T) []byte {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "bad.docx")
|
|
cmd := exec.Command("python3", "-c", "from docx import Document\nd=Document()\nd.add_paragraph('x')\nd.save("+jsonString(path)+")")
|
|
if out, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("docx: %v %s", err, out)
|
|
}
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func jsonString(s string) string {
|
|
b, _ := json.Marshal(s)
|
|
return string(b)
|
|
}
|
|
|
|
func TestFalseDeliverableFixtures(t *testing.T) {
|
|
raw, err := os.ReadFile("testdata/false_deliverables.json")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var doc struct {
|
|
Cases []struct {
|
|
Name string `json:"name"`
|
|
Body string `json:"body"`
|
|
} `json:"cases"`
|
|
}
|
|
if err := json.Unmarshal(raw, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(doc.Cases) < 5 {
|
|
t.Fatalf("cases = %d, want at least 5", len(doc.Cases))
|
|
}
|
|
for _, c := range doc.Cases {
|
|
t.Run(c.Name, func(t *testing.T) {
|
|
block, reason := deliverableGateOf(c.Body, nil, &fakeRunCtx{ctx: context.Background()})
|
|
if !block || strings.TrimSpace(reason) == "" {
|
|
t.Fatalf("gate = %v %q", block, reason)
|
|
}
|
|
turns := [][]agentllm.Event{
|
|
textDeltaTurn(c.Body),
|
|
textDeltaTurn("这一轮没有生成文件。"),
|
|
}
|
|
r := &LLMRunner{Stream: scriptStream(turns, nil), VerifyStream: passedStream()}
|
|
rc := &fakeRunCtx{ctx: context.Background(), prof: verifyOn(), verify: true}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !hasProcessNote(rc.assistant, reason) {
|
|
t.Fatalf("reason part missing %q in %+v", reason, textParts(rc.assistant))
|
|
}
|
|
})
|
|
}
|
|
}
|