diff --git a/eval/ab.go b/eval/ab.go new file mode 100644 index 0000000..d5fb11a --- /dev/null +++ b/eval/ab.go @@ -0,0 +1,175 @@ +package eval + +import ( + "context" + "fmt" + "sort" + "strings" + "time" +) + +// ab.go — 任务级 A/B 双跑 harness(Q3 WP5):同一批任务分别经两个入口 +// (direct 对话循环 / squad 编译图)执行,逐任务记录差异(不要求字节级 +// 相同),汇总为两份 EntryScore 交 DecideSwitch 裁决默认入口切换。 +// 本包保持纯机制(不依赖 runtime/squad);两侧 RunFunc 由调用方装配。 + +// ABTask 是对照批中的一个任务。 +type ABTask struct { + ID string + Goal string + // Themes 期望产出要点(短词,非整句照抄):打分用 MissingThemes, + // 差异记录用要点覆盖对比。 + Themes []string +} + +// ABOutcome 是一个入口对一个任务的执行结果(调用方 RunFunc 填写)。 +type ABOutcome struct { + // Completed 任务是否正常收口(失败/取消 = false)。 + Completed bool + // Summary 交付摘要(要点覆盖打分的对象;调用方可拼接工件正文)。 + Summary string + // FirstToken 首字延迟(入口收到任务到首个可见输出)。 + FirstToken time.Duration + // Err 失败原因(Completed=false 时填写)。 + Err string +} + +// RunFunc 是一个入口的执行侧(direct 或 squad 编译图)。 +type RunFunc func(ctx context.Context, task ABTask) (ABOutcome, error) + +// ABTaskRecord 是一个任务的双跑对照记录(差异记录单位)。 +type ABTaskRecord struct { + Task ABTask + Direct ABOutcome + Squad ABOutcome + // Diffs 差异注记(状态 / 要点覆盖 / 首字延迟),空 = 两侧一致。 + Diffs []string + // DirectPoints / SquadPoints 单任务得分(满分 100:完成 + 要点全覆盖; + // 未完成 0 分;完成但缺要点按缺失比例扣)。 + DirectPoints int + SquadPoints int +} + +// ABReport 是一批双跑的对照报告(切换裁决的证据形态)。 +type ABReport struct { + Records []ABTaskRecord + Direct EntryScore + Squad EntryScore + Decision SwitchDecision + // Text 是可附在切换 PR / 任务卡上的短报告(Report(Decision) + 差异清单)。 + Text string +} + +// RunAB 双跑同一批任务并汇总对照报告。任一侧 RunFunc 返回 error 视为该侧 +// 该任务失败(记录进 outcome.Err),不中断整批(对照要的是全批差异面)。 +func RunAB(ctx context.Context, tasks []ABTask, direct, squad RunFunc) ABReport { + rep := ABReport{} + if len(tasks) == 0 { + rep.Decision = DecideSwitch(nil, nil) + rep.Text = Report(rep.Decision) + return rep + } + var directFT, squadFT []time.Duration + var directSum, squadSum int + for _, task := range tasks { + rec := ABTaskRecord{Task: task} + rec.Direct = runOne(ctx, task, direct) + rec.Squad = runOne(ctx, task, squad) + rec.DirectPoints = scoreOutcome(rec.Direct, task.Themes) + rec.SquadPoints = scoreOutcome(rec.Squad, task.Themes) + rec.Diffs = diffRecord(rec) + rep.Records = append(rep.Records, rec) + directSum += rec.DirectPoints + squadSum += rec.SquadPoints + directFT = append(directFT, rec.Direct.FirstToken) + squadFT = append(squadFT, rec.Squad.FirstToken) + } + rep.Direct = EntryScore{Name: "direct", Score: directSum / len(tasks), FirstTokenP50: p50(directFT)} + rep.Squad = EntryScore{Name: "squad", Score: squadSum / len(tasks), FirstTokenP50: p50(squadFT)} + rep.Decision = DecideSwitch(&rep.Direct, &rep.Squad) + rep.Text = renderReport(rep) + return rep +} + +func runOne(ctx context.Context, task ABTask, fn RunFunc) ABOutcome { + if fn == nil { + return ABOutcome{Err: "入口未装配"} + } + out, err := fn(ctx, task) + if err != nil { + out.Completed = false + if out.Err == "" { + out.Err = err.Error() + } + } + return out +} + +// scoreOutcome 单任务打分:未完成 0;完成 = 100 - 缺失要点比例 × 100。 +func scoreOutcome(out ABOutcome, themes []string) int { + if !out.Completed { + return 0 + } + if len(themes) == 0 { + return 100 + } + missing := MissingThemes(out.Summary, themes) + return 100 - len(missing)*100/len(themes) +} + +// diffRecord 逐任务差异注记(状态 / 要点 / 首字延迟;不要求字节级相同)。 +func diffRecord(rec ABTaskRecord) []string { + var diffs []string + if rec.Direct.Completed != rec.Squad.Completed { + diffs = append(diffs, fmt.Sprintf("状态:direct=%v squad=%v(%s / %s)", + rec.Direct.Completed, rec.Squad.Completed, rec.Direct.Err, rec.Squad.Err)) + } + if len(rec.Task.Themes) > 0 { + dm := MissingThemes(rec.Direct.Summary, rec.Task.Themes) + sm := MissingThemes(rec.Squad.Summary, rec.Task.Themes) + if strings.Join(dm, ",") != strings.Join(sm, ",") { + diffs = append(diffs, fmt.Sprintf("要点缺失:direct=%v squad=%v", dm, sm)) + } + } + gap := rec.Squad.FirstToken - rec.Direct.FirstToken + if gap < 0 { + gap = -gap + } + if gap > time.Second { + diffs = append(diffs, fmt.Sprintf("首字延迟差:%s(direct=%s squad=%s)", + gap, rec.Direct.FirstToken, rec.Squad.FirstToken)) + } + if rec.Direct.Completed && rec.Squad.Completed && + strings.TrimSpace(rec.Direct.Summary) != strings.TrimSpace(rec.Squad.Summary) { + diffs = append(diffs, "摘要文本不同(记录在案,不要求字节级相同)") + } + return diffs +} + +func p50(ds []time.Duration) time.Duration { + if len(ds) == 0 { + return 0 + } + sorted := append([]time.Duration(nil), ds...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + return sorted[len(sorted)/2] +} + +// renderReport 报告全文:裁决 + 分数 + 逐任务差异清单。 +func renderReport(rep ABReport) string { + var b strings.Builder + b.WriteString(Report(rep.Decision)) + fmt.Fprintf(&b, "\n分数:direct=%d squad=%d;首字P50:direct=%s squad=%s\n", + rep.Direct.Score, rep.Squad.Score, rep.Direct.FirstTokenP50, rep.Squad.FirstTokenP50) + for _, rec := range rep.Records { + fmt.Fprintf(&b, "- %s:direct %d 分 / squad %d 分", rec.Task.ID, rec.DirectPoints, rec.SquadPoints) + if len(rec.Diffs) == 0 { + b.WriteString("(无差异注记)") + } + b.WriteString("\n") + for _, d := range rec.Diffs { + fmt.Fprintf(&b, " - %s\n", d) + } + } + return strings.TrimRight(b.String(), "\n") +} diff --git a/eval/ab_test.go b/eval/ab_test.go new file mode 100644 index 0000000..cfcc96f --- /dev/null +++ b/eval/ab_test.go @@ -0,0 +1,124 @@ +package eval + +import ( + "context" + "strings" + "testing" + "time" +) + +// ab_test.go — Q3 WP5:A/B 双跑 harness 机制面(差异记录 / 打分 / P50 / +// DecideSwitch 联动)。真实入口的集成对照在 internal/wfcompile 的 +// demo.brief 双跑测试。 + +func stubRun(completed bool, summary string, ft time.Duration, errText string) RunFunc { + return func(context.Context, ABTask) (ABOutcome, error) { + return ABOutcome{Completed: completed, Summary: summary, FirstToken: ft, Err: errText}, nil + } +} + +func TestRunABRecordsDiffsAndScores(t *testing.T) { + tasks := []ABTask{ + {ID: "t1", Goal: "写简报", Themes: []string{"太空电梯", "核对标记"}}, + {ID: "t2", Goal: "查资料", Themes: []string{"三来源"}}, + } + direct := func(_ context.Context, task ABTask) (ABOutcome, error) { + switch task.ID { + case "t1": + return ABOutcome{Completed: true, Summary: "关于太空电梯的简报,核对标记:P4", FirstToken: 100 * time.Millisecond}, nil + default: + return ABOutcome{Completed: true, Summary: "对照完成,三来源齐", FirstToken: 200 * time.Millisecond}, nil + } + } + squad := func(_ context.Context, task ABTask) (ABOutcome, error) { + switch task.ID { + case "t1": + // 完成但缺一个要点 → 扣分 + 差异注记。 + return ABOutcome{Completed: true, Summary: "关于太空电梯的简报", FirstToken: 100 * time.Millisecond}, nil + default: + return ABOutcome{Completed: false, Err: "编译图步骤失败", FirstToken: 0}, nil + } + } + rep := RunAB(context.Background(), tasks, direct, squad) + if len(rep.Records) != 2 { + t.Fatalf("records = %d", len(rep.Records)) + } + r1 := rep.Records[0] + if r1.DirectPoints != 100 || r1.SquadPoints != 50 { + t.Fatalf("t1 points = %d/%d", r1.DirectPoints, r1.SquadPoints) + } + if !strings.Contains(strings.Join(r1.Diffs, ";"), "要点缺失") { + t.Fatalf("t1 diffs = %v", r1.Diffs) + } + r2 := rep.Records[1] + if r2.SquadPoints != 0 || !strings.Contains(strings.Join(r2.Diffs, ";"), "状态") { + t.Fatalf("t2 = %d %v", r2.SquadPoints, r2.Diffs) + } + // 分数汇总:direct 100,squad (50+0)/2=25 → 不达标,保持 direct。 + if rep.Direct.Score != 100 || rep.Squad.Score != 25 { + t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score) + } + if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "低于") { + t.Fatalf("decision = %+v", rep.Decision) + } + if !strings.Contains(rep.Text, "保持 direct") { + t.Fatalf("report = %s", rep.Text) + } +} + +func TestRunABSwitchMetWhenEqual(t *testing.T) { + tasks := []ABTask{{ID: "t1", Goal: "g", Themes: []string{"要点"}}} + fn := stubRun(true, "覆盖要点的产出", 50*time.Millisecond, "") + rep := RunAB(context.Background(), tasks, fn, fn) + if rep.Direct.Score != 100 || rep.Squad.Score != 100 { + t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score) + } + if !rep.Decision.Met { + t.Fatalf("同分同延迟应达标:%+v", rep.Decision) + } + if len(rep.Records[0].Diffs) != 0 { + t.Fatalf("同产出应无差异注记:%v", rep.Records[0].Diffs) + } + if rep.Direct.FirstTokenP50 != 50*time.Millisecond { + t.Fatalf("p50 = %s", rep.Direct.FirstTokenP50) + } +} + +func TestRunABLatencyGate(t *testing.T) { + tasks := []ABTask{{ID: "t1", Goal: "g"}} + direct := stubRun(true, "ok", 100*time.Millisecond, "") + squad := stubRun(true, "ok", 4*time.Second, "") // 额外首字 > 3s + rep := RunAB(context.Background(), tasks, direct, squad) + if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "超过 3 秒") { + t.Fatalf("decision = %+v", rep.Decision) + } + if !strings.Contains(strings.Join(rep.Records[0].Diffs, ";"), "首字延迟差") { + t.Fatalf("diffs = %v", rep.Records[0].Diffs) + } +} + +// RunFunc 返回 error:记为该侧失败,不中断整批。 +func TestRunABToleratesRunFuncError(t *testing.T) { + tasks := []ABTask{{ID: "t1"}, {ID: "t2"}} + direct := stubRun(true, "ok", 0, "") + squad := func(_ context.Context, task ABTask) (ABOutcome, error) { + if task.ID == "t1" { + return ABOutcome{}, context.DeadlineExceeded + } + return ABOutcome{Completed: true, Summary: "ok"}, nil + } + rep := RunAB(context.Background(), tasks, direct, squad) + if rep.Records[0].SquadPoints != 0 || rep.Records[0].Squad.Err == "" { + t.Fatalf("t1 = %+v", rep.Records[0].Squad) + } + if rep.Records[1].SquadPoints != 100 { + t.Fatalf("t2 应不受 t1 失败影响:%+v", rep.Records[1]) + } +} + +func TestRunABEmptyBatch(t *testing.T) { + rep := RunAB(context.Background(), nil, nil, nil) + if rep.Decision.Met { + t.Fatal("空批不得达标(无实测分数)") + } +} diff --git a/internal/wfcompile/ab_demo_test.go b/internal/wfcompile/ab_demo_test.go new file mode 100644 index 0000000..1fbe7bd --- /dev/null +++ b/internal/wfcompile/ab_demo_test.go @@ -0,0 +1,90 @@ +package wfcompile + +import ( + "context" + "strings" + "testing" + "time" + + core "agent-system-core" + "agent-system-backend/eval" + "agent-system-backend/internal/workflow" +) + +// ab_demo_test.go — Q3 WP5 验收:同批任务(含 demo.brief 场景)经两个入口 +// 双跑并记录差异(不要求字节级相同),产出 DecideSwitch 裁决证据。 +// direct 侧 = 直连对话循环的脚本化替身(单代理直接作答);squad 侧 = +// 真实编译图执行(RunCompiled + 脚本化后端 + 桩验收流)。 + +func TestABDemoBriefDualRun(t *testing.T) { + def := parseDef(t, demoBriefJSON) + briefBody := "关于太空电梯的简报(短)。\n核对标记:P4" + + // direct 侧:直连循环一次性作答(首字即全文)。 + direct := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) { + return eval.ABOutcome{Completed: true, Summary: briefBody, FirstToken: 20 * time.Millisecond}, nil + } + // squad 侧:编译图真实执行;摘要 = 工作流汇总 + 成稿工件正文(要点覆盖 + // 面向交付物而非状态行)。 + squad := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) { + start := time.Now() + be := &seqBackend{texts: []string{briefBody}} + f := &fakeHost{ + toolNames: []string{"time_now"}, + hitl: []core.HITLResponse{ + {ChoiceID: "short"}, // length + {ChoiceID: "yes"}, // confirm + }, + } + out, err := RunCompiled(f, Deps{ + DataDir: t.TempDir(), Backends: seqResolver(t, be), DefaultBackend: "seq", + JudgeStream: verdictStream(verdictPass), + }, workflow.Launch{Def: def}, workflow.Vars{ + Input: task.Goal, Slots: map[string]string{"topic": "太空电梯"}, + }, RunOptions{}) + if err != nil { + return eval.ABOutcome{}, err + } + summary := out.Summary + for _, w := range f.writes { // 成稿工件正文并入摘要(交付物视角) + if strings.HasPrefix(w, "demo-brief.md=") { + summary += "\n" + strings.TrimPrefix(w, "demo-brief.md=") + } + } + return eval.ABOutcome{Completed: true, Summary: summary, FirstToken: time.Since(start)}, nil + } + + tasks := []eval.ABTask{{ + ID: "demo.brief", + Goal: "写一份关于太空电梯的简报", + // 要点 = 主题词 + P4 核对标记(demo.brief 的 critic 判据)。 + Themes: []string{"太空电梯", "核对标记:P4"}, + }} + rep := eval.RunAB(context.Background(), tasks, direct, squad) + rec := rep.Records[0] + if !rec.Direct.Completed || !rec.Squad.Completed { + t.Fatalf("两侧都应完成:%+v / %+v", rec.Direct, rec.Squad) + } + // 两侧都覆盖要点 → 满分;差异只在摘要形态(记录在案,不要求字节级相同)。 + if rec.DirectPoints != 100 || rec.SquadPoints != 100 { + t.Fatalf("points = %d/%d(direct=%q squad=%q)", + rec.DirectPoints, rec.SquadPoints, rec.Direct.Summary, rec.Squad.Summary) + } + var sawSummaryDiff bool + for _, d := range rec.Diffs { + if strings.Contains(d, "摘要文本不同") { + sawSummaryDiff = true + } + } + if !sawSummaryDiff { + t.Fatalf("两入口摘要形态必然不同,应记录差异:%v", rec.Diffs) + } + // 裁决是确定性的:同分 → 由首字延迟差决定;证据文本可直接入卡。 + if rep.Direct.Score != 100 || rep.Squad.Score != 100 { + t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score) + } + if rep.Text == "" || !strings.Contains(rep.Text, "分数:direct=100 squad=100") { + t.Fatalf("report = %s", rep.Text) + } + t.Logf("A/B 对照证据(demo.brief):\n%s", rep.Text) +}