Add the task-level A/B dual-run harness with demo.brief evidence (Q3 WP5)

Why: Q3 WP5 requires a same-batch dual-run harness (direct vs squad compiled graph) that records differences without demanding byte-identical output, feeding DecideSwitch evidence for the runtime.entry default-switch decision. Evidence: eval/ab_test.go mechanics (theme-based scoring, status/theme/ latency diff notes, P50, DecideSwitch wiring, RunFunc error tolerance, empty batch); wfcompile TestABDemoBriefDualRun runs the demo.brief scenario on both entries (direct = scripted single-agent answer; squad = real RunCompiled with stub backend and judge) and logs the report: both sides 100/100, summary-form difference recorded, decision '切换条件已满足…合入前需要产品确认'. Full backend suite green (SELF); CI authoritative (GATE). Impact: new eval/ab.go harness (pure mechanics, no runtime dependency); no product default is flipped - runtime.entry stays direct until product confirmation per the DecideSwitch report semantics (evidence recorded in the task card and PR). Refs: Q3 plan WP5 (2026-10-07-agent-stack-q2q3-plans.md); task card .task/feature-agent-stack-q3.md
2026-10-08 03:46:54 +00:00
parent 2e42ed83f0
commit 196c8c09f7
3 changed files with 389 additions and 0 deletions
@@ -0,0 +1,175 @@
+package eval
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// ab.go — 任务级 A/B 双跑 harness（Q3 WP5）：同一批任务分别经两个入口
+// （direct 对话循环 / squad 编译图）执行，逐任务记录差异（不要求字节级
+// 相同），汇总为两份 EntryScore 交 DecideSwitch 裁决默认入口切换。
+// 本包保持纯机制（不依赖 runtime/squad）；两侧 RunFunc 由调用方装配。
+
+// ABTask 是对照批中的一个任务。
+type ABTask struct {
+	ID   string
+	Goal string
+	// Themes 期望产出要点（短词，非整句照抄）：打分用 MissingThemes，
+	// 差异记录用要点覆盖对比。
+	Themes []string
+}
+
+// ABOutcome 是一个入口对一个任务的执行结果（调用方 RunFunc 填写）。
+type ABOutcome struct {
+	// Completed 任务是否正常收口（失败/取消 = false）。
+	Completed bool
+	// Summary 交付摘要（要点覆盖打分的对象；调用方可拼接工件正文）。
+	Summary string
+	// FirstToken 首字延迟（入口收到任务到首个可见输出）。
+	FirstToken time.Duration
+	// Err 失败原因（Completed=false 时填写）。
+	Err string
+}
+
+// RunFunc 是一个入口的执行侧（direct 或 squad 编译图）。
+type RunFunc func(ctx context.Context, task ABTask) (ABOutcome, error)
+
+// ABTaskRecord 是一个任务的双跑对照记录（差异记录单位）。
+type ABTaskRecord struct {
+	Task   ABTask
+	Direct ABOutcome
+	Squad  ABOutcome
+	// Diffs 差异注记（状态 / 要点覆盖 / 首字延迟），空 = 两侧一致。
+	Diffs []string
+	// DirectPoints / SquadPoints 单任务得分（满分 100：完成 + 要点全覆盖；
+	// 未完成 0 分；完成但缺要点按缺失比例扣）。
+	DirectPoints int
+	SquadPoints  int
+}
+
+// ABReport 是一批双跑的对照报告（切换裁决的证据形态）。
+type ABReport struct {
+	Records  []ABTaskRecord
+	Direct   EntryScore
+	Squad    EntryScore
+	Decision SwitchDecision
+	// Text 是可附在切换 PR / 任务卡上的短报告（Report(Decision) + 差异清单）。
+	Text string
+}
+
+// RunAB 双跑同一批任务并汇总对照报告。任一侧 RunFunc 返回 error 视为该侧
+// 该任务失败（记录进 outcome.Err），不中断整批（对照要的是全批差异面）。
+func RunAB(ctx context.Context, tasks []ABTask, direct, squad RunFunc) ABReport {
+	rep := ABReport{}
+	if len(tasks) == 0 {
+		rep.Decision = DecideSwitch(nil, nil)
+		rep.Text = Report(rep.Decision)
+		return rep
+	}
+	var directFT, squadFT []time.Duration
+	var directSum, squadSum int
+	for _, task := range tasks {
+		rec := ABTaskRecord{Task: task}
+		rec.Direct = runOne(ctx, task, direct)
+		rec.Squad = runOne(ctx, task, squad)
+		rec.DirectPoints = scoreOutcome(rec.Direct, task.Themes)
+		rec.SquadPoints = scoreOutcome(rec.Squad, task.Themes)
+		rec.Diffs = diffRecord(rec)
+		rep.Records = append(rep.Records, rec)
+		directSum += rec.DirectPoints
+		squadSum += rec.SquadPoints
+		directFT = append(directFT, rec.Direct.FirstToken)
+		squadFT = append(squadFT, rec.Squad.FirstToken)
+	}
+	rep.Direct = EntryScore{Name: "direct", Score: directSum / len(tasks), FirstTokenP50: p50(directFT)}
+	rep.Squad = EntryScore{Name: "squad", Score: squadSum / len(tasks), FirstTokenP50: p50(squadFT)}
+	rep.Decision = DecideSwitch(&rep.Direct, &rep.Squad)
+	rep.Text = renderReport(rep)
+	return rep
+}
+
+func runOne(ctx context.Context, task ABTask, fn RunFunc) ABOutcome {
+	if fn == nil {
+		return ABOutcome{Err: "入口未装配"}
+	}
+	out, err := fn(ctx, task)
+	if err != nil {
+		out.Completed = false
+		if out.Err == "" {
+			out.Err = err.Error()
+		}
+	}
+	return out
+}
+
+// scoreOutcome 单任务打分：未完成 0；完成 = 100 - 缺失要点比例 × 100。
+func scoreOutcome(out ABOutcome, themes []string) int {
+	if !out.Completed {
+		return 0
+	}
+	if len(themes) == 0 {
+		return 100
+	}
+	missing := MissingThemes(out.Summary, themes)
+	return 100 - len(missing)*100/len(themes)
+}
+
+// diffRecord 逐任务差异注记（状态 / 要点 / 首字延迟；不要求字节级相同）。
+func diffRecord(rec ABTaskRecord) []string {
+	var diffs []string
+	if rec.Direct.Completed != rec.Squad.Completed {
+		diffs = append(diffs, fmt.Sprintf("状态：direct=%v squad=%v（%s / %s）",
+			rec.Direct.Completed, rec.Squad.Completed, rec.Direct.Err, rec.Squad.Err))
+	}
+	if len(rec.Task.Themes) > 0 {
+		dm := MissingThemes(rec.Direct.Summary, rec.Task.Themes)
+		sm := MissingThemes(rec.Squad.Summary, rec.Task.Themes)
+		if strings.Join(dm, ",") != strings.Join(sm, ",") {
+			diffs = append(diffs, fmt.Sprintf("要点缺失：direct=%v squad=%v", dm, sm))
+		}
+	}
+	gap := rec.Squad.FirstToken - rec.Direct.FirstToken
+	if gap < 0 {
+		gap = -gap
+	}
+	if gap > time.Second {
+		diffs = append(diffs, fmt.Sprintf("首字延迟差：%s（direct=%s squad=%s）",
+			gap, rec.Direct.FirstToken, rec.Squad.FirstToken))
+	}
+	if rec.Direct.Completed && rec.Squad.Completed &&
+		strings.TrimSpace(rec.Direct.Summary) != strings.TrimSpace(rec.Squad.Summary) {
+		diffs = append(diffs, "摘要文本不同（记录在案，不要求字节级相同）")
+	}
+	return diffs
+}
+
+func p50(ds []time.Duration) time.Duration {
+	if len(ds) == 0 {
+		return 0
+	}
+	sorted := append([]time.Duration(nil), ds...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
+	return sorted[len(sorted)/2]
+}
+
+// renderReport 报告全文：裁决 + 分数 + 逐任务差异清单。
+func renderReport(rep ABReport) string {
+	var b strings.Builder
+	b.WriteString(Report(rep.Decision))
+	fmt.Fprintf(&b, "\n分数：direct=%d squad=%d；首字P50：direct=%s squad=%s\n",
+		rep.Direct.Score, rep.Squad.Score, rep.Direct.FirstTokenP50, rep.Squad.FirstTokenP50)
+	for _, rec := range rep.Records {
+		fmt.Fprintf(&b, "- %s：direct %d 分 / squad %d 分", rec.Task.ID, rec.DirectPoints, rec.SquadPoints)
+		if len(rec.Diffs) == 0 {
+			b.WriteString("（无差异注记）")
+		}
+		b.WriteString("\n")
+		for _, d := range rec.Diffs {
+			fmt.Fprintf(&b, "  - %s\n", d)
+		}
+	}
+	return strings.TrimRight(b.String(), "\n")
+}
@@ -0,0 +1,124 @@
+package eval
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+// ab_test.go — Q3 WP5：A/B 双跑 harness 机制面（差异记录 / 打分 / P50 /
+// DecideSwitch 联动）。真实入口的集成对照在 internal/wfcompile 的
+// demo.brief 双跑测试。
+
+func stubRun(completed bool, summary string, ft time.Duration, errText string) RunFunc {
+	return func(context.Context, ABTask) (ABOutcome, error) {
+		return ABOutcome{Completed: completed, Summary: summary, FirstToken: ft, Err: errText}, nil
+	}
+}
+
+func TestRunABRecordsDiffsAndScores(t *testing.T) {
+	tasks := []ABTask{
+		{ID: "t1", Goal: "写简报", Themes: []string{"太空电梯", "核对标记"}},
+		{ID: "t2", Goal: "查资料", Themes: []string{"三来源"}},
+	}
+	direct := func(_ context.Context, task ABTask) (ABOutcome, error) {
+		switch task.ID {
+		case "t1":
+			return ABOutcome{Completed: true, Summary: "关于太空电梯的简报，核对标记：P4", FirstToken: 100 * time.Millisecond}, nil
+		default:
+			return ABOutcome{Completed: true, Summary: "对照完成，三来源齐", FirstToken: 200 * time.Millisecond}, nil
+		}
+	}
+	squad := func(_ context.Context, task ABTask) (ABOutcome, error) {
+		switch task.ID {
+		case "t1":
+			// 完成但缺一个要点 → 扣分 + 差异注记。
+			return ABOutcome{Completed: true, Summary: "关于太空电梯的简报", FirstToken: 100 * time.Millisecond}, nil
+		default:
+			return ABOutcome{Completed: false, Err: "编译图步骤失败", FirstToken: 0}, nil
+		}
+	}
+	rep := RunAB(context.Background(), tasks, direct, squad)
+	if len(rep.Records) != 2 {
+		t.Fatalf("records = %d", len(rep.Records))
+	}
+	r1 := rep.Records[0]
+	if r1.DirectPoints != 100 || r1.SquadPoints != 50 {
+		t.Fatalf("t1 points = %d/%d", r1.DirectPoints, r1.SquadPoints)
+	}
+	if !strings.Contains(strings.Join(r1.Diffs, ";"), "要点缺失") {
+		t.Fatalf("t1 diffs = %v", r1.Diffs)
+	}
+	r2 := rep.Records[1]
+	if r2.SquadPoints != 0 || !strings.Contains(strings.Join(r2.Diffs, ";"), "状态") {
+		t.Fatalf("t2 = %d %v", r2.SquadPoints, r2.Diffs)
+	}
+	// 分数汇总：direct 100，squad (50+0)/2=25 → 不达标，保持 direct。
+	if rep.Direct.Score != 100 || rep.Squad.Score != 25 {
+		t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
+	}
+	if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "低于") {
+		t.Fatalf("decision = %+v", rep.Decision)
+	}
+	if !strings.Contains(rep.Text, "保持 direct") {
+		t.Fatalf("report = %s", rep.Text)
+	}
+}
+
+func TestRunABSwitchMetWhenEqual(t *testing.T) {
+	tasks := []ABTask{{ID: "t1", Goal: "g", Themes: []string{"要点"}}}
+	fn := stubRun(true, "覆盖要点的产出", 50*time.Millisecond, "")
+	rep := RunAB(context.Background(), tasks, fn, fn)
+	if rep.Direct.Score != 100 || rep.Squad.Score != 100 {
+		t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
+	}
+	if !rep.Decision.Met {
+		t.Fatalf("同分同延迟应达标：%+v", rep.Decision)
+	}
+	if len(rep.Records[0].Diffs) != 0 {
+		t.Fatalf("同产出应无差异注记：%v", rep.Records[0].Diffs)
+	}
+	if rep.Direct.FirstTokenP50 != 50*time.Millisecond {
+		t.Fatalf("p50 = %s", rep.Direct.FirstTokenP50)
+	}
+}
+
+func TestRunABLatencyGate(t *testing.T) {
+	tasks := []ABTask{{ID: "t1", Goal: "g"}}
+	direct := stubRun(true, "ok", 100*time.Millisecond, "")
+	squad := stubRun(true, "ok", 4*time.Second, "") // 额外首字 > 3s
+	rep := RunAB(context.Background(), tasks, direct, squad)
+	if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "超过 3 秒") {
+		t.Fatalf("decision = %+v", rep.Decision)
+	}
+	if !strings.Contains(strings.Join(rep.Records[0].Diffs, ";"), "首字延迟差") {
+		t.Fatalf("diffs = %v", rep.Records[0].Diffs)
+	}
+}
+
+// RunFunc 返回 error：记为该侧失败，不中断整批。
+func TestRunABToleratesRunFuncError(t *testing.T) {
+	tasks := []ABTask{{ID: "t1"}, {ID: "t2"}}
+	direct := stubRun(true, "ok", 0, "")
+	squad := func(_ context.Context, task ABTask) (ABOutcome, error) {
+		if task.ID == "t1" {
+			return ABOutcome{}, context.DeadlineExceeded
+		}
+		return ABOutcome{Completed: true, Summary: "ok"}, nil
+	}
+	rep := RunAB(context.Background(), tasks, direct, squad)
+	if rep.Records[0].SquadPoints != 0 || rep.Records[0].Squad.Err == "" {
+		t.Fatalf("t1 = %+v", rep.Records[0].Squad)
+	}
+	if rep.Records[1].SquadPoints != 100 {
+		t.Fatalf("t2 应不受 t1 失败影响：%+v", rep.Records[1])
+	}
+}
+
+func TestRunABEmptyBatch(t *testing.T) {
+	rep := RunAB(context.Background(), nil, nil, nil)
+	if rep.Decision.Met {
+		t.Fatal("空批不得达标（无实测分数）")
+	}
+}
@@ -0,0 +1,90 @@
+package wfcompile
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	core "agent-system-core"
+	"agent-system-backend/eval"
+	"agent-system-backend/internal/workflow"
+)
+
+// ab_demo_test.go — Q3 WP5 验收：同批任务（含 demo.brief 场景）经两个入口
+// 双跑并记录差异（不要求字节级相同），产出 DecideSwitch 裁决证据。
+// direct 侧 = 直连对话循环的脚本化替身（单代理直接作答）；squad 侧 =
+// 真实编译图执行（RunCompiled + 脚本化后端 + 桩验收流）。
+
+func TestABDemoBriefDualRun(t *testing.T) {
+	def := parseDef(t, demoBriefJSON)
+	briefBody := "关于太空电梯的简报（短）。\n核对标记：P4"
+
+	// direct 侧：直连循环一次性作答（首字即全文）。
+	direct := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) {
+		return eval.ABOutcome{Completed: true, Summary: briefBody, FirstToken: 20 * time.Millisecond}, nil
+	}
+	// squad 侧：编译图真实执行；摘要 = 工作流汇总 + 成稿工件正文（要点覆盖
+	// 面向交付物而非状态行）。
+	squad := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) {
+		start := time.Now()
+		be := &seqBackend{texts: []string{briefBody}}
+		f := &fakeHost{
+			toolNames: []string{"time_now"},
+			hitl: []core.HITLResponse{
+				{ChoiceID: "short"}, // length
+				{ChoiceID: "yes"},   // confirm
+			},
+		}
+		out, err := RunCompiled(f, Deps{
+			DataDir: t.TempDir(), Backends: seqResolver(t, be), DefaultBackend: "seq",
+			JudgeStream: verdictStream(verdictPass),
+		}, workflow.Launch{Def: def}, workflow.Vars{
+			Input: task.Goal, Slots: map[string]string{"topic": "太空电梯"},
+		}, RunOptions{})
+		if err != nil {
+			return eval.ABOutcome{}, err
+		}
+		summary := out.Summary
+		for _, w := range f.writes { // 成稿工件正文并入摘要（交付物视角）
+			if strings.HasPrefix(w, "demo-brief.md=") {
+				summary += "\n" + strings.TrimPrefix(w, "demo-brief.md=")
+			}
+		}
+		return eval.ABOutcome{Completed: true, Summary: summary, FirstToken: time.Since(start)}, nil
+	}
+
+	tasks := []eval.ABTask{{
+		ID:   "demo.brief",
+		Goal: "写一份关于太空电梯的简报",
+		// 要点 = 主题词 + P4 核对标记（demo.brief 的 critic 判据）。
+		Themes: []string{"太空电梯", "核对标记：P4"},
+	}}
+	rep := eval.RunAB(context.Background(), tasks, direct, squad)
+	rec := rep.Records[0]
+	if !rec.Direct.Completed || !rec.Squad.Completed {
+		t.Fatalf("两侧都应完成：%+v / %+v", rec.Direct, rec.Squad)
+	}
+	// 两侧都覆盖要点 → 满分；差异只在摘要形态（记录在案，不要求字节级相同）。
+	if rec.DirectPoints != 100 || rec.SquadPoints != 100 {
+		t.Fatalf("points = %d/%d（direct=%q squad=%q）",
+			rec.DirectPoints, rec.SquadPoints, rec.Direct.Summary, rec.Squad.Summary)
+	}
+	var sawSummaryDiff bool
+	for _, d := range rec.Diffs {
+		if strings.Contains(d, "摘要文本不同") {
+			sawSummaryDiff = true
+		}
+	}
+	if !sawSummaryDiff {
+		t.Fatalf("两入口摘要形态必然不同，应记录差异：%v", rec.Diffs)
+	}
+	// 裁决是确定性的：同分 → 由首字延迟差决定；证据文本可直接入卡。
+	if rep.Direct.Score != 100 || rep.Squad.Score != 100 {
+		t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
+	}
+	if rep.Text == "" || !strings.Contains(rep.Text, "分数：direct=100 squad=100") {
+		t.Fatalf("report = %s", rep.Text)
+	}
+	t.Logf("A/B 对照证据（demo.brief）：\n%s", rep.Text)
+}