Agent-System-Backend/eval/ab_test.go

package eval

import (
	"context"
	"strings"
	"testing"
	"time"
)

// ab_test.go — Q3 WP5：A/B 双跑 harness 机制面（差异记录 / 打分 / P50 /
// DecideSwitch 联动）。真实入口的集成对照在 internal/wfcompile 的
// demo.brief 双跑测试。

func stubRun(completed bool, summary string, ft time.Duration, errText string) RunFunc {
	return func(context.Context, ABTask) (ABOutcome, error) {
		return ABOutcome{Completed: completed, Summary: summary, FirstToken: ft, Err: errText}, nil
	}
}

func TestRunABRecordsDiffsAndScores(t *testing.T) {
	tasks := []ABTask{
		{ID: "t1", Goal: "写简报", Themes: []string{"太空电梯", "核对标记"}},
		{ID: "t2", Goal: "查资料", Themes: []string{"三来源"}},
	}
	direct := func(_ context.Context, task ABTask) (ABOutcome, error) {
		switch task.ID {
		case "t1":
			return ABOutcome{Completed: true, Summary: "关于太空电梯的简报，核对标记：P4", FirstToken: 100 * time.Millisecond}, nil
		default:
			return ABOutcome{Completed: true, Summary: "对照完成，三来源齐", FirstToken: 200 * time.Millisecond}, nil
		}
	}
	squad := func(_ context.Context, task ABTask) (ABOutcome, error) {
		switch task.ID {
		case "t1":
			// 完成但缺一个要点 → 扣分 + 差异注记。
			return ABOutcome{Completed: true, Summary: "关于太空电梯的简报", FirstToken: 100 * time.Millisecond}, nil
		default:
			return ABOutcome{Completed: false, Err: "编译图步骤失败", FirstToken: 0}, nil
		}
	}
	rep := RunAB(context.Background(), tasks, direct, squad)
	if len(rep.Records) != 2 {
		t.Fatalf("records = %d", len(rep.Records))
	}
	r1 := rep.Records[0]
	if r1.DirectPoints != 100 || r1.SquadPoints != 50 {
		t.Fatalf("t1 points = %d/%d", r1.DirectPoints, r1.SquadPoints)
	}
	if !strings.Contains(strings.Join(r1.Diffs, ";"), "要点缺失") {
		t.Fatalf("t1 diffs = %v", r1.Diffs)
	}
	r2 := rep.Records[1]
	if r2.SquadPoints != 0 || !strings.Contains(strings.Join(r2.Diffs, ";"), "状态") {
		t.Fatalf("t2 = %d %v", r2.SquadPoints, r2.Diffs)
	}
	// 分数汇总：direct 100，squad (50+0)/2=25 → 不达标，保持 direct。
	if rep.Direct.Score != 100 || rep.Squad.Score != 25 {
		t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
	}
	if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "低于") {
		t.Fatalf("decision = %+v", rep.Decision)
	}
	if !strings.Contains(rep.Text, "保持 direct") {
		t.Fatalf("report = %s", rep.Text)
	}
}

func TestRunABSwitchMetWhenEqual(t *testing.T) {
	tasks := []ABTask{{ID: "t1", Goal: "g", Themes: []string{"要点"}}}
	fn := stubRun(true, "覆盖要点的产出", 50*time.Millisecond, "")
	rep := RunAB(context.Background(), tasks, fn, fn)
	if rep.Direct.Score != 100 || rep.Squad.Score != 100 {
		t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
	}
	if !rep.Decision.Met {
		t.Fatalf("同分同延迟应达标：%+v", rep.Decision)
	}
	if len(rep.Records[0].Diffs) != 0 {
		t.Fatalf("同产出应无差异注记：%v", rep.Records[0].Diffs)
	}
	if rep.Direct.FirstTokenP50 != 50*time.Millisecond {
		t.Fatalf("p50 = %s", rep.Direct.FirstTokenP50)
	}
}

func TestRunABLatencyGate(t *testing.T) {
	tasks := []ABTask{{ID: "t1", Goal: "g"}}
	direct := stubRun(true, "ok", 100*time.Millisecond, "")
	squad := stubRun(true, "ok", 4*time.Second, "") // 额外首字 > 3s
	rep := RunAB(context.Background(), tasks, direct, squad)
	if rep.Decision.Met || !strings.Contains(rep.Decision.Reason, "超过 3 秒") {
		t.Fatalf("decision = %+v", rep.Decision)
	}
	if !strings.Contains(strings.Join(rep.Records[0].Diffs, ";"), "首字延迟差") {
		t.Fatalf("diffs = %v", rep.Records[0].Diffs)
	}
}

// RunFunc 返回 error：记为该侧失败，不中断整批。
func TestRunABToleratesRunFuncError(t *testing.T) {
	tasks := []ABTask{{ID: "t1"}, {ID: "t2"}}
	direct := stubRun(true, "ok", 0, "")
	squad := func(_ context.Context, task ABTask) (ABOutcome, error) {
		if task.ID == "t1" {
			return ABOutcome{}, context.DeadlineExceeded
		}
		return ABOutcome{Completed: true, Summary: "ok"}, nil
	}
	rep := RunAB(context.Background(), tasks, direct, squad)
	if rep.Records[0].SquadPoints != 0 || rep.Records[0].Squad.Err == "" {
		t.Fatalf("t1 = %+v", rep.Records[0].Squad)
	}
	if rep.Records[1].SquadPoints != 100 {
		t.Fatalf("t2 应不受 t1 失败影响：%+v", rep.Records[1])
	}
}

func TestRunABEmptyBatch(t *testing.T) {
	rep := RunAB(context.Background(), nil, nil, nil)
	if rep.Decision.Met {
		t.Fatal("空批不得达标（无实测分数）")
	}
}