package eval import ( "context" "fmt" "sort" "strings" "time" ) // ab.go — 任务级 A/B 双跑 harness(Q3 WP5):同一批任务分别经两个入口 // (direct 对话循环 / squad 编译图)执行,逐任务记录差异(不要求字节级 // 相同),汇总为两份 EntryScore 交 DecideSwitch 裁决默认入口切换。 // 本包保持纯机制(不依赖 runtime/squad);两侧 RunFunc 由调用方装配。 // ABTask 是对照批中的一个任务。 type ABTask struct { ID string Goal string // Themes 期望产出要点(短词,非整句照抄):打分用 MissingThemes, // 差异记录用要点覆盖对比。 Themes []string } // ABOutcome 是一个入口对一个任务的执行结果(调用方 RunFunc 填写)。 type ABOutcome struct { // Completed 任务是否正常收口(失败/取消 = false)。 Completed bool // Summary 交付摘要(要点覆盖打分的对象;调用方可拼接工件正文)。 Summary string // FirstToken 首字延迟(入口收到任务到首个可见输出)。 FirstToken time.Duration // Err 失败原因(Completed=false 时填写)。 Err string } // RunFunc 是一个入口的执行侧(direct 或 squad 编译图)。 type RunFunc func(ctx context.Context, task ABTask) (ABOutcome, error) // ABTaskRecord 是一个任务的双跑对照记录(差异记录单位)。 type ABTaskRecord struct { Task ABTask Direct ABOutcome Squad ABOutcome // Diffs 差异注记(状态 / 要点覆盖 / 首字延迟),空 = 两侧一致。 Diffs []string // DirectPoints / SquadPoints 单任务得分(满分 100:完成 + 要点全覆盖; // 未完成 0 分;完成但缺要点按缺失比例扣)。 DirectPoints int SquadPoints int } // ABReport 是一批双跑的对照报告(切换裁决的证据形态)。 type ABReport struct { Records []ABTaskRecord Direct EntryScore Squad EntryScore Decision SwitchDecision // Text 是可附在切换 PR / 任务卡上的短报告(Report(Decision) + 差异清单)。 Text string } // RunAB 双跑同一批任务并汇总对照报告。任一侧 RunFunc 返回 error 视为该侧 // 该任务失败(记录进 outcome.Err),不中断整批(对照要的是全批差异面)。 func RunAB(ctx context.Context, tasks []ABTask, direct, squad RunFunc) ABReport { rep := ABReport{} if len(tasks) == 0 { rep.Decision = DecideSwitch(nil, nil) rep.Text = Report(rep.Decision) return rep } var directFT, squadFT []time.Duration var directSum, squadSum int for _, task := range tasks { rec := ABTaskRecord{Task: task} rec.Direct = runOne(ctx, task, direct) rec.Squad = runOne(ctx, task, squad) rec.DirectPoints = scoreOutcome(rec.Direct, task.Themes) rec.SquadPoints = scoreOutcome(rec.Squad, task.Themes) rec.Diffs = diffRecord(rec) rep.Records = append(rep.Records, rec) directSum += rec.DirectPoints squadSum += rec.SquadPoints directFT = append(directFT, rec.Direct.FirstToken) squadFT = append(squadFT, rec.Squad.FirstToken) } rep.Direct = EntryScore{Name: "direct", Score: directSum / len(tasks), FirstTokenP50: p50(directFT)} rep.Squad = EntryScore{Name: "squad", Score: squadSum / len(tasks), FirstTokenP50: p50(squadFT)} rep.Decision = DecideSwitch(&rep.Direct, &rep.Squad) rep.Text = renderReport(rep) return rep } func runOne(ctx context.Context, task ABTask, fn RunFunc) ABOutcome { if fn == nil { return ABOutcome{Err: "入口未装配"} } out, err := fn(ctx, task) if err != nil { out.Completed = false if out.Err == "" { out.Err = err.Error() } } return out } // scoreOutcome 单任务打分:未完成 0;完成 = 100 - 缺失要点比例 × 100。 func scoreOutcome(out ABOutcome, themes []string) int { if !out.Completed { return 0 } if len(themes) == 0 { return 100 } missing := MissingThemes(out.Summary, themes) return 100 - len(missing)*100/len(themes) } // diffRecord 逐任务差异注记(状态 / 要点 / 首字延迟;不要求字节级相同)。 func diffRecord(rec ABTaskRecord) []string { var diffs []string if rec.Direct.Completed != rec.Squad.Completed { diffs = append(diffs, fmt.Sprintf("状态:direct=%v squad=%v(%s / %s)", rec.Direct.Completed, rec.Squad.Completed, rec.Direct.Err, rec.Squad.Err)) } if len(rec.Task.Themes) > 0 { dm := MissingThemes(rec.Direct.Summary, rec.Task.Themes) sm := MissingThemes(rec.Squad.Summary, rec.Task.Themes) if strings.Join(dm, ",") != strings.Join(sm, ",") { diffs = append(diffs, fmt.Sprintf("要点缺失:direct=%v squad=%v", dm, sm)) } } gap := rec.Squad.FirstToken - rec.Direct.FirstToken if gap < 0 { gap = -gap } if gap > time.Second { diffs = append(diffs, fmt.Sprintf("首字延迟差:%s(direct=%s squad=%s)", gap, rec.Direct.FirstToken, rec.Squad.FirstToken)) } if rec.Direct.Completed && rec.Squad.Completed && strings.TrimSpace(rec.Direct.Summary) != strings.TrimSpace(rec.Squad.Summary) { diffs = append(diffs, "摘要文本不同(记录在案,不要求字节级相同)") } return diffs } func p50(ds []time.Duration) time.Duration { if len(ds) == 0 { return 0 } sorted := append([]time.Duration(nil), ds...) sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) return sorted[len(sorted)/2] } // renderReport 报告全文:裁决 + 分数 + 逐任务差异清单。 func renderReport(rep ABReport) string { var b strings.Builder b.WriteString(Report(rep.Decision)) fmt.Fprintf(&b, "\n分数:direct=%d squad=%d;首字P50:direct=%s squad=%s\n", rep.Direct.Score, rep.Squad.Score, rep.Direct.FirstTokenP50, rep.Squad.FirstTokenP50) for _, rec := range rep.Records { fmt.Fprintf(&b, "- %s:direct %d 分 / squad %d 分", rec.Task.ID, rec.DirectPoints, rec.SquadPoints) if len(rec.Diffs) == 0 { b.WriteString("(无差异注记)") } b.WriteString("\n") for _, d := range rec.Diffs { fmt.Fprintf(&b, " - %s\n", d) } } return strings.TrimRight(b.String(), "\n") }