Agent-System/internal/agentx/abtest/judge.go

package abtest

import (
	"math"
	"sort"
	"time"
)

// Case is one comparable task. Both backends receive the same prompt and targets.
type Case struct {
	ID           string
	Kind         string
	Prompt       string
	Instructions string
	ToolProfile  string
	Targets      []string
}

// Outcome is one backend's result for one case.
type Outcome struct {
	Backend          string
	CaseID           string
	Kind             string
	Success          bool
	P0Defect         bool
	Duration         time.Duration
	PromptTokens     int
	CompletionTokens int
	TargetsPresent   int
	TargetsExpected  int
	ErrorClass       string
}

// Verdict is the replacement gate. Pass means pi-native is at least as good as pi-agent.
type Verdict struct {
	Pass          bool
	NativeSuccess float64
	RemoteSuccess float64
	NativeP0      int
	RemoteP95     time.Duration
	NativeP95     time.Duration
	Reasons       []string
}

const latencyFactor = 1.3

// Suite is the fixed, replayable comparison set.
func Suite() []Case {
	return []Case{
		{ID: "doc-brief", Kind: "doc", ToolProfile: "workspace", Targets: []string{"brief.md"},
			Instructions: "Write a short Chinese project brief.",
			Prompt:       "根据以下信息写一份项目简介，写入 brief.md：项目名「水木试点」，目标是把检验报告整理成可检索档案。"},
		{ID: "doc-outline", Kind: "doc", ToolProfile: "workspace", Targets: []string{"outline.md"},
			Instructions: "Write a markdown outline with three headings.",
			Prompt:       "为「试剂冷链运输方案」写三级大纲，写入 outline.md。"},
		{ID: "dialogue-hello", Kind: "dialogue", ToolProfile: "reasoning", Targets: []string{"reply.md"},
			Instructions: "你是项目助手，用一两句中文回答。",
			Prompt:       "你好，今天先确认项目范围。"},
		{ID: "dialogue-status", Kind: "dialogue", ToolProfile: "reasoning", Targets: []string{"reply.md"},
			Instructions: "你是项目助手，用一两句中文回答。",
			Prompt:       "当前阶段还在需求收集，请告诉我下一步该准备什么。"},
		{ID: "reason-risk", Kind: "reasoning", ToolProfile: "reasoning", Targets: []string{"reply.md"},
			Instructions: "只输出结论，不要调用工具。",
			Prompt:       "用一句话说明：缺少样本量时，为什么不能把试点结果写成确定结论。"},
		{ID: "reason-scope", Kind: "reasoning", ToolProfile: "reasoning", Targets: []string{"reply.md"},
			Instructions: "只输出结论，不要调用工具。",
			Prompt:       "用一句话区分「项目简介」和「检验方案」各自要回答的问题。"},
	}
}

// Judge applies the replacement bar: success rate not below the container backend,
// no P0 defects on pi-native, and P95 latency at most 1.3× the container backend.
func Judge(native, remote []Outcome) Verdict {
	v := Verdict{Pass: true}
	if len(native) == 0 || len(remote) == 0 {
		v.Pass = false
		v.Reasons = append(v.Reasons, "both backends need completed outcomes")
		return v
	}
	v.NativeSuccess = successRate(native)
	v.RemoteSuccess = successRate(remote)
	if v.NativeSuccess < v.RemoteSuccess {
		v.Pass = false
		v.Reasons = append(v.Reasons, "pi-native success rate is below pi-agent")
	}
	for _, o := range native {
		if o.P0Defect {
			v.NativeP0++
		}
	}
	if v.NativeP0 > 0 {
		v.Pass = false
		v.Reasons = append(v.Reasons, "pi-native has a P0 deliverable defect")
	}
	v.NativeP95 = percentile(durations(native), 0.95)
	v.RemoteP95 = percentile(durations(remote), 0.95)
	if v.RemoteP95 > 0 && float64(v.NativeP95) > latencyFactor*float64(v.RemoteP95) {
		v.Pass = false
		v.Reasons = append(v.Reasons, "pi-native P95 latency exceeds 1.3x pi-agent")
	}
	return v
}

func successRate(list []Outcome) float64 {
	if len(list) == 0 {
		return 0
	}
	ok := 0
	for _, o := range list {
		if o.Success && !o.P0Defect && o.TargetsPresent >= o.TargetsExpected && o.TargetsExpected > 0 {
			ok++
		}
	}
	return float64(ok) / float64(len(list))
}

func durations(list []Outcome) []time.Duration {
	out := make([]time.Duration, 0, len(list))
	for _, o := range list {
		if o.Success {
			out = append(out, o.Duration)
		}
	}
	sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
	return out
}

func percentile(sorted []time.Duration, p float64) time.Duration {
	if len(sorted) == 0 {
		return 0
	}
	rank := int(math.Ceil(p*float64(len(sorted)))) - 1
	if rank < 0 {
		rank = 0
	}
	if rank >= len(sorted) {
		rank = len(sorted) - 1
	}
	return sorted[rank]
}