weichen
174e59e692
Record token usage and nearest-rank latency so the replacement gate matches the written bar.
59 lines
1.7 KiB
Go
59 lines
1.7 KiB
Go
package abtest
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestJudgePassWhenNativeMatches(t *testing.T) {
|
|
native := []Outcome{
|
|
{Backend: "pi-native", Success: true, Duration: 100 * time.Millisecond, TargetsPresent: 1, TargetsExpected: 1},
|
|
{Backend: "pi-native", Success: true, Duration: 120 * time.Millisecond, TargetsPresent: 1, TargetsExpected: 1},
|
|
}
|
|
remote := []Outcome{
|
|
{Backend: "pi-agent", Success: true, Duration: 110 * time.Millisecond, TargetsPresent: 1, TargetsExpected: 1},
|
|
{Backend: "pi-agent", Success: true, Duration: 130 * time.Millisecond, TargetsPresent: 1, TargetsExpected: 1},
|
|
}
|
|
v := Judge(native, remote)
|
|
if !v.Pass {
|
|
t.Fatalf("reasons %v", v.Reasons)
|
|
}
|
|
}
|
|
|
|
func TestJudgeFailsOnSuccessP0AndLatency(t *testing.T) {
|
|
native := []Outcome{{Success: true, P0Defect: true, Duration: 2 * time.Second, TargetsPresent: 1, TargetsExpected: 1}}
|
|
remote := []Outcome{{Success: true, Duration: 100 * time.Millisecond, TargetsPresent: 1, TargetsExpected: 1}}
|
|
v := Judge(native, remote)
|
|
if v.Pass {
|
|
t.Fatal("expected fail")
|
|
}
|
|
if v.NativeP0 != 1 {
|
|
t.Fatalf("p0 %d", v.NativeP0)
|
|
}
|
|
if len(v.Reasons) < 2 {
|
|
t.Fatalf("reasons %v", v.Reasons)
|
|
}
|
|
}
|
|
|
|
func TestPercentileUsesUpperRank(t *testing.T) {
|
|
got := percentile([]time.Duration{5 * time.Second, 15 * time.Second}, 0.95)
|
|
if got != 15*time.Second {
|
|
t.Fatalf("p95 %s", got)
|
|
}
|
|
}
|
|
|
|
func TestSuiteCoversThreeKinds(t *testing.T) {
|
|
kinds := map[string]int{}
|
|
for _, c := range Suite() {
|
|
kinds[c.Kind]++
|
|
if c.Prompt == "" || len(c.Targets) == 0 || c.ToolProfile == "" {
|
|
t.Fatalf("incomplete case %#v", c)
|
|
}
|
|
}
|
|
for _, kind := range []string{"doc", "dialogue", "reasoning"} {
|
|
if kinds[kind] < 2 {
|
|
t.Fatalf("kind %s count %d", kind, kinds[kind])
|
|
}
|
|
}
|
|
}
|