weichen
174e59e692
Record token usage and nearest-rank latency so the replacement gate matches the written bar.
120 lines
3.5 KiB
Go
120 lines
3.5 KiB
Go
package abtest
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"chat-agent-system/internal/agent"
|
|
"chat-agent-system/internal/config"
|
|
)
|
|
|
|
// TestLiveDualBackend runs the fixed suite on pi-native and the container pi-agent.
|
|
// It is the replacement gate and stays skipped unless RUN_AGENTX_AB=1.
|
|
func TestLiveDualBackend(t *testing.T) {
|
|
if os.Getenv("RUN_AGENTX_AB") != "1" {
|
|
t.Skip("set RUN_AGENTX_AB=1 to compare pi-native with the container pi-agent")
|
|
}
|
|
cfg := &config.Config{
|
|
AgentXAPIBase: envFirst("AGENTX_API_BASE", "MINIMAX_ENDPOINT_URL", "https://api.minimaxi.com/v1"),
|
|
AgentXAPIKey: envFirst("AGENTX_API_KEY", "MINIMAX_API_KEY", ""),
|
|
AgentXModel: "MiniMax-M2.7-highspeed",
|
|
AgentTimeout: 3 * time.Minute,
|
|
PiAgentRunnerURL: os.Getenv("PI_AGENT_RUNNER_URL"),
|
|
PiAgentRunnerAuthToken: envFirst("PI_AGENT_RUNNER_AUTH_TOKEN", "RUNNER_AUTH_TOKEN", ""),
|
|
PiAgentModel: "MiniMax-M2.7-highspeed",
|
|
PiAgentProvider: "minimax",
|
|
}
|
|
if cfg.AgentXAPIKey == "" {
|
|
t.Fatal("AGENTX_API_KEY or MINIMAX_API_KEY is required")
|
|
}
|
|
if cfg.PiAgentRunnerURL == "" {
|
|
t.Fatal("PI_AGENT_RUNNER_URL is required so the container backend can be compared")
|
|
}
|
|
native := agent.NewPiNativeBackend(cfg)
|
|
remote := agent.NewBackend("pi-agent", cfg)
|
|
root := t.TempDir()
|
|
var nativeOut, remoteOut []Outcome
|
|
for _, c := range Suite() {
|
|
nativeOut = append(nativeOut, runCase(t, root, "pi-native", native, c))
|
|
remoteOut = append(remoteOut, runCase(t, root, "pi-agent", remote, c))
|
|
}
|
|
v := Judge(nativeOut, remoteOut)
|
|
t.Logf("native_success=%.2f remote_success=%.2f native_p95=%s remote_p95=%s p0=%d",
|
|
v.NativeSuccess, v.RemoteSuccess, v.NativeP95, v.RemoteP95, v.NativeP0)
|
|
for _, o := range append(nativeOut, remoteOut...) {
|
|
t.Logf("%s %s success=%t p0=%t targets=%d/%d dur=%s err=%s tokens=%d",
|
|
o.Backend, o.CaseID, o.Success, o.P0Defect, o.TargetsPresent, o.TargetsExpected, o.Duration, o.ErrorClass, o.PromptTokens+o.CompletionTokens)
|
|
}
|
|
if !v.Pass {
|
|
t.Fatalf("replacement gate failed: %v", v.Reasons)
|
|
}
|
|
}
|
|
|
|
func runCase(t *testing.T, root, name string, backend agent.Backend, c Case) Outcome {
|
|
t.Helper()
|
|
dir := filepath.Join(root, name, c.ID)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
start := time.Now()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
|
defer cancel()
|
|
res, err := backend.Execute(ctx, agent.Request{
|
|
WorkDir: dir,
|
|
Prompt: c.Prompt,
|
|
Instructions: c.Instructions,
|
|
Targets: c.Targets,
|
|
ToolProfile: c.ToolProfile,
|
|
Timeout: 3 * time.Minute,
|
|
})
|
|
out := Outcome{
|
|
Backend: name,
|
|
CaseID: c.ID,
|
|
Kind: c.Kind,
|
|
Duration: time.Since(start),
|
|
TargetsExpected: len(c.Targets),
|
|
}
|
|
if err != nil {
|
|
out.ErrorClass = err.Error()
|
|
out.P0Defect = true
|
|
return out
|
|
}
|
|
if res == nil {
|
|
out.ErrorClass = "nil result"
|
|
out.P0Defect = true
|
|
return out
|
|
}
|
|
out.Success = res.Success
|
|
out.ErrorClass = res.ErrorReason
|
|
out.PromptTokens = res.PromptTokens
|
|
out.CompletionTokens = res.CompletionTokens
|
|
for _, target := range c.Targets {
|
|
if len(res.Patches[target]) > 0 {
|
|
out.TargetsPresent++
|
|
}
|
|
}
|
|
if res.Success && out.TargetsPresent < out.TargetsExpected {
|
|
out.P0Defect = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
func envFirst(keys ...string) string {
|
|
fallback := ""
|
|
if len(keys) > 0 {
|
|
fallback = keys[len(keys)-1]
|
|
}
|
|
for _, key := range keys[:len(keys)-1] {
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if v := os.Getenv(key); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return fallback
|
|
}
|