weichen
45f21e84c7
Add typed jevdecide HTTP client (circuit, metrics) and coordinator DecideAssist wiring behind JEV_DECIDE_MODE (default off). Offline eval and synthetic soak evidence gate realtime/profile before assist rollout.
90 lines
2.7 KiB
Go
90 lines
2.7 KiB
Go
package coordinator
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"chat-agent-system/internal/jevdecide"
|
|
)
|
|
|
|
// TestShadowSoakReplay replays the offline corpus through shadow wiring ≥200 times.
|
|
// This is an engineering soak stand-in when live traffic is unavailable; gate 2
|
|
// still requires human review of disagreement cases in the written report.
|
|
func TestShadowSoakReplay(t *testing.T) {
|
|
if os.Getenv("RUN_JEV_SOAK") != "1" {
|
|
t.Skip("set RUN_JEV_SOAK=1 for synthetic shadow soak")
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join("..", "jevdecide", "testdata", "eval-v1.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var file struct {
|
|
Cases []struct {
|
|
ID string `json:"id"`
|
|
Suite string `json:"suite"`
|
|
Input string `json:"input"`
|
|
} `json:"cases"`
|
|
}
|
|
if err := json.Unmarshal(raw, &file); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client := jevdecide.NewClientFromEnv()
|
|
jevdecide.SetModeForTest(jevdecide.ModeShadow)
|
|
defer jevdecide.ResetModeForTest()
|
|
jevdecide.Global.Reset()
|
|
e := &CoordinatorEngine{decideAssist: client}
|
|
|
|
target := 200
|
|
n := 0
|
|
for n < target {
|
|
for _, c := range file.Cases {
|
|
if c.Suite != "realtime" && c.Suite != "profile" {
|
|
continue
|
|
}
|
|
if c.Suite == "realtime" {
|
|
_ = e.resolveRealtimeQuery(context.Background(), c.Input)
|
|
} else {
|
|
p := ProfileTask(c.Input, nil)
|
|
e.shadowProfile(context.Background(), c.Input, p)
|
|
}
|
|
n++
|
|
if n >= target {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
snap := jevdecide.Global.Snapshot()
|
|
outDir := os.Getenv("JEV_EVAL_OUT")
|
|
if outDir == "" {
|
|
outDir = filepath.Join("..", "..", ".doc", "test-reports", "assets", "2026-09-24-jev-product-shadow")
|
|
}
|
|
_ = os.MkdirAll(outDir, 0o755)
|
|
blob, _ := json.MarshalIndent(map[string]any{
|
|
"turns": n,
|
|
"metrics": snap,
|
|
"note": "synthetic soak replaying eval corpus under ModeShadow; not live traffic",
|
|
}, "", " ")
|
|
_ = os.WriteFile(filepath.Join(outDir, "p3-shadow-soak.json"), blob, 0o644)
|
|
t.Logf("soak turns=%d metrics=%v", n, snap)
|
|
if snap["jev_fail"] > int64(float64(n)*0.02) {
|
|
t.Errorf("GATE2 fail rate too high: fail=%d turns=%d", snap["jev_fail"], n)
|
|
}
|
|
}
|
|
|
|
func TestModeOffByteEquivalentRealtime(t *testing.T) {
|
|
jevdecide.SetModeForTest(jevdecide.ModeOff)
|
|
defer jevdecide.ResetModeForTest()
|
|
e := &CoordinatorEngine{decideAssist: &stubDecide{yes: true, ans: jevdecide.Answer{Choice: "yes", Confidence: 1}}}
|
|
inputs := []string{"今天北京天气怎么样", "项目进度怎么样", "型检温度指标怎么定", "美元汇率多少", "你好"}
|
|
for _, in := range inputs {
|
|
rule := IsRealtimeInformationQuery(in)
|
|
got := e.resolveRealtimeQuery(context.Background(), in)
|
|
if got != rule {
|
|
t.Fatalf("mode=off diverged for %q: rule=%v got=%v", in, rule, got)
|
|
}
|
|
}
|
|
}
|