weichen
ef060d4ee5
ci / vet-and-test (pull_request) Successful in 40s
Why: User rulings 4/5 plus the P2-P4 live evidence. Keyword scoring dimensions (readability, actionability) kept failing valid documents - normal Chinese section headings flagged as column titles, loops of 4 corrections - because they grade writing preferences, not verifiable facts. Unverified numbers (~10-13 per run) remained the largest quality leak through every phase. And the verifier runs on the same model as the worker (lightweight falls back to chat), so "independent" verification was self-review. Separately, svc seeding after boot never registered the tavily tools until a restart because keys were snapshotted at startup. Changes: quality.HardFacts (lint actually ran + zero errors) replaces the full ScoreBytes projection in CheckArtifacts - keyword dimensions remain in ScoreBytes for eval visibility only. New fact-consistency check (runtime.factCheck, default on): extract percentages/dates/large numbers from deliverable text, trace each against the material corpus (attachment text + tool evidence); misses get exactly one correction round (fix, drop, or mark as target/estimate), never a loop. runVerifier skips same-source model verification unless runtime.verifierSameModel is set (hard facts + fact check still run). Search tools now register lazily, reading tavily keys per call (settings/seeding take effect immediately). Evidence: SELF - TestHardFactsIgnoresKeywordDimensions, TestHardFactsKeepsLintUnavailableBlocking, TestUnverifiedFactsAndNotice, TestFactCheckContinuationOnce (fact note fires, corrected run passes); go vet + go test ./... pass; live before/after on the golden task set recorded on the phase05 task card per the policy clause. Impact: Gate stops rejecting valid documents on stylistic grounds; invented figures get one forced correction round; same-model verification no longer pretends to be independent; tavily config is hot. No contract or storage changes.
91 lines
3.4 KiB
Go
91 lines
3.4 KiB
Go
package wfcompile
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"agent-system-backend/eval"
|
||
"agent-system-backend/internal/workflow"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
// ab_demo_test.go — Q3 WP5 验收：同批任务（含 demo.brief 场景）经两个入口
|
||
// 双跑并记录差异（不要求字节级相同），产出 DecideSwitch 裁决证据。
|
||
// direct 侧 = 直连对话循环的脚本化替身（单代理直接作答）；squad 侧 =
|
||
// 真实编译图执行（RunCompiled + 脚本化后端 + 桩验收流）。
|
||
|
||
func TestABDemoBriefDualRun(t *testing.T) {
|
||
def := parseDef(t, demoBriefJSON)
|
||
briefBody := "关于太空电梯的简报（短）。\n核对标记：P4"
|
||
|
||
// direct 侧：直连循环一次性作答（首字即全文）。
|
||
direct := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) {
|
||
return eval.ABOutcome{Completed: true, Summary: briefBody, FirstToken: 20 * time.Millisecond}, nil
|
||
}
|
||
// squad 侧：编译图真实执行；摘要 = 工作流汇总 + 成稿工件正文（要点覆盖
|
||
// 面向交付物而非状态行）。
|
||
squad := func(_ context.Context, task eval.ABTask) (eval.ABOutcome, error) {
|
||
start := time.Now()
|
||
be := &seqBackend{texts: []string{briefBody}}
|
||
f := &fakeHost{
|
||
toolNames: []string{"time_now"},
|
||
hitl: []core.HITLResponse{
|
||
{ChoiceID: "short"}, // length
|
||
{ChoiceID: "yes"}, // confirm
|
||
},
|
||
}
|
||
out, err := RunCompiled(f, Deps{
|
||
DataDir: t.TempDir(), Backends: seqResolver(t, be), DefaultBackend: "seq",
|
||
JudgeStream: verdictStream(verdictPass),
|
||
}, workflow.Launch{Def: def}, workflow.Vars{
|
||
Input: task.Goal, Slots: map[string]string{"topic": "太空电梯"},
|
||
}, RunOptions{})
|
||
if err != nil {
|
||
return eval.ABOutcome{}, err
|
||
}
|
||
summary := out.Summary
|
||
for _, w := range f.writes { // 成稿工件正文并入摘要（交付物视角）
|
||
if strings.HasPrefix(w, "demo-brief.md=") {
|
||
summary += "\n" + strings.TrimPrefix(w, "demo-brief.md=")
|
||
}
|
||
}
|
||
return eval.ABOutcome{Completed: true, Summary: summary, FirstToken: time.Since(start)}, nil
|
||
}
|
||
|
||
tasks := []eval.ABTask{{
|
||
ID: "demo.brief",
|
||
Goal: "写一份关于太空电梯的简报",
|
||
// 要点 = 主题词 + P4 核对标记（demo.brief 的 critic 判据）。
|
||
Themes: []string{"太空电梯", "核对标记：P4"},
|
||
}}
|
||
rep := eval.RunAB(context.Background(), tasks, direct, squad)
|
||
rec := rep.Records[0]
|
||
if !rec.Direct.Completed || !rec.Squad.Completed {
|
||
t.Fatalf("两侧都应完成：%+v / %+v", rec.Direct, rec.Squad)
|
||
}
|
||
// 两侧都覆盖要点 → 满分；差异只在摘要形态（记录在案，不要求字节级相同）。
|
||
if rec.DirectPoints != 100 || rec.SquadPoints != 100 {
|
||
t.Fatalf("points = %d/%d（direct=%q squad=%q）",
|
||
rec.DirectPoints, rec.SquadPoints, rec.Direct.Summary, rec.Squad.Summary)
|
||
}
|
||
var sawSummaryDiff bool
|
||
for _, d := range rec.Diffs {
|
||
if strings.Contains(d, "摘要文本不同") {
|
||
sawSummaryDiff = true
|
||
}
|
||
}
|
||
if !sawSummaryDiff {
|
||
t.Fatalf("两入口摘要形态必然不同，应记录差异：%v", rec.Diffs)
|
||
}
|
||
// 裁决是确定性的：同分 → 由首字延迟差决定；证据文本可直接入卡。
|
||
if rep.Direct.Score != 100 || rep.Squad.Score != 100 {
|
||
t.Fatalf("scores = %d/%d", rep.Direct.Score, rep.Squad.Score)
|
||
}
|
||
if rep.Text == "" || !strings.Contains(rep.Text, "分数：direct=100 squad=100") {
|
||
t.Fatalf("report = %s", rep.Text)
|
||
}
|
||
t.Logf("A/B 对照证据（demo.brief）：\n%s", rep.Text)
|
||
}
|