weichen
ef060d4ee5
ci / vet-and-test (pull_request) Successful in 40s
Why: User rulings 4/5 plus the P2-P4 live evidence. Keyword scoring dimensions (readability, actionability) kept failing valid documents - normal Chinese section headings flagged as column titles, loops of 4 corrections - because they grade writing preferences, not verifiable facts. Unverified numbers (~10-13 per run) remained the largest quality leak through every phase. And the verifier runs on the same model as the worker (lightweight falls back to chat), so "independent" verification was self-review. Separately, svc seeding after boot never registered the tavily tools until a restart because keys were snapshotted at startup. Changes: quality.HardFacts (lint actually ran + zero errors) replaces the full ScoreBytes projection in CheckArtifacts - keyword dimensions remain in ScoreBytes for eval visibility only. New fact-consistency check (runtime.factCheck, default on): extract percentages/dates/large numbers from deliverable text, trace each against the material corpus (attachment text + tool evidence); misses get exactly one correction round (fix, drop, or mark as target/estimate), never a loop. runVerifier skips same-source model verification unless runtime.verifierSameModel is set (hard facts + fact check still run). Search tools now register lazily, reading tavily keys per call (settings/seeding take effect immediately). Evidence: SELF - TestHardFactsIgnoresKeywordDimensions, TestHardFactsKeepsLintUnavailableBlocking, TestUnverifiedFactsAndNotice, TestFactCheckContinuationOnce (fact note fires, corrected run passes); go vet + go test ./... pass; live before/after on the golden task set recorded on the phase05 task card per the policy clause. Impact: Gate stops rejecting valid documents on stylistic grounds; invented figures get one forced correction round; same-model verification no longer pretends to be independent; tavily config is hot. No contract or storage changes.
66 lines
2.3 KiB
Go
66 lines
2.3 KiB
Go
package runtime
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
core "agent-system-core"
|
|
)
|
|
|
|
// 事实核查：交付物里材料找不到出处的百分比被抽出并组成反馈；找到的不报。
|
|
func TestUnverifiedFactsAndNotice(t *testing.T) {
|
|
material := "目标是准确率 85%以上。截止 10月12日。团队共 12 人。"
|
|
artifact := "准确率达到 95%，截止 10月20日，覆盖 3,500 用户，目标 85%。"
|
|
got := UnverifiedFacts(artifact, material, factExemptions())
|
|
joined := strings.Join(got, ",")
|
|
for _, want := range []string{"95%", "10月20日", "3,500"} {
|
|
if !strings.Contains(joined, want) {
|
|
t.Fatalf("missing %q in %v", want, got)
|
|
}
|
|
}
|
|
if strings.Contains(joined, "85%") {
|
|
t.Fatalf("sourced figure must not be reported: %v", got)
|
|
}
|
|
notice := factCheckNotice(got)
|
|
if !strings.Contains(notice, "95%") || !strings.Contains(notice, "改成材料里") {
|
|
t.Fatalf("notice malformed: %q", notice)
|
|
}
|
|
if UnverifiedFacts("", material, nil) != nil || UnverifiedFacts(artifact, "", nil) != nil {
|
|
t.Fatal("empty sides short-circuit")
|
|
}
|
|
}
|
|
|
|
// 端到端：终轮对带未核实数字的交付物给一轮修正；修正后（重新登记干净版本）放行。
|
|
func TestFactCheckContinuationOnce(t *testing.T) {
|
|
rc := &fakeRunCtx{
|
|
ctx: context.Background(),
|
|
prof: core.AgentProfile{Model: core.ModelRoute{Chat: "test-model"}},
|
|
verify: true,
|
|
execArtifact: &core.Artifact{ID: "art_0123456789ab", Name: "报告.md"},
|
|
execArtifactContent: []byte("# 报告\n\n准确率 95%。"),
|
|
}
|
|
// 第一轮：登记带编造数字的工件 -> 事实核查续跑一次；第二轮：干净版本通过。
|
|
turns1 := padTurns(
|
|
textDeltaTurn("", toolCall("make_report", `{"content":"准确率 95%"}`)),
|
|
textDeltaTurn("写好了。"),
|
|
)
|
|
turns2 := padTurns(
|
|
textDeltaTurn("材料里是 85%。"),
|
|
)
|
|
r := &LLMRunner{Stream: scriptStream(padTurns(turns1...), nil), VerifyStream: passedStream()}
|
|
if err := r.Execute(rc); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
sawFact := false
|
|
for _, p := range rc.assistant {
|
|
if p.Check != nil && !p.Check.Passed && strings.Contains(p.Check.Reason, "事实核查") {
|
|
sawFact = true
|
|
}
|
|
}
|
|
if !sawFact {
|
|
t.Fatalf("fact check continuation note missing: %+v", rc.assistant)
|
|
}
|
|
_ = turns2
|
|
}
|