Add estimate-marking awareness and a second fact-check round (E3)
ci / vet-and-test (pull_request) Successful in 41s
ci / vet-and-test (pull_request) Successful in 41s
Why: The E1 live comparison was the only metric that got worse (unverifiedFacts 7 -> 12): the gate gave one correction round and then shipped bare numbers. The design (E3) calls for number sourcing with a mechanically checkable marking escape: numbers labeled as targets/assumptions are honest planning quantities, not fabricated facts, so the gate must let them pass - and the model must be told that marking is the sanctioned way out. Why: The marking decision is clause-scoped, not a fixed window - a window wrongly blessed "3,500" because a neighboring clause said "target 85%" (caught by the existing test). The second round only fires when the unverified set actually changed, because the continuation budget is shared across all final gates and a repeat block on identical tokens starves the downstream gates (also caught by an existing test). Evidence: go test ./... green; new anchors TestMarkedAsEstimate, TestUnverifiedFactsMarksPass, TestFactCheckGateRounds; the live eval gains an unverifiedUnmarkedFacts field sharing the same marking rule. Impact: base.deliverables v6 adds the hard number-sourcing rule; research/explore/general sub-agent profiles v+1 require citations for key numbers; fact gate: marked-in-clause numbers pass, round 2 escalates to "last interception" with the marking instruction, then passes through with the check line. Eval report keeps the strict series and adds the E3-series field. Refs: proposal 2026-10-10-agent-delegation-routing-sandbox.md (E3, D4); Dev task feature/agent-e3-evidence-gate.
This commit is contained in:
@@ -17,6 +17,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"agent-system-backend/internal/runtime"
|
||||
)
|
||||
|
||||
// upload 上传一个材料文件,返回 artifact id。
|
||||
@@ -377,6 +379,18 @@ func unverifiedFacts(artifactText, materialText string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// unmarkedFacts 从未核实清单里剔除已按「目标/预计/假设」标注的(与后端
|
||||
// runtime.MarkedAsEstimate 同一口径,标注词邻域 ±32 字符)。
|
||||
func unmarkedFacts(tokens []string, artifactText string) []string {
|
||||
var out []string
|
||||
for _, tok := range tokens {
|
||||
if !runtime.MarkedAsEstimate(artifactText, tok) {
|
||||
out = append(out, tok)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// finalVsArtifact 终轮回复里声称的章节(「一、…」)必须能在工件里找到。
|
||||
func finalVsArtifact(turn *turnResult, artifactText string) []string {
|
||||
if turn == nil || turn.FinalText == "" || strings.TrimSpace(artifactText) == "" {
|
||||
@@ -430,6 +444,9 @@ func renderMarkdown(rep *report) string {
|
||||
}
|
||||
if len(tk.UnverifiedFacts) > 0 {
|
||||
fmt.Fprintf(&b, "\n未核实事实(工件里出现、材料里找不到):%s\n", strings.Join(tk.UnverifiedFacts, "、"))
|
||||
if len(tk.UnverifiedUnmarkedFacts) > 0 {
|
||||
fmt.Fprintf(&b, "其中未标注(无出处且无目标/预计/假设标注,E3 闸门口径):%s\n", strings.Join(tk.UnverifiedUnmarkedFacts, "、"))
|
||||
}
|
||||
}
|
||||
if len(tk.FinalVsArtifact) > 0 {
|
||||
fmt.Fprintf(&b, "\n终轮声称但工件不存在的章节:%s\n", strings.Join(tk.FinalVsArtifact, ";"))
|
||||
|
||||
@@ -252,6 +252,9 @@ type taskResult struct {
|
||||
Turns []turnResult `json:"turns"`
|
||||
Total turnMetrics `json:"total"`
|
||||
UnverifiedFacts []string `json:"unverifiedFacts,omitempty"`
|
||||
// UnverifiedUnmarkedFacts:无出处且未按「目标/预计/假设」标注的子集
|
||||
// (E3 口径:严格口径 unverifiedFacts 保留原义以维持历史可比性)。
|
||||
UnverifiedUnmarkedFacts []string `json:"unverifiedUnmarkedFacts,omitempty"`
|
||||
FinalVsArtifact []string `json:"finalVsArtifact,omitempty"`
|
||||
}
|
||||
|
||||
@@ -307,6 +310,7 @@ func runTask(t *testing.T, c *client, projectID string, tk task, materialsDir st
|
||||
artText += downloadArtifactText(t, c, projectID, name)
|
||||
}
|
||||
tr.UnverifiedFacts = unverifiedFacts(artText, materialText)
|
||||
tr.UnverifiedUnmarkedFacts = unmarkedFacts(tr.UnverifiedFacts, artText)
|
||||
tr.FinalVsArtifact = finalVsArtifact(last, artText)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user