Present material sources once beside provided text

Why: The old source index repeated most of a three-document input and its test fixture depended on that duplicate layout.

Evidence: Source prefix, ID scope, parser revision, table, truncation and mechanism tests failed before the change and pass after; full runmgr tests pass.

Impact: Compact the material prompt without widening quote authority, and version the recipe so old results are not reused.

Refs: Agent-System-Backend#193; Agent-System-Dev#512
This commit is contained in:
2026-10-13 09:46:59 +00:00
parent f74f6a010e
commit 6ecd75870d
6 changed files with 149 additions and 24 deletions
+3 -3
View File
@@ -92,6 +92,7 @@ func buildMaterialTask(eff core.EffectiveSettings, docs []core.ParsedDocument) s
if perDoc <= 0 {
perDoc = materialPerDocDef
}
sources := materialSources(docs, perDoc)
var b strings.Builder
fmt.Fprintf(&b, "整理用户上传的 %d 份材料并输出结构化摘要。\n\n", len(docs))
b.WriteString("要求:\n")
@@ -120,19 +121,18 @@ func buildMaterialTask(eff core.EffectiveSettings, docs []core.ParsedDocument) s
cap = left
}
if runes <= cap {
b.WriteString(text)
b.WriteString(annotateProvidedMaterial(d, text, sources))
b.WriteString("\n")
left -= runes
continue
}
b.WriteString(string([]rune(text)[:cap]))
b.WriteString(annotateProvidedMaterial(d, string([]rune(text)[:cap]), sources))
b.WriteString("\n…(超长截断,余下部分用 knowledge_search 检索本文件名)\n")
left -= cap
if left <= 0 {
left = 0
}
}
b.WriteString(materialSources(docs, perDoc).prompt())
return b.String()
}
+36 -16
View File
@@ -20,6 +20,8 @@ type materialSource struct {
Ref core.TaskSourceRef
Text string
Provided string
Start int // byte offsets in strings.TrimSpace(ParsedDocument.PlainText())
End int
}
type materialSourceRegistry map[string]materialSource
@@ -81,7 +83,7 @@ func materialSources(docs []core.ParsedDocument, perDoc int) materialSourceRegis
providedBlock := text[start:end]
sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%d", d.Meta.Origin.ArtifactID, d.Meta.Origin.ContentHash, index)))
id := "src_" + hex.EncodeToString(sum[:12])
r[id] = materialSource{Ref: core.TaskSourceRef{ID: id, ArtifactID: d.Meta.Origin.ArtifactID, ContentHash: d.Meta.Origin.ContentHash, ParserRevision: d.Meta.Origin.ParserRevision, BlockIndex: index, Label: fmt.Sprintf("%s · 段落%d", d.Meta.Filename, index+1), Availability: "available"}, Text: full, Provided: providedBlock}
r[id] = materialSource{Ref: core.TaskSourceRef{ID: id, ArtifactID: d.Meta.Origin.ArtifactID, ContentHash: d.Meta.Origin.ContentHash, ParserRevision: d.Meta.Origin.ParserRevision, BlockIndex: index, Label: fmt.Sprintf("%s · 段落%d", d.Meta.Filename, index+1), Availability: "available"}, Text: full, Provided: providedBlock, Start: start, End: end}
}
}
return r
@@ -95,24 +97,42 @@ func (r materialSourceRegistry) validate(q core.TaskSourceQuote) (core.TaskSourc
ref.Quote = q.Quote
return ref, nil
}
func (r materialSourceRegistry) prompt() string {
keys := make([]string, 0, len(r))
for k := range r {
keys = append(keys, k)
func annotateProvidedMaterial(d core.ParsedDocument, provided string, sources materialSourceRegistry) string {
if provided == "" || d.Meta.Origin == nil {
return provided
}
sort.Strings(keys)
var b strings.Builder
if len(keys) > 0 {
b.WriteString("\n\n=== 可引用的已提供来源(不含未提供的全文) ===\n")
type marker struct {
start int
id string
block int
}
for _, id := range keys {
src := r[id]
hint := []rune(src.Provided)
if len(hint) > 180 {
hint = hint[:180]
markers := make([]marker, 0)
for id, src := range sources {
if src.Ref.ArtifactID != d.Meta.Origin.ArtifactID || src.Ref.ContentHash != d.Meta.Origin.ContentHash || src.Ref.ParserRevision != d.Meta.Origin.ParserRevision || src.Start < 0 || src.End > len(provided) || src.Start >= src.End {
continue
}
fmt.Fprintf(&b, "sourceId=%s blockIndex=%d %s\n引用定位片段:%q\n", id, src.Ref.BlockIndex, src.Ref.Label, string(hint))
if provided[src.Start:src.End] != src.Provided {
continue
}
markers = append(markers, marker{start: src.Start, id: id, block: src.Ref.BlockIndex})
}
sort.Slice(markers, func(i, j int) bool {
if markers[i].start != markers[j].start {
return markers[i].start < markers[j].start
}
return markers[i].block < markers[j].block
})
var b strings.Builder
cursor := 0
for _, mark := range markers {
if mark.start < cursor {
continue
}
b.WriteString(provided[cursor:mark.start])
fmt.Fprintf(&b, "sourceId=%s blockIndex=%d\n", mark.id, mark.block)
cursor = mark.start
}
b.WriteString(provided[cursor:])
return b.String()
}
@@ -121,7 +141,7 @@ func (c *runCtx) materialRevision() string {
if perDoc <= 0 {
perDoc = materialPerDocDef
}
recipe := "material-recipe.v7"
recipe := "material-recipe.v8"
if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil {
if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil {
recipe += "\n" + body
+88 -4
View File
@@ -2,11 +2,91 @@ package runmgr
import (
core "agent-system-core"
"regexp"
"strings"
"testing"
"time"
)
func TestMaterialSourceInlineAnnotationsPreserveProvidedPrefix(t *testing.T) {
d := materialDocs()[0]
d.Meta.Origin = &core.DocumentOrigin{ArtifactID: "art1", ContentHash: strings.Repeat("a", 64), ParserRevision: "docx1"}
docs := []core.ParsedDocument{d}
provided := strings.TrimSpace(d.PlainText())
got := annotateProvidedMaterial(d, provided, materialSources(docs, 48000))
markers := regexp.MustCompile(`(?m)^sourceId=src_[a-f0-9]+ blockIndex=[0-9]+\n`)
if !markers.MatchString(got) {
t.Fatal("provided block has no source marker")
}
if markers.ReplaceAllString(got, "") != provided {
t.Fatal("source annotations changed the provided material bytes")
}
}
func TestMaterialSourceInlineAnnotationsNeverExposeHiddenBlocks(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "资料.txt", Origin: &core.DocumentOrigin{ArtifactID: "art2", ContentHash: strings.Repeat("b", 64), ParserRevision: "txt1"}}, Blocks: []core.DocBlock{
{Type: core.BlockPara, Text: "上线六月"},
{Type: core.BlockPara, Text: "隐藏的审批结论"},
}}
provided := "上线六月"
got := annotateProvidedMaterial(d, provided, materialSources([]core.ParsedDocument{d}, 4))
if strings.Count(got, "sourceId=") != 1 || strings.Contains(got, "隐藏的审批结论") {
t.Fatalf("unprovided block received a marker or text: %q", got)
}
if strings.Count(got, provided) != 1 {
t.Fatalf("provided prefix was duplicated: %q", got)
}
}
func TestMaterialSourceInlineAnnotationsPreserveTableAndUntrustedText(t *testing.T) {
trusted := core.ParsedDocument{Meta: core.DocMeta{Filename: "预算.xlsx", Origin: &core.DocumentOrigin{ArtifactID: "table", ContentHash: strings.Repeat("d", 64), ParserRevision: "xlsx1"}}, Blocks: []core.DocBlock{{Type: core.BlockTable, Rows: [][]string{{"项目", "预算"}, {"一期", "50万元"}}}}}
provided := strings.TrimSpace(trusted.PlainText())
got := annotateProvidedMaterial(trusted, provided, materialSources([]core.ParsedDocument{trusted}, 48000))
markers := regexp.MustCompile(`(?m)^sourceId=src_[a-f0-9]+ blockIndex=[0-9]+\n`)
if markers.ReplaceAllString(got, "") != provided || !strings.Contains(got, "一期\t50万元") {
t.Fatalf("table text changed while adding a source marker: %q", got)
}
untrusted := core.ParsedDocument{Meta: core.DocMeta{Filename: "无来源.txt"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "不得生成引用ID"}}}
plain := strings.TrimSpace(untrusted.PlainText())
if got := annotateProvidedMaterial(untrusted, plain, materialSources([]core.ParsedDocument{untrusted}, 48000)); got != plain {
t.Fatalf("document without Origin gained a source marker: %q", got)
}
}
func TestMaterialSourceInlineAnnotationsKeepOnlyPartialLastBlock(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "资料.txt", Origin: &core.DocumentOrigin{ArtifactID: "partial", ContentHash: strings.Repeat("e", 64), ParserRevision: "txt1"}}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}, {Type: core.BlockPara, Text: "隐藏的审批结论"}}}
provided := string([]rune(strings.TrimSpace(d.PlainText()))[:7])
got := annotateProvidedMaterial(d, provided, materialSources([]core.ParsedDocument{d}, 7))
markers := regexp.MustCompile(`(?m)^sourceId=src_[a-f0-9]+ blockIndex=[0-9]+\n`)
if markers.ReplaceAllString(got, "") != provided || strings.Contains(got, "藏") {
t.Fatalf("partial block exposed unavailable text: %q", got)
}
}
func TestMaterialSourceInlineAnnotationsRejectDifferentParserRevision(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "同一工件.txt", Origin: &core.DocumentOrigin{ArtifactID: "same", ContentHash: strings.Repeat("f", 64), ParserRevision: "parser-old"}}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}}}
oldSources := materialSources([]core.ParsedDocument{d}, 48000)
d.Meta.Origin.ParserRevision = "parser-new"
provided := strings.TrimSpace(d.PlainText())
if got := annotateProvidedMaterial(d, provided, oldSources); got != provided {
t.Fatalf("old parser source ID leaked into new parsing: %q", got)
}
}
func TestMaterialSourceInlineAnnotationsMatchRegistryIDs(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "多段.txt", Origin: &core.DocumentOrigin{ArtifactID: "multi", ContentHash: strings.Repeat("9", 64), ParserRevision: "txt1"}}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "第一段"}, {Type: core.BlockPara, Text: "第二段"}, {Type: core.BlockPara, Text: "第三段"}}}
sources := materialSources([]core.ParsedDocument{d}, 48000)
got := annotateProvidedMaterial(d, strings.TrimSpace(d.PlainText()), sources)
if strings.Count(got, "sourceId=") != len(sources) {
t.Fatalf("annotation count=%d registry=%d", strings.Count(got, "sourceId="), len(sources))
}
for id := range sources {
if strings.Count(got, "sourceId="+id+" ") != 1 {
t.Fatalf("registered source %s missing or duplicated", id)
}
}
}
func TestMaterialKeyDetectsEqualLengthEdits(t *testing.T) {
a := []core.ParsedDocument{{Meta: core.DocMeta{Filename: "决定.txt", Parser: "builtin:md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}}}}
b := []core.ParsedDocument{{Meta: a[0].Meta, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线七月"}}}}
@@ -66,8 +146,12 @@ func TestMaterialKeyChangesWithParserAndWarnings(t *testing.T) {
}
}
func TestMaterialSourceDuplicateTextDoesNotAuthorizeUnprovidedBlock(t *testing.T){
d:=core.ParsedDocument{Meta:core.DocMeta{Filename:"重复.txt",Origin:&core.DocumentOrigin{ArtifactID:"a",ContentHash:"h",ParserRevision:"txt"}},Blocks:[]core.DocBlock{{Type:core.BlockPara,Text:"相同内容"},{Type:core.BlockPara,Text:"相同内容"}}}
registry:=materialSources([]core.ParsedDocument{d},4)
for _,src:=range registry{if src.Ref.BlockIndex==1{t.Fatal("unprovided duplicate block authorized")}}
func TestMaterialSourceDuplicateTextDoesNotAuthorizeUnprovidedBlock(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "重复.txt", Origin: &core.DocumentOrigin{ArtifactID: "a", ContentHash: "h", ParserRevision: "txt"}}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "相同内容"}, {Type: core.BlockPara, Text: "相同内容"}}}
registry := materialSources([]core.ParsedDocument{d}, 4)
for _, src := range registry {
if src.Ref.BlockIndex == 1 {
t.Fatal("unprovided duplicate block authorized")
}
}
}
+8
View File
@@ -171,6 +171,14 @@ func TestBuildMaterialTaskInlineBudget(t *testing.T) {
}
}
func TestBuildMaterialTaskShowsSourceOnce(t *testing.T) {
d := core.ParsedDocument{Meta: core.DocMeta{Filename: "来源.txt", Parser: "builtin:txt", Origin: &core.DocumentOrigin{ArtifactID: "art-source", ContentHash: strings.Repeat("c", 64), ParserRevision: "txt1"}}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "独特材料事实XYZ"}}}
task := buildMaterialTask(core.EffectiveSettings{}, []core.ParsedDocument{d})
if strings.Count(task, "独特材料事实XYZ") != 1 || !strings.Contains(task, "sourceId=src_") {
t.Fatalf("material and source were not presented once: %q", task)
}
}
func TestMaterialKeyStableAcrossOrder(t *testing.T) {
docs := materialDocs()
a := materialKey(docs)
@@ -12,11 +12,20 @@ import (
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
agentllm "agent-native/runtime/llm"
)
func TestMechanismSourceExtractionAfterSourceOnce(t *testing.T) {
id := "src_" + strings.Repeat("a", 24)
got := mechanismCheckpointSources("sourceId=" + id + " blockIndex=0\n上线六月。验收条件需要补齐")
if len(got) != 1 || got[0][1] != id || got[0][2] != "0" || !strings.Contains(got[0][3], "上线六月") {
t.Fatalf("inline source unavailable to mechanism fixture: %v", got)
}
}
func newEvaluationEventStore(t *testing.T) (string, *sql.DB) {
t.Helper()
dir := t.TempDir()
+5 -1
View File
@@ -83,7 +83,7 @@ func evalMechanismStream(ctx context.Context, model string, msgs []agentllm.Mess
count++
}
}
matches := regexp.MustCompile(`sourceId=(src_[a-f0-9]+) blockIndex=(\d+)[^\n]*\n引用定位片段:([^\n]*)`).FindAllStringSubmatch(sourceText, -1)
matches := mechanismCheckpointSources(sourceText)
matches = filterCheckpointSources(matches)
if count < 2 && len(matches) > 0 {
var block int
@@ -461,3 +461,7 @@ func filterCheckpointSources(matches [][]string) [][]string {
}
return out
}
func mechanismCheckpointSources(sourceText string) [][]string {
return regexp.MustCompile(`(?m)^sourceId=(src_[a-f0-9]+) blockIndex=(\d+)\n([^\n]*)`).FindAllStringSubmatch(sourceText, -1)
}