Identify materials by content and register grounded source blocks

Why: equal-length edits and parser changes must not reuse prior results.

Evidence: content identity and provided-source tests RED to GREEN; backend suite recorded through evidence.

Impact: trusted optional document origins, content keys and source catalog; no new routing or DB schema.

Refs: U2/U3 Task2

internal/runmgr/material.go

@@ -10,6 +10,7 @@ package runmgr
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"log"
 	"sort"
@@ -39,15 +40,26 @@ const materialPendingNote = "系统此前已为这批材料启动材料整理后
 
 const materialDoneNote = "这批材料此前已整理完成（见收件箱「材料整理」卡片的结果）。不要重复派发整理任务；如需正式交付物（摘要/评审/需求文档/方案等），基于已有整理结果直接推进。"
 
-// materialKey 内容去重键：排序后的 文件名+解析字符数 参与哈希——同一批文件
-// 重发（新的消息、新的附件切片）得到同键；内容变化则视为新材料。
+// materialKey identifies content and parsing, never timestamps or upload IDs.
 func materialKey(docs []core.ParsedDocument) string {
 	parts := make([]string, 0, len(docs))
 	for _, d := range docs {
-		parts = append(parts, fmt.Sprintf("%s|%d", d.Meta.Filename, utf8.RuneCountInString(d.PlainText())))
+		hash, parserRev := "", ""
+		if d.Meta.Origin != nil {
+			hash = d.Meta.Origin.ContentHash
+			parserRev = d.Meta.Origin.ParserRevision
+		}
+		// Full blocks retain images, table structure and coverage warnings.
+		raw, _ := json.Marshal(struct {
+			Filename, MIME, Parser, ParserRevision, OriginalHash string
+			Blocks                                               []core.DocBlock
+			Warnings                                             []string
+		}{d.Meta.Filename, d.Meta.MIME, d.Meta.Parser, parserRev, hash, d.Blocks, d.Warnings})
+		parts = append(parts, string(raw))
 	}
 	sort.Strings(parts)
-	sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
+	raw, _ := json.Marshal(parts)
+	sum := sha256.Sum256(raw)
 	return hex.EncodeToString(sum[:])
 }
 
@@ -116,6 +128,7 @@ func buildMaterialTask(eff core.EffectiveSettings, docs []core.ParsedDocument) s
 			left = 0
 		}
 	}
+	b.WriteString(materialSources(docs, perDoc).prompt())
 	return b.String()
 }
 
@@ -125,7 +138,7 @@ func (m *Manager) materialIntake(rc *runCtx) string {
 	if !rc.eff.GetBool("runtime.materialIntake", true) || len(rc.attachments) == 0 {
 		return ""
 	}
-	if note, existing := m.materialExisting(rc, materialKey(rc.attachments)); existing {
+	if note, existing := m.materialExisting(rc, rc.materialRevision()); existing {
 		return note
 	}
 	userText := messageText(rc.userMessage)
@@ -133,7 +146,7 @@ func (m *Manager) materialIntake(rc *runCtx) string {
 	if userText != "" && materialChars(rc.attachments) < threshold {
 		return "" // 文字+小材料：模型先答，回合结束兜底
 	}
-	return m.materialSpawn(rc, materialKey(rc.attachments))
+	return m.materialSpawn(rc, rc.materialRevision())
 }
 
 // ensureMaterial stop 兜底入口（幂等：只在首个调用实际派发并返回说明）。
@@ -146,7 +159,7 @@ func (c *runCtx) ensureMaterial() string {
 	if len(c.attachments) == 0 || !c.eff.GetBool("runtime.materialIntake", true) {
 		return ""
 	}
-	key := materialKey(c.attachments)
+	key := c.materialRevision()
 	if note, existing := c.mgr.materialExisting(c, key); existing {
 		c.materialNote = note
 		return note

internal/runmgr/material_sources.go

@@ -0,0 +1,128 @@
+package runmgr
+
+import (
+	"agent-system-backend/internal/runtime"
+	core "agent-system-core"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+type materialInputItem struct {
+	ArtifactID   core.ID
+	Parsed       core.ParsedDocument
+	OriginalHash string
+}
+type materialSource struct {
+	Ref      core.TaskSourceRef
+	Text     string
+	Provided string
+}
+type materialSourceRegistry map[string]materialSource
+
+func materialInputRevision(items []materialInputItem, processingRevision string) string {
+	docs := make([]core.ParsedDocument, 0, len(items))
+	for _, item := range items {
+		d := item.Parsed
+		origin := core.DocumentOrigin{}
+		if d.Meta.Origin != nil {
+			origin = *d.Meta.Origin
+		}
+		origin.ContentHash = item.OriginalHash
+		d.Meta.Origin = &origin
+		docs = append(docs, d)
+	}
+	sum := sha256.Sum256([]byte(materialKey(docs) + "\n" + processingRevision))
+	return hex.EncodeToString(sum[:])
+}
+func materialSources(docs []core.ParsedDocument, perDoc int) materialSourceRegistry {
+	r := materialSourceRegistry{}
+	left := materialInlineTotal
+	if perDoc <= 0 {
+		perDoc = materialPerDocDef
+	}
+	for _, d := range docs {
+		cap := perDoc
+		if cap > left {
+			cap = left
+		}
+		text := strings.TrimSpace(d.PlainText())
+		rr := []rune(text)
+		if len(rr) < cap {
+			cap = len(rr)
+		}
+		provided := string(rr[:cap])
+		left -= cap
+		if d.Meta.Origin == nil || d.Meta.Origin.ArtifactID == "" || d.Meta.Origin.ContentHash == "" {
+			continue
+		}
+		for index, block := range d.Blocks {
+			full := strings.TrimSpace((core.ParsedDocument{Blocks: []core.DocBlock{block}}).PlainText())
+			if full == "" || !strings.Contains(provided, full) {
+				continue
+			}
+			sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%d", d.Meta.Origin.ArtifactID, d.Meta.Origin.ContentHash, index)))
+			id := "src_" + hex.EncodeToString(sum[:12])
+			r[id] = materialSource{Ref: core.TaskSourceRef{ID: id, ArtifactID: d.Meta.Origin.ArtifactID, ContentHash: d.Meta.Origin.ContentHash, ParserRevision: d.Meta.Origin.ParserRevision, BlockIndex: index, Label: fmt.Sprintf("%s · 段落%d", d.Meta.Filename, index+1), Availability: "available"}, Text: full, Provided: provided}
+		}
+	}
+	return r
+}
+func (r materialSourceRegistry) validate(q core.TaskSourceQuote) (core.TaskSourceRef, error) {
+	src, ok := r[q.SourceID]
+	if !ok || q.BlockIndex != src.Ref.BlockIndex || strings.TrimSpace(q.Quote) == "" || !strings.Contains(src.Text, q.Quote) || !strings.Contains(src.Provided, q.Quote) {
+		return core.TaskSourceRef{}, fmt.Errorf("checkpoint.source: quote not in provided block")
+	}
+	ref := src.Ref
+	ref.Quote = q.Quote
+	return ref, nil
+}
+func (r materialSourceRegistry) prompt() string {
+	keys := make([]string, 0, len(r))
+	for k := range r {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	var b strings.Builder
+	if len(keys) > 0 {
+		b.WriteString("\n\n=== 可引用的已提供来源（不含未提供的全文） ===\n")
+	}
+	for _, id := range keys {
+		src := r[id]
+		fmt.Fprintf(&b, "sourceId=%s blockIndex=%d %s\n", id, src.Ref.BlockIndex, src.Ref.Label)
+	}
+	return b.String()
+}
+
+func (c *runCtx) materialRevision() string {
+	perDoc := int(c.eff.GetNumber(materialPerDocKey, materialPerDocDef))
+	if perDoc <= 0 {
+		perDoc = materialPerDocDef
+	}
+	recipe := "material-recipe.v3"
+	if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil {
+		if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil {
+			recipe += "\n" + body
+		}
+		if spec, ok := runner.Prompts.SubagentSpec(c.run.Owner, materialSubagentID); ok {
+			raw, _ := json.Marshal(spec)
+			recipe += "\n" + string(raw)
+		}
+	}
+	if perDoc == materialPerDocDef && recipe == "material-recipe.v3" {
+		return materialKey(c.attachments)
+	}
+	items := make([]materialInputItem, 0, len(c.attachments))
+	for _, d := range c.attachments {
+		item := materialInputItem{Parsed: d}
+		if d.Meta.Origin != nil {
+			item.ArtifactID = d.Meta.Origin.ArtifactID
+			item.OriginalHash = d.Meta.Origin.ContentHash
+		}
+		items = append(items, item)
+	}
+	return materialInputRevision(items, fmt.Sprintf("%s\nperDoc=%d,total=%d", recipe, perDoc, materialInlineTotal))
+}

internal/runmgr/material_sources_test.go

@@ -0,0 +1,53 @@
+package runmgr
+
+import (
+ "strings"
+	core "agent-system-core"
+	"testing"
+	"time"
+)
+
+func TestMaterialKeyDetectsEqualLengthEdits(t *testing.T) {
+	a := []core.ParsedDocument{{Meta: core.DocMeta{Filename: "决定.txt", Parser: "builtin:md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}}}}
+	b := []core.ParsedDocument{{Meta: a[0].Meta, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线七月"}}}}
+	if materialKey(a) == materialKey(b) {
+		t.Fatal("same length edit reused a prior material result")
+	}
+}
+
+func TestMaterialSourceQuoteRequiresActualProvidedBlock(t *testing.T) {
+ docs:=[]core.ParsedDocument{{Meta:core.DocMeta{Filename:"决定.txt",Parser:"builtin:md",Origin:&core.DocumentOrigin{ArtifactID:"art1",ContentHash:strings.Repeat("a",64),ParserRevision:"ir1"}},Blocks:[]core.DocBlock{{Type:core.BlockPara,Text:"上线六月"},{Type:core.BlockPara,Text:"预算未定"}}}}
+ sources:=materialSources(docs,5)
+ if len(sources)==0 {t.Fatal("provided source was not registered")}
+ for id,src:=range sources {
+  if src.Ref.BlockIndex!=0 {continue}
+  got,err:=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:0,Quote:"上线六月"})
+  if err!=nil||got.ArtifactID!="art1"{t.Fatal("real source rejected",err)}
+  if _,err=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:1,Quote:"上线六月"});err==nil {t.Fatal("wrong block accepted")}
+  if _,err=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:0,Quote:"上线七月"});err==nil {t.Fatal("invented quote accepted")}
+ }
+ if _,err:=sources.validate(core.TaskSourceQuote{SourceID:"foreign",Quote:"上线六月"});err==nil{t.Fatal("foreign source accepted")}
+}
+func TestMaterialInputRevisionIncludesProcessingVersion(t *testing.T) {
+ items:=[]materialInputItem{{Parsed:materialDocs()[0],OriginalHash:strings.Repeat("a",64)}}
+ if materialInputRevision(items,"recipe1")==materialInputRevision(items,"recipe2"){t.Fatal("processing change reused result")}
+}
+func TestMaterialKeyChangesWithParserAndWarnings(t *testing.T) {
+	a := materialDocs()
+	b := materialDocs()
+	b[0].Meta.Parser = "different-parser"
+	if materialKey(a) == materialKey(b) {
+		t.Fatal("parser version ignored")
+	}
+	b = materialDocs()
+	b[0].Warnings = []string{"未解析图片"}
+	if materialKey(a) == materialKey(b) {
+		t.Fatal("changed coverage warning ignored")
+	}
+	b = materialDocs()
+	b[0].Meta.ParsedAt = time.Now()
+	b[0].Meta.Origin = &core.DocumentOrigin{ArtifactID: "new-upload-id"}
+	if materialKey(a) != materialKey(b) {
+		t.Fatal("transport identity/time should not change content identity")
+	}
+}

internal/server/server.go

@@ -4,6 +4,8 @@ package server
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -124,7 +126,7 @@ func (s *Server) authorizeProjectAction(owner string, projectID core.ID, action
 }
 
 // authorizeSharedResource EXT-8b 资源级读判定（U9）：V2 注册时按六要素咨询
-//（Actor=认证会话、Action/ResourceType/ResourceID 由调用点受信构造、
+// （Actor=认证会话、Action/ResourceType/ResourceID 由调用点受信构造、
 // Owner/ProjectID 从受信存储元数据解析；出错/字段缺失一律 false，fail-closed
 // 且不泄露存在性）；V2 未注册回落 authorizeSharedProject（现行兼容）。
 func (s *Server) authorizeSharedResource(actor, action, resourceType string, resourceID, owner, projectID core.ID) bool {
@@ -829,6 +831,8 @@ func (s *Server) postMessage(w http.ResponseWriter, r *http.Request) {
 			inputs = append(inputs, runmgr.WorkspaceFile{Name: art.Name, Content: content})
 		}
 		doc := ingest.Parse(art.Name, art.MIME, content)
+		hash := sha256.Sum256(content)
+		doc.Meta.Origin = &core.DocumentOrigin{ArtifactID: art.ID, ContentHash: hex.EncodeToString(hash[:]), ParserRevision: "parsed-ir.v1/" + doc.Meta.Parser}
 		parsedDocs = append(parsedDocs, doc)
 		parts = append(parts, core.Part{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: art.ID, Name: art.Name, Kind: art.Kind}})
 	}