diff --git a/internal/runmgr/material.go b/internal/runmgr/material.go index fc7d300..0a31f4a 100644 --- a/internal/runmgr/material.go +++ b/internal/runmgr/material.go @@ -10,6 +10,7 @@ package runmgr import ( "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "log" "sort" @@ -39,15 +40,26 @@ const materialPendingNote = "系统此前已为这批材料启动材料整理后 const materialDoneNote = "这批材料此前已整理完成(见收件箱「材料整理」卡片的结果)。不要重复派发整理任务;如需正式交付物(摘要/评审/需求文档/方案等),基于已有整理结果直接推进。" -// materialKey 内容去重键:排序后的 文件名+解析字符数 参与哈希——同一批文件 -// 重发(新的消息、新的附件切片)得到同键;内容变化则视为新材料。 +// materialKey identifies content and parsing, never timestamps or upload IDs. func materialKey(docs []core.ParsedDocument) string { parts := make([]string, 0, len(docs)) for _, d := range docs { - parts = append(parts, fmt.Sprintf("%s|%d", d.Meta.Filename, utf8.RuneCountInString(d.PlainText()))) + hash, parserRev := "", "" + if d.Meta.Origin != nil { + hash = d.Meta.Origin.ContentHash + parserRev = d.Meta.Origin.ParserRevision + } + // Full blocks retain images, table structure and coverage warnings. + raw, _ := json.Marshal(struct { + Filename, MIME, Parser, ParserRevision, OriginalHash string + Blocks []core.DocBlock + Warnings []string + }{d.Meta.Filename, d.Meta.MIME, d.Meta.Parser, parserRev, hash, d.Blocks, d.Warnings}) + parts = append(parts, string(raw)) } sort.Strings(parts) - sum := sha256.Sum256([]byte(strings.Join(parts, "\n"))) + raw, _ := json.Marshal(parts) + sum := sha256.Sum256(raw) return hex.EncodeToString(sum[:]) } @@ -116,6 +128,7 @@ func buildMaterialTask(eff core.EffectiveSettings, docs []core.ParsedDocument) s left = 0 } } + b.WriteString(materialSources(docs, perDoc).prompt()) return b.String() } @@ -125,7 +138,7 @@ func (m *Manager) materialIntake(rc *runCtx) string { if !rc.eff.GetBool("runtime.materialIntake", true) || len(rc.attachments) == 0 { return "" } - if note, existing := m.materialExisting(rc, materialKey(rc.attachments)); existing { + if note, existing := m.materialExisting(rc, rc.materialRevision()); existing { return note } userText := messageText(rc.userMessage) @@ -133,7 +146,7 @@ func (m *Manager) materialIntake(rc *runCtx) string { if userText != "" && materialChars(rc.attachments) < threshold { return "" // 文字+小材料:模型先答,回合结束兜底 } - return m.materialSpawn(rc, materialKey(rc.attachments)) + return m.materialSpawn(rc, rc.materialRevision()) } // ensureMaterial stop 兜底入口(幂等:只在首个调用实际派发并返回说明)。 @@ -146,7 +159,7 @@ func (c *runCtx) ensureMaterial() string { if len(c.attachments) == 0 || !c.eff.GetBool("runtime.materialIntake", true) { return "" } - key := materialKey(c.attachments) + key := c.materialRevision() if note, existing := c.mgr.materialExisting(c, key); existing { c.materialNote = note return note diff --git a/internal/runmgr/material_sources.go b/internal/runmgr/material_sources.go new file mode 100644 index 0000000..0fb2298 --- /dev/null +++ b/internal/runmgr/material_sources.go @@ -0,0 +1,128 @@ +package runmgr + +import ( + "agent-system-backend/internal/runtime" + core "agent-system-core" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" +) + +type materialInputItem struct { + ArtifactID core.ID + Parsed core.ParsedDocument + OriginalHash string +} +type materialSource struct { + Ref core.TaskSourceRef + Text string + Provided string +} +type materialSourceRegistry map[string]materialSource + +func materialInputRevision(items []materialInputItem, processingRevision string) string { + docs := make([]core.ParsedDocument, 0, len(items)) + for _, item := range items { + d := item.Parsed + origin := core.DocumentOrigin{} + if d.Meta.Origin != nil { + origin = *d.Meta.Origin + } + origin.ContentHash = item.OriginalHash + d.Meta.Origin = &origin + docs = append(docs, d) + } + sum := sha256.Sum256([]byte(materialKey(docs) + "\n" + processingRevision)) + return hex.EncodeToString(sum[:]) +} +func materialSources(docs []core.ParsedDocument, perDoc int) materialSourceRegistry { + r := materialSourceRegistry{} + left := materialInlineTotal + if perDoc <= 0 { + perDoc = materialPerDocDef + } + for _, d := range docs { + cap := perDoc + if cap > left { + cap = left + } + text := strings.TrimSpace(d.PlainText()) + rr := []rune(text) + if len(rr) < cap { + cap = len(rr) + } + provided := string(rr[:cap]) + left -= cap + if d.Meta.Origin == nil || d.Meta.Origin.ArtifactID == "" || d.Meta.Origin.ContentHash == "" { + continue + } + for index, block := range d.Blocks { + full := strings.TrimSpace((core.ParsedDocument{Blocks: []core.DocBlock{block}}).PlainText()) + if full == "" || !strings.Contains(provided, full) { + continue + } + sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%d", d.Meta.Origin.ArtifactID, d.Meta.Origin.ContentHash, index))) + id := "src_" + hex.EncodeToString(sum[:12]) + r[id] = materialSource{Ref: core.TaskSourceRef{ID: id, ArtifactID: d.Meta.Origin.ArtifactID, ContentHash: d.Meta.Origin.ContentHash, ParserRevision: d.Meta.Origin.ParserRevision, BlockIndex: index, Label: fmt.Sprintf("%s · 段落%d", d.Meta.Filename, index+1), Availability: "available"}, Text: full, Provided: provided} + } + } + return r +} +func (r materialSourceRegistry) validate(q core.TaskSourceQuote) (core.TaskSourceRef, error) { + src, ok := r[q.SourceID] + if !ok || q.BlockIndex != src.Ref.BlockIndex || strings.TrimSpace(q.Quote) == "" || !strings.Contains(src.Text, q.Quote) || !strings.Contains(src.Provided, q.Quote) { + return core.TaskSourceRef{}, fmt.Errorf("checkpoint.source: quote not in provided block") + } + ref := src.Ref + ref.Quote = q.Quote + return ref, nil +} +func (r materialSourceRegistry) prompt() string { + keys := make([]string, 0, len(r)) + for k := range r { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + if len(keys) > 0 { + b.WriteString("\n\n=== 可引用的已提供来源(不含未提供的全文) ===\n") + } + for _, id := range keys { + src := r[id] + fmt.Fprintf(&b, "sourceId=%s blockIndex=%d %s\n", id, src.Ref.BlockIndex, src.Ref.Label) + } + return b.String() +} + +func (c *runCtx) materialRevision() string { + perDoc := int(c.eff.GetNumber(materialPerDocKey, materialPerDocDef)) + if perDoc <= 0 { + perDoc = materialPerDocDef + } + recipe := "material-recipe.v3" + if runner, ok := c.mgr.subRunner.(*runtime.SubLLMRunner); ok && runner.Prompts != nil { + if body, err := runner.Prompts.AgentPrompt(c.run.Owner, materialSubagentID); err == nil { + recipe += "\n" + body + } + if spec, ok := runner.Prompts.SubagentSpec(c.run.Owner, materialSubagentID); ok { + raw, _ := json.Marshal(spec) + recipe += "\n" + string(raw) + } + } + if perDoc == materialPerDocDef && recipe == "material-recipe.v3" { + return materialKey(c.attachments) + } + items := make([]materialInputItem, 0, len(c.attachments)) + for _, d := range c.attachments { + item := materialInputItem{Parsed: d} + if d.Meta.Origin != nil { + item.ArtifactID = d.Meta.Origin.ArtifactID + item.OriginalHash = d.Meta.Origin.ContentHash + } + items = append(items, item) + } + return materialInputRevision(items, fmt.Sprintf("%s\nperDoc=%d,total=%d", recipe, perDoc, materialInlineTotal)) +} diff --git a/internal/runmgr/material_sources_test.go b/internal/runmgr/material_sources_test.go new file mode 100644 index 0000000..0cd96d3 --- /dev/null +++ b/internal/runmgr/material_sources_test.go @@ -0,0 +1,53 @@ +package runmgr + +import ( + "strings" + core "agent-system-core" + "testing" + "time" +) + +func TestMaterialKeyDetectsEqualLengthEdits(t *testing.T) { + a := []core.ParsedDocument{{Meta: core.DocMeta{Filename: "决定.txt", Parser: "builtin:md"}, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}}}} + b := []core.ParsedDocument{{Meta: a[0].Meta, Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线七月"}}}} + if materialKey(a) == materialKey(b) { + t.Fatal("same length edit reused a prior material result") + } +} + +func TestMaterialSourceQuoteRequiresActualProvidedBlock(t *testing.T) { + docs:=[]core.ParsedDocument{{Meta:core.DocMeta{Filename:"决定.txt",Parser:"builtin:md",Origin:&core.DocumentOrigin{ArtifactID:"art1",ContentHash:strings.Repeat("a",64),ParserRevision:"ir1"}},Blocks:[]core.DocBlock{{Type:core.BlockPara,Text:"上线六月"},{Type:core.BlockPara,Text:"预算未定"}}}} + sources:=materialSources(docs,5) + if len(sources)==0 {t.Fatal("provided source was not registered")} + for id,src:=range sources { + if src.Ref.BlockIndex!=0 {continue} + got,err:=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:0,Quote:"上线六月"}) + if err!=nil||got.ArtifactID!="art1"{t.Fatal("real source rejected",err)} + if _,err=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:1,Quote:"上线六月"});err==nil {t.Fatal("wrong block accepted")} + if _,err=sources.validate(core.TaskSourceQuote{SourceID:id,BlockIndex:0,Quote:"上线七月"});err==nil {t.Fatal("invented quote accepted")} + } + if _,err:=sources.validate(core.TaskSourceQuote{SourceID:"foreign",Quote:"上线六月"});err==nil{t.Fatal("foreign source accepted")} +} +func TestMaterialInputRevisionIncludesProcessingVersion(t *testing.T) { + items:=[]materialInputItem{{Parsed:materialDocs()[0],OriginalHash:strings.Repeat("a",64)}} + if materialInputRevision(items,"recipe1")==materialInputRevision(items,"recipe2"){t.Fatal("processing change reused result")} +} +func TestMaterialKeyChangesWithParserAndWarnings(t *testing.T) { + a := materialDocs() + b := materialDocs() + b[0].Meta.Parser = "different-parser" + if materialKey(a) == materialKey(b) { + t.Fatal("parser version ignored") + } + b = materialDocs() + b[0].Warnings = []string{"未解析图片"} + if materialKey(a) == materialKey(b) { + t.Fatal("changed coverage warning ignored") + } + b = materialDocs() + b[0].Meta.ParsedAt = time.Now() + b[0].Meta.Origin = &core.DocumentOrigin{ArtifactID: "new-upload-id"} + if materialKey(a) != materialKey(b) { + t.Fatal("transport identity/time should not change content identity") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 3b70bb6..2cdacd7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -4,6 +4,8 @@ package server import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -124,7 +126,7 @@ func (s *Server) authorizeProjectAction(owner string, projectID core.ID, action } // authorizeSharedResource EXT-8b 资源级读判定(U9):V2 注册时按六要素咨询 -//(Actor=认证会话、Action/ResourceType/ResourceID 由调用点受信构造、 +// (Actor=认证会话、Action/ResourceType/ResourceID 由调用点受信构造、 // Owner/ProjectID 从受信存储元数据解析;出错/字段缺失一律 false,fail-closed // 且不泄露存在性);V2 未注册回落 authorizeSharedProject(现行兼容)。 func (s *Server) authorizeSharedResource(actor, action, resourceType string, resourceID, owner, projectID core.ID) bool { @@ -829,6 +831,8 @@ func (s *Server) postMessage(w http.ResponseWriter, r *http.Request) { inputs = append(inputs, runmgr.WorkspaceFile{Name: art.Name, Content: content}) } doc := ingest.Parse(art.Name, art.MIME, content) + hash := sha256.Sum256(content) + doc.Meta.Origin = &core.DocumentOrigin{ArtifactID: art.ID, ContentHash: hex.EncodeToString(hash[:]), ParserRevision: "parsed-ir.v1/" + doc.Meta.Parser} parsedDocs = append(parsedDocs, doc) parts = append(parts, core.Part{Type: core.PartArtifactRef, Artifact: &core.ArtifactRef{ArtifactID: art.ID, Name: art.Name, Kind: art.Kind}}) }