From 495e72adacee8beac4b8f3bc041beb8caf2ec26a Mon Sep 17 00:00:00 2001 From: weichen Date: Wed, 14 Oct 2026 02:24:47 +0000 Subject: [PATCH] Scope material search fallback to current attachments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: combined filename and Chinese-keyword queries returned no hits even when the uploaded material was present, delaying useful background findings. Evidence: combined-query, owner/project, duplicate, resource-policy, version-drift and excerpt RED→GREEN tests; SQLite and isolated PostgreSQL pass; Backend full vet and tests pass with the Agent-Native pin. Impact: only a no-hit material subagent query may read a verified current-task attachment snapshot; general knowledge retrieval remains unchanged. Refs: U3 r0.4 design; internal/runmgr/material_retrieve.go --- internal/runmgr/material_retrieve.go | 147 ++++++++++++++++++++++ internal/runmgr/material_retrieve_test.go | 133 ++++++++++++++++++++ internal/runmgr/runmgr.go | 14 ++- 3 files changed, 289 insertions(+), 5 deletions(-) create mode 100644 internal/runmgr/material_retrieve.go create mode 100644 internal/runmgr/material_retrieve_test.go diff --git a/internal/runmgr/material_retrieve.go b/internal/runmgr/material_retrieve.go new file mode 100644 index 0000000..0d680fd --- /dev/null +++ b/internal/runmgr/material_retrieve.go @@ -0,0 +1,147 @@ +package runmgr + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + + core "agent-system-core" +) + +// materialRetrievalResult keeps ordinary retrieval authoritative. A no-hit +// material child may fall back only to the attachments frozen for this task. +func (c *runCtx) materialRetrievalResult(q core.RetrievalQuery, hits []core.RetrievalHit, err error) ([]core.RetrievalHit, error) { + if err != nil || len(hits) != 0 || c.subAgent != materialSubagentID { + return hits, err + } + return c.materialAttachmentFallback(q) +} + +func (c *runCtx) materialAttachmentFallback(q core.RetrievalQuery) ([]core.RetrievalHit, error) { + if c.task.cardID == "" || c.run.Owner == "" || c.run.ProjectID == "" || + q.ProjectID != c.run.ProjectID || strings.TrimSpace(q.Text) == "" { + return nil, nil + } + if len(q.Sources) > 0 { + allowed := false + for _, source := range q.Sources { + allowed = allowed || source == "upload" + } + if !allowed { + return nil, nil + } + } + var matched *core.ParsedDocument + for i := range c.attachments { + name := strings.TrimSpace(c.attachments[i].Meta.Filename) + if name == "" || !strings.Contains(q.Text, name) { + continue + } + if matched != nil { + return nil, nil // duplicate or multiple filenames: never guess + } + matched = &c.attachments[i] + } + if matched == nil || matched.Meta.Origin == nil { + return nil, nil + } + origin := matched.Meta.Origin + if origin.ArtifactID == "" || origin.ContentHash == "" || origin.ParserRevision == "" { + return nil, nil + } + card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID) + if err != nil { + return nil, err + } + s := card.Task + if s == nil || s.Terminal() || s.TaskID != c.task.taskID || + s.AttemptID != c.task.attemptID || s.RunID != c.run.ID || + s.OwnerInstanceID != c.task.instanceID || s.ExecutionEpoch != c.task.epoch { + return nil, nil + } + bound := 0 + for _, input := range s.Inputs { + if input.ArtifactID == string(origin.ArtifactID) && input.ContentHash == origin.ContentHash && + input.ParserRevision == origin.ParserRevision { + bound++ + } + } + if bound != 1 { + return nil, nil + } + a, err := c.mgr.st.GetArtifact(c.run.Owner, string(origin.ArtifactID)) + if err != nil { + return nil, err + } + if a.ProjectID != c.run.ProjectID || a.Name != matched.Meta.Filename { + return nil, nil + } + data, err := c.mgr.st.ArtifactContent(c.run.Owner, string(origin.ArtifactID)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if hex.EncodeToString(sum[:]) != origin.ContentHash { + return nil, nil + } + after, err := c.mgr.st.GetArtifact(c.run.Owner, string(origin.ArtifactID)) + if err != nil || !after.UpdatedAt.Equal(a.UpdatedAt) { + return nil, err + } + full := strings.TrimSpace(matched.PlainText()) + if full == "" { + return nil, nil + } + limit := c.prof.Context.MaxToolResultChars + if limit <= 0 { + limit = int(c.eff.GetNumber("context.maxToolResultChars", 20000)) + } + if limit <= 0 || limit > 4000 { + limit = 4000 + } + doc := core.KnowledgeDoc{Owner: c.run.Owner, ProjectID: c.run.ProjectID, + Source: "upload", Title: matched.Meta.Filename, Ref: string(origin.ArtifactID), + Chunk: materialExcerpt(full, q.Text, matched.Meta.Filename, c.attachments, origin.ArtifactID, c.eff, limit), + CreatedAt: a.CreatedAt} + if c.mgr.knowledgeResourceAllow != nil && !c.mgr.knowledgeResourceAllow(string(c.run.Owner), doc) { + return nil, nil + } + return []core.RetrievalHit{{Doc: doc, Score: 1, Kind: "fulltext"}}, nil +} + +func materialExcerpt(full, query, filename string, docs []core.ParsedDocument, id core.ID, eff core.EffectiveSettings, limit int) string { + runes := []rune(full) + start := 0 + matchedTerm := false + terms := strings.Fields(strings.TrimSpace(strings.Replace(query, filename, "", 1))) + for _, term := range terms { + if at := strings.Index(full, term); at >= 0 { + matchedTerm = true + start = len([]rune(full[:at])) - 80 + if start < 0 { + start = 0 + } + break + } + } + if !matchedTerm { + perDoc := int(eff.GetNumber(materialPerDocKey, materialPerDocDef)) + if perDoc <= 0 { + perDoc = materialPerDocDef + } + left := materialInlineTotal + for _, d := range docs { + count := len([]rune(strings.TrimSpace(d.PlainText()))) + provided := min(perDoc, left, count) + if d.Meta.Origin != nil && d.Meta.Origin.ArtifactID == id { + if provided < len(runes) { + start = provided + } + break + } + left -= provided + } + } + end := min(start+limit, len(runes)) + return string(runes[start:end]) +} diff --git a/internal/runmgr/material_retrieve_test.go b/internal/runmgr/material_retrieve_test.go new file mode 100644 index 0000000..83ab803 --- /dev/null +++ b/internal/runmgr/material_retrieve_test.go @@ -0,0 +1,133 @@ +package runmgr + +import ( + "strings" + "testing" + + core "agent-system-core" +) + +func materialRetrievalContext(t *testing.T) *runCtx { + t.Helper() + c := checkpointEnvironment(t) + card, err := c.mgr.st.GetCard(c.run.Owner, c.task.cardID) + if err != nil || card.Task == nil || len(card.Task.Inputs) != 1 { + t.Fatalf("trusted material input missing: card=%+v err=%v", card, err) + } + input := card.Task.Inputs[0] + c.attachments = []core.ParsedDocument{{ + Meta: core.DocMeta{Filename: "决定.txt", Parser: "builtin:txt", Origin: &core.DocumentOrigin{ + ArtifactID: core.ID(input.ArtifactID), ContentHash: input.ContentHash, ParserRevision: input.ParserRevision, + }}, + Blocks: []core.DocBlock{{Type: core.BlockPara, Text: "上线六月"}}, + }} + return c +} + +func TestMaterialAttachmentFallbackQuery(t *testing.T) { + c := materialRetrievalContext(t) + hits, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if err != nil || len(hits) != 1 { + t.Fatalf("current material was not recovered from a combined query: hits=%+v err=%v", hits, err) + } + hit := hits[0].Doc + if hit.Ref != string(c.attachments[0].Meta.Origin.ArtifactID) || + hit.ProjectID != c.run.ProjectID || hit.Owner != c.run.Owner || + hit.Title != "决定.txt" || !strings.Contains(hit.Chunk, "上线六月") { + t.Fatalf("fallback lost the current attachment identity or text: %+v", hit) + } +} + +func TestMaterialAttachmentFallbackScope(t *testing.T) { + for _, tc := range []struct { + name string + change func(*runCtx) + }{ + {"other-subagent", func(c *runCtx) { c.subAgent = "agent.sub.general" }}, + {"other-owner", func(c *runCtx) { c.run.Owner = "another-owner" }}, + {"other-project", func(c *runCtx) { c.run.ProjectID = "another-project" }}, + {"duplicate-name", func(c *runCtx) { + other := c.attachments[0] + origin := *other.Meta.Origin + origin.ArtifactID = "art_other" + other.Meta.Origin = &origin + c.attachments = append(c.attachments, other) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + c := materialRetrievalContext(t) + before, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if err != nil || len(before) != 1 { + t.Fatalf("control material was unavailable before scope change: hits=%+v err=%v", before, err) + } + tc.change(c) + hits, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if len(hits) != 0 { + t.Fatalf("out-of-scope attachment leaked: hits=%+v err=%v", hits, err) + } + }) + } +} + +func TestMaterialAttachmentFallbackVersionDrift(t *testing.T) { + for _, tc := range []struct { + name string + change func(*testing.T, *runCtx) + }{ + {"changed-content", func(t *testing.T, c *runCtx) { + t.Helper() + if err := c.mgr.st.UpdateArtifactContent(string(c.run.Owner), string(c.attachments[0].Meta.Origin.ArtifactID), []byte("上线七月")); err != nil { + t.Fatal(err) + } + }}, + {"changed-parser-revision", func(_ *testing.T, c *runCtx) { + c.attachments[0].Meta.Origin.ParserRevision = "different-parser" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + c := materialRetrievalContext(t) + before, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if err != nil || len(before) != 1 { + t.Fatalf("control material was unavailable before version change: hits=%+v err=%v", before, err) + } + tc.change(t, c) + hits, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if len(hits) != 0 { + t.Fatalf("stale material was returned: hits=%+v err=%v", hits, err) + } + }) + } +} + +func TestMaterialAttachmentFallbackRespectsResourceAllow(t *testing.T) { + c := materialRetrievalContext(t) + c.mgr.SetKnowledgeResourceAllow(func(owner string, doc core.KnowledgeDoc) bool { + return owner == string(c.run.Owner) && doc.Ref == string(c.attachments[0].Meta.Origin.ArtifactID) + }) + allowed, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if err != nil || len(allowed) != 1 { + t.Fatalf("authorized current material was unavailable: hits=%+v err=%v", allowed, err) + } + c.mgr.SetKnowledgeResourceAllow(func(string, core.KnowledgeDoc) bool { return false }) + denied, err := c.Retrieve(core.RetrievalQuery{Text: "决定.txt 上线 日期"}) + if err != nil || len(denied) != 0 { + t.Fatalf("resource policy was bypassed by fallback: hits=%+v err=%v", denied, err) + } +} + +func TestMaterialExcerptKeepsMatchedBeginning(t *testing.T) { + full := "上线六月" + strings.Repeat("附", 20) + "后续结论" + docs := []core.ParsedDocument{{ + Meta: core.DocMeta{Filename: "长文.txt", Origin: &core.DocumentOrigin{ArtifactID: "art_long"}}, + Blocks: []core.DocBlock{{Type: core.BlockPara, Text: full}}, + }} + eff := core.EffectiveSettings{"context": map[string]any{"maxAttachmentChars": float64(4)}} + got := materialExcerpt(full, "长文.txt 上线", "长文.txt", docs, "art_long", eff, 10) + if want := "上线六月" + strings.Repeat("附", 6); got != want { + t.Fatalf("matched start lost or exceeded the result limit: got %q want %q", got, want) + } + got = materialExcerpt(full, "长文.txt", "长文.txt", docs, "art_long", eff, 10) + if want := strings.Repeat("附", 10); got != want { + t.Fatalf("filename-only retrieval did not start after the inline prefix: got %q want %q", got, want) + } +} diff --git a/internal/runmgr/runmgr.go b/internal/runmgr/runmgr.go index 63b74be..26acbd5 100644 --- a/internal/runmgr/runmgr.go +++ b/internal/runmgr/runmgr.go @@ -982,7 +982,8 @@ func (c *runCtx) Retrieve(q core.RetrievalQuery) ([]core.RetrievalHit, error) { if rs, ok := c.mgr.retriever.(interface { RetrieveAuthorized(context.Context, core.ID, core.RetrievalQuery, func(core.KnowledgeDoc) bool) ([]core.RetrievalHit, error) }); ok { - return rs.RetrieveAuthorized(c.ctx, c.run.Owner, q, allow) + hits, err := rs.RetrieveAuthorized(c.ctx, c.run.Owner, q, allow) + return c.materialRetrievalResult(q, hits, err) } hits, err := c.mgr.st.SearchKnowledge(c.run.Owner, q) if err != nil { @@ -1002,19 +1003,22 @@ func (c *runCtx) Retrieve(q core.RetrievalQuery) ([]core.RetrievalHit, error) { if len(out) > q.TopK { out = out[:q.TopK] } - return out, nil + return c.materialRetrievalResult(q, out, nil) } if c.mgr.retriever != nil { if c.mgr.sharedKnowledgeAllow != nil { if rs, ok := c.mgr.retriever.(sharedRetriever); ok { - return rs.RetrieveShared(c.ctx, c.run.Owner, q, func(docOwner, projectID core.ID) bool { + hits, err := rs.RetrieveShared(c.ctx, c.run.Owner, q, func(docOwner, projectID core.ID) bool { return c.mgr.sharedKnowledgeAllow(string(c.run.Owner), projectID) }) + return c.materialRetrievalResult(q, hits, err) } } - return c.mgr.retriever.Retrieve(c.ctx, c.run.Owner, q) + hits, err := c.mgr.retriever.Retrieve(c.ctx, c.run.Owner, q) + return c.materialRetrievalResult(q, hits, err) } - return c.mgr.st.SearchKnowledge(c.run.Owner, q) + hits, err := c.mgr.st.SearchKnowledge(c.run.Owner, q) + return c.materialRetrievalResult(q, hits, err) } // ExecuteTool 执行画像策略：deny→错误；confirm→HITL confirm 卡片；执行→截断转工件。