From 1fb59e12efbf06d6dee0147fc1730fb62d538694 Mon Sep 17 00:00:00 2001 From: weichen Date: Mon, 21 Sep 2026 13:40:39 +0000 Subject: [PATCH] =?UTF-8?q?feat(bizdev):=20=E6=88=90=E5=8A=9F=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E7=9A=84=E6=9D=90=E6=96=99=E4=B8=80=E5=BE=8B=E5=86=99?= =?UTF-8?q?=E5=85=A5=20material=5Fchunks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 大小阈值只决定全文注入还是目录加检索片段，不再决定是否切块。上传后的异步解析同样跳过 60KB 门槛。 Co-authored-by: Cursor --- .../2026-09-21-force-material-chunk-index.md | 48 +++++++++++++++++++ internal/bizdev/materials.go | 13 ++++- internal/bizdev/materials_index_test.go | 45 +++++++++++++++++ internal/materialindex/pipeline.go | 6 +-- 4 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 .doc/ops/short-term/2026-09-21-force-material-chunk-index.md create mode 100644 internal/bizdev/materials_index_test.go diff --git a/.doc/ops/short-term/2026-09-21-force-material-chunk-index.md b/.doc/ops/short-term/2026-09-21-force-material-chunk-index.md new file mode 100644 index 00000000..98332196 --- /dev/null +++ b/.doc/ops/short-term/2026-09-21-force-material-chunk-index.md @@ -0,0 +1,48 @@ +--- +title: "成功解析的材料一律切块入库" +status: in_progress +created: 2026-09-21 +updated: 2026-09-21 +owner: Cursor +branch: feature/force-material-chunk-index-2026-09-21 +worktree: .worktrees/force-material-chunk-index-2026-09-21 +base_dev_sha: a241e838150f218f3a09d3523f93291b847d45fd +write_scope: [internal/bizdev/materials.go, internal/bizdev/materials_index_test.go, internal/materialindex/pipeline.go, .doc/ops/CURRENT.md, .doc/test-reports/] +head_sha: +deployment: pending +cleanup: pending +mutex: bizdev+materialindex +--- + +# 成功解析的材料一律切块入库 + +## 目标 + +项目 6579 的 `doc_proj6579_bd_intake` 正式正文改为候选 535；每份成功解析的材料都写入 `material_chunks`，小文档仍全文注入。 + +## 范围 + +- `materialToItems` 在解析成功后无条件 `IndexChunks`，大小阈值只决定注入方式。 +- 上传后的 `EnqueueAsync` 同样不再用 60KB 门槛跳过切块。 + +## 非目标 + +- 不改职责矩阵，不晋升 test/main，不改 `production_ready`。 +- 不替换已过期的 MinerU 精度 token。 + +## 步骤 + +1. [x] 编制人一次性会话确认候选 535，并撤销会话 +2. [ ] 切块调用移出大小分支 +3. [ ] 用项目 6579 已上传 PDF 复核 `material_chunks` 与检索 +4. [ ] 单测与报告 + +## 验收标准 + +- [ ] 正式正文不再含「二进制压缩流」 +- [ ] `material_chunks` 对该 PDF 有行且 embedding 非空 +- [ ] `go test -short ./internal/mineru/ ./internal/materialindex/ ./internal/bizdev/` 通过 + +## 相关链接 + +- 报告：`.doc/test-reports/2026-09-21-mineru-doc-parsing-test-report.md` diff --git a/internal/bizdev/materials.go b/internal/bizdev/materials.go index 3283c54f..4641b2d7 100644 --- a/internal/bizdev/materials.go +++ b/internal/bizdev/materials.go @@ -17,7 +17,8 @@ import ( // ResolveMaterials 将请求材料 + 项目 CAS 资产解析为可注入工作区的 squad 材料。 // 项目内全部有效资产自动纳入（含私密文档）；显式附件优先并去重。 -// 已配置 MinerU 时优先使用解析缓存；大文档改为目录 + RAG 片段注入。 +// 已配置 MinerU 时优先使用解析缓存。解析成功的材料一律切块写入 material_chunks； +// 正文不超过全文阈值时仍全文注入，超过阈值时改为目录 + RAG 片段注入。 func (s *Service) ResolveMaterials(ctx context.Context, req TaskRunRequest) ([]squad.MaterialItem, map[string]any, error) { manifest := map[string]any{ "explicit_materials": len(req.Materials), @@ -173,6 +174,15 @@ func (s *Service) materialToItems( base = "material.bin" } + // 切块入库与注入方式分开：任何成功解析的正文都写入 material_chunks，失败只记日志。 + if s.materialIndex != nil { + if err := s.materialIndex.IndexChunks(ctx, projectID, blobSHA, base, text); err != nil { + slog.Warn("material chunk index failed", "file", base, "project_id", projectID, "err", err) + } else { + meta["chunks_indexed"] = true + } + } + // 小文档：全文注入，文件名改为 .md 避免 workspace 二次按扩展名刮串冲掉页锚点 if len(text) <= fullMax { outName := textMaterialName(base) @@ -191,7 +201,6 @@ func (s *Service) materialToItems( toc := materialindex.BuildTOC(base, text) ragBody := "" if s.materialIndex != nil { - _ = s.materialIndex.IndexChunks(ctx, projectID, blobSHA, base, text) hits, searchErr := s.materialIndex.Search(ctx, projectID, blobSHA, ragQuery, 0) if searchErr == nil { ragBody = materialindex.FormatRAGMarkdown(base, hits) diff --git a/internal/bizdev/materials_index_test.go b/internal/bizdev/materials_index_test.go new file mode 100644 index 00000000..c4cc4af0 --- /dev/null +++ b/internal/bizdev/materials_index_test.go @@ -0,0 +1,45 @@ +package bizdev + +import ( + "context" + "strings" + "testing" + + "chat-agent-system/internal/config" + "chat-agent-system/internal/materialindex" +) + +func TestMaterialToItems_SmallDocStillFulltextAfterIndex(t *testing.T) { + svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{}, nil)} + items, meta, err := svc.materialToItems(context.Background(), "note.txt", "abc123", []byte("能量参数是 2.5 bar"), 0, 0, 7, "能量") + if err != nil { + t.Fatal(err) + } + if meta["inject_mode"] != "fulltext" { + t.Fatalf("small doc inject_mode=%v", meta["inject_mode"]) + } + if meta["chunks_indexed"] != true { + t.Fatalf("small doc must attempt chunk index, meta=%v", meta) + } + if len(items) != 1 || !strings.Contains(string(items[0].Content), "能量参数是 2.5 bar") { + t.Fatalf("full text missing: %+v", items) + } +} + +func TestMaterialToItems_LargeDocIndexesBeforeRAGInject(t *testing.T) { + svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{MaterialFulltextMaxBytes: 8}, nil)} + body := []byte("能量参数是 2.5 bar，并且还有更多正文用来越过全文阈值。") + items, meta, err := svc.materialToItems(context.Background(), "note.txt", "def456", body, 0, 0, 7, "能量") + if err != nil { + t.Fatal(err) + } + if meta["chunks_indexed"] != true { + t.Fatalf("large doc must attempt chunk index, meta=%v", meta) + } + if meta["inject_mode"] != "rag" && meta["inject_mode"] != "rag_degraded" { + t.Fatalf("large doc must not use fulltext, inject_mode=%v", meta["inject_mode"]) + } + if len(items) != 2 { + t.Fatalf("want toc+rag items, got %d", len(items)) + } +} diff --git a/internal/materialindex/pipeline.go b/internal/materialindex/pipeline.go index 3f06639c..1fbcdad8 100644 --- a/internal/materialindex/pipeline.go +++ b/internal/materialindex/pipeline.go @@ -284,10 +284,8 @@ func (p *Pipeline) EnqueueAsync(fileName string, blobSHA string, data []byte, pr if err != nil || !ok || strings.TrimSpace(md) == "" { return } - if utf8.RuneCountInString(md) > 0 && len(md) > p.FulltextMaxBytes() { - if idxErr := p.IndexChunks(ctx, projectID, blobSHA, fileName, md); idxErr != nil { - slog.Warn("materialindex index chunks failed", "sha", blobSHA[:min(12, len(blobSHA))], "error", idxErr) - } + if idxErr := p.IndexChunks(ctx, projectID, blobSHA, fileName, md); idxErr != nil { + slog.Warn("materialindex index chunks failed", "sha", blobSHA[:min(12, len(blobSHA))], "error", idxErr) } }() return true