feat(bizdev): 成功解析的材料一律写入 material_chunks
大小阈值只决定全文注入还是目录加检索片段,不再决定是否切块。上传后的异步解析同样跳过 60KB 门槛。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,48 @@
|
|||||||
|
---
|
||||||
|
title: "成功解析的材料一律切块入库"
|
||||||
|
status: in_progress
|
||||||
|
created: 2026-09-21
|
||||||
|
updated: 2026-09-21
|
||||||
|
owner: Cursor
|
||||||
|
branch: feature/force-material-chunk-index-2026-09-21
|
||||||
|
worktree: .worktrees/force-material-chunk-index-2026-09-21
|
||||||
|
base_dev_sha: a241e838150f218f3a09d3523f93291b847d45fd
|
||||||
|
write_scope: [internal/bizdev/materials.go, internal/bizdev/materials_index_test.go, internal/materialindex/pipeline.go, .doc/ops/CURRENT.md, .doc/test-reports/]
|
||||||
|
head_sha:
|
||||||
|
deployment: pending
|
||||||
|
cleanup: pending
|
||||||
|
mutex: bizdev+materialindex
|
||||||
|
---
|
||||||
|
|
||||||
|
# 成功解析的材料一律切块入库
|
||||||
|
|
||||||
|
## 目标
|
||||||
|
|
||||||
|
项目 6579 的 `doc_proj6579_bd_intake` 正式正文改为候选 535;每份成功解析的材料都写入 `material_chunks`,小文档仍全文注入。
|
||||||
|
|
||||||
|
## 范围
|
||||||
|
|
||||||
|
- `materialToItems` 在解析成功后无条件 `IndexChunks`,大小阈值只决定注入方式。
|
||||||
|
- 上传后的 `EnqueueAsync` 同样不再用 60KB 门槛跳过切块。
|
||||||
|
|
||||||
|
## 非目标
|
||||||
|
|
||||||
|
- 不改职责矩阵,不晋升 test/main,不改 `production_ready`。
|
||||||
|
- 不替换已过期的 MinerU 精度 token。
|
||||||
|
|
||||||
|
## 步骤
|
||||||
|
|
||||||
|
1. [x] 编制人一次性会话确认候选 535,并撤销会话
|
||||||
|
2. [ ] 切块调用移出大小分支
|
||||||
|
3. [ ] 用项目 6579 已上传 PDF 复核 `material_chunks` 与检索
|
||||||
|
4. [ ] 单测与报告
|
||||||
|
|
||||||
|
## 验收标准
|
||||||
|
|
||||||
|
- [ ] 正式正文不再含「二进制压缩流」
|
||||||
|
- [ ] `material_chunks` 对该 PDF 有行且 embedding 非空
|
||||||
|
- [ ] `go test -short ./internal/mineru/ ./internal/materialindex/ ./internal/bizdev/` 通过
|
||||||
|
|
||||||
|
## 相关链接
|
||||||
|
|
||||||
|
- 报告:`.doc/test-reports/2026-09-21-mineru-doc-parsing-test-report.md`
|
||||||
@@ -17,7 +17,8 @@ import (
|
|||||||
|
|
||||||
// ResolveMaterials 将请求材料 + 项目 CAS 资产解析为可注入工作区的 squad 材料。
|
// ResolveMaterials 将请求材料 + 项目 CAS 资产解析为可注入工作区的 squad 材料。
|
||||||
// 项目内全部有效资产自动纳入(含私密文档);显式附件优先并去重。
|
// 项目内全部有效资产自动纳入(含私密文档);显式附件优先并去重。
|
||||||
// 已配置 MinerU 时优先使用解析缓存;大文档改为目录 + RAG 片段注入。
|
// 已配置 MinerU 时优先使用解析缓存。解析成功的材料一律切块写入 material_chunks;
|
||||||
|
// 正文不超过全文阈值时仍全文注入,超过阈值时改为目录 + RAG 片段注入。
|
||||||
func (s *Service) ResolveMaterials(ctx context.Context, req TaskRunRequest) ([]squad.MaterialItem, map[string]any, error) {
|
func (s *Service) ResolveMaterials(ctx context.Context, req TaskRunRequest) ([]squad.MaterialItem, map[string]any, error) {
|
||||||
manifest := map[string]any{
|
manifest := map[string]any{
|
||||||
"explicit_materials": len(req.Materials),
|
"explicit_materials": len(req.Materials),
|
||||||
@@ -173,6 +174,15 @@ func (s *Service) materialToItems(
|
|||||||
base = "material.bin"
|
base = "material.bin"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 切块入库与注入方式分开:任何成功解析的正文都写入 material_chunks,失败只记日志。
|
||||||
|
if s.materialIndex != nil {
|
||||||
|
if err := s.materialIndex.IndexChunks(ctx, projectID, blobSHA, base, text); err != nil {
|
||||||
|
slog.Warn("material chunk index failed", "file", base, "project_id", projectID, "err", err)
|
||||||
|
} else {
|
||||||
|
meta["chunks_indexed"] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 小文档:全文注入,文件名改为 .md 避免 workspace 二次按扩展名刮串冲掉页锚点
|
// 小文档:全文注入,文件名改为 .md 避免 workspace 二次按扩展名刮串冲掉页锚点
|
||||||
if len(text) <= fullMax {
|
if len(text) <= fullMax {
|
||||||
outName := textMaterialName(base)
|
outName := textMaterialName(base)
|
||||||
@@ -191,7 +201,6 @@ func (s *Service) materialToItems(
|
|||||||
toc := materialindex.BuildTOC(base, text)
|
toc := materialindex.BuildTOC(base, text)
|
||||||
ragBody := ""
|
ragBody := ""
|
||||||
if s.materialIndex != nil {
|
if s.materialIndex != nil {
|
||||||
_ = s.materialIndex.IndexChunks(ctx, projectID, blobSHA, base, text)
|
|
||||||
hits, searchErr := s.materialIndex.Search(ctx, projectID, blobSHA, ragQuery, 0)
|
hits, searchErr := s.materialIndex.Search(ctx, projectID, blobSHA, ragQuery, 0)
|
||||||
if searchErr == nil {
|
if searchErr == nil {
|
||||||
ragBody = materialindex.FormatRAGMarkdown(base, hits)
|
ragBody = materialindex.FormatRAGMarkdown(base, hits)
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package bizdev
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"chat-agent-system/internal/config"
|
||||||
|
"chat-agent-system/internal/materialindex"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMaterialToItems_SmallDocStillFulltextAfterIndex(t *testing.T) {
|
||||||
|
svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{}, nil)}
|
||||||
|
items, meta, err := svc.materialToItems(context.Background(), "note.txt", "abc123", []byte("能量参数是 2.5 bar"), 0, 0, 7, "能量")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if meta["inject_mode"] != "fulltext" {
|
||||||
|
t.Fatalf("small doc inject_mode=%v", meta["inject_mode"])
|
||||||
|
}
|
||||||
|
if meta["chunks_indexed"] != true {
|
||||||
|
t.Fatalf("small doc must attempt chunk index, meta=%v", meta)
|
||||||
|
}
|
||||||
|
if len(items) != 1 || !strings.Contains(string(items[0].Content), "能量参数是 2.5 bar") {
|
||||||
|
t.Fatalf("full text missing: %+v", items)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaterialToItems_LargeDocIndexesBeforeRAGInject(t *testing.T) {
|
||||||
|
svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{MaterialFulltextMaxBytes: 8}, nil)}
|
||||||
|
body := []byte("能量参数是 2.5 bar,并且还有更多正文用来越过全文阈值。")
|
||||||
|
items, meta, err := svc.materialToItems(context.Background(), "note.txt", "def456", body, 0, 0, 7, "能量")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if meta["chunks_indexed"] != true {
|
||||||
|
t.Fatalf("large doc must attempt chunk index, meta=%v", meta)
|
||||||
|
}
|
||||||
|
if meta["inject_mode"] != "rag" && meta["inject_mode"] != "rag_degraded" {
|
||||||
|
t.Fatalf("large doc must not use fulltext, inject_mode=%v", meta["inject_mode"])
|
||||||
|
}
|
||||||
|
if len(items) != 2 {
|
||||||
|
t.Fatalf("want toc+rag items, got %d", len(items))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -284,10 +284,8 @@ func (p *Pipeline) EnqueueAsync(fileName string, blobSHA string, data []byte, pr
|
|||||||
if err != nil || !ok || strings.TrimSpace(md) == "" {
|
if err != nil || !ok || strings.TrimSpace(md) == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if utf8.RuneCountInString(md) > 0 && len(md) > p.FulltextMaxBytes() {
|
if idxErr := p.IndexChunks(ctx, projectID, blobSHA, fileName, md); idxErr != nil {
|
||||||
if idxErr := p.IndexChunks(ctx, projectID, blobSHA, fileName, md); idxErr != nil {
|
slog.Warn("materialindex index chunks failed", "sha", blobSHA[:min(12, len(blobSHA))], "error", idxErr)
|
||||||
slog.Warn("materialindex index chunks failed", "sha", blobSHA[:min(12, len(blobSHA))], "error", idxErr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
return true
|
return true
|
||||||
|
|||||||
Reference in New Issue
Block a user