weichenandCursor
9ee890950a
Prefer MinerU markdown, inject fulltext or toc+rag for large docs, and wire kb.Embedder into CAS instead of nil. Co-authored-by: Cursor <cursoragent@cursor.com>
288 lines
7.9 KiB
Go
288 lines
7.9 KiB
Go
package bizdev
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"errors"
|
||
"fmt"
|
||
"path/filepath"
|
||
"strings"
|
||
|
||
"chat-agent-system/internal/materialindex"
|
||
"chat-agent-system/internal/squad"
|
||
"chat-agent-system/internal/storage"
|
||
)
|
||
|
||
// ResolveMaterials 将请求材料 + 项目 CAS 资产解析为可注入工作区的 squad 材料。
|
||
// 项目内全部有效资产自动纳入（含私密文档）；显式附件优先并去重。
|
||
// 已配置 MinerU 时优先使用解析缓存；大文档改为目录 + RAG 片段注入。
|
||
func (s *Service) ResolveMaterials(ctx context.Context, req TaskRunRequest) ([]squad.MaterialItem, map[string]any, error) {
|
||
manifest := map[string]any{
|
||
"explicit_materials": len(req.Materials),
|
||
}
|
||
byKey := make(map[string]squad.MaterialItem)
|
||
|
||
addItem := func(item squad.MaterialItem) error {
|
||
if strings.TrimSpace(item.Filename) == "" {
|
||
return errors.New("material filename required")
|
||
}
|
||
if len(item.Content) == 0 {
|
||
return fmt.Errorf("material %q has no extractable content", item.Filename)
|
||
}
|
||
key := item.Filename
|
||
if item.AssetID > 0 {
|
||
key = fmt.Sprintf("asset:%d", item.AssetID)
|
||
}
|
||
byKey[key] = item
|
||
return nil
|
||
}
|
||
|
||
ragQuery := buildMaterialRAGQuery(req)
|
||
|
||
// 1. 项目资产池（不要求用户本轮再次附加）
|
||
if s.cas != nil && req.ProjectID > 0 {
|
||
refs, err := s.cas.ListProjectRefs(ctx, req.ProjectID)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
var assetEntries []map[string]any
|
||
for _, ref := range refs {
|
||
data, loadedRef, err := s.cas.ReadRefBytes(ctx, ref.ID)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
items, meta, err := s.materialToItems(ctx, loadedRef.FileName, loadedRef.BlobSHA256, data, loadedRef.ID, loadedRef.UserID, req.ProjectID, ragQuery)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
for _, item := range items {
|
||
_ = addItem(item)
|
||
}
|
||
entry := map[string]any{
|
||
"asset_id": loadedRef.ID,
|
||
"filename": loadedRef.FileName,
|
||
"sha256": loadedRef.BlobSHA256,
|
||
"owner_user_id": loadedRef.UserID,
|
||
"file_category": loadedRef.FileCategory,
|
||
"source": "project_asset_pool",
|
||
}
|
||
for k, v := range meta {
|
||
entry[k] = v
|
||
}
|
||
assetEntries = append(assetEntries, entry)
|
||
}
|
||
manifest["project_assets"] = assetEntries
|
||
}
|
||
|
||
// 2. 显式附件（asset_id 优先，其次 inline content）
|
||
for _, m := range req.Materials {
|
||
if m.AssetID > 0 && s.cas != nil {
|
||
data, ref, err := s.cas.ReadRefBytes(ctx, m.AssetID)
|
||
if err != nil {
|
||
return nil, nil, fmt.Errorf("load attachment asset_id=%d: %w", m.AssetID, err)
|
||
}
|
||
if ref.ProjectID > 0 && req.ProjectID > 0 && ref.ProjectID != req.ProjectID {
|
||
return nil, nil, fmt.Errorf("asset_id=%d belongs to another project", m.AssetID)
|
||
}
|
||
filename := m.Filename
|
||
if filename == "" {
|
||
filename = ref.FileName
|
||
}
|
||
items, _, err := s.materialToItems(ctx, filename, ref.BlobSHA256, data, ref.ID, ref.UserID, req.ProjectID, ragQuery)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
for _, item := range items {
|
||
if err := addItem(item); err != nil {
|
||
return nil, nil, err
|
||
}
|
||
}
|
||
continue
|
||
}
|
||
|
||
filename := strings.TrimSpace(m.Filename)
|
||
if filename == "" {
|
||
return nil, nil, errors.New("material filename required when asset_id absent")
|
||
}
|
||
if len(m.Content) == 0 {
|
||
return nil, nil, fmt.Errorf("material %q has empty content and no asset_id", filename)
|
||
}
|
||
sum := sha256.Sum256(m.Content)
|
||
blobSHA := hex.EncodeToString(sum[:])
|
||
items, _, err := s.materialToItems(ctx, filepath.Base(filename), blobSHA, m.Content, 0, 0, req.ProjectID, ragQuery)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
for _, item := range items {
|
||
if err := addItem(item); err != nil {
|
||
return nil, nil, err
|
||
}
|
||
}
|
||
}
|
||
|
||
out := make([]squad.MaterialItem, 0, len(byKey))
|
||
for _, item := range byKey {
|
||
out = append(out, item)
|
||
}
|
||
manifest["resolved_count"] = len(out)
|
||
return out, manifest, nil
|
||
}
|
||
|
||
func (s *Service) materialToItems(
|
||
ctx context.Context,
|
||
fileName, blobSHA string,
|
||
data []byte,
|
||
assetID, ownerID, projectID int64,
|
||
ragQuery string,
|
||
) ([]squad.MaterialItem, map[string]any, error) {
|
||
meta := map[string]any{"parse_engine": "local"}
|
||
text := ""
|
||
fromMinerU := false
|
||
var err error
|
||
|
||
if s.materialIndex != nil {
|
||
text, fromMinerU, err = s.materialIndex.EnsureParsed(ctx, fileName, blobSHA, data)
|
||
if err != nil {
|
||
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
|
||
}
|
||
if fromMinerU {
|
||
meta["parse_engine"] = "mineru"
|
||
}
|
||
} else {
|
||
text, err = squad.ExtractTextWithAnchors(fileName, data)
|
||
if err != nil {
|
||
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
|
||
}
|
||
}
|
||
text = strings.TrimSpace(text)
|
||
if text == "" {
|
||
return nil, nil, fmt.Errorf("material %q extracted empty text", fileName)
|
||
}
|
||
|
||
fullMax := 60 * 1024
|
||
if s.materialIndex != nil {
|
||
fullMax = s.materialIndex.FulltextMaxBytes()
|
||
}
|
||
|
||
base := filepath.Base(fileName)
|
||
if base == "" || base == "." {
|
||
base = "material.bin"
|
||
}
|
||
|
||
// 小文档：全文注入，文件名改为 .md 避免 workspace 二次按扩展名刮串冲掉页锚点
|
||
if len(text) <= fullMax {
|
||
outName := textMaterialName(base)
|
||
meta["inject_mode"] = "fulltext"
|
||
return []squad.MaterialItem{{
|
||
Filename: outName,
|
||
Content: []byte(text),
|
||
AssetID: assetID,
|
||
SHA256: blobSHA,
|
||
OwnerID: ownerID,
|
||
}}, meta, nil
|
||
}
|
||
|
||
// 大文档：目录 + RAG 片段
|
||
meta["inject_mode"] = "rag"
|
||
toc := materialindex.BuildTOC(base, text)
|
||
ragBody := ""
|
||
if s.materialIndex != nil {
|
||
_ = s.materialIndex.IndexChunks(ctx, projectID, blobSHA, base, text)
|
||
hits, searchErr := s.materialIndex.Search(ctx, projectID, blobSHA, ragQuery, 0)
|
||
if searchErr == nil {
|
||
ragBody = materialindex.FormatRAGMarkdown(base, hits)
|
||
meta["rag_hits"] = len(hits)
|
||
}
|
||
}
|
||
if strings.TrimSpace(ragBody) == "" {
|
||
// 无向量服务时退回截断全文前缀，避免任务完全无材料
|
||
prefix := text
|
||
if len(prefix) > fullMax {
|
||
prefix = prefix[:fullMax] + "\n\n…（文档过长，已截断；配置 Embedding 后可启用分块检索）\n"
|
||
}
|
||
ragBody = "# 材料摘要（降级）\n\n" + prefix
|
||
meta["inject_mode"] = "rag_degraded"
|
||
}
|
||
|
||
stem := strings.TrimSuffix(base, filepath.Ext(base))
|
||
if stem == "" {
|
||
stem = "material"
|
||
}
|
||
return []squad.MaterialItem{
|
||
{
|
||
Filename: stem + ".toc.md",
|
||
Content: []byte(toc),
|
||
AssetID: assetID,
|
||
SHA256: blobSHA,
|
||
OwnerID: ownerID,
|
||
},
|
||
{
|
||
Filename: stem + ".rag.md",
|
||
Content: []byte(ragBody),
|
||
AssetID: assetID,
|
||
SHA256: blobSHA,
|
||
OwnerID: ownerID,
|
||
},
|
||
}, meta, nil
|
||
}
|
||
|
||
func textMaterialName(base string) string {
|
||
ext := strings.ToLower(filepath.Ext(base))
|
||
switch ext {
|
||
case ".txt", ".md", ".json", ".yaml", ".yml", ".csv", ".log":
|
||
return base
|
||
default:
|
||
return base + ".md"
|
||
}
|
||
}
|
||
|
||
func buildMaterialRAGQuery(req TaskRunRequest) string {
|
||
parts := []string{req.TaskKey, req.Title, req.ProjectName, req.DeviceType, req.IntendedUse, req.CustomerName}
|
||
if req.Fields != nil {
|
||
for _, k := range []string{"project_name", "device_type", "intended_use", "customer_name"} {
|
||
if v := strings.TrimSpace(req.Fields[k]); v != "" {
|
||
parts = append(parts, v)
|
||
}
|
||
}
|
||
}
|
||
var b strings.Builder
|
||
for _, p := range parts {
|
||
p = strings.TrimSpace(p)
|
||
if p == "" {
|
||
continue
|
||
}
|
||
if b.Len() > 0 {
|
||
b.WriteByte(' ')
|
||
}
|
||
b.WriteString(p)
|
||
}
|
||
q := b.String()
|
||
if q == "" {
|
||
return "项目材料 关键需求 技术方案参数"
|
||
}
|
||
return q
|
||
}
|
||
|
||
// ValidateTaskRunIdentity 拒绝缺失项目/会话标识的任务启动。
|
||
func ValidateTaskRunIdentity(sessionID string, projectID int64) error {
|
||
if strings.TrimSpace(sessionID) == "" {
|
||
return errors.New("session_id required for task run")
|
||
}
|
||
if projectID <= 0 {
|
||
return errors.New("project_id required for task run")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// MaterialInputFromRef 由 CAS 引用构造任务材料输入（供上传 API 测试）。
|
||
func MaterialInputFromRef(ref *storage.FileReference) MaterialInput {
|
||
if ref == nil {
|
||
return MaterialInput{}
|
||
}
|
||
return MaterialInput{
|
||
Filename: ref.FileName,
|
||
AssetID: ref.ID,
|
||
}
|
||
}
|