diff --git a/.doc/system-design/module-boundaries.yaml b/.doc/system-design/module-boundaries.yaml index 26828d7b..eed60736 100644 --- a/.doc/system-design/module-boundaries.yaml +++ b/.doc/system-design/module-boundaries.yaml @@ -27,8 +27,8 @@ imports: - docduty bizdev: - agent - - authz - app/uow + - authz - config - conflictresolve - contracts/vocabulary @@ -50,6 +50,7 @@ imports: - squad - storage - stream + - textextract - workflowdef - workspace chat: @@ -214,6 +215,7 @@ imports: - kb - mineru - squad + - textextract maturity: [] mineru: [] notification: [] @@ -272,23 +274,25 @@ imports: - storage - stream - stream/transport + - supplementimpact - swboot - sysadmin - taskrun + - textextract - traceimpact - upload - workflowdef - workflowscope - workspace - - supplementimpact squad: - agent - config - quote - stream - template - - workspace + - textextract - workflowdef + - workspace storage: - config stream: @@ -306,6 +310,7 @@ imports: - projectworkflow - stream - workspace + textextract: [] template: - workflowdef upload: [] diff --git a/internal/bizdev/materials.go b/internal/bizdev/materials.go index 4641b2d7..9ee75d69 100644 --- a/internal/bizdev/materials.go +++ b/internal/bizdev/materials.go @@ -12,6 +12,7 @@ import ( "chat-agent-system/internal/materialindex" "chat-agent-system/internal/squad" + "chat-agent-system/internal/textextract" "chat-agent-system/internal/storage" ) @@ -154,7 +155,7 @@ func (s *Service) materialToItems( meta["parse_engine"] = "mineru" } } else { - text, err = squad.ExtractTextWithAnchors(fileName, data) + text, err = textextract.ExtractTextWithAnchors(fileName, data) if err != nil { return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err) } diff --git a/internal/materialindex/pipeline.go b/internal/materialindex/pipeline.go index aa7d34bb..e0485324 100644 --- a/internal/materialindex/pipeline.go +++ b/internal/materialindex/pipeline.go @@ -18,7 +18,7 @@ import ( "chat-agent-system/internal/config" "chat-agent-system/internal/kb" "chat-agent-system/internal/mineru" - "chat-agent-system/internal/squad" + "chat-agent-system/internal/textextract" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -233,7 +233,7 @@ func (p *Pipeline) ResetForReparse(ctx context.Context, blobSHA string) bool { // fallback text from squad.ExtractTextWithAnchors (may be empty). func (p *Pipeline) EnsureParsed(ctx context.Context, fileName string, blobSHA string, data []byte) (markdown string, fromMinerU bool, err error) { fallback := func() (string, bool, error) { - txt, e := squad.ExtractTextWithAnchors(fileName, data) + txt, e := textextract.ExtractTextWithAnchors(fileName, data) return strings.TrimSpace(txt), false, e } if p == nil { diff --git a/internal/server/chat_document_reader.go b/internal/server/chat_document_reader.go index d46fd3a3..eda843a9 100644 --- a/internal/server/chat_document_reader.go +++ b/internal/server/chat_document_reader.go @@ -15,7 +15,7 @@ import ( "chat-agent-system/internal/docgov" "chat-agent-system/internal/intent" "chat-agent-system/internal/projectinstance" - "chat-agent-system/internal/squad" + "chat-agent-system/internal/textextract" "chat-agent-system/internal/storage" "chat-agent-system/internal/stream" "chat-agent-system/internal/workflowscope" @@ -179,7 +179,7 @@ func (s *Server) resolveLatestUploadDocument(ctx context.Context, projectID int6 } markdown = strings.TrimSpace(md) } else { - md, extractErr := squad.ExtractTextWithAnchors(ref.FileName, data) + md, extractErr := textextract.ExtractTextWithAnchors(ref.FileName, data) if extractErr != nil { return nil, extractErr } diff --git a/internal/squad/context.go b/internal/squad/context.go index 8158341f..6351a359 100644 --- a/internal/squad/context.go +++ b/internal/squad/context.go @@ -1,6 +1,7 @@ package squad import ( + "chat-agent-system/internal/textextract" "encoding/json" "fmt" "os" @@ -145,7 +146,7 @@ func (cm *ContextManager) InjectMaterials(wsDir string, materials []MaterialItem name = "material.txt" } - textContent, err := ExtractTextWithAnchors(name, m.Content) + textContent, err := textextract.ExtractTextWithAnchors(name, m.Content) if err != nil || strings.TrimSpace(textContent) == "" { textContent = string(m.Content) } diff --git a/internal/squad/extract.go b/internal/squad/extract.go index 0752cc9e..64f25d64 100644 --- a/internal/squad/extract.go +++ b/internal/squad/extract.go @@ -1,183 +1,12 @@ package squad import ( - "archive/zip" - "bytes" - "encoding/xml" "fmt" - "io" - "path/filepath" "strings" - "unicode/utf8" "chat-agent-system/internal/workflowdef" ) -// ExtractTextWithAnchors 从不同格式材料中提取纯文本，并自动注入物理页码锚点 -func ExtractTextWithAnchors(filename string, data []byte) (string, error) { - if len(data) == 0 { - return "", nil - } - - ext := strings.ToLower(filepath.Ext(filename)) - switch ext { - case ".txt", ".md", ".json", ".yaml", ".yml": - return string(data), nil - case ".docx": - txt, err := extractTextFromDOCX(data) - if err == nil && len(strings.TrimSpace(txt)) > 0 { - return injectPageAnchorsForText(txt), nil - } - // 降级：从二进制中提取可见字符 - return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil - default: - // 通用降级 - return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil - } -} - -// injectPageAnchorsForText 将连续文本按约 1200 字符切分，并注入 锚点 -func injectPageAnchorsForText(raw string) string { - raw = strings.TrimSpace(raw) - if raw == "" { - return "" - } - - // 若原始文本中包含换页符 \f - if strings.Contains(raw, "\f") { - pages := strings.Split(raw, "\f") - var sb strings.Builder - for i, page := range pages { - pTrim := strings.TrimSpace(page) - if pTrim == "" { - continue - } - sb.WriteString(fmt.Sprintf("\n\n%s\n", i+1, pTrim)) - } - return strings.TrimSpace(sb.String()) - } - - // 否则按段落累加约 1200 字符为一页 - paragraphs := strings.Split(raw, "\n") - var sb strings.Builder - currentPage := 1 - currentLen := 0 - - sb.WriteString(fmt.Sprintf("\n", currentPage)) - for _, p := range paragraphs { - pTrim := strings.TrimRight(p, "\r\t ") - pLen := utf8.RuneCountInString(pTrim) - if currentLen > 0 && currentLen+pLen > 1200 { - currentPage++ - sb.WriteString(fmt.Sprintf("\n\n", currentPage)) - currentLen = 0 - } - sb.WriteString(pTrim) - sb.WriteString("\n") - currentLen += pLen + 1 - } - - return strings.TrimSpace(sb.String()) -} - -// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本 -func extractTextFromDOCX(data []byte) (string, error) { - r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) - if err != nil { - return "", err - } - - var docFile *zip.File - for _, f := range r.File { - if f.Name == "word/document.xml" { - docFile = f - break - } - } - if docFile == nil { - return "", fmt.Errorf("word/document.xml not found in docx") - } - - rc, err := docFile.Open() - if err != nil { - return "", err - } - defer rc.Close() - - xmlBytes, err := io.ReadAll(rc) - if err != nil { - return "", err - } - - decoder := xml.NewDecoder(bytes.NewReader(xmlBytes)) - var sb strings.Builder - inParagraph := false - - for { - tok, err := decoder.Token() - if err != nil { - break - } - switch elem := tok.(type) { - case xml.StartElement: - if elem.Name.Local == "p" { - inParagraph = true - } else if elem.Name.Local == "br" { - for _, attr := range elem.Attr { - if attr.Name.Local == "type" && attr.Value == "page" { - sb.WriteString("\f") - } - } - } else if elem.Name.Local == "lastRenderedPageBreak" { - sb.WriteString("\f") - } - case xml.EndElement: - if elem.Name.Local == "p" { - if inParagraph { - sb.WriteString("\n") - inParagraph = false - } - } - case xml.CharData: - sb.Write(elem) - } - } - - return sb.String(), nil -} - -// extractAsciiUtf8Strings 降级算法：从二进制流中提取连续的有效可读 UTF-8 文本 -func extractAsciiUtf8Strings(data []byte) string { - var sb strings.Builder - var cur bytes.Buffer - - flush := func() { - if cur.Len() >= 4 { - s := cur.String() - s = strings.TrimSpace(s) - if s != "" { - sb.WriteString(s) - sb.WriteString("\n") - } - } - cur.Reset() - } - - for i := 0; i < len(data); { - r, size := utf8.DecodeRune(data[i:]) - if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') { - cur.WriteRune(r) - i += size - } else { - flush() - i++ - } - } - flush() - - return sb.String() -} - // ExtractAdvisorBriefing 从 output.md 与黑板事实中提取 B 轨业务参谋决策锦囊 func ExtractAdvisorBriefing(taskKey string, markdownBody string, fields map[string]string) *AdvisorBriefing { if fields == nil { diff --git a/internal/squad/workspace_context_build.go b/internal/squad/workspace_context_build.go index 487908b4..e256bc06 100644 --- a/internal/squad/workspace_context_build.go +++ b/internal/squad/workspace_context_build.go @@ -1,6 +1,7 @@ package squad import ( + "chat-agent-system/internal/textextract" "fmt" "os" "path/filepath" @@ -68,7 +69,7 @@ func (e *WorkspaceEngine) buildContextSystem(workDir string, req WorkspacePrepar } // 自动提取纯文本并注入页码锚点 - textContent, err := ExtractTextWithAnchors(name, m.Content) + textContent, err := textextract.ExtractTextWithAnchors(name, m.Content) if err != nil || strings.TrimSpace(textContent) == "" { textContent = string(m.Content) } diff --git a/internal/textextract/extract.go b/internal/textextract/extract.go new file mode 100644 index 00000000..b5dcef68 --- /dev/null +++ b/internal/textextract/extract.go @@ -0,0 +1,178 @@ +package textextract + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path/filepath" + "strings" + "unicode/utf8" + +) + +// ExtractTextWithAnchors 从不同格式材料中提取纯文本，并自动注入物理页码锚点 +func ExtractTextWithAnchors(filename string, data []byte) (string, error) { + if len(data) == 0 { + return "", nil + } + + ext := strings.ToLower(filepath.Ext(filename)) + switch ext { + case ".txt", ".md", ".json", ".yaml", ".yml": + return string(data), nil + case ".docx": + txt, err := extractTextFromDOCX(data) + if err == nil && len(strings.TrimSpace(txt)) > 0 { + return injectPageAnchorsForText(txt), nil + } + // 降级：从二进制中提取可见字符 + return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil + default: + // 通用降级 + return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil + } +} + +// injectPageAnchorsForText 将连续文本按约 1200 字符切分，并注入 锚点 +func injectPageAnchorsForText(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + + // 若原始文本中包含换页符 \f + if strings.Contains(raw, "\f") { + pages := strings.Split(raw, "\f") + var sb strings.Builder + for i, page := range pages { + pTrim := strings.TrimSpace(page) + if pTrim == "" { + continue + } + sb.WriteString(fmt.Sprintf("\n\n%s\n", i+1, pTrim)) + } + return strings.TrimSpace(sb.String()) + } + + // 否则按段落累加约 1200 字符为一页 + paragraphs := strings.Split(raw, "\n") + var sb strings.Builder + currentPage := 1 + currentLen := 0 + + sb.WriteString(fmt.Sprintf("\n", currentPage)) + for _, p := range paragraphs { + pTrim := strings.TrimRight(p, "\r\t ") + pLen := utf8.RuneCountInString(pTrim) + if currentLen > 0 && currentLen+pLen > 1200 { + currentPage++ + sb.WriteString(fmt.Sprintf("\n\n", currentPage)) + currentLen = 0 + } + sb.WriteString(pTrim) + sb.WriteString("\n") + currentLen += pLen + 1 + } + + return strings.TrimSpace(sb.String()) +} + +// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本 +func extractTextFromDOCX(data []byte) (string, error) { + r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + return "", err + } + + var docFile *zip.File + for _, f := range r.File { + if f.Name == "word/document.xml" { + docFile = f + break + } + } + if docFile == nil { + return "", fmt.Errorf("word/document.xml not found in docx") + } + + rc, err := docFile.Open() + if err != nil { + return "", err + } + defer rc.Close() + + xmlBytes, err := io.ReadAll(rc) + if err != nil { + return "", err + } + + decoder := xml.NewDecoder(bytes.NewReader(xmlBytes)) + var sb strings.Builder + inParagraph := false + + for { + tok, err := decoder.Token() + if err != nil { + break + } + switch elem := tok.(type) { + case xml.StartElement: + if elem.Name.Local == "p" { + inParagraph = true + } else if elem.Name.Local == "br" { + for _, attr := range elem.Attr { + if attr.Name.Local == "type" && attr.Value == "page" { + sb.WriteString("\f") + } + } + } else if elem.Name.Local == "lastRenderedPageBreak" { + sb.WriteString("\f") + } + case xml.EndElement: + if elem.Name.Local == "p" { + if inParagraph { + sb.WriteString("\n") + inParagraph = false + } + } + case xml.CharData: + sb.Write(elem) + } + } + + return sb.String(), nil +} + +// extractAsciiUtf8Strings 降级算法：从二进制流中提取连续的有效可读 UTF-8 文本 +func extractAsciiUtf8Strings(data []byte) string { + var sb strings.Builder + var cur bytes.Buffer + + flush := func() { + if cur.Len() >= 4 { + s := cur.String() + s = strings.TrimSpace(s) + if s != "" { + sb.WriteString(s) + sb.WriteString("\n") + } + } + cur.Reset() + } + + for i := 0; i < len(data); { + r, size := utf8.DecodeRune(data[i:]) + if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') { + cur.WriteRune(r) + i += size + } else { + flush() + i++ + } + } + flush() + + return sb.String() +}