refactor(extract): move text extraction to internal/textextract
KTD1 third step: material parsing returns to its own capability. - ExtractTextWithAnchors + docx/anchor helpers move from squad/extract.go to internal/textextract (pure stdlib, no workflowdef dep) - squad/extract.go keeps only ExtractAdvisorBriefing (squad domain logic) - callers updated: squad context builders, bizdev materials, server chat_document_reader, materialindex pipeline - reverse dependency materialindex -> squad eliminated (KTD1 explicit goal) - module-boundaries.yaml: textextract declared + 4 edges - verified: build/vet clean, arch green, textextract/squad/materialindex/ bizdev tests green; server pre-existing env-dependent failure unchanged (issue #2, fails with and without this change)
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"chat-agent-system/internal/materialindex"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
"chat-agent-system/internal/storage"
|
||||
)
|
||||
|
||||
@@ -154,7 +155,7 @@ func (s *Service) materialToItems(
|
||||
meta["parse_engine"] = "mineru"
|
||||
}
|
||||
} else {
|
||||
text, err = squad.ExtractTextWithAnchors(fileName, data)
|
||||
text, err = textextract.ExtractTextWithAnchors(fileName, data)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"chat-agent-system/internal/config"
|
||||
"chat-agent-system/internal/kb"
|
||||
"chat-agent-system/internal/mineru"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -233,7 +233,7 @@ func (p *Pipeline) ResetForReparse(ctx context.Context, blobSHA string) bool {
|
||||
// fallback text from squad.ExtractTextWithAnchors (may be empty).
|
||||
func (p *Pipeline) EnsureParsed(ctx context.Context, fileName string, blobSHA string, data []byte) (markdown string, fromMinerU bool, err error) {
|
||||
fallback := func() (string, bool, error) {
|
||||
txt, e := squad.ExtractTextWithAnchors(fileName, data)
|
||||
txt, e := textextract.ExtractTextWithAnchors(fileName, data)
|
||||
return strings.TrimSpace(txt), false, e
|
||||
}
|
||||
if p == nil {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"chat-agent-system/internal/docgov"
|
||||
"chat-agent-system/internal/intent"
|
||||
"chat-agent-system/internal/projectinstance"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
"chat-agent-system/internal/storage"
|
||||
"chat-agent-system/internal/stream"
|
||||
"chat-agent-system/internal/workflowscope"
|
||||
@@ -179,7 +179,7 @@ func (s *Server) resolveLatestUploadDocument(ctx context.Context, projectID int6
|
||||
}
|
||||
markdown = strings.TrimSpace(md)
|
||||
} else {
|
||||
md, extractErr := squad.ExtractTextWithAnchors(ref.FileName, data)
|
||||
md, extractErr := textextract.ExtractTextWithAnchors(ref.FileName, data)
|
||||
if extractErr != nil {
|
||||
return nil, extractErr
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"chat-agent-system/internal/textextract"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -145,7 +146,7 @@ func (cm *ContextManager) InjectMaterials(wsDir string, materials []MaterialItem
|
||||
name = "material.txt"
|
||||
}
|
||||
|
||||
textContent, err := ExtractTextWithAnchors(name, m.Content)
|
||||
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
|
||||
if err != nil || strings.TrimSpace(textContent) == "" {
|
||||
textContent = string(m.Content)
|
||||
}
|
||||
|
||||
@@ -1,183 +1,12 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"chat-agent-system/internal/workflowdef"
|
||||
)
|
||||
|
||||
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
|
||||
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
case ".txt", ".md", ".json", ".yaml", ".yml":
|
||||
return string(data), nil
|
||||
case ".docx":
|
||||
txt, err := extractTextFromDOCX(data)
|
||||
if err == nil && len(strings.TrimSpace(txt)) > 0 {
|
||||
return injectPageAnchorsForText(txt), nil
|
||||
}
|
||||
// 降级:从二进制中提取可见字符
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
default:
|
||||
// 通用降级
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
|
||||
func injectPageAnchorsForText(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 若原始文本中包含换页符 \f
|
||||
if strings.Contains(raw, "\f") {
|
||||
pages := strings.Split(raw, "\f")
|
||||
var sb strings.Builder
|
||||
for i, page := range pages {
|
||||
pTrim := strings.TrimSpace(page)
|
||||
if pTrim == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// 否则按段落累加约 1200 字符为一页
|
||||
paragraphs := strings.Split(raw, "\n")
|
||||
var sb strings.Builder
|
||||
currentPage := 1
|
||||
currentLen := 0
|
||||
|
||||
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
|
||||
for _, p := range paragraphs {
|
||||
pTrim := strings.TrimRight(p, "\r\t ")
|
||||
pLen := utf8.RuneCountInString(pTrim)
|
||||
if currentLen > 0 && currentLen+pLen > 1200 {
|
||||
currentPage++
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
|
||||
currentLen = 0
|
||||
}
|
||||
sb.WriteString(pTrim)
|
||||
sb.WriteString("\n")
|
||||
currentLen += pLen + 1
|
||||
}
|
||||
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
|
||||
func extractTextFromDOCX(data []byte) (string, error) {
|
||||
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var docFile *zip.File
|
||||
for _, f := range r.File {
|
||||
if f.Name == "word/document.xml" {
|
||||
docFile = f
|
||||
break
|
||||
}
|
||||
}
|
||||
if docFile == nil {
|
||||
return "", fmt.Errorf("word/document.xml not found in docx")
|
||||
}
|
||||
|
||||
rc, err := docFile.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
xmlBytes, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
|
||||
var sb strings.Builder
|
||||
inParagraph := false
|
||||
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch elem := tok.(type) {
|
||||
case xml.StartElement:
|
||||
if elem.Name.Local == "p" {
|
||||
inParagraph = true
|
||||
} else if elem.Name.Local == "br" {
|
||||
for _, attr := range elem.Attr {
|
||||
if attr.Name.Local == "type" && attr.Value == "page" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
}
|
||||
} else if elem.Name.Local == "lastRenderedPageBreak" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem.Name.Local == "p" {
|
||||
if inParagraph {
|
||||
sb.WriteString("\n")
|
||||
inParagraph = false
|
||||
}
|
||||
}
|
||||
case xml.CharData:
|
||||
sb.Write(elem)
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
|
||||
func extractAsciiUtf8Strings(data []byte) string {
|
||||
var sb strings.Builder
|
||||
var cur bytes.Buffer
|
||||
|
||||
flush := func() {
|
||||
if cur.Len() >= 4 {
|
||||
s := cur.String()
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" {
|
||||
sb.WriteString(s)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
cur.Reset()
|
||||
}
|
||||
|
||||
for i := 0; i < len(data); {
|
||||
r, size := utf8.DecodeRune(data[i:])
|
||||
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
|
||||
cur.WriteRune(r)
|
||||
i += size
|
||||
} else {
|
||||
flush()
|
||||
i++
|
||||
}
|
||||
}
|
||||
flush()
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// ExtractAdvisorBriefing 从 output.md 与黑板事实中提取 B 轨业务参谋决策锦囊
|
||||
func ExtractAdvisorBriefing(taskKey string, markdownBody string, fields map[string]string) *AdvisorBriefing {
|
||||
if fields == nil {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"chat-agent-system/internal/textextract"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -68,7 +69,7 @@ func (e *WorkspaceEngine) buildContextSystem(workDir string, req WorkspacePrepar
|
||||
}
|
||||
|
||||
// 自动提取纯文本并注入页码锚点 <!-- page N -->
|
||||
textContent, err := ExtractTextWithAnchors(name, m.Content)
|
||||
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
|
||||
if err != nil || strings.TrimSpace(textContent) == "" {
|
||||
textContent = string(m.Content)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package textextract
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
)
|
||||
|
||||
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
|
||||
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
case ".txt", ".md", ".json", ".yaml", ".yml":
|
||||
return string(data), nil
|
||||
case ".docx":
|
||||
txt, err := extractTextFromDOCX(data)
|
||||
if err == nil && len(strings.TrimSpace(txt)) > 0 {
|
||||
return injectPageAnchorsForText(txt), nil
|
||||
}
|
||||
// 降级:从二进制中提取可见字符
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
default:
|
||||
// 通用降级
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
|
||||
func injectPageAnchorsForText(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 若原始文本中包含换页符 \f
|
||||
if strings.Contains(raw, "\f") {
|
||||
pages := strings.Split(raw, "\f")
|
||||
var sb strings.Builder
|
||||
for i, page := range pages {
|
||||
pTrim := strings.TrimSpace(page)
|
||||
if pTrim == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// 否则按段落累加约 1200 字符为一页
|
||||
paragraphs := strings.Split(raw, "\n")
|
||||
var sb strings.Builder
|
||||
currentPage := 1
|
||||
currentLen := 0
|
||||
|
||||
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
|
||||
for _, p := range paragraphs {
|
||||
pTrim := strings.TrimRight(p, "\r\t ")
|
||||
pLen := utf8.RuneCountInString(pTrim)
|
||||
if currentLen > 0 && currentLen+pLen > 1200 {
|
||||
currentPage++
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
|
||||
currentLen = 0
|
||||
}
|
||||
sb.WriteString(pTrim)
|
||||
sb.WriteString("\n")
|
||||
currentLen += pLen + 1
|
||||
}
|
||||
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
|
||||
func extractTextFromDOCX(data []byte) (string, error) {
|
||||
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var docFile *zip.File
|
||||
for _, f := range r.File {
|
||||
if f.Name == "word/document.xml" {
|
||||
docFile = f
|
||||
break
|
||||
}
|
||||
}
|
||||
if docFile == nil {
|
||||
return "", fmt.Errorf("word/document.xml not found in docx")
|
||||
}
|
||||
|
||||
rc, err := docFile.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
xmlBytes, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
|
||||
var sb strings.Builder
|
||||
inParagraph := false
|
||||
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch elem := tok.(type) {
|
||||
case xml.StartElement:
|
||||
if elem.Name.Local == "p" {
|
||||
inParagraph = true
|
||||
} else if elem.Name.Local == "br" {
|
||||
for _, attr := range elem.Attr {
|
||||
if attr.Name.Local == "type" && attr.Value == "page" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
}
|
||||
} else if elem.Name.Local == "lastRenderedPageBreak" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem.Name.Local == "p" {
|
||||
if inParagraph {
|
||||
sb.WriteString("\n")
|
||||
inParagraph = false
|
||||
}
|
||||
}
|
||||
case xml.CharData:
|
||||
sb.Write(elem)
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
|
||||
func extractAsciiUtf8Strings(data []byte) string {
|
||||
var sb strings.Builder
|
||||
var cur bytes.Buffer
|
||||
|
||||
flush := func() {
|
||||
if cur.Len() >= 4 {
|
||||
s := cur.String()
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" {
|
||||
sb.WriteString(s)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
cur.Reset()
|
||||
}
|
||||
|
||||
for i := 0; i < len(data); {
|
||||
r, size := utf8.DecodeRune(data[i:])
|
||||
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
|
||||
cur.WriteRune(r)
|
||||
i += size
|
||||
} else {
|
||||
flush()
|
||||
i++
|
||||
}
|
||||
}
|
||||
flush()
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
Reference in New Issue
Block a user