refactor(extract): move text extraction to internal/textextract
CI / go-vet (pull_request) Successful in 12s
CI / short-tests-report (pull_request) Failing after 36s
CI / backup-restore-drill (pull_request) Successful in 24s
CI / web (pull_request) Successful in 32s

KTD1 third step: material parsing returns to its own capability.
- ExtractTextWithAnchors + docx/anchor helpers move from squad/extract.go
  to internal/textextract (pure stdlib, no workflowdef dep)
- squad/extract.go keeps only ExtractAdvisorBriefing (squad domain logic)
- callers updated: squad context builders, bizdev materials, server
  chat_document_reader, materialindex pipeline
- reverse dependency materialindex -> squad eliminated (KTD1 explicit goal)
- module-boundaries.yaml: textextract declared + 4 edges
- verified: build/vet clean, arch green, textextract/squad/materialindex/
  bizdev tests green; server pre-existing env-dependent failure unchanged
  (issue #2, fails with and without this change)
This commit is contained in:
2026-09-29 06:11:20 +00:00
parent b8e9e75c99
commit e47cfd8f4e
8 changed files with 196 additions and 181 deletions
+2 -1
View File
@@ -12,6 +12,7 @@ import (
"chat-agent-system/internal/materialindex"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"chat-agent-system/internal/storage"
)
@@ -154,7 +155,7 @@ func (s *Service) materialToItems(
meta["parse_engine"] = "mineru"
}
} else {
text, err = squad.ExtractTextWithAnchors(fileName, data)
text, err = textextract.ExtractTextWithAnchors(fileName, data)
if err != nil {
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
}
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"chat-agent-system/internal/config"
"chat-agent-system/internal/kb"
"chat-agent-system/internal/mineru"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -233,7 +233,7 @@ func (p *Pipeline) ResetForReparse(ctx context.Context, blobSHA string) bool {
// fallback text from squad.ExtractTextWithAnchors (may be empty).
func (p *Pipeline) EnsureParsed(ctx context.Context, fileName string, blobSHA string, data []byte) (markdown string, fromMinerU bool, err error) {
fallback := func() (string, bool, error) {
txt, e := squad.ExtractTextWithAnchors(fileName, data)
txt, e := textextract.ExtractTextWithAnchors(fileName, data)
return strings.TrimSpace(txt), false, e
}
if p == nil {
+2 -2
View File
@@ -15,7 +15,7 @@ import (
"chat-agent-system/internal/docgov"
"chat-agent-system/internal/intent"
"chat-agent-system/internal/projectinstance"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"chat-agent-system/internal/storage"
"chat-agent-system/internal/stream"
"chat-agent-system/internal/workflowscope"
@@ -179,7 +179,7 @@ func (s *Server) resolveLatestUploadDocument(ctx context.Context, projectID int6
}
markdown = strings.TrimSpace(md)
} else {
md, extractErr := squad.ExtractTextWithAnchors(ref.FileName, data)
md, extractErr := textextract.ExtractTextWithAnchors(ref.FileName, data)
if extractErr != nil {
return nil, extractErr
}
+2 -1
View File
@@ -1,6 +1,7 @@
package squad
import (
"chat-agent-system/internal/textextract"
"encoding/json"
"fmt"
"os"
@@ -145,7 +146,7 @@ func (cm *ContextManager) InjectMaterials(wsDir string, materials []MaterialItem
name = "material.txt"
}
textContent, err := ExtractTextWithAnchors(name, m.Content)
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
if err != nil || strings.TrimSpace(textContent) == "" {
textContent = string(m.Content)
}
-171
View File
@@ -1,183 +1,12 @@
package squad
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path/filepath"
"strings"
"unicode/utf8"
"chat-agent-system/internal/workflowdef"
)
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
if len(data) == 0 {
return "", nil
}
ext := strings.ToLower(filepath.Ext(filename))
switch ext {
case ".txt", ".md", ".json", ".yaml", ".yml":
return string(data), nil
case ".docx":
txt, err := extractTextFromDOCX(data)
if err == nil && len(strings.TrimSpace(txt)) > 0 {
return injectPageAnchorsForText(txt), nil
}
// 降级:从二进制中提取可见字符
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
default:
// 通用降级
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
}
}
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
func injectPageAnchorsForText(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
// 若原始文本中包含换页符 \f
if strings.Contains(raw, "\f") {
pages := strings.Split(raw, "\f")
var sb strings.Builder
for i, page := range pages {
pTrim := strings.TrimSpace(page)
if pTrim == "" {
continue
}
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
}
return strings.TrimSpace(sb.String())
}
// 否则按段落累加约 1200 字符为一页
paragraphs := strings.Split(raw, "\n")
var sb strings.Builder
currentPage := 1
currentLen := 0
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
for _, p := range paragraphs {
pTrim := strings.TrimRight(p, "\r\t ")
pLen := utf8.RuneCountInString(pTrim)
if currentLen > 0 && currentLen+pLen > 1200 {
currentPage++
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
currentLen = 0
}
sb.WriteString(pTrim)
sb.WriteString("\n")
currentLen += pLen + 1
}
return strings.TrimSpace(sb.String())
}
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
func extractTextFromDOCX(data []byte) (string, error) {
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return "", err
}
var docFile *zip.File
for _, f := range r.File {
if f.Name == "word/document.xml" {
docFile = f
break
}
}
if docFile == nil {
return "", fmt.Errorf("word/document.xml not found in docx")
}
rc, err := docFile.Open()
if err != nil {
return "", err
}
defer rc.Close()
xmlBytes, err := io.ReadAll(rc)
if err != nil {
return "", err
}
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
var sb strings.Builder
inParagraph := false
for {
tok, err := decoder.Token()
if err != nil {
break
}
switch elem := tok.(type) {
case xml.StartElement:
if elem.Name.Local == "p" {
inParagraph = true
} else if elem.Name.Local == "br" {
for _, attr := range elem.Attr {
if attr.Name.Local == "type" && attr.Value == "page" {
sb.WriteString("\f")
}
}
} else if elem.Name.Local == "lastRenderedPageBreak" {
sb.WriteString("\f")
}
case xml.EndElement:
if elem.Name.Local == "p" {
if inParagraph {
sb.WriteString("\n")
inParagraph = false
}
}
case xml.CharData:
sb.Write(elem)
}
}
return sb.String(), nil
}
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
func extractAsciiUtf8Strings(data []byte) string {
var sb strings.Builder
var cur bytes.Buffer
flush := func() {
if cur.Len() >= 4 {
s := cur.String()
s = strings.TrimSpace(s)
if s != "" {
sb.WriteString(s)
sb.WriteString("\n")
}
}
cur.Reset()
}
for i := 0; i < len(data); {
r, size := utf8.DecodeRune(data[i:])
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
cur.WriteRune(r)
i += size
} else {
flush()
i++
}
}
flush()
return sb.String()
}
// ExtractAdvisorBriefing 从 output.md 与黑板事实中提取 B 轨业务参谋决策锦囊
func ExtractAdvisorBriefing(taskKey string, markdownBody string, fields map[string]string) *AdvisorBriefing {
if fields == nil {
+2 -1
View File
@@ -1,6 +1,7 @@
package squad
import (
"chat-agent-system/internal/textextract"
"fmt"
"os"
"path/filepath"
@@ -68,7 +69,7 @@ func (e *WorkspaceEngine) buildContextSystem(workDir string, req WorkspacePrepar
}
// 自动提取纯文本并注入页码锚点 <!-- page N -->
textContent, err := ExtractTextWithAnchors(name, m.Content)
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
if err != nil || strings.TrimSpace(textContent) == "" {
textContent = string(m.Content)
}
+178
View File
@@ -0,0 +1,178 @@
package textextract
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path/filepath"
"strings"
"unicode/utf8"
)
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
if len(data) == 0 {
return "", nil
}
ext := strings.ToLower(filepath.Ext(filename))
switch ext {
case ".txt", ".md", ".json", ".yaml", ".yml":
return string(data), nil
case ".docx":
txt, err := extractTextFromDOCX(data)
if err == nil && len(strings.TrimSpace(txt)) > 0 {
return injectPageAnchorsForText(txt), nil
}
// 降级:从二进制中提取可见字符
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
default:
// 通用降级
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
}
}
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
func injectPageAnchorsForText(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
// 若原始文本中包含换页符 \f
if strings.Contains(raw, "\f") {
pages := strings.Split(raw, "\f")
var sb strings.Builder
for i, page := range pages {
pTrim := strings.TrimSpace(page)
if pTrim == "" {
continue
}
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
}
return strings.TrimSpace(sb.String())
}
// 否则按段落累加约 1200 字符为一页
paragraphs := strings.Split(raw, "\n")
var sb strings.Builder
currentPage := 1
currentLen := 0
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
for _, p := range paragraphs {
pTrim := strings.TrimRight(p, "\r\t ")
pLen := utf8.RuneCountInString(pTrim)
if currentLen > 0 && currentLen+pLen > 1200 {
currentPage++
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
currentLen = 0
}
sb.WriteString(pTrim)
sb.WriteString("\n")
currentLen += pLen + 1
}
return strings.TrimSpace(sb.String())
}
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
func extractTextFromDOCX(data []byte) (string, error) {
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return "", err
}
var docFile *zip.File
for _, f := range r.File {
if f.Name == "word/document.xml" {
docFile = f
break
}
}
if docFile == nil {
return "", fmt.Errorf("word/document.xml not found in docx")
}
rc, err := docFile.Open()
if err != nil {
return "", err
}
defer rc.Close()
xmlBytes, err := io.ReadAll(rc)
if err != nil {
return "", err
}
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
var sb strings.Builder
inParagraph := false
for {
tok, err := decoder.Token()
if err != nil {
break
}
switch elem := tok.(type) {
case xml.StartElement:
if elem.Name.Local == "p" {
inParagraph = true
} else if elem.Name.Local == "br" {
for _, attr := range elem.Attr {
if attr.Name.Local == "type" && attr.Value == "page" {
sb.WriteString("\f")
}
}
} else if elem.Name.Local == "lastRenderedPageBreak" {
sb.WriteString("\f")
}
case xml.EndElement:
if elem.Name.Local == "p" {
if inParagraph {
sb.WriteString("\n")
inParagraph = false
}
}
case xml.CharData:
sb.Write(elem)
}
}
return sb.String(), nil
}
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
func extractAsciiUtf8Strings(data []byte) string {
var sb strings.Builder
var cur bytes.Buffer
flush := func() {
if cur.Len() >= 4 {
s := cur.String()
s = strings.TrimSpace(s)
if s != "" {
sb.WriteString(s)
sb.WriteString("\n")
}
}
cur.Reset()
}
for i := 0; i < len(data); {
r, size := utf8.DecodeRune(data[i:])
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
cur.WriteRune(r)
i += size
} else {
flush()
i++
}
}
flush()
return sb.String()
}