package textextract import ( "archive/zip" "bytes" "encoding/xml" "fmt" "io" "path/filepath" "strings" "unicode/utf8" ) // ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 func ExtractTextWithAnchors(filename string, data []byte) (string, error) { if len(data) == 0 { return "", nil } ext := strings.ToLower(filepath.Ext(filename)) switch ext { case ".txt", ".md", ".json", ".yaml", ".yml": return string(data), nil case ".docx": txt, err := extractTextFromDOCX(data) if err == nil && len(strings.TrimSpace(txt)) > 0 { return injectPageAnchorsForText(txt), nil } // 降级:从二进制中提取可见字符 return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil default: // 通用降级 return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil } } // injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 锚点 func injectPageAnchorsForText(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { return "" } // 若原始文本中包含换页符 \f if strings.Contains(raw, "\f") { pages := strings.Split(raw, "\f") var sb strings.Builder for i, page := range pages { pTrim := strings.TrimSpace(page) if pTrim == "" { continue } sb.WriteString(fmt.Sprintf("\n\n%s\n", i+1, pTrim)) } return strings.TrimSpace(sb.String()) } // 否则按段落累加约 1200 字符为一页 paragraphs := strings.Split(raw, "\n") var sb strings.Builder currentPage := 1 currentLen := 0 sb.WriteString(fmt.Sprintf("\n", currentPage)) for _, p := range paragraphs { pTrim := strings.TrimRight(p, "\r\t ") pLen := utf8.RuneCountInString(pTrim) if currentLen > 0 && currentLen+pLen > 1200 { currentPage++ sb.WriteString(fmt.Sprintf("\n\n", currentPage)) currentLen = 0 } sb.WriteString(pTrim) sb.WriteString("\n") currentLen += pLen + 1 } return strings.TrimSpace(sb.String()) } // extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本 func extractTextFromDOCX(data []byte) (string, error) { r, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) if err != nil { return "", err } var docFile *zip.File for _, f := range r.File { if f.Name == "word/document.xml" { docFile = f break } } if docFile == nil { return "", fmt.Errorf("word/document.xml not found in docx") } rc, err := docFile.Open() if err != nil { return "", err } defer rc.Close() xmlBytes, err := io.ReadAll(rc) if err != nil { return "", err } decoder := xml.NewDecoder(bytes.NewReader(xmlBytes)) var sb strings.Builder inParagraph := false for { tok, err := decoder.Token() if err != nil { break } switch elem := tok.(type) { case xml.StartElement: if elem.Name.Local == "p" { inParagraph = true } else if elem.Name.Local == "br" { for _, attr := range elem.Attr { if attr.Name.Local == "type" && attr.Value == "page" { sb.WriteString("\f") } } } else if elem.Name.Local == "lastRenderedPageBreak" { sb.WriteString("\f") } case xml.EndElement: if elem.Name.Local == "p" { if inParagraph { sb.WriteString("\n") inParagraph = false } } case xml.CharData: sb.Write(elem) } } return sb.String(), nil } // extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本 func extractAsciiUtf8Strings(data []byte) string { var sb strings.Builder var cur bytes.Buffer flush := func() { if cur.Len() >= 4 { s := cur.String() s = strings.TrimSpace(s) if s != "" { sb.WriteString(s) sb.WriteString("\n") } } cur.Reset() } for i := 0; i < len(data); { r, size := utf8.DecodeRune(data[i:]) if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') { cur.WriteRune(r) i += size } else { flush() i++ } } flush() return sb.String() }