Agent-System/internal/textextract/extract.go

package textextract

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"unicode/utf8"

)

// ExtractTextWithAnchors 从不同格式材料中提取纯文本，并自动注入物理页码锚点 <!-- page N -->
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
	if len(data) == 0 {
		return "", nil
	}

	ext := strings.ToLower(filepath.Ext(filename))
	switch ext {
	case ".txt", ".md", ".json", ".yaml", ".yml":
		return string(data), nil
	case ".docx":
		txt, err := extractTextFromDOCX(data)
		if err == nil && len(strings.TrimSpace(txt)) > 0 {
			return injectPageAnchorsForText(txt), nil
		}
		// 降级：从二进制中提取可见字符
		return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
	default:
		// 通用降级
		return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
	}
}

// injectPageAnchorsForText 将连续文本按约 1200 字符切分，并注入 <!-- page N --> 锚点
func injectPageAnchorsForText(raw string) string {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return ""
	}

	// 若原始文本中包含换页符 \f
	if strings.Contains(raw, "\f") {
		pages := strings.Split(raw, "\f")
		var sb strings.Builder
		for i, page := range pages {
			pTrim := strings.TrimSpace(page)
			if pTrim == "" {
				continue
			}
			sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
		}
		return strings.TrimSpace(sb.String())
	}

	// 否则按段落累加约 1200 字符为一页
	paragraphs := strings.Split(raw, "\n")
	var sb strings.Builder
	currentPage := 1
	currentLen := 0

	sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
	for _, p := range paragraphs {
		pTrim := strings.TrimRight(p, "\r\t ")
		pLen := utf8.RuneCountInString(pTrim)
		if currentLen > 0 && currentLen+pLen > 1200 {
			currentPage++
			sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
			currentLen = 0
		}
		sb.WriteString(pTrim)
		sb.WriteString("\n")
		currentLen += pLen + 1
	}

	return strings.TrimSpace(sb.String())
}

// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
func extractTextFromDOCX(data []byte) (string, error) {
	r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return "", err
	}

	var docFile *zip.File
	for _, f := range r.File {
		if f.Name == "word/document.xml" {
			docFile = f
			break
		}
	}
	if docFile == nil {
		return "", fmt.Errorf("word/document.xml not found in docx")
	}

	rc, err := docFile.Open()
	if err != nil {
		return "", err
	}
	defer rc.Close()

	xmlBytes, err := io.ReadAll(rc)
	if err != nil {
		return "", err
	}

	decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
	var sb strings.Builder
	inParagraph := false

	for {
		tok, err := decoder.Token()
		if err != nil {
			break
		}
		switch elem := tok.(type) {
		case xml.StartElement:
			if elem.Name.Local == "p" {
				inParagraph = true
			} else if elem.Name.Local == "br" {
				for _, attr := range elem.Attr {
					if attr.Name.Local == "type" && attr.Value == "page" {
						sb.WriteString("\f")
					}
				}
			} else if elem.Name.Local == "lastRenderedPageBreak" {
				sb.WriteString("\f")
			}
		case xml.EndElement:
			if elem.Name.Local == "p" {
				if inParagraph {
					sb.WriteString("\n")
					inParagraph = false
				}
			}
		case xml.CharData:
			sb.Write(elem)
		}
	}

	return sb.String(), nil
}

// extractAsciiUtf8Strings 降级算法：从二进制流中提取连续的有效可读 UTF-8 文本
func extractAsciiUtf8Strings(data []byte) string {
	var sb strings.Builder
	var cur bytes.Buffer

	flush := func() {
		if cur.Len() >= 4 {
			s := cur.String()
			s = strings.TrimSpace(s)
			if s != "" {
				sb.WriteString(s)
				sb.WriteString("\n")
			}
		}
		cur.Reset()
	}

	for i := 0; i < len(data); {
		r, size := utf8.DecodeRune(data[i:])
		if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
			cur.WriteRune(r)
			i += size
		} else {
			flush()
			i++
		}
	}
	flush()

	return sb.String()
}