Agent-System-Backend/internal/ingest/ingest.go

// Package ingest 实现文档摄取与解析（4.11）：上传/工件 → ParsedDocument IR。
// 主路线为内置代码解析器（确定性、纯 Go、可独立测试）；MinerU 深度解析与
// agent 驱动解析为远期档位（ParserProvider 注册位已留）。
package ingest

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"time"

	core "agent-system-core"
	pdflib "github.com/ledongthuc/pdf"
)

// Parse 按扩展名分派内置解析器；无法解析时返回带 warning 的空文档（显式失败，不静默）。
func Parse(filename, mime string, content []byte) core.ParsedDocument {
	doc := core.ParsedDocument{Meta: core.DocMeta{
		Filename: filename, MIME: mime, Parser: "builtin", ParsedAt: time.Now().UTC(),
	}}
	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
	switch ext {
	case "md", "markdown":
		doc.Meta.Parser = "builtin:md"
		doc.Blocks = parseMarkdown(string(content))
		doc.Meta.Confidence = 1.0
	case "txt", "text", "log", "csv":
		doc.Meta.Parser = "builtin:txt"
		doc.Blocks = parsePlain(string(content))
		doc.Meta.Confidence = 1.0
	case "docx":
		doc.Meta.Parser = "builtin:docx"
		blocks, revised, err := parseDocxRevised(content)
		switch {
		case err != nil:
			doc.Warnings = append(doc.Warnings, "docx 解析失败："+err.Error())
			doc.Meta.Confidence = 0.3
		case revised:
			doc.Blocks = blocks
			doc.Warnings = append(doc.Warnings, docxRevisionWarning)
			doc.Meta.Confidence = 0.85
		default:
			doc.Blocks = blocks
			doc.Meta.Confidence = 0.92
		}
	case "pdf":
		doc.Meta.Parser = "builtin:pdf"
		blocks, err := parsePDF(content)
		if err != nil {
			doc.Warnings = append(doc.Warnings, "pdf 文本层解析失败："+err.Error()+"（扫描件需远期 MinerU 档）")
			doc.Meta.Confidence = 0.2
		} else {
			doc.Blocks = blocks
			doc.Meta.Confidence = 0.85
		}
	case "xlsx", "xlsm":
		doc.Meta.Parser = "builtin:xlsx"
		blocks, warns := parseXlsx(content)
		doc.Blocks = blocks
		doc.Warnings = append(doc.Warnings, warns...)
		switch {
		case len(blocks) == 0:
			doc.Meta.Confidence = 0.3
		case len(warns) > 0:
			doc.Meta.Confidence = 0.6 // 超限截断：内容可用但注明不完整
		default:
			doc.Meta.Confidence = 0.9
		}
	case "pptx", "pptm", "ppsx":
		doc.Meta.Parser = "builtin:pptx"
		blocks, warns := parsePptx(content)
		doc.Blocks = blocks
		doc.Warnings = append(doc.Warnings, warns...)
		switch {
		case len(blocks) == 0:
			doc.Meta.Confidence = 0.3
		case len(warns) > 0:
			doc.Meta.Confidence = 0.6
		default:
			doc.Meta.Confidence = 0.9
		}
	case "doc", "xls", "ppt":
		doc.Meta.Parser = "builtin:legacy-office"
		doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析：请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
	default:
		if isImage(ext) {
			doc.Warnings = append(doc.Warnings, "图片不走文本解析（OCR 为 MinerU 深度档，远期）")
		} else {
			// 未知类型按纯文本尝试（二进制内容会产生乱码 warning）。
			text := string(content)
			if !isMostlyPrintable(text) {
				doc.Warnings = append(doc.Warnings, fmt.Sprintf("未知类型 .%s 且内容非文本，未解析", ext))
			} else {
				doc.Blocks = parsePlain(text)
				doc.Meta.Confidence = 0.6
				doc.Warnings = append(doc.Warnings, "未知类型按纯文本解析")
			}
		}
	}
	return doc
}

// ---- md / txt ----

func parseMarkdown(s string) []core.DocBlock {
	var blocks []core.DocBlock
	var para []string
	flush := func() {
		if len(para) > 0 {
			blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: strings.Join(para, "\n")})
			para = nil
		}
	}
	for _, line := range strings.Split(s, "\n") {
		t := strings.TrimSpace(line)
		switch {
		case t == "":
			flush()
		case strings.HasPrefix(t, "#"):
			flush()
			level := 0
			for level < len(t) && t[level] == '#' {
				level++
			}
			blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: level, Text: strings.TrimSpace(t[level:])})
		case strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* "):
			flush()
			if n := len(blocks); n > 0 && blocks[n-1].Type == core.BlockList {
				blocks[n-1].Items = append(blocks[n-1].Items, strings.TrimSpace(t[2:]))
			} else {
				blocks = append(blocks, core.DocBlock{Type: core.BlockList, Items: []string{strings.TrimSpace(t[2:])}})
			}
		case strings.HasPrefix(t, "```"):
			flush()
			blocks = append(blocks, core.DocBlock{Type: core.BlockCode, Text: ""}) // 简化：代码块内容并入口令块
		default:
			para = append(para, t)
		}
	}
	flush()
	return blocks
}

func parsePlain(s string) []core.DocBlock {
	var blocks []core.DocBlock
	var para []string
	flush := func() {
		if len(para) > 0 {
			blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: strings.Join(para, "\n")})
			para = nil
		}
	}
	for _, line := range strings.Split(s, "\n") {
		if strings.TrimSpace(line) == "" {
			flush()
			continue
		}
		para = append(para, strings.TrimRight(line, "\r"))
	}
	flush()
	return blocks
}

// ---- docx（OOXML：zip 内 word/document.xml）----

// xpar 是解码后的段落。文字按文档顺序从任意深度的 w:t 收集，w:ins、
// w:hyperlink、w:fldSimple、w:sdt 等包裹层内的文字因此不会丢；被删除或
// 移走的原文（w:del、w:moveFrom）、域代码（w:instrText）与兼容回退分支
// （mc:Fallback，与 mc:Choice 内容重复）不进正文。
type xpar struct {
	Style   string
	Text    string
	Revised bool // 段内含修订标记（插入/删除/移动）
}

// docxSkipText 子树里的文字不属于当前正文。
var docxSkipText = map[string]bool{
	"del": true, "moveFrom": true, "instrText": true, "delInstrText": true, "Fallback": true,
}

// docxRevision 出现即说明文档带修订。
var docxRevision = map[string]bool{"ins": true, "del": true, "moveFrom": true, "moveTo": true}

// UnmarshalXML 按 token 流遍历段落子树。按固定层级（p/r/t）映射只认 w:p
// 的直接子级 w:r，修订插入、超链接与域结果里的文字会被静默跳过。
func (p *xpar) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	var sb strings.Builder
	skip := 0
	for {
		tok, err := d.Token()
		if err != nil {
			return err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			name := t.Name.Local
			if docxRevision[name] {
				p.Revised = true
			}
			switch {
			case docxSkipText[name]:
				skip++
			case skip > 0:
			case name == "pStyle" && p.Style == "":
				p.Style = attrAnyNS(t, "val")
			case name == "t":
				var s string
				if err := d.DecodeElement(&s, &t); err != nil {
					return err
				}
				sb.WriteString(s)
			}
		case xml.EndElement:
			if docxSkipText[t.Name.Local] && skip > 0 {
				skip--
			}
			if t.Name == start.Name {
				p.Text = sb.String()
				return nil
			}
		}
	}
}

// attrAnyNS 按本地名取属性值（w:val 与无前缀 val 都算）。
func attrAnyNS(st xml.StartElement, name string) string {
	for _, a := range st.Attr {
		if a.Name.Local == name {
			return a.Value
		}
	}
	return ""
}

type xtbl struct {
	Rows []xtr `xml:"tr"`
}

type xtr struct {
	Cells []xtc `xml:"tc"`
}

type xtc struct {
	Paragraphs []xpar `xml:"p"`
}

func parseDocx(content []byte) ([]core.DocBlock, error) {
	blocks, _, err := parseDocxRevised(content)
	return blocks, err
}

// parseDocxRevised 同 parseDocx，另报告文档是否带修订标记。
func parseDocxRevised(content []byte) ([]core.DocBlock, bool, error) {
	zr, err := openZip(content)
	if err != nil {
		return nil, false, fmt.Errorf("open zip: %w", err)
	}
	docXML, err := zipPart(zr, "word/document.xml")
	if err != nil {
		return nil, false, err
	}
	return parseDocxBody(docXML)
}

// docxRevisionWarning 文档带修订时的显式告警（SR-A5：不静默）。
const docxRevisionWarning = "docx 含修订标记：正文按「接受全部修订」提取（插入保留、删除不计入）；审阅修订过程请查看原件"

// parseDocxBody 按文档顺序解码 body 的段落与表格：token 流上遇到 p/tbl 就地
// DecodeElement（整元素消费，单元格内嵌 p 不会重复入列），表格与前后段落保持原位。
// 段落可能包在 w:ins、w:sdt 等容器里，所以任意深度的 p 与 tbl 都识别；
// 返回值 revised 报告是否遇到修订标记。
func parseDocxBody(docXML []byte) (blocks []core.DocBlock, revised bool, err error) {
	dec := xml.NewDecoder(bytes.NewReader(docXML))
	skip := 0
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, revised, fmt.Errorf("decode document.xml: %w", err)
		}
		if end, ok := tok.(xml.EndElement); ok {
			if docxSkipText[end.Name.Local] && skip > 0 {
				skip--
			}
			continue
		}
		start, ok := tok.(xml.StartElement)
		if !ok {
			continue
		}
		name := start.Name.Local
		if docxRevision[name] {
			revised = true
		}
		if docxSkipText[name] {
			skip++
			continue
		}
		if skip > 0 {
			continue
		}
		switch name {
		case "p":
			var p xpar
			if err := dec.DecodeElement(&p, &start); err != nil {
				return nil, revised, fmt.Errorf("decode paragraph: %w", err)
			}
			revised = revised || p.Revised
			if strings.TrimSpace(p.Text) == "" {
				continue
			}
			if lvl, ok := headingLevel(p.Style); ok {
				blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: lvl, Text: p.Text})
			} else {
				blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: p.Text})
			}
		case "tbl":
			var tbl xtbl
			if err := dec.DecodeElement(&tbl, &start); err != nil {
				return nil, revised, fmt.Errorf("decode table: %w", err)
			}
			rows, tblRevised := tableRows(tbl)
			revised = revised || tblRevised
			if len(rows) > 0 {
				blocks = append(blocks, core.DocBlock{Type: core.BlockTable, Rows: rows})
			}
		}
	}
	return blocks, revised, nil
}

func tableRows(tbl xtbl) ([][]string, bool) {
	var rows [][]string
	revised := false
	for _, tr := range tbl.Rows {
		row := []string{}
		for _, tc := range tr.Cells {
			cell := []string{}
			for _, p := range tc.Paragraphs {
				revised = revised || p.Revised
				cell = append(cell, p.Text)
			}
			row = append(row, strings.Join(cell, " "))
		}
		rows = append(rows, row)
	}
	return rows, revised
}

func headingLevel(style string) (int, bool) {
	s := strings.ToLower(style)
	switch {
	case strings.Contains(s, "heading1") || s == "1":
		return 1, true
	case strings.Contains(s, "heading2") || s == "2":
		return 2, true
	case strings.Contains(s, "heading3") || s == "3" || strings.Contains(s, "title"):
		return 3, true
	}
	return 0, false
}

// ---- pdf（文本层提取）----

func parsePDF(content []byte) ([]core.DocBlock, error) {
	r, err := pdflib.NewReader(bytes.NewReader(content), int64(len(content)))
	if err != nil {
		return nil, err
	}
	var blocks []core.DocBlock
	for i := 1; i <= r.NumPage(); i++ {
		page := r.Page(i)
		if page.V.IsNull() {
			continue
		}
		text, err := page.GetPlainText(nil)
		if err != nil {
			continue
		}
		for _, para := range strings.Split(text, "\n\n") {
			if t := strings.TrimSpace(para); t != "" {
				blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: t})
			}
		}
	}
	if len(blocks) == 0 {
		return nil, fmt.Errorf("无文本层（可能是扫描件）")
	}
	return blocks, nil
}

// ---- 辅助 ----

func isImage(ext string) bool {
	switch ext {
	case "png", "jpg", "jpeg", "gif", "webp", "bmp", "svg":
		return true
	}
	return false
}

func isMostlyPrintable(s string) bool {
	if len(s) == 0 {
		return true
	}
	nonPrint := 0
	for _, r := range s {
		if r == 0xFFFD || (r < 0x20 && r != '\n' && r != '\r' && r != '\t') {
			nonPrint++
		}
	}
	return nonPrint*10 < len(s)
}

// legacyOfficeTarget 旧版二进制 Office 格式的现代等价格式（提示转存用）。
func legacyOfficeTarget(ext string) string {
	switch ext {
	case "doc":
		return "docx"
	case "xls":
		return "xlsx"
	case "ppt":
		return "pptx"
	}
	return ext
}

// ParsersDescription 返回解析器注册表描述（来源标记；设置页/文档用）。
func ParsersDescription() []core.ParserDesc {
	return []core.ParserDesc{
		{ID: "builtin:md", Formats: []string{"md", "markdown"}, Source: core.SourceBuiltin},
		{ID: "builtin:txt", Formats: []string{"txt", "text", "csv", "log"}, Source: core.SourceBuiltin},
		{ID: "builtin:docx", Formats: []string{"docx"}, Source: core.SourceBuiltin},
		{ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层；扫描件为远期 MinerU 深度档"},
		{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
		{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
		{ID: "builtin:legacy-office", Formats: []string{"doc", "xls", "ppt"}, Source: core.SourceBuiltin, Note: "不解析；提示转存现代格式后重传"},
		{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选：深度解析与交叉验证（默认不部署）"},
	}
}