Agent-System-Backend/internal/ingest/xlsx.go

package ingest

// xlsx/xlsm（SpreadsheetML）内置解析器。工作表顺序取 workbook.xml 的 sheets
// 列表（target 经关系部件解析，支持相对/绝对路径）；共享字符串含富文本 run
//（注音 rPh 无对应解码字段，天然排除）；单元格覆盖 inlineStr/str/b/e/n，
// 公式取缓存值、无缓存回显公式；日期按数字格式（内置 ID 与自定义记号）转
// ISO，1900/1904 双日期系统；稀疏单元格按引用就位。纯标准库。

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"math"
	"strconv"
	"strings"
	"time"

	core "agent-system-core"
)

// 解析上限（包级变量，测试可覆盖）。
var (
	xlsxMaxCells = 200000
	xlsxMaxCols  = 1024
)

type xlsxSheetRef struct {
	Name    string
	State   string
	Path    string
	IsSheet bool
}

func parseXlsx(content []byte) ([]core.DocBlock, []string) {
	zr, err := openZip(content)
	if err != nil {
		return nil, []string{"xlsx 解析失败：" + err.Error()}
	}
	sheets, date1904, warns := xlsxWorkbookSheets(zr)
	if len(warns) > 0 && sheets == nil {
		return nil, warns
	}
	shared := xlsxSharedStrings(zr)
	dateStyles := xlsxDateStyles(zr)
	var blocks []core.DocBlock
	cells := 0
	for _, sh := range sheets {
		if !sh.IsSheet || sh.Path == "" {
			continue // chartsheet 等非工作表部件
		}
		data, err := zipPart(zr, sh.Path)
		if err != nil {
			warns = append(warns, fmt.Sprintf("工作表 %s 读取失败：%v", sh.Name, err))
			continue
		}
		table, used, trunc, wide, sheetWarn := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells)
		cells = used
		if sheetWarn != "" {
			warns = append(warns, fmt.Sprintf("工作表 %s：%s", sh.Name, sheetWarn))
		}
		if wide {
			warns = append(warns, fmt.Sprintf("工作表 %s 含超过 %d 列的稀疏单元格，已跳过该列（其余内容不受影响）", sh.Name, xlsxMaxCols))
		}
		if table == nil {
			continue // 空表（读取失败/坏 XML 已单独告警）
		}
		title := sh.Name
		if sh.State == "hidden" || sh.State == "veryHidden" {
			title += "（隐藏）"
		}
		blocks = append(blocks,
			core.DocBlock{Type: core.BlockHeading, Level: 2, Text: title},
			*table,
		)
		if trunc {
			warns = append(warns, fmt.Sprintf("单元格总数超过 %d 上限，工作表 %s 之后未解析", xlsxMaxCells, sh.Name))
			break
		}
	}
	if len(blocks) == 0 && len(warns) == 0 {
		warns = append(warns, "xlsx 无可解析的工作表内容")
	}
	return blocks, warns
}

// xlsxWorkbookSheets 读 workbook.xml：工作表顺序、隐藏状态、1904 日期系统。
func xlsxWorkbookSheets(zr *zip.Reader) (sheets []xlsxSheetRef, date1904 bool, warns []string) {
	data, err := zipPart(zr, "xl/workbook.xml")
	if err != nil {
		return nil, false, []string{"xlsx 解析失败：" + err.Error()}
	}
	rels, err := parseRels(zr, "xl/workbook.xml")
	if err != nil {
		return nil, false, []string{"xlsx 解析失败：" + err.Error()}
	}
	dec := xml.NewDecoder(bytes.NewReader(data))
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return sheets, date1904, append(warns, "workbook.xml 解码失败："+err.Error())
		}
		st, ok := tok.(xml.StartElement)
		if !ok {
			continue
		}
		switch st.Name.Local {
		case "workbookPr":
			if v := attrLocal(st, "date1904"); v == "1" || strings.EqualFold(v, "true") {
				date1904 = true
			}
		case "sheet":
			ref := xlsxSheetRef{Name: attrLocal(st, "name"), State: attrLocal(st, "state")}
			if rel, ok := rels[ridAttr(st)]; ok {
				ref.Path = resolveRelTarget("xl", rel.Target)
				ref.IsSheet = relIsType(rel, "/worksheet")
			} else {
				warns = append(warns, fmt.Sprintf("工作表 %s 的关系缺失", ref.Name))
			}
			sheets = append(sheets, ref)
		}
	}
	return sheets, date1904, warns
}

// xlsxSharedStrings 读共享字符串表；富文本按 run 拼接，rPh 注音不进结果。
func xlsxSharedStrings(zr *zip.Reader) []string {
	data, err := zipPart(zr, "xl/sharedStrings.xml")
	if err != nil {
		return nil
	}
	dec := xml.NewDecoder(bytes.NewReader(data))
	var out []string
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return out
		}
		st, ok := tok.(xml.StartElement)
		if !ok || st.Name.Local != "si" {
			continue
		}
		var si struct {
			T    string `xml:"t"`
			Runs []struct {
				T string `xml:"t"`
			} `xml:"r"`
		}
		if err := dec.DecodeElement(&si, &st); err != nil {
			continue
		}
		if si.T != "" || len(si.Runs) == 0 {
			out = append(out, si.T)
			continue
		}
		parts := make([]string, 0, len(si.Runs))
		for _, r := range si.Runs {
			parts = append(parts, r.T)
		}
		out = append(out, strings.Join(parts, ""))
	}
	return out
}

// xlsxDateStyles 返回按日期呈现的样式索引集合（cellXfs 下标）。
func xlsxDateStyles(zr *zip.Reader) map[int]bool {
	data, err := zipPart(zr, "xl/styles.xml")
	if err != nil {
		return nil
	}
	custom := map[int]string{}
	date := map[int]bool{}
	dec := xml.NewDecoder(bytes.NewReader(data))
	inCellXfs := false
	xfIdx := 0
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return date
		}
		switch t := tok.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "numFmt":
				id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
				custom[id] = attrLocal(t, "formatCode")
			case "cellXfs":
				inCellXfs = true
				xfIdx = 0
			case "xf":
				if !inCellXfs {
					continue
				}
				id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
				if xlsxFmtIsDate(id, custom) {
					date[xfIdx] = true
				}
				xfIdx++
			}
		case xml.EndElement:
			if t.Name.Local == "cellXfs" {
				inCellXfs = false
			}
		}
	}
	return date
}

// xlsxFmtIsDate：自定义格式按记号判断；内置 ID 覆盖常见日期/时间区间。
func xlsxFmtIsDate(id int, custom map[int]string) bool {
	if code, ok := custom[id]; ok {
		return formatCodeIsDate(code)
	}
	switch {
	case id >= 14 && id <= 22, id >= 27 && id <= 36,
		id >= 45 && id <= 47, id >= 50 && id <= 58:
		return true
	}
	return false
}

// formatCodeIsDate 剥掉引号字面量与方括号节（[Red]、[DBNum1]、[$-409] 等）后，
// 剩余字母含 y/d/h/s/m 记号即按日期处理。
func formatCodeIsDate(code string) bool {
	var b strings.Builder
	inQuote, inBracket := false, false
	runes := []rune(code)
	for i := 0; i < len(runes); i++ {
		r := runes[i]
		switch {
		case r == '"':
			inQuote = !inQuote
		case inQuote:
		case r == '[':
			inBracket = true
		case r == ']':
			inBracket = false
		case inBracket:
		case r == '\\':
			i++ // 转义字符：连同下一字符跳过
		default:
			b.WriteRune(r)
		}
	}
	return strings.ContainsAny(strings.ToLower(b.String()), "ydhsm")
}

type xlsxCellEl struct {
	Ref string `xml:"r,attr"`
	T   string `xml:"t,attr"`
	S   string `xml:"s,attr"`
	IS  *struct {
		T string `xml:"t"`
	} `xml:"is"`
	F string `xml:"f"`
	V string `xml:"v"`
}

type xlsxRowEl struct {
	Cells []xlsxCellEl `xml:"c"`
}

// xlsxSheetBlocks 解析单张工作表；返回表格块、累计单元格数、是否触达单元格
// 上限、是否跳过超宽列、以及坏 XML 的告警（显式失败，不静默）。无内容行
// 返回 nil 表。
func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool, bool, string) {
	dec := xml.NewDecoder(bytes.NewReader(data))
	var rows [][]string
	count := cellsUsed
	truncated := false
	wide := false
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, count, truncated, wide, "XML 解码失败：" + err.Error()
		}
		st, ok := tok.(xml.StartElement)
		if !ok || st.Name.Local != "row" {
			continue
		}
		var re xlsxRowEl
		if err := dec.DecodeElement(&re, &st); err != nil {
			continue
		}
		cells := map[int]string{}
		maxCol := -1
		for _, c := range re.Cells {
			if count >= xlsxMaxCells {
				truncated = true
				break
			}
			count++
			col := xlsxColIndex(c.Ref)
			if col < 0 {
				col = maxCol + 1 // 无引用：按出现顺序接续
			}
			if col >= xlsxMaxCols {
				// 稀疏样式单元格常落在远端列（XFD 区）：跳过该格并告警，
				// 不中断本表其余内容，也不阻断后续工作表。
				wide = true
				continue
			}
			cells[col] = xlsxCellText(c, shared, dateStyles, date1904)
			if col > maxCol {
				maxCol = col
			}
		}
		if len(cells) == 0 {
			continue
		}
		full := make([]string, maxCol+1)
		for i, v := range cells {
			full[i] = v
		}
		for len(full) > 0 && strings.TrimSpace(full[len(full)-1]) == "" {
			full = full[:len(full)-1]
		}
		if len(full) > 0 {
			rows = append(rows, full)
		}
	}
	if len(rows) == 0 {
		return nil, count, truncated, wide, ""
	}
	return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated, wide, ""
}

// xlsxCellText 把单元格转成展示文本。
func xlsxCellText(c xlsxCellEl, shared []string, dateStyles map[int]bool, date1904 bool) string {
	text := ""
	switch c.T {
	case "s":
		if idx, err := strconv.Atoi(c.V); err == nil && idx >= 0 && idx < len(shared) {
			text = shared[idx]
		}
	case "inlineStr":
		if c.IS != nil {
			text = c.IS.T
		}
	case "b":
		if c.V == "1" || strings.EqualFold(c.V, "true") {
			text = "TRUE"
		} else {
			text = "FALSE"
		}
	case "e":
		text = c.V
	default: // n 或空：数字 / 公式缓存值
		text = c.V
		if sIdx, err := strconv.Atoi(c.S); err == nil && dateStyles[sIdx] && c.V != "" {
			if f, err := strconv.ParseFloat(c.V, 64); err == nil {
				text = xlsxSerialString(f, date1904)
			}
		}
	}
	if text == "" && c.F != "" {
		return "=" + c.F // 无缓存值的公式：回显公式本身
	}
	return text
}

// xlsxSerialString Excel 日期序列值 → ISO 字符串；带小数部分时输出到秒。
func xlsxSerialString(serial float64, date1904 bool) string {
	days := math.Floor(serial)
	frac := serial - days
	var t time.Time
	switch {
	case date1904:
		t = time.Date(1904, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
	case days >= 61:
		t = time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
	default: // 1–60：兼容 Excel 的 1900 闰年 bug 区段
		t = time.Date(1899, 12, 31, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
	}
	if frac > 0 {
		return t.Add(time.Duration(frac * 86400 * float64(time.Second))).Round(time.Second).Format("2006-01-02 15:04:05")
	}
	return t.Format("2006-01-02")
}

// xlsxColIndex "BC12" → 0 起列号；无引用或非法返回 -1。
func xlsxColIndex(ref string) int {
	if ref == "" {
		return -1
	}
	i := 0
	for i < len(ref) {
		ch := ref[i]
		if (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') {
			i++
			continue
		}
		break
	}
	if i == 0 || i == len(ref) {
		return -1
	}
	for _, ch := range ref[i:] {
		if ch < '0' || ch > '9' {
			return -1
		}
	}
	n := 0
	for _, ch := range ref[:i] {
		c := ch
		if c >= 'a' && c <= 'z' {
			c = c - 'a' + 'A'
		}
		n = n*26 + int(c-'A') + 1
	}
	return n - 1
}