Agent-System-Backend/internal/ingest/ooxml_test.go

package ingest

// OOXML（xlsx/pptx）解析器测试：手工构造最小 zip 部件，覆盖顺序解析、富文本、
// 公式、日期、稀疏列、隐藏表、组合形状、AlternateContent、表格合并、备注、
// 上限与失败路径。真实生成器的轮测见 ooxml_python_test.go。

import (
	"archive/zip"
	"bytes"
	"sort"
	"strings"
	"testing"

	core "agent-system-core"
)

// zipBytes 按名称构造 zip（键排序保证确定性）。
func zipBytes(t *testing.T, files map[string]string) []byte {
	t.Helper()
	var buf bytes.Buffer
	zw := zip.NewWriter(&buf)
	names := make([]string, 0, len(files))
	for n := range files {
		names = append(names, n)
	}
	sort.Strings(names)
	for _, n := range names {
		w, err := zw.Create(n)
		if err != nil {
			t.Fatal(err)
		}
		if _, err := w.Write([]byte(files[n])); err != nil {
			t.Fatal(err)
		}
	}
	if err := zw.Close(); err != nil {
		t.Fatal(err)
	}
	return buf.Bytes()
}

const relTypeNS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"

func xlsxFixture() map[string]string {
	return map[string]string{
		"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="` + relTypeNS + `">
<sheets>
<sheet name="报价" sheetId="1" r:id="rId1"/>
<sheet name="草稿" sheetId="2" state="hidden" r:id="rId2"/>
</sheets></workbook>`,
		"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="` + relTypeNS + `/worksheet" Target="worksheets/sheet1.xml"/>
<Relationship Id="rId2" Type="` + relTypeNS + `/worksheet" Target="/xl/worksheets/sheet2.xml"/>
</Relationships>`,
		// shared[0]=物料；shared[1]=富文本"单价"（rPh 注音必须排除）。
		"xl/sharedStrings.xml": `<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<si><t>物料</t></si>
<si><rPh sb="0" eb="2"><t>たんか</t></rPh><r><rPr lang="ja"/><t>单价</t></r></si>
</sst>`,
		"xl/styles.xml": `<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<numFmts><numFmt numFmtId="164" formatCode="yyyy&quot;年&quot;m&quot;月&quot;d&quot;日&quot;"/></numFmts>
<cellXfs count="3">
<xf numFmtId="0"/>
<xf numFmtId="14" applyNumberFormat="1"/>
<xf numFmtId="164" applyNumberFormat="1"/>
</cellXfs></styleSheet>`,
		// A1=物料 B1=单价（富文本） A2=公式缓存 10 B2=无缓存公式 C2=日期(内置14)=2024-01-01
		// D2=TRUE E2=#DIV/0! F2=内联 G2=自定义日期格式（2024-01-02） 稀疏行 C4=物料。
		"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>
<row r="2"><c r="A2"><f>B9*2</f><v>10</v></c><c r="B2"><f>B9*3</f></c><c r="C2" s="1"><v>45292</v></c><c r="D2" t="b"><v>1</v></c><c r="E2" t="e"><v>#DIV/0!</v></c><c r="F2" t="inlineStr"><is><t>内联</t></is></c><c r="G2" s="2"><v>45293</v></c></row>
<row r="4"><c r="C4" t="s"><v>0</v></c></row>
</sheetData></worksheet>`,
		"xl/worksheets/sheet2.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row r="1"><c r="A1"><v>7</v></c></row></sheetData></worksheet>`,
	}
}

func findTable(doc core.ParsedDocument, after int) *core.DocBlock {
	for i := after; i < len(doc.Blocks); i++ {
		if doc.Blocks[i].Type == core.BlockTable {
			return &doc.Blocks[i]
		}
	}
	return nil
}

func TestParseXlsxBuiltFromParts(t *testing.T) {
	doc := Parse("报价单.xlsx", "", zipBytes(t, xlsxFixture()))
	if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 {
		t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
	}
	if doc.Meta.Confidence != 0.9 {
		t.Fatalf("confidence: %v", doc.Meta.Confidence)
	}
	// 工作表顺序取 workbook.xml（报价在前），隐藏表标注；绝对 target /xl/… 也能定位。
	if doc.Blocks[0].Type != core.BlockHeading || doc.Blocks[0].Text != "报价" {
		t.Fatalf("first block: %+v", doc.Blocks[0])
	}
	heads := []string{}
	for _, b := range doc.Blocks {
		if b.Type == core.BlockHeading {
			heads = append(heads, b.Text)
		}
	}
	if strings.Join(heads, "|") != "报价|草稿（隐藏）" {
		t.Fatalf("headings: %v", heads)
	}
	tbl := findTable(doc, 0)
	if tbl == nil || len(tbl.Rows) != 3 {
		t.Fatalf("table: %+v", tbl)
	}
	row1, row2, row4 := tbl.Rows[0], tbl.Rows[1], tbl.Rows[2]
	if row1[0] != "物料" || row1[1] != "单价" {
		t.Errorf("header row (phonetic must be excluded): %v", row1)
	}
	want := []string{"10", "=B9*3", "2024-01-01", "TRUE", "#DIV/0!", "内联", "2024-01-02"}
	if len(row2) != len(want) {
		t.Fatalf("row2: %v", row2)
	}
	for i, w := range want {
		if row2[i] != w {
			t.Errorf("row2[%d]: got %q want %q", i, row2[i], w)
		}
	}
	// 稀疏单元格按引用就位：C4 → 第三列。
	if len(row4) != 3 || row4[0] != "" || row4[1] != "" || row4[2] != "物料" {
		t.Errorf("sparse row: %v", row4)
	}
	if !strings.Contains(doc.PlainText(), "2024-01-01") {
		t.Error("PlainText missing date text")
	}
}

func TestParseXlsxEmptySheetSkipped(t *testing.T) {
	files := xlsxFixture()
	files["xl/worksheets/sheet2.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1"/></sheetData></worksheet>`
	doc := Parse("x.xlsx", "", zipBytes(t, files))
	heads := 0
	for _, b := range doc.Blocks {
		if b.Type == core.BlockHeading {
			heads++
		}
	}
	if heads != 1 || len(doc.Warnings) != 0 {
		t.Fatalf("empty sheet must be skipped: blocks=%+v warnings=%v", doc.Blocks, doc.Warnings)
	}
}

func TestParseXlsxInvalid(t *testing.T) {
	doc := Parse("坏.xlsx", "", []byte("not a zip"))
	if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
		t.Fatalf("invalid xlsx should fail visibly: %+v %v", doc.Meta, doc.Warnings)
	}
	files := xlsxFixture()
	delete(files, "xl/workbook.xml")
	doc = Parse("无工作簿.xlsx", "", zipBytes(t, files))
	if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
		t.Fatalf("missing workbook.xml should fail visibly: %v", doc.Warnings)
	}
}

func TestParseXlsxCellCap(t *testing.T) {
	old := xlsxMaxCells
	xlsxMaxCells = 3
	defer func() { xlsxMaxCells = old }()
	doc := Parse("超限.xlsx", "", zipBytes(t, xlsxFixture()))
	if len(doc.Blocks) == 0 {
		t.Fatal("truncated parse should keep partial blocks")
	}
	if len(doc.Warnings) == 0 || doc.Meta.Confidence != 0.6 {
		t.Fatalf("truncation must warn and lower confidence: %v %v", doc.Meta.Confidence, doc.Warnings)
	}
	if !strings.Contains(strings.Join(doc.Warnings, ";"), "上限") {
		t.Errorf("warnings: %v", doc.Warnings)
	}
}

func TestParseXlsxPartSizeCap(t *testing.T) {
	old := ooxmlPartLimit
	ooxmlPartLimit = 16
	defer func() { ooxmlPartLimit = old }()
	doc := Parse("炸弹.xlsx", "", zipBytes(t, xlsxFixture()))
	if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
		t.Fatalf("oversized part must fail visibly: %+v %v", doc.Meta, doc.Warnings)
	}
}

func TestParseXlsxDate1904(t *testing.T) {
	files := xlsxFixture()
	files["xl/workbook.xml"] = strings.Replace(files["xl/workbook.xml"],
		"<sheets>", `<workbookPr date1904="1"/><sheets>`, 1)
	files["xl/worksheets/sheet1.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row r="1"><c r="A1" s="1"><v>1</v></c><c r="B1" s="1"><v>0.520833333333333</v></c></row></sheetData></worksheet>`
	doc := Parse("mac.xlsx", "", zipBytes(t, files))
	tbl := findTable(doc, 0)
	// 1904 历法：serial 0 = 1904-01-01，serial 1 = 1904-01-02。
	if tbl == nil || tbl.Rows[0][0] != "1904-01-02" {
		t.Fatalf("1904 epoch: %+v", tbl)
	}
	if !strings.Contains(tbl.Rows[0][1], "1904-01-01 12:30") {
		t.Errorf("time fraction: %q", tbl.Rows[0][1])
	}
}

const pNS = "http://schemas.openxmlformats.org/presentationml/2006/main"
const aNS = "http://schemas.openxmlformats.org/drawingml/2006/main"

func sp(phType, text string) string {
	ph := ""
	if phType != "" {
		ph = `<p:nvPr><p:ph type="` + phType + `"/></p:nvPr>`
	} else {
		ph = `<p:nvPr/>`
	}
	return `<p:sp><p:nvSpPr><p:cNvPr id="2" name="S"/><p:cNvSpPr/></p:nvSpPr>` + ph + `<p:spPr/>` +
		`<p:txBody><a:bodyPr/><a:p><a:r><a:t>` + text + `</a:t></a:r></a:p></p:txBody></p:sp>`
}

func pptxFixture() map[string]string {
	presRel := func(id, target string) string {
		return `<Relationship Id="` + id + `" Type="` + relTypeNS + `/slide" Target="` + target + `"/>`
	}
	slide1 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `" xmlns:r="` + relTypeNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>
` + sp("title", "封面标题") + `
` + sp("body", "") + `
<p:sp><p:nvSpPr><p:cNvPr id="4" name="B"/><p:cNvSpPr/></p:nvSpPr><p:nvPr/><p:spPr/><p:txBody><a:bodyPr/>
<a:p><a:r><a:t>第一行</a:t></a:r><a:br/><a:r><a:t>第二行</a:t></a:r></a:p></p:txBody></p:sp>
<p:grpSp><p:nvGrpSpPr/><p:grpSpPr/>` + sp("", "组合内文本") + `</p:grpSp>
<mc:AlternateContent xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006">
<mc:Choice Requires="p14">` + sp("", "选内文本") + `</mc:Choice>
<mc:Fallback>` + sp("", "弃内文本") + `</mc:Fallback>
</mc:AlternateContent>
<p:sp><p:nvSpPr><p:cNvPr id="6" name="N"/><p:cNvSpPr/></p:nvSpPr><p:nvPr><p:ph type="sldNum"/></p:nvPr><p:spPr/><p:txBody><a:p><a:fld id="{F1}"><a:t>3</a:t></a:fld></a:p></p:txBody></p:sp>
<p:graphicFrame><p:nvGraphicFramePr/><p:xfrm/><a:graphic><a:graphicData uri="` + aNS + `/table"><a:tbl>
<a:tr h="1"><a:tc gridSpan="2"><a:txBody><a:p><a:r><a:t>合并头</a:t></a:r></a:p></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>列三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
<a:tr h="1"><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>值三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
</a:tbl></a:graphicData></a:graphic></p:graphicFrame>
</p:spTree></p:cSld></p:sld>`
	slide2 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "第二页要点") + `
</p:spTree></p:cSld></p:sld>`
	return map[string]string{
		"ppt/presentation.xml": `<p:presentation xmlns:p="` + pNS + `" xmlns:r="` + relTypeNS + `"><p:sldIdLst>
<p:sldId id="256" r:id="rIdA"/><p:sldId id="257" r:id="rIdB"/>
</p:sldIdLst></p:presentation>`,
		"ppt/_rels/presentation.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">` +
			presRel("rIdA", "slides/slide1.xml") + presRel("rIdB", "slides/slide2.xml") + `</Relationships>`,
		"ppt/slides/slide1.xml":            slide1,
		"ppt/slides/slide2.xml":            slide2,
		"ppt/slides/_rels/slide1.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rIdN" Type="` + relTypeNS + `/notesSlide" Target="../notesSlides/notesSlide1.xml"/></Relationships>`,
		// slide2 的备注故意缺失（无 rels 文件）。
		"ppt/notesSlides/notesSlide1.xml": `<p:notes xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "补充说明") + `
</p:spTree></p:cSld></p:notes>`,
	}
}

func TestParsePptxBuiltFromParts(t *testing.T) {
	doc := Parse("演示.pptx", "", zipBytes(t, pptxFixture()))
	if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 || doc.Meta.Confidence != 0.9 {
		t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
	}
	var heads []string
	for _, b := range doc.Blocks {
		if b.Type == core.BlockHeading {
			heads = append(heads, b.Text)
		}
	}
	// sldIdLst 顺序：slide1（有标题）在前；slide2 无标题占位符 → 无标题兜底。
	if len(heads) != 2 || heads[0] != "幻灯片 1：封面标题" || heads[1] != "幻灯片 2" {
		t.Fatalf("headings: %v", heads)
	}
	plain := doc.PlainText()
	for _, want := range []string{"第一行\n第二行", "组合内文本", "选内文本", "备注：补充说明"} {
		if !strings.Contains(plain, want) {
			t.Errorf("missing %q in plain text", want)
		}
	}
	if strings.Contains(plain, "弃内文本") {
		t.Error("mc:Fallback content must be skipped")
	}
	// 页码占位符与字段文本不出现。
	if strings.Contains(plain, "\n3\n") {
		t.Error("slide number placeholder text leaked")
	}
	// 表格：gridSpan 占位 + hMerge 续格留空，保持三列对齐。
	tbl := findTable(doc, 0)
	if tbl == nil || len(tbl.Rows) != 2 {
		t.Fatalf("table: %+v", tbl)
	}
	if strings.Join(tbl.Rows[0], "|") != "合并头||列三" {
		t.Errorf("row1: %v", tbl.Rows[0])
	}
	if strings.Join(tbl.Rows[1], "|") != "||值三" {
		t.Errorf("row2: %v", tbl.Rows[1])
	}
}

func TestParsePptxInvalidAndEmpty(t *testing.T) {
	doc := Parse("坏.pptx", "", []byte("garbage"))
	if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
		t.Fatalf("invalid pptx: %+v %v", doc.Meta, doc.Warnings)
	}
	files := pptxFixture()
	files["ppt/presentation.xml"] = `<p:presentation xmlns:p="` + pNS + `"></p:presentation>`
	doc = Parse("空演示.pptx", "", zipBytes(t, files))
	if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
		t.Fatalf("no slides: %v", doc.Warnings)
	}
}

func TestParseLegacyOfficeHint(t *testing.T) {
	for ext, want := range map[string]string{
		"doc": "docx", "xls": "xlsx", "ppt": "pptx",
	} {
		doc := Parse("旧档."+ext, "", []byte{0xD0, 0xCF})
		if doc.Meta.Parser != "builtin:legacy-office" {
			t.Fatalf("%s parser: %+v", ext, doc.Meta)
		}
		if len(doc.Blocks) != 0 || len(doc.Warnings) != 1 {
			t.Fatalf("%s blocks/warnings: %+v %v", ext, doc.Blocks, doc.Warnings)
		}
		if !strings.Contains(doc.Warnings[0], want) {
			t.Errorf("%s hint should mention %s: %v", ext, want, doc.Warnings[0])
		}
	}
}

// 注册表与分派行为一致：每个已注册格式的解析器 ID 与 Parse 分派结果一致，
// 且 builtin 解析器覆盖的扩展名都出现在注册表里。
func TestParsersDescriptionMatchesDispatch(t *testing.T) {
	byFormat := map[string]string{}
	for _, d := range ParsersDescription() {
		if d.Source != core.SourceBuiltin {
			continue
		}
		for _, f := range d.Formats {
			byFormat[f] = d.ID
		}
	}
	check := map[string]string{
		"md": "builtin:md", "markdown": "builtin:md",
		"txt": "builtin:txt", "text": "builtin:txt", "csv": "builtin:txt", "log": "builtin:txt",
		"docx": "builtin:docx", "pdf": "builtin:pdf",
		"xlsx": "builtin:xlsx", "xlsm": "builtin:xlsx",
		"pptx": "builtin:pptx", "pptm": "builtin:pptx", "ppsx": "builtin:pptx",
		"doc": "builtin:legacy-office", "xls": "builtin:legacy-office", "ppt": "builtin:legacy-office",
	}
	for ext, wantID := range check {
		got, ok := byFormat[ext]
		if !ok {
			t.Errorf("format .%s missing from registry", ext)
			continue
		}
		doc := Parse("a."+ext, "", []byte("x"))
		if doc.Meta.Parser != wantID || got != wantID {
			t.Errorf(".%s: registry=%s dispatch=%s want=%s", ext, got, doc.Meta.Parser, wantID)
		}
	}
}

// TestParseXlsxWideColumnSkipsCellNotSheets：稀疏样式单元格落在远端列（XFD 区，
// 列序 ≥1024）只跳过该格并告警——不误报单元格上限、不丢弃后续工作表。
func TestParseXlsxWideColumnSkipsCellNotSheets(t *testing.T) {
	files := xlsxFixture()
	// 第一表加一个列引用 XFD1（16384 列）的样式单元格；第二表（隐藏的"草稿"）必须照常解析。
	files["xl/worksheets/sheet1.xml"] = strings.Replace(files["xl/worksheets/sheet1.xml"],
		"</row>\n<row r=\"4\"", "</row>\n<row r=\"3\"><c r=\"XFD3\" t=\"inlineStr\"><is><t>边角样式</t></is></c></row>\n<row r=\"4\"", 1)
	doc := Parse("宽表.xlsx", "", zipBytes(t, files))
	heads := []string{}
	for _, b := range doc.Blocks {
		if b.Type == core.BlockHeading {
			heads = append(heads, b.Text)
		}
	}
	if strings.Join(heads, "|") != "报价|草稿（隐藏）" {
		t.Fatalf("later sheets must survive a wide column: headings=%v warnings=%v", heads, doc.Warnings)
	}
	joined := strings.Join(doc.Warnings, ";")
	if !strings.Contains(joined, "超过 1024 列") {
		t.Fatalf("wide-column warning missing: %v", doc.Warnings)
	}
	if strings.Contains(joined, "单元格总数") {
		t.Fatalf("cell-cap warning must not fire for a wide column: %v", doc.Warnings)
	}
	if doc.Meta.Confidence != 0.6 {
		t.Fatalf("partial-content confidence expected 0.6: %v", doc.Meta.Confidence)
	}
	// 被跳过的远端列内容不得混进表格。
	for _, b := range doc.Blocks {
		if b.Type == core.BlockTable {
			for _, row := range b.Rows {
				for _, cell := range row {
					if strings.Contains(cell, "边角样式") {
						t.Fatalf("skipped wide-column cell leaked into table: %v", row)
					}
				}
			}
		}
	}
}

// TestParseXlsxBadSheetXMLWarns：单张工作表 XML 损坏必须显式告警（其余表照常），
// 与提案「显式失败，不静默」一致。
func TestParseXlsxBadSheetXMLWarns(t *testing.T) {
	files := xlsxFixture()
	files["xl/worksheets/sheet1.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row`
	doc := Parse("坏表.xlsx", "", zipBytes(t, files))
	if len(doc.Warnings) == 0 {
		t.Fatal("malformed sheet XML must warn")
	}
	joined := strings.Join(doc.Warnings, ";")
	if !strings.Contains(joined, "工作表 报价") || !strings.Contains(joined, "XML 解码失败") {
		t.Fatalf("warning should name the sheet and the failure: %v", doc.Warnings)
	}
	// 第二张表照常解析。
	found := false
	for _, b := range doc.Blocks {
		if b.Type == core.BlockHeading && b.Text == "草稿（隐藏）" {
			found = true
		}
	}
	if !found {
		t.Fatalf("healthy sheets must still parse: %+v", doc.Blocks)
	}
	if doc.Meta.Confidence != 0.6 {
		t.Fatalf("partial parse confidence expected 0.6: %v", doc.Meta.Confidence)
	}
}