Skip wide sparse columns without dropping later xlsx sheets
ci / vet-and-test (pull_request) Successful in 41s
ci / vet-and-test (pull_request) Successful in 41s
Why: review of EXT-5 found a column-cap cell (common styled XFD cells) wrongly reported the 200k-cell cap and aborted every remaining worksheet; malformed sheet XML was also dropped silently, contradicting the explicit-failure contract; the legacy-office parser id was missing from the registry. Evidence: local GOWORK=off go test ./... 29 packages pass with new regressions (wide column keeps later sheets, bad sheet XML warns, registry maps doc/xls/ppt); go vet clean; recorded as SELF on the Dev task card. Impact: xlsxSheetBlocks returns wide/badXML signals; wide cells skip with a per-sheet warning, cell-cap truncation unchanged; builtin:legacy-office added to ParsersDescription. Refs: review of backend 727f370; documents proposal 2026-10-06-ext-5-host-parse-document sect.2.
This commit is contained in:
@@ -361,6 +361,7 @@ func ParsersDescription() []core.ParserDesc {
|
||||
{ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层;扫描件为远期 MinerU 深度档"},
|
||||
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:legacy-office", Formats: []string{"doc", "xls", "ppt"}, Source: core.SourceBuiltin, Note: "不解析;提示转存现代格式后重传"},
|
||||
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,6 +347,7 @@ func TestParsersDescriptionMatchesDispatch(t *testing.T) {
|
||||
"docx": "builtin:docx", "pdf": "builtin:pdf",
|
||||
"xlsx": "builtin:xlsx", "xlsm": "builtin:xlsx",
|
||||
"pptx": "builtin:pptx", "pptm": "builtin:pptx", "ppsx": "builtin:pptx",
|
||||
"doc": "builtin:legacy-office", "xls": "builtin:legacy-office", "ppt": "builtin:legacy-office",
|
||||
}
|
||||
for ext, wantID := range check {
|
||||
got, ok := byFormat[ext]
|
||||
@@ -360,3 +361,72 @@ func TestParsersDescriptionMatchesDispatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseXlsxWideColumnSkipsCellNotSheets:稀疏样式单元格落在远端列(XFD 区,
|
||||
// 列序 ≥1024)只跳过该格并告警——不误报单元格上限、不丢弃后续工作表。
|
||||
func TestParseXlsxWideColumnSkipsCellNotSheets(t *testing.T) {
|
||||
files := xlsxFixture()
|
||||
// 第一表加一个列引用 XFD1(16384 列)的样式单元格;第二表(隐藏的"草稿")必须照常解析。
|
||||
files["xl/worksheets/sheet1.xml"] = strings.Replace(files["xl/worksheets/sheet1.xml"],
|
||||
"</row>\n<row r=\"4\"", "</row>\n<row r=\"3\"><c r=\"XFD3\" t=\"inlineStr\"><is><t>边角样式</t></is></c></row>\n<row r=\"4\"", 1)
|
||||
doc := Parse("宽表.xlsx", "", zipBytes(t, files))
|
||||
heads := []string{}
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads = append(heads, b.Text)
|
||||
}
|
||||
}
|
||||
if strings.Join(heads, "|") != "报价|草稿(隐藏)" {
|
||||
t.Fatalf("later sheets must survive a wide column: headings=%v warnings=%v", heads, doc.Warnings)
|
||||
}
|
||||
joined := strings.Join(doc.Warnings, ";")
|
||||
if !strings.Contains(joined, "超过 1024 列") {
|
||||
t.Fatalf("wide-column warning missing: %v", doc.Warnings)
|
||||
}
|
||||
if strings.Contains(joined, "单元格总数") {
|
||||
t.Fatalf("cell-cap warning must not fire for a wide column: %v", doc.Warnings)
|
||||
}
|
||||
if doc.Meta.Confidence != 0.6 {
|
||||
t.Fatalf("partial-content confidence expected 0.6: %v", doc.Meta.Confidence)
|
||||
}
|
||||
// 被跳过的远端列内容不得混进表格。
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockTable {
|
||||
for _, row := range b.Rows {
|
||||
for _, cell := range row {
|
||||
if strings.Contains(cell, "边角样式") {
|
||||
t.Fatalf("skipped wide-column cell leaked into table: %v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseXlsxBadSheetXMLWarns:单张工作表 XML 损坏必须显式告警(其余表照常),
|
||||
// 与提案「显式失败,不静默」一致。
|
||||
func TestParseXlsxBadSheetXMLWarns(t *testing.T) {
|
||||
files := xlsxFixture()
|
||||
files["xl/worksheets/sheet1.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row`
|
||||
doc := Parse("坏表.xlsx", "", zipBytes(t, files))
|
||||
if len(doc.Warnings) == 0 {
|
||||
t.Fatal("malformed sheet XML must warn")
|
||||
}
|
||||
joined := strings.Join(doc.Warnings, ";")
|
||||
if !strings.Contains(joined, "工作表 报价") || !strings.Contains(joined, "XML 解码失败") {
|
||||
t.Fatalf("warning should name the sheet and the failure: %v", doc.Warnings)
|
||||
}
|
||||
// 第二张表照常解析。
|
||||
found := false
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading && b.Text == "草稿(隐藏)" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("healthy sheets must still parse: %+v", doc.Blocks)
|
||||
}
|
||||
if doc.Meta.Confidence != 0.6 {
|
||||
t.Fatalf("partial parse confidence expected 0.6: %v", doc.Meta.Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
+19
-9
@@ -55,10 +55,16 @@ func parseXlsx(content []byte) ([]core.DocBlock, []string) {
|
||||
warns = append(warns, fmt.Sprintf("工作表 %s 读取失败:%v", sh.Name, err))
|
||||
continue
|
||||
}
|
||||
table, used, trunc := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells)
|
||||
table, used, trunc, wide, sheetWarn := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells)
|
||||
cells = used
|
||||
if sheetWarn != "" {
|
||||
warns = append(warns, fmt.Sprintf("工作表 %s:%s", sh.Name, sheetWarn))
|
||||
}
|
||||
if wide {
|
||||
warns = append(warns, fmt.Sprintf("工作表 %s 含超过 %d 列的稀疏单元格,已跳过该列(其余内容不受影响)", sh.Name, xlsxMaxCols))
|
||||
}
|
||||
if table == nil {
|
||||
continue // 空表
|
||||
continue // 空表(读取失败/坏 XML 已单独告警)
|
||||
}
|
||||
title := sh.Name
|
||||
if sh.State == "hidden" || sh.State == "veryHidden" {
|
||||
@@ -264,20 +270,22 @@ type xlsxRowEl struct {
|
||||
Cells []xlsxCellEl `xml:"c"`
|
||||
}
|
||||
|
||||
// xlsxSheetBlocks 解析单张工作表;返回表格块、累计单元格数、是否触达上限。
|
||||
// 无内容行返回 nil 表。
|
||||
func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool) {
|
||||
// xlsxSheetBlocks 解析单张工作表;返回表格块、累计单元格数、是否触达单元格
|
||||
// 上限、是否跳过超宽列、以及坏 XML 的告警(显式失败,不静默)。无内容行
|
||||
// 返回 nil 表。
|
||||
func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool, bool, string) {
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
var rows [][]string
|
||||
count := cellsUsed
|
||||
truncated := false
|
||||
wide := false
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, count, false
|
||||
return nil, count, truncated, wide, "XML 解码失败:" + err.Error()
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok || st.Name.Local != "row" {
|
||||
@@ -300,7 +308,9 @@ func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date
|
||||
col = maxCol + 1 // 无引用:按出现顺序接续
|
||||
}
|
||||
if col >= xlsxMaxCols {
|
||||
truncated = true
|
||||
// 稀疏样式单元格常落在远端列(XFD 区):跳过该格并告警,
|
||||
// 不中断本表其余内容,也不阻断后续工作表。
|
||||
wide = true
|
||||
continue
|
||||
}
|
||||
cells[col] = xlsxCellText(c, shared, dateStyles, date1904)
|
||||
@@ -323,9 +333,9 @@ func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date
|
||||
}
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, count, truncated
|
||||
return nil, count, truncated, wide, ""
|
||||
}
|
||||
return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated
|
||||
return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated, wide, ""
|
||||
}
|
||||
|
||||
// xlsxCellText 把单元格转成展示文本。
|
||||
|
||||
Reference in New Issue
Block a user