From 727f3702d0646dbe5cbc7b55344d7aa264a23191 Mon Sep 17 00:00:00 2001 From: weichen Date: Tue, 6 Oct 2026 14:59:38 +0000 Subject: [PATCH] Expose builtin document parsing through the host handle Why: downstream parsers are unreachable via extension.Host and the base lacked xlsx/pptx parsers; MS1 needs Excel and PPT ingest (EXT-5 request, documents proposal 2026-10-06-ext-5-host-parse-document). Evidence: local GOWORK=off go test ./... all 29 packages pass incl. new ooxml_test (hand-built parts), ooxml_python_test (openpyxl/python-pptx round-trip), inproc owner-scope and limit tests, xlsx upload e2e; go vet clean; recorded as SELF on the Dev task card, PR CI to confirm. Impact: Host gains ParseDocument (pure compute; errors only on ctx cancel or >100MiB input); builtin:xlsx (sheet order via workbook.xml rels, rich-text shared strings, formula cache, 1900/1904 dates, sparse cells) and builtin:pptx (sldIdLst order, titles, group shapes, AlternateContent Choice, tables, notes) added with 200k-cell/64MiB-part caps; legacy doc/xls/ppt get an actionable hint; ParsersDescription gains xlsx/pptx and the missing text ext; surface snapshot now records exported interface methods; docx part reads share the 64MiB cap. Refs: documents 2026-10-06-ext-5-host-parse-document.md; EXT-1 host.go precedent. --- internal/app/hostcaps.go | 24 +- internal/app/inproc_host_test.go | 111 +++++++ internal/ingest/ingest.go | 66 +++-- internal/ingest/ooxml.go | 140 +++++++++ internal/ingest/ooxml_python_test.go | 140 +++++++++ internal/ingest/ooxml_test.go | 362 +++++++++++++++++++++++ internal/ingest/pptx.go | 291 +++++++++++++++++++ internal/ingest/xlsx.go | 415 +++++++++++++++++++++++++++ internal/server/e2e_test.go | 73 +++++ pkg/extension/host.go | 10 +- pkg/extension/surface.txt | 17 ++ pkg/extension/surface_test.go | 11 + 12 files changed, 1640 insertions(+), 20 deletions(-) create mode 100644 internal/ingest/ooxml.go create mode 100644 internal/ingest/ooxml_python_test.go create mode 100644 internal/ingest/ooxml_test.go create mode 100644 internal/ingest/pptx.go create mode 100644 internal/ingest/xlsx.go diff --git a/internal/app/hostcaps.go b/internal/app/hostcaps.go index 05a941f..99d00a7 100644 --- a/internal/app/hostcaps.go +++ b/internal/app/hostcaps.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "agent-system-backend/internal/ingest" "agent-system-backend/internal/rag" "agent-system-backend/internal/runtime" "agent-system-backend/internal/settings" @@ -39,6 +40,21 @@ func (c hostcaps) complete(ctx context.Context, owner, prompt string) (string, e return completeChat(ctx, route.Base, route.Key, route.Model, prompt) } +// maxParseInput 与上传端点的读取上限同口径（server.upload 的 100<<20）。 +const maxParseInput = 100 << 20 + +// parse 走基座内置解析器（EXT-5）：纯计算、不落 owner 数据；不可解析由 +// Warnings 显式表达，只有取消与超限是 error。 +func (c hostcaps) parse(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) { + if err := ctx.Err(); err != nil { + return core.ParsedDocument{}, err + } + if len(content) > maxParseInput { + return core.ParsedDocument{}, fmt.Errorf("parse input %d bytes exceeds %d limit", len(content), maxParseInput) + } + return ingest.Parse(filename, mime, content), nil +} + // inprocHost 是进程内插件命令的 owner 作用域宿主句柄（extension.Host 实现）。 // 每次分发按请求 Owner 新建；所有能力调用都以该 owner 为第一参数，插件无法越权。 type inprocHost struct { @@ -75,7 +91,7 @@ func (h inprocHost) KnowledgeSearch(ctx context.Context, projectID core.ID, text // KnowledgeWrite 强制 owner 归属：Owner 改写为本次分发 owner，Source 为空补 // "plugin:"；embed=false 只写全文行，embed=true 走 IndexChunks 建向量 -//（embedding 失败降级纯全文，不阻断写入）。 +// （embedding 失败降级纯全文，不阻断写入）。 func (h inprocHost) KnowledgeWrite(ctx context.Context, docs []core.KnowledgeDoc, embed bool) error { if len(docs) == 0 { return nil @@ -114,3 +130,9 @@ func (h inprocHost) ArtifactGet(artifactID string) (core.Artifact, error) { func (h inprocHost) ArtifactContent(artifactID string) ([]byte, error) { return h.caps.st.ArtifactContent(h.owner, artifactID) } + +// ParseDocument 内置文档解析（EXT-5）：委托 hostcaps.parse；纯计算不经存储， +// owner 作用域由调用方经 ArtifactContent 取字节时保证。 +func (h inprocHost) ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) { + return h.caps.parse(ctx, filename, mime, content) +} diff --git a/internal/app/inproc_host_test.go b/internal/app/inproc_host_test.go index ea9026a..acb6a39 100644 --- a/internal/app/inproc_host_test.go +++ b/internal/app/inproc_host_test.go @@ -1,6 +1,8 @@ package app import ( + "archive/zip" + "bytes" "context" "encoding/json" "path/filepath" @@ -193,3 +195,112 @@ func TestInprocHostKnowledgeWriteEmbedHash(t *testing.T) { t.Fatalf("vector owner leak: %+v", bhits) } } + +// TestInprocHostParseDocumentOwnerScope：EXT-5 解析方法的 owner 作用域—— +// alice 经句柄读自己的工件并解析；bob 用同一工件 ID 在 ArtifactContent 处 +// 即被拒（解析是纯计算，作用域边界在取原件一环）。 +func TestInprocHostParseDocumentOwnerScope(t *testing.T) { + extension.ResetCompiled() + t.Cleanup(extension.ResetCompiled) + extension.RegisterCompiled(extension.InProc{ + ID: "tst", + Register: func(reg extension.Registry) error { + return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.parse","title":"解析"}`), + func(ctx context.Context, h extension.Host, req extension.Request) (any, error) { + id, _ := req.Params["artifactId"].(string) + content, err := h.ArtifactContent(id) + if err != nil { + return nil, err + } + doc, err := h.ParseDocument(ctx, "报价.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", content) + if err != nil { + return nil, err + } + return map[string]any{ + "parser": doc.Meta.Parser, + "warnings": len(doc.Warnings), + "blocks": len(doc.Blocks), + }, nil + }) + }, + }) + bus, st := newInprocTestApp(t, nil) + + art := core.Artifact{ProjectID: "p1", Owner: "alice", Name: "报价.xlsx", Kind: "document"} + if err := st.CreateArtifact(&art, testXLSXBytes(t)); err != nil { + t.Fatal(err) + } + artID := art.ID + + resp := bus.Dispatch(context.Background(), cmdbus.Request{ + ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "alice", Entrance: "chat", + }) + if resp.Receipt.State != agentcommand.StateCommitted { + t.Fatalf("alice dispatch: %+v %s", resp.Receipt, resp.Receipt.Reason) + } + m, _ := resp.Payload.(map[string]any) + if m["parser"] != "builtin:xlsx" || m["warnings"] != 0 || m["blocks"] == 0 { + t.Fatalf("alice payload: %+v", m) + } + + // bob 同一工件 ID：取原件即被拒（跨 owner 不可见）。 + resp = bus.Dispatch(context.Background(), cmdbus.Request{ + ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "bob", Entrance: "chat", + }) + if resp.Receipt.State == agentcommand.StateCommitted { + t.Fatalf("bob must not read alice's artifact: %+v", resp.Payload) + } +} + +// TestInprocHostParseDocumentLimits：仅有的两条 error 路径——ctx 取消与输入 +// 超过 100MiB 上限；不可解析格式仍是带 Warnings 的显式失败。 +func TestInprocHostParseDocumentLimits(t *testing.T) { + st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + h := inprocHost{caps: hostcaps{st: st}, pluginID: "tst", owner: "alice"} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := h.ParseDocument(ctx, "a.xlsx", "", []byte("x")); err == nil { + t.Fatal("canceled ctx must error") + } + if _, err := h.ParseDocument(context.Background(), "a.xlsx", "", make([]byte, maxParseInput+1)); err == nil { + t.Fatal("oversized input must error") + } + doc, err := h.ParseDocument(context.Background(), "旧.xls", "", []byte{0xD0, 0xCF}) + if err != nil { + t.Fatalf("legacy format is a visible failure, not an error: %v", err) + } + if doc.Meta.Parser != "builtin:legacy-office" || len(doc.Warnings) != 1 { + t.Fatalf("legacy doc: %+v %v", doc.Meta, doc.Warnings) + } +} + +// testXLSXBytes 构造最小可用 xlsx（单表两行），供句柄测试解析。 +func testXLSXBytes(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + files := map[string]string{ + "xl/workbook.xml": ``, + "xl/_rels/workbook.xml.rels": ``, + "xl/worksheets/sheet1.xml": ` +物料单价 +支架12.5 +`, + } + for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(files[name])) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index fe254c3..168c7dc 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -4,7 +4,6 @@ package ingest import ( - "archive/zip" "bytes" "encoding/xml" "fmt" @@ -52,6 +51,35 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument { doc.Blocks = blocks doc.Meta.Confidence = 0.85 } + case "xlsx", "xlsm": + doc.Meta.Parser = "builtin:xlsx" + blocks, warns := parseXlsx(content) + doc.Blocks = blocks + doc.Warnings = append(doc.Warnings, warns...) + switch { + case len(blocks) == 0: + doc.Meta.Confidence = 0.3 + case len(warns) > 0: + doc.Meta.Confidence = 0.6 // 超限截断：内容可用但注明不完整 + default: + doc.Meta.Confidence = 0.9 + } + case "pptx", "pptm", "ppsx": + doc.Meta.Parser = "builtin:pptx" + blocks, warns := parsePptx(content) + doc.Blocks = blocks + doc.Warnings = append(doc.Warnings, warns...) + switch { + case len(blocks) == 0: + doc.Meta.Confidence = 0.3 + case len(warns) > 0: + doc.Meta.Confidence = 0.6 + default: + doc.Meta.Confidence = 0.9 + } + case "doc", "xls", "ppt": + doc.Meta.Parser = "builtin:legacy-office" + doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析：请转存为 ."+legacyOfficeTarget(ext)+" 后重传") default: if isImage(ext) { doc.Warnings = append(doc.Warnings, "图片不走文本解析（OCR 为 MinerU 深度档，远期）") @@ -164,24 +192,13 @@ type xtc struct { } func parseDocx(content []byte) ([]core.DocBlock, error) { - zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content))) + zr, err := openZip(content) if err != nil { return nil, fmt.Errorf("open zip: %w", err) } - var docXML []byte - for _, f := range zr.File { - if f.Name == "word/document.xml" { - rc, err := f.Open() - if err != nil { - return nil, err - } - docXML, _ = io.ReadAll(rc) - rc.Close() - break - } - } - if docXML == nil { - return nil, fmt.Errorf("word/document.xml not found") + docXML, err := zipPart(zr, "word/document.xml") + if err != nil { + return nil, err } return parseDocxBody(docXML) } @@ -322,13 +339,28 @@ func isMostlyPrintable(s string) bool { return nonPrint*10 < len(s) } +// legacyOfficeTarget 旧版二进制 Office 格式的现代等价格式（提示转存用）。 +func legacyOfficeTarget(ext string) string { + switch ext { + case "doc": + return "docx" + case "xls": + return "xlsx" + case "ppt": + return "pptx" + } + return ext +} + // ParsersDescription 返回解析器注册表描述（来源标记；设置页/文档用）。 func ParsersDescription() []core.ParserDesc { return []core.ParserDesc{ {ID: "builtin:md", Formats: []string{"md", "markdown"}, Source: core.SourceBuiltin}, - {ID: "builtin:txt", Formats: []string{"txt", "csv", "log"}, Source: core.SourceBuiltin}, + {ID: "builtin:txt", Formats: []string{"txt", "text", "csv", "log"}, Source: core.SourceBuiltin}, {ID: "builtin:docx", Formats: []string{"docx"}, Source: core.SourceBuiltin}, {ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层；扫描件为远期 MinerU 深度档"}, + {ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin}, + {ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin}, {ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选：深度解析与交叉验证（默认不部署）"}, } } diff --git a/internal/ingest/ooxml.go b/internal/ingest/ooxml.go new file mode 100644 index 0000000..d4ef83d --- /dev/null +++ b/internal/ingest/ooxml.go @@ -0,0 +1,140 @@ +package ingest + +// OOXML 公共件：zip 部件读取（解压上限防 zip 炸弹）、关系部件解析、 +// r:id 属性与相对路径归一。docx/xlsx/pptx 解析器共用；纯标准库。 + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path" + "strings" +) + +// ooxmlPartLimit 单个 zip 部件的解压上限（测试可覆盖）。 +var ooxmlPartLimit int64 = 64 << 20 + +func openZip(content []byte) (*zip.Reader, error) { + return zip.NewReader(bytes.NewReader(content), int64(len(content))) +} + +// zipPart 按名称读取部件；解压超过 ooxmlPartLimit 视为异常拒绝。 +func zipPart(zr *zip.Reader, name string) ([]byte, error) { + for _, f := range zr.File { + if f.Name != name { + continue + } + rc, err := f.Open() + if err != nil { + return nil, err + } + b, err := io.ReadAll(io.LimitReader(rc, ooxmlPartLimit+1)) + rc.Close() + if err != nil { + return nil, err + } + if int64(len(b)) > ooxmlPartLimit { + return nil, fmt.Errorf("part %s exceeds %d bytes decompression limit", name, ooxmlPartLimit) + } + return b, nil + } + return nil, fmt.Errorf("part %s not found", name) +} + +// relsPath 计算部件对应的关系部件名（xl/workbook.xml → xl/_rels/workbook.xml.rels）。 +func relsPath(part string) string { + dir, base := path.Split(part) + return dir + "_rels/" + base + ".rels" +} + +type ooxmlRel struct { + ID, Target, Type string +} + +// parseRels 解析部件的关系表；无关系部件返回空表（不是错误）。 +func parseRels(zr *zip.Reader, part string) (map[string]ooxmlRel, error) { + data, err := zipPart(zr, relsPath(part)) + if err != nil { + return map[string]ooxmlRel{}, nil + } + dec := xml.NewDecoder(bytes.NewReader(data)) + out := map[string]ooxmlRel{} + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + st, ok := tok.(xml.StartElement) + if !ok || st.Name.Local != "Relationship" { + continue + } + r := ooxmlRel{} + for _, a := range st.Attr { + switch a.Name.Local { + case "Id": + r.ID = a.Value + case "Target": + r.Target = a.Value + case "Type": + r.Type = a.Value + } + } + if r.ID != "" { + out[r.ID] = r + } + } + return out, nil +} + +// resolveRelTarget 把关系 target 解析为 zip 部件名：绝对 target（/xl/…）去 +// 前导斜杠；相对 target 相对 partDir；归一化 ../ 与 ./ 段。 +func resolveRelTarget(partDir, target string) string { + t := target + if !strings.HasPrefix(target, "/") { + t = partDir + "/" + target + } + var seg []string + for _, s := range strings.Split(t, "/") { + switch s { + case "", ".": + case "..": + if len(seg) > 0 { + seg = seg[:len(seg)-1] + } + default: + seg = append(seg, s) + } + } + return strings.Join(seg, "/") +} + +// ridAttr 取 r:id：局部名 id 且带命名空间前缀（Transitional 与 Strict OOXML +// 的关系引用都满足该形态；无前缀的 id 是别的编号，不误取）。 +func ridAttr(st xml.StartElement) string { + for _, a := range st.Attr { + if a.Name.Local == "id" && a.Name.Space != "" { + return a.Value + } + } + return "" +} + +// attrLocal 取无命名空间前缀的属性值。 +func attrLocal(st xml.StartElement, name string) string { + for _, a := range st.Attr { + if a.Name.Local == name && a.Name.Space == "" { + return a.Value + } + } + return "" +} + +// relIsType 判断关系类型是否以标准关系类型后缀结尾（/worksheet、/slide 等）。 +func relIsType(rel ooxmlRel, suffix string) bool { + return strings.HasSuffix(rel.Type, suffix) +} diff --git a/internal/ingest/ooxml_python_test.go b/internal/ingest/ooxml_python_test.go new file mode 100644 index 0000000..74885b2 --- /dev/null +++ b/internal/ingest/ooxml_python_test.go @@ -0,0 +1,140 @@ +package ingest + +// 真实生成器轮测：openpyxl / python-pptx 现场生成 xlsx/pptx 再解析，防止 +// 手工夹具只验证"自己写的 XML"。生成器缺失时 t.Skip（诚实出口）；CI 已装 +// 这两个包（.gitea/workflows/ci.yaml 安装 python-docx python-pptx openpyxl）。 + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + core "agent-system-core" +) + +func pythonModuleAvailable(t *testing.T, module string) bool { + t.Helper() + if _, err := exec.LookPath("python3"); err != nil { + return false + } + return exec.Command("python3", "-c", "import "+module).Run() == nil +} + +// runPythonScript 把脚本写到临时目录执行；失败即 t.Fatal（脚本属测试自身）。 +func runPythonScript(t *testing.T, script string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "gen.py") + if err := os.WriteFile(path, []byte(script), 0o644); err != nil { + t.Fatal(err) + } + out, err := exec.Command("python3", path).CombinedOutput() + if err != nil { + t.Fatalf("python generator failed: %v\n%s", err, out) + } +} + +func TestParseXlsxFromOpenpyxl(t *testing.T) { + if !pythonModuleAvailable(t, "openpyxl") { + t.Skip("python3 openpyxl not available") + } + dir := t.TempDir() + file := filepath.Join(dir, "报价单.xlsx") + runPythonScript(t, ` +import openpyxl +from datetime import datetime +wb = openpyxl.Workbook() +ws = wb.active +ws.title = "报价" +ws.append(["物料", "单价", "日期"]) +ws.append(["支架", 12.5, datetime(2024, 1, 1)]) +ws["C2"].number_format = "yyyy-mm-dd" +ws["D2"] = "=A2&\"-x\"" +ws["D3"] = True +wb.create_sheet("说明").append(["备注行"]) +wb.save(r"`+file+`") +`) + + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + doc := Parse(file, "", data) + if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 { + t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings) + } + var heads []string + for _, b := range doc.Blocks { + if b.Type == core.BlockHeading { + heads = append(heads, b.Text) + } + } + if len(heads) != 2 || heads[0] != "报价" || heads[1] != "说明" { + t.Fatalf("headings: %v", heads) + } + plain := doc.PlainText() + for _, want := range []string{"物料", "支架", "12.5", "2024-01-01"} { + if !strings.Contains(plain, want) { + t.Errorf("missing %q in: %s", want, plain) + } + } + // openpyxl 公式不带缓存值：回显公式本身（可检索、不冒充结果）。 + if !strings.Contains(plain, "=A2") { + t.Errorf("formula without cache should echo: %s", plain) + } + if !strings.Contains(plain, "TRUE") { + t.Errorf("boolean cell should render TRUE: %s", plain) + } +} + +func TestParsePptxFromPythonPptx(t *testing.T) { + if !pythonModuleAvailable(t, "pptx") { + t.Skip("python3 pptx not available") + } + dir := t.TempDir() + file := filepath.Join(dir, "方案评审.pptx") + runPythonScript(t, ` +from pptx import Presentation +from pptx.util import Inches +prs = Presentation() +s1 = prs.slides.add_slide(prs.slide_layouts[0]) +s1.shapes.title.text = "评审结论" +s1.placeholders[1].text = "通过" +s2 = prs.slides.add_slide(prs.slide_layouts[5]) +s2.shapes.title.text = "报价明细" +tbl = s2.shapes.add_table(2, 2, Inches(1), Inches(2), Inches(6), Inches(1)).table +tbl.cell(0, 0).text = "物料" +tbl.cell(0, 1).text = "单价" +tbl.cell(1, 0).text = "支架" +tbl.cell(1, 1).text = "12.5" +notes = s1.notes_slide.notes_text_frame +notes.text = "需要补充法规依据" +prs.save(r"`+file+`") +`) + + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + doc := Parse(file, "", data) + if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 { + t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings) + } + var heads []string + for _, b := range doc.Blocks { + if b.Type == core.BlockHeading { + heads = append(heads, b.Text) + } + } + if len(heads) != 2 || !strings.HasPrefix(heads[0], "幻灯片 1：评审结论") || !strings.HasPrefix(heads[1], "幻灯片 2：报价明细") { + t.Fatalf("headings: %v", heads) + } + plain := doc.PlainText() + for _, want := range []string{"通过", "物料", "支架", "12.5", "备注：需要补充法规依据"} { + if !strings.Contains(plain, want) { + t.Errorf("missing %q in: %s", want, plain) + } + } +} diff --git a/internal/ingest/ooxml_test.go b/internal/ingest/ooxml_test.go new file mode 100644 index 0000000..a854eea --- /dev/null +++ b/internal/ingest/ooxml_test.go @@ -0,0 +1,362 @@ +package ingest + +// OOXML（xlsx/pptx）解析器测试：手工构造最小 zip 部件，覆盖顺序解析、富文本、 +// 公式、日期、稀疏列、隐藏表、组合形状、AlternateContent、表格合并、备注、 +// 上限与失败路径。真实生成器的轮测见 ooxml_python_test.go。 + +import ( + "archive/zip" + "bytes" + "sort" + "strings" + "testing" + + core "agent-system-core" +) + +// zipBytes 按名称构造 zip（键排序保证确定性）。 +func zipBytes(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + names := make([]string, 0, len(files)) + for n := range files { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + w, err := zw.Create(n) + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(files[n])); err != nil { + t.Fatal(err) + } + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +const relTypeNS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + +func xlsxFixture() map[string]string { + return map[string]string{ + "xl/workbook.xml": ` + + + +`, + "xl/_rels/workbook.xml.rels": ` + + +`, + // shared[0]=物料；shared[1]=富文本"单价"（rPh 注音必须排除）。 + "xl/sharedStrings.xml": ` +物料 +たんか单价 +`, + "xl/styles.xml": ` + + + + + +`, + // A1=物料 B1=单价（富文本） A2=公式缓存 10 B2=无缓存公式 C2=日期(内置14)=2024-01-01 + // D2=TRUE E2=#DIV/0! F2=内联 G2=自定义日期格式（2024-01-02） 稀疏行 C4=物料。 + "xl/worksheets/sheet1.xml": ` + +01 +B9*210B9*3452921#DIV/0!内联45293 +0 +`, + "xl/worksheets/sheet2.xml": ` +7`, + } +} + +func findTable(doc core.ParsedDocument, after int) *core.DocBlock { + for i := after; i < len(doc.Blocks); i++ { + if doc.Blocks[i].Type == core.BlockTable { + return &doc.Blocks[i] + } + } + return nil +} + +func TestParseXlsxBuiltFromParts(t *testing.T) { + doc := Parse("报价单.xlsx", "", zipBytes(t, xlsxFixture())) + if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 { + t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings) + } + if doc.Meta.Confidence != 0.9 { + t.Fatalf("confidence: %v", doc.Meta.Confidence) + } + // 工作表顺序取 workbook.xml（报价在前），隐藏表标注；绝对 target /xl/… 也能定位。 + if doc.Blocks[0].Type != core.BlockHeading || doc.Blocks[0].Text != "报价" { + t.Fatalf("first block: %+v", doc.Blocks[0]) + } + heads := []string{} + for _, b := range doc.Blocks { + if b.Type == core.BlockHeading { + heads = append(heads, b.Text) + } + } + if strings.Join(heads, "|") != "报价|草稿（隐藏）" { + t.Fatalf("headings: %v", heads) + } + tbl := findTable(doc, 0) + if tbl == nil || len(tbl.Rows) != 3 { + t.Fatalf("table: %+v", tbl) + } + row1, row2, row4 := tbl.Rows[0], tbl.Rows[1], tbl.Rows[2] + if row1[0] != "物料" || row1[1] != "单价" { + t.Errorf("header row (phonetic must be excluded): %v", row1) + } + want := []string{"10", "=B9*3", "2024-01-01", "TRUE", "#DIV/0!", "内联", "2024-01-02"} + if len(row2) != len(want) { + t.Fatalf("row2: %v", row2) + } + for i, w := range want { + if row2[i] != w { + t.Errorf("row2[%d]: got %q want %q", i, row2[i], w) + } + } + // 稀疏单元格按引用就位：C4 → 第三列。 + if len(row4) != 3 || row4[0] != "" || row4[1] != "" || row4[2] != "物料" { + t.Errorf("sparse row: %v", row4) + } + if !strings.Contains(doc.PlainText(), "2024-01-01") { + t.Error("PlainText missing date text") + } +} + +func TestParseXlsxEmptySheetSkipped(t *testing.T) { + files := xlsxFixture() + files["xl/worksheets/sheet2.xml"] = `` + doc := Parse("x.xlsx", "", zipBytes(t, files)) + heads := 0 + for _, b := range doc.Blocks { + if b.Type == core.BlockHeading { + heads++ + } + } + if heads != 1 || len(doc.Warnings) != 0 { + t.Fatalf("empty sheet must be skipped: blocks=%+v warnings=%v", doc.Blocks, doc.Warnings) + } +} + +func TestParseXlsxInvalid(t *testing.T) { + doc := Parse("坏.xlsx", "", []byte("not a zip")) + if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 { + t.Fatalf("invalid xlsx should fail visibly: %+v %v", doc.Meta, doc.Warnings) + } + files := xlsxFixture() + delete(files, "xl/workbook.xml") + doc = Parse("无工作簿.xlsx", "", zipBytes(t, files)) + if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 { + t.Fatalf("missing workbook.xml should fail visibly: %v", doc.Warnings) + } +} + +func TestParseXlsxCellCap(t *testing.T) { + old := xlsxMaxCells + xlsxMaxCells = 3 + defer func() { xlsxMaxCells = old }() + doc := Parse("超限.xlsx", "", zipBytes(t, xlsxFixture())) + if len(doc.Blocks) == 0 { + t.Fatal("truncated parse should keep partial blocks") + } + if len(doc.Warnings) == 0 || doc.Meta.Confidence != 0.6 { + t.Fatalf("truncation must warn and lower confidence: %v %v", doc.Meta.Confidence, doc.Warnings) + } + if !strings.Contains(strings.Join(doc.Warnings, ";"), "上限") { + t.Errorf("warnings: %v", doc.Warnings) + } +} + +func TestParseXlsxPartSizeCap(t *testing.T) { + old := ooxmlPartLimit + ooxmlPartLimit = 16 + defer func() { ooxmlPartLimit = old }() + doc := Parse("炸弹.xlsx", "", zipBytes(t, xlsxFixture())) + if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 { + t.Fatalf("oversized part must fail visibly: %+v %v", doc.Meta, doc.Warnings) + } +} + +func TestParseXlsxDate1904(t *testing.T) { + files := xlsxFixture() + files["xl/workbook.xml"] = strings.Replace(files["xl/workbook.xml"], + "", ``, 1) + files["xl/worksheets/sheet1.xml"] = ` +10.520833333333333` + doc := Parse("mac.xlsx", "", zipBytes(t, files)) + tbl := findTable(doc, 0) + // 1904 历法：serial 0 = 1904-01-01，serial 1 = 1904-01-02。 + if tbl == nil || tbl.Rows[0][0] != "1904-01-02" { + t.Fatalf("1904 epoch: %+v", tbl) + } + if !strings.Contains(tbl.Rows[0][1], "1904-01-01 12:30") { + t.Errorf("time fraction: %q", tbl.Rows[0][1]) + } +} + +const pNS = "http://schemas.openxmlformats.org/presentationml/2006/main" +const aNS = "http://schemas.openxmlformats.org/drawingml/2006/main" + +func sp(phType, text string) string { + ph := "" + if phType != "" { + ph = `` + } else { + ph = `` + } + return `` + ph + `` + + `` + text + `` +} + +func pptxFixture() map[string]string { + presRel := func(id, target string) string { + return `` + } + slide1 := ` + +` + sp("title", "封面标题") + ` +` + sp("body", "") + ` + +第一行第二行 +` + sp("", "组合内文本") + ` + +` + sp("", "选内文本") + ` +` + sp("", "弃内文本") + ` + +3 + +合并头列三 +值三 + +` + slide2 := ` +` + sp("body", "第二页要点") + ` +` + return map[string]string{ + "ppt/presentation.xml": ` + +`, + "ppt/_rels/presentation.xml.rels": `` + + presRel("rIdA", "slides/slide1.xml") + presRel("rIdB", "slides/slide2.xml") + ``, + "ppt/slides/slide1.xml": slide1, + "ppt/slides/slide2.xml": slide2, + "ppt/slides/_rels/slide1.xml.rels": ``, + // slide2 的备注故意缺失（无 rels 文件）。 + "ppt/notesSlides/notesSlide1.xml": ` +` + sp("body", "补充说明") + ` +`, + } +} + +func TestParsePptxBuiltFromParts(t *testing.T) { + doc := Parse("演示.pptx", "", zipBytes(t, pptxFixture())) + if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 || doc.Meta.Confidence != 0.9 { + t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings) + } + var heads []string + for _, b := range doc.Blocks { + if b.Type == core.BlockHeading { + heads = append(heads, b.Text) + } + } + // sldIdLst 顺序：slide1（有标题）在前；slide2 无标题占位符 → 无标题兜底。 + if len(heads) != 2 || heads[0] != "幻灯片 1：封面标题" || heads[1] != "幻灯片 2" { + t.Fatalf("headings: %v", heads) + } + plain := doc.PlainText() + for _, want := range []string{"第一行\n第二行", "组合内文本", "选内文本", "备注：补充说明"} { + if !strings.Contains(plain, want) { + t.Errorf("missing %q in plain text", want) + } + } + if strings.Contains(plain, "弃内文本") { + t.Error("mc:Fallback content must be skipped") + } + // 页码占位符与字段文本不出现。 + if strings.Contains(plain, "\n3\n") { + t.Error("slide number placeholder text leaked") + } + // 表格：gridSpan 占位 + hMerge 续格留空，保持三列对齐。 + tbl := findTable(doc, 0) + if tbl == nil || len(tbl.Rows) != 2 { + t.Fatalf("table: %+v", tbl) + } + if strings.Join(tbl.Rows[0], "|") != "合并头||列三" { + t.Errorf("row1: %v", tbl.Rows[0]) + } + if strings.Join(tbl.Rows[1], "|") != "||值三" { + t.Errorf("row2: %v", tbl.Rows[1]) + } +} + +func TestParsePptxInvalidAndEmpty(t *testing.T) { + doc := Parse("坏.pptx", "", []byte("garbage")) + if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 { + t.Fatalf("invalid pptx: %+v %v", doc.Meta, doc.Warnings) + } + files := pptxFixture() + files["ppt/presentation.xml"] = `` + doc = Parse("空演示.pptx", "", zipBytes(t, files)) + if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 { + t.Fatalf("no slides: %v", doc.Warnings) + } +} + +func TestParseLegacyOfficeHint(t *testing.T) { + for ext, want := range map[string]string{ + "doc": "docx", "xls": "xlsx", "ppt": "pptx", + } { + doc := Parse("旧档."+ext, "", []byte{0xD0, 0xCF}) + if doc.Meta.Parser != "builtin:legacy-office" { + t.Fatalf("%s parser: %+v", ext, doc.Meta) + } + if len(doc.Blocks) != 0 || len(doc.Warnings) != 1 { + t.Fatalf("%s blocks/warnings: %+v %v", ext, doc.Blocks, doc.Warnings) + } + if !strings.Contains(doc.Warnings[0], want) { + t.Errorf("%s hint should mention %s: %v", ext, want, doc.Warnings[0]) + } + } +} + +// 注册表与分派行为一致：每个已注册格式的解析器 ID 与 Parse 分派结果一致， +// 且 builtin 解析器覆盖的扩展名都出现在注册表里。 +func TestParsersDescriptionMatchesDispatch(t *testing.T) { + byFormat := map[string]string{} + for _, d := range ParsersDescription() { + if d.Source != core.SourceBuiltin { + continue + } + for _, f := range d.Formats { + byFormat[f] = d.ID + } + } + check := map[string]string{ + "md": "builtin:md", "markdown": "builtin:md", + "txt": "builtin:txt", "text": "builtin:txt", "csv": "builtin:txt", "log": "builtin:txt", + "docx": "builtin:docx", "pdf": "builtin:pdf", + "xlsx": "builtin:xlsx", "xlsm": "builtin:xlsx", + "pptx": "builtin:pptx", "pptm": "builtin:pptx", "ppsx": "builtin:pptx", + } + for ext, wantID := range check { + got, ok := byFormat[ext] + if !ok { + t.Errorf("format .%s missing from registry", ext) + continue + } + doc := Parse("a."+ext, "", []byte("x")) + if doc.Meta.Parser != wantID || got != wantID { + t.Errorf(".%s: registry=%s dispatch=%s want=%s", ext, got, doc.Meta.Parser, wantID) + } + } +} diff --git a/internal/ingest/pptx.go b/internal/ingest/pptx.go new file mode 100644 index 0000000..d63cf2e --- /dev/null +++ b/internal/ingest/pptx.go @@ -0,0 +1,291 @@ +package ingest + +// pptx/pptm/ppsx（PresentationML）内置解析器。幻灯片顺序取 presentation.xml +// 的 sldIdLst；标题占位符（title/ctrTitle）产标题块，正文形状按段落输出 +//（a:br 换行），组合形状经同一 token 走查天然递归；mc:AlternateContent 只取 +// Choice 分支；graphicFrame 内的表格按 tr/tc 输出（gridSpan/hMerge 占位保列 +// 对齐）；备注页正文输出「备注：…」；页码/日期/页脚占位符跳过。图片与 +// SmartArt 不解析。纯标准库。 + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "path" + "strconv" + "strings" + + core "agent-system-core" +) + +func parsePptx(content []byte) ([]core.DocBlock, []string) { + zr, err := openZip(content) + if err != nil { + return nil, []string{"pptx 解析失败：" + err.Error()} + } + slides, warns := pptxSlideOrder(zr) + if slides == nil && len(warns) > 0 { + return nil, warns + } + var blocks []core.DocBlock + for i, sp := range slides { + n := i + 1 + data, err := zipPart(zr, sp) + if err != nil { + warns = append(warns, fmt.Sprintf("幻灯片 %d 读取失败：%v", n, err)) + blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: fmt.Sprintf("幻灯片 %d", n)}) + continue + } + title, paras, tables := pptxSlideBlocks(data) + heading := fmt.Sprintf("幻灯片 %d", n) + if title != "" { + heading += "：" + title + } + blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: heading}) + blocks = append(blocks, paras...) + blocks = append(blocks, tables...) + if note := pptxSlideNotes(zr, sp); note != "" { + blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: "备注：" + note}) + } + } + if len(blocks) == 0 && len(warns) == 0 { + warns = append(warns, "pptx 无可解析的幻灯片内容") + } + return blocks, warns +} + +// pptxSlideOrder 读 presentation.xml 的 sldIdLst → 幻灯片部件名（按顺序）。 +func pptxSlideOrder(zr *zip.Reader) ([]string, []string) { + data, err := zipPart(zr, "ppt/presentation.xml") + if err != nil { + return nil, []string{"pptx 解析失败：" + err.Error()} + } + rels, err := parseRels(zr, "ppt/presentation.xml") + if err != nil { + return nil, []string{"pptx 解析失败：" + err.Error()} + } + dec := xml.NewDecoder(bytes.NewReader(data)) + var out, warns []string + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return out, append(warns, "presentation.xml 解码失败："+err.Error()) + } + st, ok := tok.(xml.StartElement) + if !ok || st.Name.Local != "sldId" { + continue + } + rel, ok := rels[ridAttr(st)] + if !ok || !relIsType(rel, "/slide") { + warns = append(warns, "幻灯片关系缺失或类型不符") + continue + } + out = append(out, resolveRelTarget("ppt", rel.Target)) + } + return out, warns +} + +// pptxSlideBlocks 解析一张幻灯片：标题、正文段落、表格块。 +// 表格块统一附在段落后（图形框与正文的精确穿插对检索无价值）。 +func pptxSlideBlocks(data []byte) (title string, paras []core.DocBlock, tables []core.DocBlock) { + dec := xml.NewDecoder(bytes.NewReader(data)) + var stack []string + has := func(name string) bool { + for _, s := range stack { + if s == name { + return true + } + } + return false + } + skip := 0 // mc:Fallback 子树深度 + // 形状状态 + spDepth := -1 + phSkip := false + isTitle := false + var spParas []string + var para strings.Builder + paraOpen := false + // 文本收集目标 + textInPara := false + textInCell := false + // 表格状态 + var tblRows [][]string + var tblRow []string + var cellBuf strings.Builder + cellOpen := false + gridSpan := 1 + cellHM := false + + finishPara := func() { + if !paraOpen { + return + } + paraOpen = false + if s := strings.TrimRight(para.String(), "\n"); strings.TrimSpace(s) != "" { + spParas = append(spParas, strings.TrimSpace(s)) + } + para.Reset() + } + + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return "", paras, tables + } + switch t := tok.(type) { + case xml.StartElement: + if skip > 0 { + skip++ + continue + } + local := t.Name.Local + switch local { + case "Fallback": + if has("AlternateContent") { + skip = 1 + continue + } + case "sp": + spDepth = len(stack) + phSkip, isTitle, spParas = false, false, nil + case "ph": + if spDepth >= 0 && len(stack) > spDepth { + switch attrLocal(t, "type") { + case "title", "ctrTitle": + isTitle = true + case "dt", "ftr", "sldNum": + phSkip = true + } + } + case "p": + if spDepth >= 0 && has("txBody") && !has("tbl") { + finishPara() + paraOpen = true + para.Reset() + } else if cellOpen { + if cellBuf.Len() > 0 { + cellBuf.WriteByte('\n') + } + } + case "br": + if paraOpen { + para.WriteByte('\n') + } + case "t": + if has("fld") { + break // 字段（页码/日期）不取 + } + textInPara = paraOpen + textInCell = cellOpen && !paraOpen + case "tc": + if has("tbl") { + cellBuf.Reset() + cellOpen = true + gridSpan, _ = strconv.Atoi(attrLocal(t, "gridSpan")) + if gridSpan < 1 { + gridSpan = 1 + } + hm := attrLocal(t, "hMerge") + cellHM = hm == "1" || strings.EqualFold(hm, "true") + } + } + stack = append(stack, local) + case xml.CharData: + if textInPara { + para.Write(t) + } else if textInCell { + cellBuf.Write(t) + } + case xml.EndElement: + if skip > 0 { + skip-- + continue + } + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + switch t.Name.Local { + case "t": + textInPara, textInCell = false, false + case "p": + finishPara() + case "sp": + if spDepth >= 0 && len(stack) == spDepth { + if !phSkip && len(spParas) > 0 { + if isTitle && title == "" { + title = strings.Join(spParas, " ") + } else { + for _, p := range spParas { + paras = append(paras, core.DocBlock{Type: core.BlockPara, Text: p}) + } + } + } + spDepth = -1 + phSkip, isTitle, spParas = false, false, nil + } + case "tc": + if cellOpen { + text := strings.TrimSpace(cellBuf.String()) + if cellHM { + text = "" // 横向合并的续格：占位保列对齐 + } + tblRow = append(tblRow, text) + for i := 1; i < gridSpan; i++ { + tblRow = append(tblRow, "") // 合并占位保列对齐 + } + cellOpen = false + } + case "tr": + if len(tblRow) > 0 { + for len(tblRow) > 0 && tblRow[len(tblRow)-1] == "" { + tblRow = tblRow[:len(tblRow)-1] + } + if len(tblRow) > 0 { + tblRows = append(tblRows, tblRow) + } + tblRow = nil + } + case "tbl": + if len(tblRows) > 0 { + tables = append(tables, core.DocBlock{Type: core.BlockTable, Rows: tblRows}) + } + tblRows = nil + } + } + } + finishPara() + return title, paras, tables +} + +// pptxSlideNotes 读幻灯片的备注页正文；无备注返回空串。 +func pptxSlideNotes(zr *zip.Reader, slidePart string) string { + rels, err := parseRels(zr, slidePart) + if err != nil { + return "" + } + for _, rel := range rels { + if !relIsType(rel, "/notesSlide") { + continue + } + data, err := zipPart(zr, resolveRelTarget(path.Dir(slidePart), rel.Target)) + if err != nil { + return "" + } + _, paras, _ := pptxSlideBlocks(data) + var lines []string + for _, p := range paras { + lines = append(lines, p.Text) + } + return strings.Join(lines, "\n") + } + return "" +} diff --git a/internal/ingest/xlsx.go b/internal/ingest/xlsx.go new file mode 100644 index 0000000..0c0bf1e --- /dev/null +++ b/internal/ingest/xlsx.go @@ -0,0 +1,415 @@ +package ingest + +// xlsx/xlsm（SpreadsheetML）内置解析器。工作表顺序取 workbook.xml 的 sheets +// 列表（target 经关系部件解析，支持相对/绝对路径）；共享字符串含富文本 run +//（注音 rPh 无对应解码字段，天然排除）；单元格覆盖 inlineStr/str/b/e/n， +// 公式取缓存值、无缓存回显公式；日期按数字格式（内置 ID 与自定义记号）转 +// ISO，1900/1904 双日期系统；稀疏单元格按引用就位。纯标准库。 + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + core "agent-system-core" +) + +// 解析上限（包级变量，测试可覆盖）。 +var ( + xlsxMaxCells = 200000 + xlsxMaxCols = 1024 +) + +type xlsxSheetRef struct { + Name string + State string + Path string + IsSheet bool +} + +func parseXlsx(content []byte) ([]core.DocBlock, []string) { + zr, err := openZip(content) + if err != nil { + return nil, []string{"xlsx 解析失败：" + err.Error()} + } + sheets, date1904, warns := xlsxWorkbookSheets(zr) + if len(warns) > 0 && sheets == nil { + return nil, warns + } + shared := xlsxSharedStrings(zr) + dateStyles := xlsxDateStyles(zr) + var blocks []core.DocBlock + cells := 0 + for _, sh := range sheets { + if !sh.IsSheet || sh.Path == "" { + continue // chartsheet 等非工作表部件 + } + data, err := zipPart(zr, sh.Path) + if err != nil { + warns = append(warns, fmt.Sprintf("工作表 %s 读取失败：%v", sh.Name, err)) + continue + } + table, used, trunc := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells) + cells = used + if table == nil { + continue // 空表 + } + title := sh.Name + if sh.State == "hidden" || sh.State == "veryHidden" { + title += "（隐藏）" + } + blocks = append(blocks, + core.DocBlock{Type: core.BlockHeading, Level: 2, Text: title}, + *table, + ) + if trunc { + warns = append(warns, fmt.Sprintf("单元格总数超过 %d 上限，工作表 %s 之后未解析", xlsxMaxCells, sh.Name)) + break + } + } + if len(blocks) == 0 && len(warns) == 0 { + warns = append(warns, "xlsx 无可解析的工作表内容") + } + return blocks, warns +} + +// xlsxWorkbookSheets 读 workbook.xml：工作表顺序、隐藏状态、1904 日期系统。 +func xlsxWorkbookSheets(zr *zip.Reader) (sheets []xlsxSheetRef, date1904 bool, warns []string) { + data, err := zipPart(zr, "xl/workbook.xml") + if err != nil { + return nil, false, []string{"xlsx 解析失败：" + err.Error()} + } + rels, err := parseRels(zr, "xl/workbook.xml") + if err != nil { + return nil, false, []string{"xlsx 解析失败：" + err.Error()} + } + dec := xml.NewDecoder(bytes.NewReader(data)) + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return sheets, date1904, append(warns, "workbook.xml 解码失败："+err.Error()) + } + st, ok := tok.(xml.StartElement) + if !ok { + continue + } + switch st.Name.Local { + case "workbookPr": + if v := attrLocal(st, "date1904"); v == "1" || strings.EqualFold(v, "true") { + date1904 = true + } + case "sheet": + ref := xlsxSheetRef{Name: attrLocal(st, "name"), State: attrLocal(st, "state")} + if rel, ok := rels[ridAttr(st)]; ok { + ref.Path = resolveRelTarget("xl", rel.Target) + ref.IsSheet = relIsType(rel, "/worksheet") + } else { + warns = append(warns, fmt.Sprintf("工作表 %s 的关系缺失", ref.Name)) + } + sheets = append(sheets, ref) + } + } + return sheets, date1904, warns +} + +// xlsxSharedStrings 读共享字符串表；富文本按 run 拼接，rPh 注音不进结果。 +func xlsxSharedStrings(zr *zip.Reader) []string { + data, err := zipPart(zr, "xl/sharedStrings.xml") + if err != nil { + return nil + } + dec := xml.NewDecoder(bytes.NewReader(data)) + var out []string + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return out + } + st, ok := tok.(xml.StartElement) + if !ok || st.Name.Local != "si" { + continue + } + var si struct { + T string `xml:"t"` + Runs []struct { + T string `xml:"t"` + } `xml:"r"` + } + if err := dec.DecodeElement(&si, &st); err != nil { + continue + } + if si.T != "" || len(si.Runs) == 0 { + out = append(out, si.T) + continue + } + parts := make([]string, 0, len(si.Runs)) + for _, r := range si.Runs { + parts = append(parts, r.T) + } + out = append(out, strings.Join(parts, "")) + } + return out +} + +// xlsxDateStyles 返回按日期呈现的样式索引集合（cellXfs 下标）。 +func xlsxDateStyles(zr *zip.Reader) map[int]bool { + data, err := zipPart(zr, "xl/styles.xml") + if err != nil { + return nil + } + custom := map[int]string{} + date := map[int]bool{} + dec := xml.NewDecoder(bytes.NewReader(data)) + inCellXfs := false + xfIdx := 0 + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return date + } + switch t := tok.(type) { + case xml.StartElement: + switch t.Name.Local { + case "numFmt": + id, _ := strconv.Atoi(attrLocal(t, "numFmtId")) + custom[id] = attrLocal(t, "formatCode") + case "cellXfs": + inCellXfs = true + xfIdx = 0 + case "xf": + if !inCellXfs { + continue + } + id, _ := strconv.Atoi(attrLocal(t, "numFmtId")) + if xlsxFmtIsDate(id, custom) { + date[xfIdx] = true + } + xfIdx++ + } + case xml.EndElement: + if t.Name.Local == "cellXfs" { + inCellXfs = false + } + } + } + return date +} + +// xlsxFmtIsDate：自定义格式按记号判断；内置 ID 覆盖常见日期/时间区间。 +func xlsxFmtIsDate(id int, custom map[int]string) bool { + if code, ok := custom[id]; ok { + return formatCodeIsDate(code) + } + switch { + case id >= 14 && id <= 22, id >= 27 && id <= 36, + id >= 45 && id <= 47, id >= 50 && id <= 58: + return true + } + return false +} + +// formatCodeIsDate 剥掉引号字面量与方括号节（[Red]、[DBNum1]、[$-409] 等）后， +// 剩余字母含 y/d/h/s/m 记号即按日期处理。 +func formatCodeIsDate(code string) bool { + var b strings.Builder + inQuote, inBracket := false, false + runes := []rune(code) + for i := 0; i < len(runes); i++ { + r := runes[i] + switch { + case r == '"': + inQuote = !inQuote + case inQuote: + case r == '[': + inBracket = true + case r == ']': + inBracket = false + case inBracket: + case r == '\\': + i++ // 转义字符：连同下一字符跳过 + default: + b.WriteRune(r) + } + } + return strings.ContainsAny(strings.ToLower(b.String()), "ydhsm") +} + +type xlsxCellEl struct { + Ref string `xml:"r,attr"` + T string `xml:"t,attr"` + S string `xml:"s,attr"` + IS *struct { + T string `xml:"t"` + } `xml:"is"` + F string `xml:"f"` + V string `xml:"v"` +} + +type xlsxRowEl struct { + Cells []xlsxCellEl `xml:"c"` +} + +// xlsxSheetBlocks 解析单张工作表；返回表格块、累计单元格数、是否触达上限。 +// 无内容行返回 nil 表。 +func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool) { + dec := xml.NewDecoder(bytes.NewReader(data)) + var rows [][]string + count := cellsUsed + truncated := false + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return nil, count, false + } + st, ok := tok.(xml.StartElement) + if !ok || st.Name.Local != "row" { + continue + } + var re xlsxRowEl + if err := dec.DecodeElement(&re, &st); err != nil { + continue + } + cells := map[int]string{} + maxCol := -1 + for _, c := range re.Cells { + if count >= xlsxMaxCells { + truncated = true + break + } + count++ + col := xlsxColIndex(c.Ref) + if col < 0 { + col = maxCol + 1 // 无引用：按出现顺序接续 + } + if col >= xlsxMaxCols { + truncated = true + continue + } + cells[col] = xlsxCellText(c, shared, dateStyles, date1904) + if col > maxCol { + maxCol = col + } + } + if len(cells) == 0 { + continue + } + full := make([]string, maxCol+1) + for i, v := range cells { + full[i] = v + } + for len(full) > 0 && strings.TrimSpace(full[len(full)-1]) == "" { + full = full[:len(full)-1] + } + if len(full) > 0 { + rows = append(rows, full) + } + } + if len(rows) == 0 { + return nil, count, truncated + } + return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated +} + +// xlsxCellText 把单元格转成展示文本。 +func xlsxCellText(c xlsxCellEl, shared []string, dateStyles map[int]bool, date1904 bool) string { + text := "" + switch c.T { + case "s": + if idx, err := strconv.Atoi(c.V); err == nil && idx >= 0 && idx < len(shared) { + text = shared[idx] + } + case "inlineStr": + if c.IS != nil { + text = c.IS.T + } + case "b": + if c.V == "1" || strings.EqualFold(c.V, "true") { + text = "TRUE" + } else { + text = "FALSE" + } + case "e": + text = c.V + default: // n 或空：数字 / 公式缓存值 + text = c.V + if sIdx, err := strconv.Atoi(c.S); err == nil && dateStyles[sIdx] && c.V != "" { + if f, err := strconv.ParseFloat(c.V, 64); err == nil { + text = xlsxSerialString(f, date1904) + } + } + } + if text == "" && c.F != "" { + return "=" + c.F // 无缓存值的公式：回显公式本身 + } + return text +} + +// xlsxSerialString Excel 日期序列值 → ISO 字符串；带小数部分时输出到秒。 +func xlsxSerialString(serial float64, date1904 bool) string { + days := math.Floor(serial) + frac := serial - days + var t time.Time + switch { + case date1904: + t = time.Date(1904, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days)) + case days >= 61: + t = time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days)) + default: // 1–60：兼容 Excel 的 1900 闰年 bug 区段 + t = time.Date(1899, 12, 31, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days)) + } + if frac > 0 { + return t.Add(time.Duration(frac * 86400 * float64(time.Second))).Round(time.Second).Format("2006-01-02 15:04:05") + } + return t.Format("2006-01-02") +} + +// xlsxColIndex "BC12" → 0 起列号；无引用或非法返回 -1。 +func xlsxColIndex(ref string) int { + if ref == "" { + return -1 + } + i := 0 + for i < len(ref) { + ch := ref[i] + if (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') { + i++ + continue + } + break + } + if i == 0 || i == len(ref) { + return -1 + } + for _, ch := range ref[i:] { + if ch < '0' || ch > '9' { + return -1 + } + } + n := 0 + for _, ch := range ref[:i] { + c := ch + if c >= 'a' && c <= 'z' { + c = c - 'a' + 'A' + } + n = n*26 + int(c-'A') + 1 + } + return n - 1 +} diff --git a/internal/server/e2e_test.go b/internal/server/e2e_test.go index 2f3eb2e..cd1b188 100644 --- a/internal/server/e2e_test.go +++ b/internal/server/e2e_test.go @@ -1199,3 +1199,76 @@ func TestSettingsDottedKeyAndSecretMask(t *testing.T) { t.Fatalf("apiKey mask = %v", chat["apiKey"]) } } + +// TestUploadXlsxParsedAndIndexed：xlsx 上传走内置解析器（builtin:xlsx、无警告）， +// 单元格文本进入知识检索（EXT-5 验收路径）。 +func TestUploadXlsxParsedAndIndexed(t *testing.T) { + srv, _ := newTestServer(t) + h := srv.Handler() + projectID, _ := setupConversation(t, h) + + xlsx := buildTestXLSX(t) + body := &bytes.Buffer{} + mw := multipart.NewWriter(body) + fw, _ := mw.CreateFormFile("file", "报价单.xlsx") + fw.Write(xlsx) + mw.Close() + req := httptest.NewRequest("POST", "/api/projects/"+projectID+"/uploads", body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("upload status %d: %s", rec.Code, rec.Body.String()) + } + up := map[string]any{} + json.Unmarshal(rec.Body.Bytes(), &up) + parsed, _ := up["parsed"].(map[string]any) + if parsed == nil { + t.Fatalf("upload response missing parsed: %s", rec.Body.String()) + } + meta, _ := parsed["meta"].(map[string]any) + if meta["parser"] != "builtin:xlsx" { + t.Fatalf("parser: %v", meta["parser"]) + } + if w, ok := parsed["warnings"].([]any); ok && len(w) > 0 { + t.Fatalf("xlsx upload should parse without warnings: %v", w) + } + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + code, raw := doAny(t, h, "GET", "/api/knowledge/search?q=%E6%94%AF%E6%9E%B6", nil) + if code == 200 { + if hits, ok := raw.([]any); ok && len(hits) > 0 { + return + } + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("knowledge search found nothing for xlsx cell text") +} + +// buildTestXLSX 构造最小可用 xlsx（inlineStr 单元格）。 +func buildTestXLSX(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + files := map[string]string{ + "xl/workbook.xml": ``, + "xl/_rels/workbook.xml.rels": ``, + "xl/worksheets/sheet1.xml": ` +物料 +支架 +`, + } + for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} { + w, err := zw.Create(name) + if err != nil { + t.Fatal(err) + } + w.Write([]byte(files[name])) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} diff --git a/pkg/extension/host.go b/pkg/extension/host.go index 4397a88..8e880ee 100644 --- a/pkg/extension/host.go +++ b/pkg/extension/host.go @@ -20,9 +20,9 @@ type Request struct { // Host 句柄传入，处理器经句柄访问基座能力，不得自行越权持有 owner 凭证。 type Handler func(ctx context.Context, h Host, req Request) (any, error) -// Host 是基座在分发时构造的 owner 作用域宿主句柄（EXT-1 最小集）。 +// Host 是基座在分发时构造的 owner 作用域宿主句柄（EXT-1 最小集 + EXT-5 解析）。 // 所有能力的可见范围都以本次分发的 Owner 为准；artifacts 只读 -//（写入属完整集，另行评估）。实现由基座 internal 提供。 +// （写入属完整集，另行评估）。实现由基座 internal 提供。 type Host interface { // PluginID 返回注册处理器的插件标识（形如 rdm）。 PluginID() string @@ -52,4 +52,10 @@ type Host interface { // ArtifactGet/ArtifactContent 工件只读（R1-3 ingest 读原件）。 ArtifactGet(artifactID string) (core.Artifact, error) ArtifactContent(artifactID string) ([]byte, error) + + // ParseDocument 用基座内置解析器把文档字节解析为 ParsedDocument（EXT-5）。 + // 纯计算，不读写 owner 数据——要解析工件原件先经 ArtifactContent 取字节， + // owner 作用域由它保证。按扩展名分派；不可解析返回带 Warnings 的文档 + //（显式失败），只有 ctx 取消与输入超过 100MiB 两种情况返回 error。 + ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) } diff --git a/pkg/extension/surface.txt b/pkg/extension/surface.txt index 79f0298..834d0ed 100644 --- a/pkg/extension/surface.txt +++ b/pkg/extension/surface.txt @@ -2,6 +2,19 @@ Compiled ErrNotLinked Handler Host +Host.ArtifactContent +Host.ArtifactGet +Host.Audit +Host.Complete +Host.KVGet +Host.KVPut +Host.KnowledgeSearch +Host.KnowledgeWrite +Host.Owner +Host.ParseDocument +Host.PluginID +Host.TaskCreate +Host.TaskUpdate HostClient InProc Main @@ -9,6 +22,10 @@ NewHostClient Options RegisterCompiled Registry +Registry.AddCommand +Registry.AddCommandHandler +Registry.AddSkill +Registry.Note Request ResetCompiled SetRunner diff --git a/pkg/extension/surface_test.go b/pkg/extension/surface_test.go index ee2ae47..a435b65 100644 --- a/pkg/extension/surface_test.go +++ b/pkg/extension/surface_test.go @@ -37,6 +37,17 @@ func TestSurfaceSnapshot(t *testing.T) { case *ast.TypeSpec: if s.Name.IsExported() { names = append(names, s.Name.Name) + // 导出接口同时记录其方法：只加方法、不改顶层名的 + // 变更同样必须更新 surface.txt（EXT-5 起执行）。 + if it, ok := s.Type.(*ast.InterfaceType); ok { + for _, m := range it.Methods.List { + for _, id := range m.Names { + if id.IsExported() { + names = append(names, s.Name.Name+"."+id.Name) + } + } + } + } } case *ast.ValueSpec: for _, name := range s.Names {