Expose builtin document parsing through the host handle #67

Merged
weichen merged 1 commits from feature/ext-5-host-parse into main 2026-10-06 15:14:21 +00:00
12 changed files with 1640 additions and 20 deletions
+23 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"agent-system-backend/internal/ingest"
"agent-system-backend/internal/rag"
"agent-system-backend/internal/runtime"
"agent-system-backend/internal/settings"
@@ -39,6 +40,21 @@ func (c hostcaps) complete(ctx context.Context, owner, prompt string) (string, e
return completeChat(ctx, route.Base, route.Key, route.Model, prompt)
}
// maxParseInput 与上传端点的读取上限同口径(server.upload 的 100<<20)。
const maxParseInput = 100 << 20
// parse 走基座内置解析器(EXT-5):纯计算、不落 owner 数据;不可解析由
// Warnings 显式表达,只有取消与超限是 error。
func (c hostcaps) parse(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) {
if err := ctx.Err(); err != nil {
return core.ParsedDocument{}, err
}
if len(content) > maxParseInput {
return core.ParsedDocument{}, fmt.Errorf("parse input %d bytes exceeds %d limit", len(content), maxParseInput)
}
return ingest.Parse(filename, mime, content), nil
}
// inprocHost 是进程内插件命令的 owner 作用域宿主句柄(extension.Host 实现)。
// 每次分发按请求 Owner 新建;所有能力调用都以该 owner 为第一参数,插件无法越权。
type inprocHost struct {
@@ -75,7 +91,7 @@ func (h inprocHost) KnowledgeSearch(ctx context.Context, projectID core.ID, text
// KnowledgeWrite 强制 owner 归属:Owner 改写为本次分发 owner,Source 为空补
// "plugin:<id>";embed=false 只写全文行,embed=true 走 IndexChunks 建向量
//(embedding 失败降级纯全文,不阻断写入)。
// (embedding 失败降级纯全文,不阻断写入)。
func (h inprocHost) KnowledgeWrite(ctx context.Context, docs []core.KnowledgeDoc, embed bool) error {
if len(docs) == 0 {
return nil
@@ -114,3 +130,9 @@ func (h inprocHost) ArtifactGet(artifactID string) (core.Artifact, error) {
func (h inprocHost) ArtifactContent(artifactID string) ([]byte, error) {
return h.caps.st.ArtifactContent(h.owner, artifactID)
}
// ParseDocument 内置文档解析(EXT-5):委托 hostcaps.parse;纯计算不经存储,
// owner 作用域由调用方经 ArtifactContent 取字节时保证。
func (h inprocHost) ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) {
return h.caps.parse(ctx, filename, mime, content)
}
+111
View File
@@ -1,6 +1,8 @@
package app
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"path/filepath"
@@ -193,3 +195,112 @@ func TestInprocHostKnowledgeWriteEmbedHash(t *testing.T) {
t.Fatalf("vector owner leak: %+v", bhits)
}
}
// TestInprocHostParseDocumentOwnerScope:EXT-5 解析方法的 owner 作用域——
// alice 经句柄读自己的工件并解析;bob 用同一工件 ID 在 ArtifactContent 处
// 即被拒(解析是纯计算,作用域边界在取原件一环)。
func TestInprocHostParseDocumentOwnerScope(t *testing.T) {
extension.ResetCompiled()
t.Cleanup(extension.ResetCompiled)
extension.RegisterCompiled(extension.InProc{
ID: "tst",
Register: func(reg extension.Registry) error {
return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.parse","title":"解析"}`),
func(ctx context.Context, h extension.Host, req extension.Request) (any, error) {
id, _ := req.Params["artifactId"].(string)
content, err := h.ArtifactContent(id)
if err != nil {
return nil, err
}
doc, err := h.ParseDocument(ctx, "报价.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", content)
if err != nil {
return nil, err
}
return map[string]any{
"parser": doc.Meta.Parser,
"warnings": len(doc.Warnings),
"blocks": len(doc.Blocks),
}, nil
})
},
})
bus, st := newInprocTestApp(t, nil)
art := core.Artifact{ProjectID: "p1", Owner: "alice", Name: "报价.xlsx", Kind: "document"}
if err := st.CreateArtifact(&art, testXLSXBytes(t)); err != nil {
t.Fatal(err)
}
artID := art.ID
resp := bus.Dispatch(context.Background(), cmdbus.Request{
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "alice", Entrance: "chat",
})
if resp.Receipt.State != agentcommand.StateCommitted {
t.Fatalf("alice dispatch: %+v %s", resp.Receipt, resp.Receipt.Reason)
}
m, _ := resp.Payload.(map[string]any)
if m["parser"] != "builtin:xlsx" || m["warnings"] != 0 || m["blocks"] == 0 {
t.Fatalf("alice payload: %+v", m)
}
// bob 同一工件 ID:取原件即被拒(跨 owner 不可见)。
resp = bus.Dispatch(context.Background(), cmdbus.Request{
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "bob", Entrance: "chat",
})
if resp.Receipt.State == agentcommand.StateCommitted {
t.Fatalf("bob must not read alice's artifact: %+v", resp.Payload)
}
}
// TestInprocHostParseDocumentLimits:仅有的两条 error 路径——ctx 取消与输入
// 超过 100MiB 上限;不可解析格式仍是带 Warnings 的显式失败。
func TestInprocHostParseDocumentLimits(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
h := inprocHost{caps: hostcaps{st: st}, pluginID: "tst", owner: "alice"}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := h.ParseDocument(ctx, "a.xlsx", "", []byte("x")); err == nil {
t.Fatal("canceled ctx must error")
}
if _, err := h.ParseDocument(context.Background(), "a.xlsx", "", make([]byte, maxParseInput+1)); err == nil {
t.Fatal("oversized input must error")
}
doc, err := h.ParseDocument(context.Background(), "旧.xls", "", []byte{0xD0, 0xCF})
if err != nil {
t.Fatalf("legacy format is a visible failure, not an error: %v", err)
}
if doc.Meta.Parser != "builtin:legacy-office" || len(doc.Warnings) != 1 {
t.Fatalf("legacy doc: %+v %v", doc.Meta, doc.Warnings)
}
}
// testXLSXBytes 构造最小可用 xlsx(单表两行),供句柄测试解析。
func testXLSXBytes(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
files := map[string]string{
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="报价" sheetId="1" r:id="rId1"/></sheets></workbook>`,
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>`,
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>
<row r="1"><c r="A1" t="inlineStr"><is><t>物料</t></is></c><c r="B1" t="inlineStr"><is><t>单价</t></is></c></row>
<row r="2"><c r="A2" t="inlineStr"><is><t>支架</t></is></c><c r="B2"><v>12.5</v></c></row>
</sheetData></worksheet>`,
}
for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
w.Write([]byte(files[name]))
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
+49 -17
View File
@@ -4,7 +4,6 @@
package ingest
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
@@ -52,6 +51,35 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument {
doc.Blocks = blocks
doc.Meta.Confidence = 0.85
}
case "xlsx", "xlsm":
doc.Meta.Parser = "builtin:xlsx"
blocks, warns := parseXlsx(content)
doc.Blocks = blocks
doc.Warnings = append(doc.Warnings, warns...)
switch {
case len(blocks) == 0:
doc.Meta.Confidence = 0.3
case len(warns) > 0:
doc.Meta.Confidence = 0.6 // 超限截断:内容可用但注明不完整
default:
doc.Meta.Confidence = 0.9
}
case "pptx", "pptm", "ppsx":
doc.Meta.Parser = "builtin:pptx"
blocks, warns := parsePptx(content)
doc.Blocks = blocks
doc.Warnings = append(doc.Warnings, warns...)
switch {
case len(blocks) == 0:
doc.Meta.Confidence = 0.3
case len(warns) > 0:
doc.Meta.Confidence = 0.6
default:
doc.Meta.Confidence = 0.9
}
case "doc", "xls", "ppt":
doc.Meta.Parser = "builtin:legacy-office"
doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析:请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
default:
if isImage(ext) {
doc.Warnings = append(doc.Warnings, "图片不走文本解析(OCR 为 MinerU 深度档,远期)")
@@ -164,24 +192,13 @@ type xtc struct {
}
func parseDocx(content []byte) ([]core.DocBlock, error) {
zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
zr, err := openZip(content)
if err != nil {
return nil, fmt.Errorf("open zip: %w", err)
}
var docXML []byte
for _, f := range zr.File {
if f.Name == "word/document.xml" {
rc, err := f.Open()
if err != nil {
return nil, err
}
docXML, _ = io.ReadAll(rc)
rc.Close()
break
}
}
if docXML == nil {
return nil, fmt.Errorf("word/document.xml not found")
docXML, err := zipPart(zr, "word/document.xml")
if err != nil {
return nil, err
}
return parseDocxBody(docXML)
}
@@ -322,13 +339,28 @@ func isMostlyPrintable(s string) bool {
return nonPrint*10 < len(s)
}
// legacyOfficeTarget 旧版二进制 Office 格式的现代等价格式(提示转存用)。
func legacyOfficeTarget(ext string) string {
switch ext {
case "doc":
return "docx"
case "xls":
return "xlsx"
case "ppt":
return "pptx"
}
return ext
}
// ParsersDescription 返回解析器注册表描述(来源标记;设置页/文档用)。
func ParsersDescription() []core.ParserDesc {
return []core.ParserDesc{
{ID: "builtin:md", Formats: []string{"md", "markdown"}, Source: core.SourceBuiltin},
{ID: "builtin:txt", Formats: []string{"txt", "csv", "log"}, Source: core.SourceBuiltin},
{ID: "builtin:txt", Formats: []string{"txt", "text", "csv", "log"}, Source: core.SourceBuiltin},
{ID: "builtin:docx", Formats: []string{"docx"}, Source: core.SourceBuiltin},
{ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层;扫描件为远期 MinerU 深度档"},
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
}
}
+140
View File
@@ -0,0 +1,140 @@
package ingest
// OOXML 公共件:zip 部件读取(解压上限防 zip 炸弹)、关系部件解析、
// r:id 属性与相对路径归一。docx/xlsx/pptx 解析器共用;纯标准库。
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path"
"strings"
)
// ooxmlPartLimit 单个 zip 部件的解压上限(测试可覆盖)。
var ooxmlPartLimit int64 = 64 << 20
func openZip(content []byte) (*zip.Reader, error) {
return zip.NewReader(bytes.NewReader(content), int64(len(content)))
}
// zipPart 按名称读取部件;解压超过 ooxmlPartLimit 视为异常拒绝。
func zipPart(zr *zip.Reader, name string) ([]byte, error) {
for _, f := range zr.File {
if f.Name != name {
continue
}
rc, err := f.Open()
if err != nil {
return nil, err
}
b, err := io.ReadAll(io.LimitReader(rc, ooxmlPartLimit+1))
rc.Close()
if err != nil {
return nil, err
}
if int64(len(b)) > ooxmlPartLimit {
return nil, fmt.Errorf("part %s exceeds %d bytes decompression limit", name, ooxmlPartLimit)
}
return b, nil
}
return nil, fmt.Errorf("part %s not found", name)
}
// relsPath 计算部件对应的关系部件名(xl/workbook.xml → xl/_rels/workbook.xml.rels)。
func relsPath(part string) string {
dir, base := path.Split(part)
return dir + "_rels/" + base + ".rels"
}
type ooxmlRel struct {
ID, Target, Type string
}
// parseRels 解析部件的关系表;无关系部件返回空表(不是错误)。
func parseRels(zr *zip.Reader, part string) (map[string]ooxmlRel, error) {
data, err := zipPart(zr, relsPath(part))
if err != nil {
return map[string]ooxmlRel{}, nil
}
dec := xml.NewDecoder(bytes.NewReader(data))
out := map[string]ooxmlRel{}
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
st, ok := tok.(xml.StartElement)
if !ok || st.Name.Local != "Relationship" {
continue
}
r := ooxmlRel{}
for _, a := range st.Attr {
switch a.Name.Local {
case "Id":
r.ID = a.Value
case "Target":
r.Target = a.Value
case "Type":
r.Type = a.Value
}
}
if r.ID != "" {
out[r.ID] = r
}
}
return out, nil
}
// resolveRelTarget 把关系 target 解析为 zip 部件名:绝对 target(/xl/…)去
// 前导斜杠;相对 target 相对 partDir;归一化 ../ 与 ./ 段。
func resolveRelTarget(partDir, target string) string {
t := target
if !strings.HasPrefix(target, "/") {
t = partDir + "/" + target
}
var seg []string
for _, s := range strings.Split(t, "/") {
switch s {
case "", ".":
case "..":
if len(seg) > 0 {
seg = seg[:len(seg)-1]
}
default:
seg = append(seg, s)
}
}
return strings.Join(seg, "/")
}
// ridAttr 取 r:id:局部名 id 且带命名空间前缀(Transitional 与 Strict OOXML
// 的关系引用都满足该形态;无前缀的 id 是别的编号,不误取)。
func ridAttr(st xml.StartElement) string {
for _, a := range st.Attr {
if a.Name.Local == "id" && a.Name.Space != "" {
return a.Value
}
}
return ""
}
// attrLocal 取无命名空间前缀的属性值。
func attrLocal(st xml.StartElement, name string) string {
for _, a := range st.Attr {
if a.Name.Local == name && a.Name.Space == "" {
return a.Value
}
}
return ""
}
// relIsType 判断关系类型是否以标准关系类型后缀结尾(/worksheet、/slide 等)。
func relIsType(rel ooxmlRel, suffix string) bool {
return strings.HasSuffix(rel.Type, suffix)
}
+140
View File
@@ -0,0 +1,140 @@
package ingest
// 真实生成器轮测:openpyxl / python-pptx 现场生成 xlsx/pptx 再解析,防止
// 手工夹具只验证"自己写的 XML"。生成器缺失时 t.Skip(诚实出口);CI 已装
// 这两个包(.gitea/workflows/ci.yaml 安装 python-docx python-pptx openpyxl)。
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
core "agent-system-core"
)
func pythonModuleAvailable(t *testing.T, module string) bool {
t.Helper()
if _, err := exec.LookPath("python3"); err != nil {
return false
}
return exec.Command("python3", "-c", "import "+module).Run() == nil
}
// runPythonScript 把脚本写到临时目录执行;失败即 t.Fatal(脚本属测试自身)。
func runPythonScript(t *testing.T, script string) {
t.Helper()
dir := t.TempDir()
path := filepath.Join(dir, "gen.py")
if err := os.WriteFile(path, []byte(script), 0o644); err != nil {
t.Fatal(err)
}
out, err := exec.Command("python3", path).CombinedOutput()
if err != nil {
t.Fatalf("python generator failed: %v\n%s", err, out)
}
}
func TestParseXlsxFromOpenpyxl(t *testing.T) {
if !pythonModuleAvailable(t, "openpyxl") {
t.Skip("python3 openpyxl not available")
}
dir := t.TempDir()
file := filepath.Join(dir, "报价单.xlsx")
runPythonScript(t, `
import openpyxl
from datetime import datetime
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "报价"
ws.append(["物料", "单价", "日期"])
ws.append(["支架", 12.5, datetime(2024, 1, 1)])
ws["C2"].number_format = "yyyy-mm-dd"
ws["D2"] = "=A2&\"-x\""
ws["D3"] = True
wb.create_sheet("说明").append(["备注行"])
wb.save(r"`+file+`")
`)
data, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
doc := Parse(file, "", data)
if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 {
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
}
var heads []string
for _, b := range doc.Blocks {
if b.Type == core.BlockHeading {
heads = append(heads, b.Text)
}
}
if len(heads) != 2 || heads[0] != "报价" || heads[1] != "说明" {
t.Fatalf("headings: %v", heads)
}
plain := doc.PlainText()
for _, want := range []string{"物料", "支架", "12.5", "2024-01-01"} {
if !strings.Contains(plain, want) {
t.Errorf("missing %q in: %s", want, plain)
}
}
// openpyxl 公式不带缓存值:回显公式本身(可检索、不冒充结果)。
if !strings.Contains(plain, "=A2") {
t.Errorf("formula without cache should echo: %s", plain)
}
if !strings.Contains(plain, "TRUE") {
t.Errorf("boolean cell should render TRUE: %s", plain)
}
}
func TestParsePptxFromPythonPptx(t *testing.T) {
if !pythonModuleAvailable(t, "pptx") {
t.Skip("python3 pptx not available")
}
dir := t.TempDir()
file := filepath.Join(dir, "方案评审.pptx")
runPythonScript(t, `
from pptx import Presentation
from pptx.util import Inches
prs = Presentation()
s1 = prs.slides.add_slide(prs.slide_layouts[0])
s1.shapes.title.text = "评审结论"
s1.placeholders[1].text = "通过"
s2 = prs.slides.add_slide(prs.slide_layouts[5])
s2.shapes.title.text = "报价明细"
tbl = s2.shapes.add_table(2, 2, Inches(1), Inches(2), Inches(6), Inches(1)).table
tbl.cell(0, 0).text = "物料"
tbl.cell(0, 1).text = "单价"
tbl.cell(1, 0).text = "支架"
tbl.cell(1, 1).text = "12.5"
notes = s1.notes_slide.notes_text_frame
notes.text = "需要补充法规依据"
prs.save(r"`+file+`")
`)
data, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
doc := Parse(file, "", data)
if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 {
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
}
var heads []string
for _, b := range doc.Blocks {
if b.Type == core.BlockHeading {
heads = append(heads, b.Text)
}
}
if len(heads) != 2 || !strings.HasPrefix(heads[0], "幻灯片 1:评审结论") || !strings.HasPrefix(heads[1], "幻灯片 2:报价明细") {
t.Fatalf("headings: %v", heads)
}
plain := doc.PlainText()
for _, want := range []string{"通过", "物料", "支架", "12.5", "备注:需要补充法规依据"} {
if !strings.Contains(plain, want) {
t.Errorf("missing %q in: %s", want, plain)
}
}
}
+362
View File
@@ -0,0 +1,362 @@
package ingest
// OOXML(xlsx/pptx)解析器测试:手工构造最小 zip 部件,覆盖顺序解析、富文本、
// 公式、日期、稀疏列、隐藏表、组合形状、AlternateContent、表格合并、备注、
// 上限与失败路径。真实生成器的轮测见 ooxml_python_test.go。
import (
"archive/zip"
"bytes"
"sort"
"strings"
"testing"
core "agent-system-core"
)
// zipBytes 按名称构造 zip(键排序保证确定性)。
func zipBytes(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
names := make([]string, 0, len(files))
for n := range files {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
w, err := zw.Create(n)
if err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte(files[n])); err != nil {
t.Fatal(err)
}
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
const relTypeNS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
func xlsxFixture() map[string]string {
return map[string]string{
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="` + relTypeNS + `">
<sheets>
<sheet name="报价" sheetId="1" r:id="rId1"/>
<sheet name="草稿" sheetId="2" state="hidden" r:id="rId2"/>
</sheets></workbook>`,
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
<Relationship Id="rId1" Type="` + relTypeNS + `/worksheet" Target="worksheets/sheet1.xml"/>
<Relationship Id="rId2" Type="` + relTypeNS + `/worksheet" Target="/xl/worksheets/sheet2.xml"/>
</Relationships>`,
// shared[0]=物料;shared[1]=富文本"单价"(rPh 注音必须排除)。
"xl/sharedStrings.xml": `<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<si><t>物料</t></si>
<si><rPh sb="0" eb="2"><t>たんか</t></rPh><r><rPr lang="ja"/><t>单价</t></r></si>
</sst>`,
"xl/styles.xml": `<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<numFmts><numFmt numFmtId="164" formatCode="yyyy&quot;年&quot;m&quot;月&quot;d&quot;日&quot;"/></numFmts>
<cellXfs count="3">
<xf numFmtId="0"/>
<xf numFmtId="14" applyNumberFormat="1"/>
<xf numFmtId="164" applyNumberFormat="1"/>
</cellXfs></styleSheet>`,
// A1=物料 B1=单价(富文本) A2=公式缓存 10 B2=无缓存公式 C2=日期(内置14)=2024-01-01
// D2=TRUE E2=#DIV/0! F2=内联 G2=自定义日期格式(2024-01-02) 稀疏行 C4=物料。
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData>
<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>
<row r="2"><c r="A2"><f>B9*2</f><v>10</v></c><c r="B2"><f>B9*3</f></c><c r="C2" s="1"><v>45292</v></c><c r="D2" t="b"><v>1</v></c><c r="E2" t="e"><v>#DIV/0!</v></c><c r="F2" t="inlineStr"><is><t>内联</t></is></c><c r="G2" s="2"><v>45293</v></c></row>
<row r="4"><c r="C4" t="s"><v>0</v></c></row>
</sheetData></worksheet>`,
"xl/worksheets/sheet2.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row r="1"><c r="A1"><v>7</v></c></row></sheetData></worksheet>`,
}
}
func findTable(doc core.ParsedDocument, after int) *core.DocBlock {
for i := after; i < len(doc.Blocks); i++ {
if doc.Blocks[i].Type == core.BlockTable {
return &doc.Blocks[i]
}
}
return nil
}
func TestParseXlsxBuiltFromParts(t *testing.T) {
doc := Parse("报价单.xlsx", "", zipBytes(t, xlsxFixture()))
if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 {
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
}
if doc.Meta.Confidence != 0.9 {
t.Fatalf("confidence: %v", doc.Meta.Confidence)
}
// 工作表顺序取 workbook.xml(报价在前),隐藏表标注;绝对 target /xl/… 也能定位。
if doc.Blocks[0].Type != core.BlockHeading || doc.Blocks[0].Text != "报价" {
t.Fatalf("first block: %+v", doc.Blocks[0])
}
heads := []string{}
for _, b := range doc.Blocks {
if b.Type == core.BlockHeading {
heads = append(heads, b.Text)
}
}
if strings.Join(heads, "|") != "报价|草稿(隐藏)" {
t.Fatalf("headings: %v", heads)
}
tbl := findTable(doc, 0)
if tbl == nil || len(tbl.Rows) != 3 {
t.Fatalf("table: %+v", tbl)
}
row1, row2, row4 := tbl.Rows[0], tbl.Rows[1], tbl.Rows[2]
if row1[0] != "物料" || row1[1] != "单价" {
t.Errorf("header row (phonetic must be excluded): %v", row1)
}
want := []string{"10", "=B9*3", "2024-01-01", "TRUE", "#DIV/0!", "内联", "2024-01-02"}
if len(row2) != len(want) {
t.Fatalf("row2: %v", row2)
}
for i, w := range want {
if row2[i] != w {
t.Errorf("row2[%d]: got %q want %q", i, row2[i], w)
}
}
// 稀疏单元格按引用就位:C4 → 第三列。
if len(row4) != 3 || row4[0] != "" || row4[1] != "" || row4[2] != "物料" {
t.Errorf("sparse row: %v", row4)
}
if !strings.Contains(doc.PlainText(), "2024-01-01") {
t.Error("PlainText missing date text")
}
}
func TestParseXlsxEmptySheetSkipped(t *testing.T) {
files := xlsxFixture()
files["xl/worksheets/sheet2.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1"/></sheetData></worksheet>`
doc := Parse("x.xlsx", "", zipBytes(t, files))
heads := 0
for _, b := range doc.Blocks {
if b.Type == core.BlockHeading {
heads++
}
}
if heads != 1 || len(doc.Warnings) != 0 {
t.Fatalf("empty sheet must be skipped: blocks=%+v warnings=%v", doc.Blocks, doc.Warnings)
}
}
func TestParseXlsxInvalid(t *testing.T) {
doc := Parse("坏.xlsx", "", []byte("not a zip"))
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
t.Fatalf("invalid xlsx should fail visibly: %+v %v", doc.Meta, doc.Warnings)
}
files := xlsxFixture()
delete(files, "xl/workbook.xml")
doc = Parse("无工作簿.xlsx", "", zipBytes(t, files))
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
t.Fatalf("missing workbook.xml should fail visibly: %v", doc.Warnings)
}
}
func TestParseXlsxCellCap(t *testing.T) {
old := xlsxMaxCells
xlsxMaxCells = 3
defer func() { xlsxMaxCells = old }()
doc := Parse("超限.xlsx", "", zipBytes(t, xlsxFixture()))
if len(doc.Blocks) == 0 {
t.Fatal("truncated parse should keep partial blocks")
}
if len(doc.Warnings) == 0 || doc.Meta.Confidence != 0.6 {
t.Fatalf("truncation must warn and lower confidence: %v %v", doc.Meta.Confidence, doc.Warnings)
}
if !strings.Contains(strings.Join(doc.Warnings, ";"), "上限") {
t.Errorf("warnings: %v", doc.Warnings)
}
}
func TestParseXlsxPartSizeCap(t *testing.T) {
old := ooxmlPartLimit
ooxmlPartLimit = 16
defer func() { ooxmlPartLimit = old }()
doc := Parse("炸弹.xlsx", "", zipBytes(t, xlsxFixture()))
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
t.Fatalf("oversized part must fail visibly: %+v %v", doc.Meta, doc.Warnings)
}
}
func TestParseXlsxDate1904(t *testing.T) {
files := xlsxFixture()
files["xl/workbook.xml"] = strings.Replace(files["xl/workbook.xml"],
"<sheets>", `<workbookPr date1904="1"/><sheets>`, 1)
files["xl/worksheets/sheet1.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<sheetData><row r="1"><c r="A1" s="1"><v>1</v></c><c r="B1" s="1"><v>0.520833333333333</v></c></row></sheetData></worksheet>`
doc := Parse("mac.xlsx", "", zipBytes(t, files))
tbl := findTable(doc, 0)
// 1904 历法:serial 0 = 1904-01-01,serial 1 = 1904-01-02。
if tbl == nil || tbl.Rows[0][0] != "1904-01-02" {
t.Fatalf("1904 epoch: %+v", tbl)
}
if !strings.Contains(tbl.Rows[0][1], "1904-01-01 12:30") {
t.Errorf("time fraction: %q", tbl.Rows[0][1])
}
}
const pNS = "http://schemas.openxmlformats.org/presentationml/2006/main"
const aNS = "http://schemas.openxmlformats.org/drawingml/2006/main"
func sp(phType, text string) string {
ph := ""
if phType != "" {
ph = `<p:nvPr><p:ph type="` + phType + `"/></p:nvPr>`
} else {
ph = `<p:nvPr/>`
}
return `<p:sp><p:nvSpPr><p:cNvPr id="2" name="S"/><p:cNvSpPr/></p:nvSpPr>` + ph + `<p:spPr/>` +
`<p:txBody><a:bodyPr/><a:p><a:r><a:t>` + text + `</a:t></a:r></a:p></p:txBody></p:sp>`
}
func pptxFixture() map[string]string {
presRel := func(id, target string) string {
return `<Relationship Id="` + id + `" Type="` + relTypeNS + `/slide" Target="` + target + `"/>`
}
slide1 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `" xmlns:r="` + relTypeNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>
` + sp("title", "封面标题") + `
` + sp("body", "") + `
<p:sp><p:nvSpPr><p:cNvPr id="4" name="B"/><p:cNvSpPr/></p:nvSpPr><p:nvPr/><p:spPr/><p:txBody><a:bodyPr/>
<a:p><a:r><a:t>第一行</a:t></a:r><a:br/><a:r><a:t>第二行</a:t></a:r></a:p></p:txBody></p:sp>
<p:grpSp><p:nvGrpSpPr/><p:grpSpPr/>` + sp("", "组合内文本") + `</p:grpSp>
<mc:AlternateContent xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006">
<mc:Choice Requires="p14">` + sp("", "选内文本") + `</mc:Choice>
<mc:Fallback>` + sp("", "弃内文本") + `</mc:Fallback>
</mc:AlternateContent>
<p:sp><p:nvSpPr><p:cNvPr id="6" name="N"/><p:cNvSpPr/></p:nvSpPr><p:nvPr><p:ph type="sldNum"/></p:nvPr><p:spPr/><p:txBody><a:p><a:fld id="{F1}"><a:t>3</a:t></a:fld></a:p></p:txBody></p:sp>
<p:graphicFrame><p:nvGraphicFramePr/><p:xfrm/><a:graphic><a:graphicData uri="` + aNS + `/table"><a:tbl>
<a:tr h="1"><a:tc gridSpan="2"><a:txBody><a:p><a:r><a:t>合并头</a:t></a:r></a:p></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>列三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
<a:tr h="1"><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>值三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
</a:tbl></a:graphicData></a:graphic></p:graphicFrame>
</p:spTree></p:cSld></p:sld>`
slide2 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "第二页要点") + `
</p:spTree></p:cSld></p:sld>`
return map[string]string{
"ppt/presentation.xml": `<p:presentation xmlns:p="` + pNS + `" xmlns:r="` + relTypeNS + `"><p:sldIdLst>
<p:sldId id="256" r:id="rIdA"/><p:sldId id="257" r:id="rIdB"/>
</p:sldIdLst></p:presentation>`,
"ppt/_rels/presentation.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">` +
presRel("rIdA", "slides/slide1.xml") + presRel("rIdB", "slides/slide2.xml") + `</Relationships>`,
"ppt/slides/slide1.xml": slide1,
"ppt/slides/slide2.xml": slide2,
"ppt/slides/_rels/slide1.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rIdN" Type="` + relTypeNS + `/notesSlide" Target="../notesSlides/notesSlide1.xml"/></Relationships>`,
// slide2 的备注故意缺失(无 rels 文件)。
"ppt/notesSlides/notesSlide1.xml": `<p:notes xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "补充说明") + `
</p:spTree></p:cSld></p:notes>`,
}
}
func TestParsePptxBuiltFromParts(t *testing.T) {
doc := Parse("演示.pptx", "", zipBytes(t, pptxFixture()))
if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 || doc.Meta.Confidence != 0.9 {
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
}
var heads []string
for _, b := range doc.Blocks {
if b.Type == core.BlockHeading {
heads = append(heads, b.Text)
}
}
// sldIdLst 顺序:slide1(有标题)在前;slide2 无标题占位符 → 无标题兜底。
if len(heads) != 2 || heads[0] != "幻灯片 1:封面标题" || heads[1] != "幻灯片 2" {
t.Fatalf("headings: %v", heads)
}
plain := doc.PlainText()
for _, want := range []string{"第一行\n第二行", "组合内文本", "选内文本", "备注:补充说明"} {
if !strings.Contains(plain, want) {
t.Errorf("missing %q in plain text", want)
}
}
if strings.Contains(plain, "弃内文本") {
t.Error("mc:Fallback content must be skipped")
}
// 页码占位符与字段文本不出现。
if strings.Contains(plain, "\n3\n") {
t.Error("slide number placeholder text leaked")
}
// 表格:gridSpan 占位 + hMerge 续格留空,保持三列对齐。
tbl := findTable(doc, 0)
if tbl == nil || len(tbl.Rows) != 2 {
t.Fatalf("table: %+v", tbl)
}
if strings.Join(tbl.Rows[0], "|") != "合并头||列三" {
t.Errorf("row1: %v", tbl.Rows[0])
}
if strings.Join(tbl.Rows[1], "|") != "||值三" {
t.Errorf("row2: %v", tbl.Rows[1])
}
}
func TestParsePptxInvalidAndEmpty(t *testing.T) {
doc := Parse("坏.pptx", "", []byte("garbage"))
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
t.Fatalf("invalid pptx: %+v %v", doc.Meta, doc.Warnings)
}
files := pptxFixture()
files["ppt/presentation.xml"] = `<p:presentation xmlns:p="` + pNS + `"></p:presentation>`
doc = Parse("空演示.pptx", "", zipBytes(t, files))
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
t.Fatalf("no slides: %v", doc.Warnings)
}
}
func TestParseLegacyOfficeHint(t *testing.T) {
for ext, want := range map[string]string{
"doc": "docx", "xls": "xlsx", "ppt": "pptx",
} {
doc := Parse("旧档."+ext, "", []byte{0xD0, 0xCF})
if doc.Meta.Parser != "builtin:legacy-office" {
t.Fatalf("%s parser: %+v", ext, doc.Meta)
}
if len(doc.Blocks) != 0 || len(doc.Warnings) != 1 {
t.Fatalf("%s blocks/warnings: %+v %v", ext, doc.Blocks, doc.Warnings)
}
if !strings.Contains(doc.Warnings[0], want) {
t.Errorf("%s hint should mention %s: %v", ext, want, doc.Warnings[0])
}
}
}
// 注册表与分派行为一致:每个已注册格式的解析器 ID 与 Parse 分派结果一致,
// 且 builtin 解析器覆盖的扩展名都出现在注册表里。
func TestParsersDescriptionMatchesDispatch(t *testing.T) {
byFormat := map[string]string{}
for _, d := range ParsersDescription() {
if d.Source != core.SourceBuiltin {
continue
}
for _, f := range d.Formats {
byFormat[f] = d.ID
}
}
check := map[string]string{
"md": "builtin:md", "markdown": "builtin:md",
"txt": "builtin:txt", "text": "builtin:txt", "csv": "builtin:txt", "log": "builtin:txt",
"docx": "builtin:docx", "pdf": "builtin:pdf",
"xlsx": "builtin:xlsx", "xlsm": "builtin:xlsx",
"pptx": "builtin:pptx", "pptm": "builtin:pptx", "ppsx": "builtin:pptx",
}
for ext, wantID := range check {
got, ok := byFormat[ext]
if !ok {
t.Errorf("format .%s missing from registry", ext)
continue
}
doc := Parse("a."+ext, "", []byte("x"))
if doc.Meta.Parser != wantID || got != wantID {
t.Errorf(".%s: registry=%s dispatch=%s want=%s", ext, got, doc.Meta.Parser, wantID)
}
}
}
+291
View File
@@ -0,0 +1,291 @@
package ingest
// pptx/pptm/ppsx(PresentationML)内置解析器。幻灯片顺序取 presentation.xml
// 的 sldIdLst;标题占位符(title/ctrTitle)产标题块,正文形状按段落输出
//(a:br 换行),组合形状经同一 token 走查天然递归;mc:AlternateContent 只取
// Choice 分支;graphicFrame 内的表格按 tr/tc 输出(gridSpan/hMerge 占位保列
// 对齐);备注页正文输出「备注:…」;页码/日期/页脚占位符跳过。图片与
// SmartArt 不解析。纯标准库。
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path"
"strconv"
"strings"
core "agent-system-core"
)
func parsePptx(content []byte) ([]core.DocBlock, []string) {
zr, err := openZip(content)
if err != nil {
return nil, []string{"pptx 解析失败:" + err.Error()}
}
slides, warns := pptxSlideOrder(zr)
if slides == nil && len(warns) > 0 {
return nil, warns
}
var blocks []core.DocBlock
for i, sp := range slides {
n := i + 1
data, err := zipPart(zr, sp)
if err != nil {
warns = append(warns, fmt.Sprintf("幻灯片 %d 读取失败:%v", n, err))
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: fmt.Sprintf("幻灯片 %d", n)})
continue
}
title, paras, tables := pptxSlideBlocks(data)
heading := fmt.Sprintf("幻灯片 %d", n)
if title != "" {
heading += ":" + title
}
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: heading})
blocks = append(blocks, paras...)
blocks = append(blocks, tables...)
if note := pptxSlideNotes(zr, sp); note != "" {
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: "备注:" + note})
}
}
if len(blocks) == 0 && len(warns) == 0 {
warns = append(warns, "pptx 无可解析的幻灯片内容")
}
return blocks, warns
}
// pptxSlideOrder 读 presentation.xml 的 sldIdLst → 幻灯片部件名(按顺序)。
func pptxSlideOrder(zr *zip.Reader) ([]string, []string) {
data, err := zipPart(zr, "ppt/presentation.xml")
if err != nil {
return nil, []string{"pptx 解析失败:" + err.Error()}
}
rels, err := parseRels(zr, "ppt/presentation.xml")
if err != nil {
return nil, []string{"pptx 解析失败:" + err.Error()}
}
dec := xml.NewDecoder(bytes.NewReader(data))
var out, warns []string
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return out, append(warns, "presentation.xml 解码失败:"+err.Error())
}
st, ok := tok.(xml.StartElement)
if !ok || st.Name.Local != "sldId" {
continue
}
rel, ok := rels[ridAttr(st)]
if !ok || !relIsType(rel, "/slide") {
warns = append(warns, "幻灯片关系缺失或类型不符")
continue
}
out = append(out, resolveRelTarget("ppt", rel.Target))
}
return out, warns
}
// pptxSlideBlocks 解析一张幻灯片:标题、正文段落、表格块。
// 表格块统一附在段落后(图形框与正文的精确穿插对检索无价值)。
func pptxSlideBlocks(data []byte) (title string, paras []core.DocBlock, tables []core.DocBlock) {
dec := xml.NewDecoder(bytes.NewReader(data))
var stack []string
has := func(name string) bool {
for _, s := range stack {
if s == name {
return true
}
}
return false
}
skip := 0 // mc:Fallback 子树深度
// 形状状态
spDepth := -1
phSkip := false
isTitle := false
var spParas []string
var para strings.Builder
paraOpen := false
// 文本收集目标
textInPara := false
textInCell := false
// 表格状态
var tblRows [][]string
var tblRow []string
var cellBuf strings.Builder
cellOpen := false
gridSpan := 1
cellHM := false
finishPara := func() {
if !paraOpen {
return
}
paraOpen = false
if s := strings.TrimRight(para.String(), "\n"); strings.TrimSpace(s) != "" {
spParas = append(spParas, strings.TrimSpace(s))
}
para.Reset()
}
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return "", paras, tables
}
switch t := tok.(type) {
case xml.StartElement:
if skip > 0 {
skip++
continue
}
local := t.Name.Local
switch local {
case "Fallback":
if has("AlternateContent") {
skip = 1
continue
}
case "sp":
spDepth = len(stack)
phSkip, isTitle, spParas = false, false, nil
case "ph":
if spDepth >= 0 && len(stack) > spDepth {
switch attrLocal(t, "type") {
case "title", "ctrTitle":
isTitle = true
case "dt", "ftr", "sldNum":
phSkip = true
}
}
case "p":
if spDepth >= 0 && has("txBody") && !has("tbl") {
finishPara()
paraOpen = true
para.Reset()
} else if cellOpen {
if cellBuf.Len() > 0 {
cellBuf.WriteByte('\n')
}
}
case "br":
if paraOpen {
para.WriteByte('\n')
}
case "t":
if has("fld") {
break // 字段(页码/日期)不取
}
textInPara = paraOpen
textInCell = cellOpen && !paraOpen
case "tc":
if has("tbl") {
cellBuf.Reset()
cellOpen = true
gridSpan, _ = strconv.Atoi(attrLocal(t, "gridSpan"))
if gridSpan < 1 {
gridSpan = 1
}
hm := attrLocal(t, "hMerge")
cellHM = hm == "1" || strings.EqualFold(hm, "true")
}
}
stack = append(stack, local)
case xml.CharData:
if textInPara {
para.Write(t)
} else if textInCell {
cellBuf.Write(t)
}
case xml.EndElement:
if skip > 0 {
skip--
continue
}
if len(stack) > 0 {
stack = stack[:len(stack)-1]
}
switch t.Name.Local {
case "t":
textInPara, textInCell = false, false
case "p":
finishPara()
case "sp":
if spDepth >= 0 && len(stack) == spDepth {
if !phSkip && len(spParas) > 0 {
if isTitle && title == "" {
title = strings.Join(spParas, " ")
} else {
for _, p := range spParas {
paras = append(paras, core.DocBlock{Type: core.BlockPara, Text: p})
}
}
}
spDepth = -1
phSkip, isTitle, spParas = false, false, nil
}
case "tc":
if cellOpen {
text := strings.TrimSpace(cellBuf.String())
if cellHM {
text = "" // 横向合并的续格:占位保列对齐
}
tblRow = append(tblRow, text)
for i := 1; i < gridSpan; i++ {
tblRow = append(tblRow, "") // 合并占位保列对齐
}
cellOpen = false
}
case "tr":
if len(tblRow) > 0 {
for len(tblRow) > 0 && tblRow[len(tblRow)-1] == "" {
tblRow = tblRow[:len(tblRow)-1]
}
if len(tblRow) > 0 {
tblRows = append(tblRows, tblRow)
}
tblRow = nil
}
case "tbl":
if len(tblRows) > 0 {
tables = append(tables, core.DocBlock{Type: core.BlockTable, Rows: tblRows})
}
tblRows = nil
}
}
}
finishPara()
return title, paras, tables
}
// pptxSlideNotes 读幻灯片的备注页正文;无备注返回空串。
func pptxSlideNotes(zr *zip.Reader, slidePart string) string {
rels, err := parseRels(zr, slidePart)
if err != nil {
return ""
}
for _, rel := range rels {
if !relIsType(rel, "/notesSlide") {
continue
}
data, err := zipPart(zr, resolveRelTarget(path.Dir(slidePart), rel.Target))
if err != nil {
return ""
}
_, paras, _ := pptxSlideBlocks(data)
var lines []string
for _, p := range paras {
lines = append(lines, p.Text)
}
return strings.Join(lines, "\n")
}
return ""
}
+415
View File
@@ -0,0 +1,415 @@
package ingest
// xlsx/xlsm(SpreadsheetML)内置解析器。工作表顺序取 workbook.xml 的 sheets
// 列表(target 经关系部件解析,支持相对/绝对路径);共享字符串含富文本 run
//(注音 rPh 无对应解码字段,天然排除);单元格覆盖 inlineStr/str/b/e/n,
// 公式取缓存值、无缓存回显公式;日期按数字格式(内置 ID 与自定义记号)转
// ISO,1900/1904 双日期系统;稀疏单元格按引用就位。纯标准库。
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"math"
"strconv"
"strings"
"time"
core "agent-system-core"
)
// 解析上限(包级变量,测试可覆盖)。
var (
xlsxMaxCells = 200000
xlsxMaxCols = 1024
)
type xlsxSheetRef struct {
Name string
State string
Path string
IsSheet bool
}
func parseXlsx(content []byte) ([]core.DocBlock, []string) {
zr, err := openZip(content)
if err != nil {
return nil, []string{"xlsx 解析失败:" + err.Error()}
}
sheets, date1904, warns := xlsxWorkbookSheets(zr)
if len(warns) > 0 && sheets == nil {
return nil, warns
}
shared := xlsxSharedStrings(zr)
dateStyles := xlsxDateStyles(zr)
var blocks []core.DocBlock
cells := 0
for _, sh := range sheets {
if !sh.IsSheet || sh.Path == "" {
continue // chartsheet 等非工作表部件
}
data, err := zipPart(zr, sh.Path)
if err != nil {
warns = append(warns, fmt.Sprintf("工作表 %s 读取失败:%v", sh.Name, err))
continue
}
table, used, trunc := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells)
cells = used
if table == nil {
continue // 空表
}
title := sh.Name
if sh.State == "hidden" || sh.State == "veryHidden" {
title += "(隐藏)"
}
blocks = append(blocks,
core.DocBlock{Type: core.BlockHeading, Level: 2, Text: title},
*table,
)
if trunc {
warns = append(warns, fmt.Sprintf("单元格总数超过 %d 上限,工作表 %s 之后未解析", xlsxMaxCells, sh.Name))
break
}
}
if len(blocks) == 0 && len(warns) == 0 {
warns = append(warns, "xlsx 无可解析的工作表内容")
}
return blocks, warns
}
// xlsxWorkbookSheets 读 workbook.xml:工作表顺序、隐藏状态、1904 日期系统。
func xlsxWorkbookSheets(zr *zip.Reader) (sheets []xlsxSheetRef, date1904 bool, warns []string) {
data, err := zipPart(zr, "xl/workbook.xml")
if err != nil {
return nil, false, []string{"xlsx 解析失败:" + err.Error()}
}
rels, err := parseRels(zr, "xl/workbook.xml")
if err != nil {
return nil, false, []string{"xlsx 解析失败:" + err.Error()}
}
dec := xml.NewDecoder(bytes.NewReader(data))
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return sheets, date1904, append(warns, "workbook.xml 解码失败:"+err.Error())
}
st, ok := tok.(xml.StartElement)
if !ok {
continue
}
switch st.Name.Local {
case "workbookPr":
if v := attrLocal(st, "date1904"); v == "1" || strings.EqualFold(v, "true") {
date1904 = true
}
case "sheet":
ref := xlsxSheetRef{Name: attrLocal(st, "name"), State: attrLocal(st, "state")}
if rel, ok := rels[ridAttr(st)]; ok {
ref.Path = resolveRelTarget("xl", rel.Target)
ref.IsSheet = relIsType(rel, "/worksheet")
} else {
warns = append(warns, fmt.Sprintf("工作表 %s 的关系缺失", ref.Name))
}
sheets = append(sheets, ref)
}
}
return sheets, date1904, warns
}
// xlsxSharedStrings 读共享字符串表;富文本按 run 拼接,rPh 注音不进结果。
func xlsxSharedStrings(zr *zip.Reader) []string {
data, err := zipPart(zr, "xl/sharedStrings.xml")
if err != nil {
return nil
}
dec := xml.NewDecoder(bytes.NewReader(data))
var out []string
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return out
}
st, ok := tok.(xml.StartElement)
if !ok || st.Name.Local != "si" {
continue
}
var si struct {
T string `xml:"t"`
Runs []struct {
T string `xml:"t"`
} `xml:"r"`
}
if err := dec.DecodeElement(&si, &st); err != nil {
continue
}
if si.T != "" || len(si.Runs) == 0 {
out = append(out, si.T)
continue
}
parts := make([]string, 0, len(si.Runs))
for _, r := range si.Runs {
parts = append(parts, r.T)
}
out = append(out, strings.Join(parts, ""))
}
return out
}
// xlsxDateStyles 返回按日期呈现的样式索引集合(cellXfs 下标)。
func xlsxDateStyles(zr *zip.Reader) map[int]bool {
data, err := zipPart(zr, "xl/styles.xml")
if err != nil {
return nil
}
custom := map[int]string{}
date := map[int]bool{}
dec := xml.NewDecoder(bytes.NewReader(data))
inCellXfs := false
xfIdx := 0
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return date
}
switch t := tok.(type) {
case xml.StartElement:
switch t.Name.Local {
case "numFmt":
id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
custom[id] = attrLocal(t, "formatCode")
case "cellXfs":
inCellXfs = true
xfIdx = 0
case "xf":
if !inCellXfs {
continue
}
id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
if xlsxFmtIsDate(id, custom) {
date[xfIdx] = true
}
xfIdx++
}
case xml.EndElement:
if t.Name.Local == "cellXfs" {
inCellXfs = false
}
}
}
return date
}
// xlsxFmtIsDate:自定义格式按记号判断;内置 ID 覆盖常见日期/时间区间。
func xlsxFmtIsDate(id int, custom map[int]string) bool {
if code, ok := custom[id]; ok {
return formatCodeIsDate(code)
}
switch {
case id >= 14 && id <= 22, id >= 27 && id <= 36,
id >= 45 && id <= 47, id >= 50 && id <= 58:
return true
}
return false
}
// formatCodeIsDate 剥掉引号字面量与方括号节([Red]、[DBNum1]、[$-409] 等)后,
// 剩余字母含 y/d/h/s/m 记号即按日期处理。
func formatCodeIsDate(code string) bool {
var b strings.Builder
inQuote, inBracket := false, false
runes := []rune(code)
for i := 0; i < len(runes); i++ {
r := runes[i]
switch {
case r == '"':
inQuote = !inQuote
case inQuote:
case r == '[':
inBracket = true
case r == ']':
inBracket = false
case inBracket:
case r == '\\':
i++ // 转义字符:连同下一字符跳过
default:
b.WriteRune(r)
}
}
return strings.ContainsAny(strings.ToLower(b.String()), "ydhsm")
}
type xlsxCellEl struct {
Ref string `xml:"r,attr"`
T string `xml:"t,attr"`
S string `xml:"s,attr"`
IS *struct {
T string `xml:"t"`
} `xml:"is"`
F string `xml:"f"`
V string `xml:"v"`
}
type xlsxRowEl struct {
Cells []xlsxCellEl `xml:"c"`
}
// xlsxSheetBlocks 解析单张工作表;返回表格块、累计单元格数、是否触达上限。
// 无内容行返回 nil 表。
func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool) {
dec := xml.NewDecoder(bytes.NewReader(data))
var rows [][]string
count := cellsUsed
truncated := false
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return nil, count, false
}
st, ok := tok.(xml.StartElement)
if !ok || st.Name.Local != "row" {
continue
}
var re xlsxRowEl
if err := dec.DecodeElement(&re, &st); err != nil {
continue
}
cells := map[int]string{}
maxCol := -1
for _, c := range re.Cells {
if count >= xlsxMaxCells {
truncated = true
break
}
count++
col := xlsxColIndex(c.Ref)
if col < 0 {
col = maxCol + 1 // 无引用:按出现顺序接续
}
if col >= xlsxMaxCols {
truncated = true
continue
}
cells[col] = xlsxCellText(c, shared, dateStyles, date1904)
if col > maxCol {
maxCol = col
}
}
if len(cells) == 0 {
continue
}
full := make([]string, maxCol+1)
for i, v := range cells {
full[i] = v
}
for len(full) > 0 && strings.TrimSpace(full[len(full)-1]) == "" {
full = full[:len(full)-1]
}
if len(full) > 0 {
rows = append(rows, full)
}
}
if len(rows) == 0 {
return nil, count, truncated
}
return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated
}
// xlsxCellText 把单元格转成展示文本。
func xlsxCellText(c xlsxCellEl, shared []string, dateStyles map[int]bool, date1904 bool) string {
text := ""
switch c.T {
case "s":
if idx, err := strconv.Atoi(c.V); err == nil && idx >= 0 && idx < len(shared) {
text = shared[idx]
}
case "inlineStr":
if c.IS != nil {
text = c.IS.T
}
case "b":
if c.V == "1" || strings.EqualFold(c.V, "true") {
text = "TRUE"
} else {
text = "FALSE"
}
case "e":
text = c.V
default: // n 或空:数字 / 公式缓存值
text = c.V
if sIdx, err := strconv.Atoi(c.S); err == nil && dateStyles[sIdx] && c.V != "" {
if f, err := strconv.ParseFloat(c.V, 64); err == nil {
text = xlsxSerialString(f, date1904)
}
}
}
if text == "" && c.F != "" {
return "=" + c.F // 无缓存值的公式:回显公式本身
}
return text
}
// xlsxSerialString Excel 日期序列值 → ISO 字符串;带小数部分时输出到秒。
func xlsxSerialString(serial float64, date1904 bool) string {
days := math.Floor(serial)
frac := serial - days
var t time.Time
switch {
case date1904:
t = time.Date(1904, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
case days >= 61:
t = time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
default: // 1–60:兼容 Excel 的 1900 闰年 bug 区段
t = time.Date(1899, 12, 31, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
}
if frac > 0 {
return t.Add(time.Duration(frac * 86400 * float64(time.Second))).Round(time.Second).Format("2006-01-02 15:04:05")
}
return t.Format("2006-01-02")
}
// xlsxColIndex "BC12" → 0 起列号;无引用或非法返回 -1。
func xlsxColIndex(ref string) int {
if ref == "" {
return -1
}
i := 0
for i < len(ref) {
ch := ref[i]
if (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') {
i++
continue
}
break
}
if i == 0 || i == len(ref) {
return -1
}
for _, ch := range ref[i:] {
if ch < '0' || ch > '9' {
return -1
}
}
n := 0
for _, ch := range ref[:i] {
c := ch
if c >= 'a' && c <= 'z' {
c = c - 'a' + 'A'
}
n = n*26 + int(c-'A') + 1
}
return n - 1
}
+73
View File
@@ -1199,3 +1199,76 @@ func TestSettingsDottedKeyAndSecretMask(t *testing.T) {
t.Fatalf("apiKey mask = %v", chat["apiKey"])
}
}
// TestUploadXlsxParsedAndIndexed:xlsx 上传走内置解析器(builtin:xlsx、无警告),
// 单元格文本进入知识检索(EXT-5 验收路径)。
func TestUploadXlsxParsedAndIndexed(t *testing.T) {
srv, _ := newTestServer(t)
h := srv.Handler()
projectID, _ := setupConversation(t, h)
xlsx := buildTestXLSX(t)
body := &bytes.Buffer{}
mw := multipart.NewWriter(body)
fw, _ := mw.CreateFormFile("file", "报价单.xlsx")
fw.Write(xlsx)
mw.Close()
req := httptest.NewRequest("POST", "/api/projects/"+projectID+"/uploads", body)
req.Header.Set("Content-Type", mw.FormDataContentType())
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != 200 {
t.Fatalf("upload status %d: %s", rec.Code, rec.Body.String())
}
up := map[string]any{}
json.Unmarshal(rec.Body.Bytes(), &up)
parsed, _ := up["parsed"].(map[string]any)
if parsed == nil {
t.Fatalf("upload response missing parsed: %s", rec.Body.String())
}
meta, _ := parsed["meta"].(map[string]any)
if meta["parser"] != "builtin:xlsx" {
t.Fatalf("parser: %v", meta["parser"])
}
if w, ok := parsed["warnings"].([]any); ok && len(w) > 0 {
t.Fatalf("xlsx upload should parse without warnings: %v", w)
}
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
code, raw := doAny(t, h, "GET", "/api/knowledge/search?q=%E6%94%AF%E6%9E%B6", nil)
if code == 200 {
if hits, ok := raw.([]any); ok && len(hits) > 0 {
return
}
}
time.Sleep(100 * time.Millisecond)
}
t.Fatal("knowledge search found nothing for xlsx cell text")
}
// buildTestXLSX 构造最小可用 xlsx(inlineStr 单元格)。
func buildTestXLSX(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
files := map[string]string{
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="报价" sheetId="1" r:id="rId1"/></sheets></workbook>`,
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>`,
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>
<row r="1"><c r="A1" t="inlineStr"><is><t>物料</t></is></c></row>
<row r="2"><c r="A2" t="inlineStr"><is><t>支架</t></is></c></row>
</sheetData></worksheet>`,
}
for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} {
w, err := zw.Create(name)
if err != nil {
t.Fatal(err)
}
w.Write([]byte(files[name]))
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
+8 -2
View File
@@ -20,9 +20,9 @@ type Request struct {
// Host 句柄传入,处理器经句柄访问基座能力,不得自行越权持有 owner 凭证。
type Handler func(ctx context.Context, h Host, req Request) (any, error)
// Host 是基座在分发时构造的 owner 作用域宿主句柄(EXT-1 最小集)。
// Host 是基座在分发时构造的 owner 作用域宿主句柄(EXT-1 最小集 + EXT-5 解析)。
// 所有能力的可见范围都以本次分发的 Owner 为准;artifacts 只读
//(写入属完整集,另行评估)。实现由基座 internal 提供。
// (写入属完整集,另行评估)。实现由基座 internal 提供。
type Host interface {
// PluginID 返回注册处理器的插件标识(形如 rdm)。
PluginID() string
@@ -52,4 +52,10 @@ type Host interface {
// ArtifactGet/ArtifactContent 工件只读(R1-3 ingest 读原件)。
ArtifactGet(artifactID string) (core.Artifact, error)
ArtifactContent(artifactID string) ([]byte, error)
// ParseDocument 用基座内置解析器把文档字节解析为 ParsedDocument(EXT-5)。
// 纯计算,不读写 owner 数据——要解析工件原件先经 ArtifactContent 取字节,
// owner 作用域由它保证。按扩展名分派;不可解析返回带 Warnings 的文档
//(显式失败),只有 ctx 取消与输入超过 100MiB 两种情况返回 error。
ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error)
}
+17
View File
@@ -2,6 +2,19 @@ Compiled
ErrNotLinked
Handler
Host
Host.ArtifactContent
Host.ArtifactGet
Host.Audit
Host.Complete
Host.KVGet
Host.KVPut
Host.KnowledgeSearch
Host.KnowledgeWrite
Host.Owner
Host.ParseDocument
Host.PluginID
Host.TaskCreate
Host.TaskUpdate
HostClient
InProc
Main
@@ -9,6 +22,10 @@ NewHostClient
Options
RegisterCompiled
Registry
Registry.AddCommand
Registry.AddCommandHandler
Registry.AddSkill
Registry.Note
Request
ResetCompiled
SetRunner
+11
View File
@@ -37,6 +37,17 @@ func TestSurfaceSnapshot(t *testing.T) {
case *ast.TypeSpec:
if s.Name.IsExported() {
names = append(names, s.Name.Name)
// 导出接口同时记录其方法:只加方法、不改顶层名的
// 变更同样必须更新 surface.txt(EXT-5 起执行)。
if it, ok := s.Type.(*ast.InterfaceType); ok {
for _, m := range it.Methods.List {
for _, id := range m.Names {
if id.IsExported() {
names = append(names, s.Name.Name+"."+id.Name)
}
}
}
}
}
case *ast.ValueSpec:
for _, name := range s.Names {