Compare commits
2
Commits
cb26e4c947
...
727f3702d0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
727f3702d0 | ||
|
|
f9212d01b7 |
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"agent-system-backend/internal/ingest"
|
||||
"agent-system-backend/internal/rag"
|
||||
"agent-system-backend/internal/runtime"
|
||||
"agent-system-backend/internal/settings"
|
||||
@@ -39,6 +40,21 @@ func (c hostcaps) complete(ctx context.Context, owner, prompt string) (string, e
|
||||
return completeChat(ctx, route.Base, route.Key, route.Model, prompt)
|
||||
}
|
||||
|
||||
// maxParseInput 与上传端点的读取上限同口径(server.upload 的 100<<20)。
|
||||
const maxParseInput = 100 << 20
|
||||
|
||||
// parse 走基座内置解析器(EXT-5):纯计算、不落 owner 数据;不可解析由
|
||||
// Warnings 显式表达,只有取消与超限是 error。
|
||||
func (c hostcaps) parse(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return core.ParsedDocument{}, err
|
||||
}
|
||||
if len(content) > maxParseInput {
|
||||
return core.ParsedDocument{}, fmt.Errorf("parse input %d bytes exceeds %d limit", len(content), maxParseInput)
|
||||
}
|
||||
return ingest.Parse(filename, mime, content), nil
|
||||
}
|
||||
|
||||
// inprocHost 是进程内插件命令的 owner 作用域宿主句柄(extension.Host 实现)。
|
||||
// 每次分发按请求 Owner 新建;所有能力调用都以该 owner 为第一参数,插件无法越权。
|
||||
type inprocHost struct {
|
||||
@@ -75,7 +91,7 @@ func (h inprocHost) KnowledgeSearch(ctx context.Context, projectID core.ID, text
|
||||
|
||||
// KnowledgeWrite 强制 owner 归属:Owner 改写为本次分发 owner,Source 为空补
|
||||
// "plugin:<id>";embed=false 只写全文行,embed=true 走 IndexChunks 建向量
|
||||
//(embedding 失败降级纯全文,不阻断写入)。
|
||||
// (embedding 失败降级纯全文,不阻断写入)。
|
||||
func (h inprocHost) KnowledgeWrite(ctx context.Context, docs []core.KnowledgeDoc, embed bool) error {
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
@@ -114,3 +130,9 @@ func (h inprocHost) ArtifactGet(artifactID string) (core.Artifact, error) {
|
||||
func (h inprocHost) ArtifactContent(artifactID string) ([]byte, error) {
|
||||
return h.caps.st.ArtifactContent(h.owner, artifactID)
|
||||
}
|
||||
|
||||
// ParseDocument 内置文档解析(EXT-5):委托 hostcaps.parse;纯计算不经存储,
|
||||
// owner 作用域由调用方经 ArtifactContent 取字节时保证。
|
||||
func (h inprocHost) ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error) {
|
||||
return h.caps.parse(ctx, filename, mime, content)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
@@ -193,3 +195,112 @@ func TestInprocHostKnowledgeWriteEmbedHash(t *testing.T) {
|
||||
t.Fatalf("vector owner leak: %+v", bhits)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInprocHostParseDocumentOwnerScope:EXT-5 解析方法的 owner 作用域——
|
||||
// alice 经句柄读自己的工件并解析;bob 用同一工件 ID 在 ArtifactContent 处
|
||||
// 即被拒(解析是纯计算,作用域边界在取原件一环)。
|
||||
func TestInprocHostParseDocumentOwnerScope(t *testing.T) {
|
||||
extension.ResetCompiled()
|
||||
t.Cleanup(extension.ResetCompiled)
|
||||
extension.RegisterCompiled(extension.InProc{
|
||||
ID: "tst",
|
||||
Register: func(reg extension.Registry) error {
|
||||
return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.parse","title":"解析"}`),
|
||||
func(ctx context.Context, h extension.Host, req extension.Request) (any, error) {
|
||||
id, _ := req.Params["artifactId"].(string)
|
||||
content, err := h.ArtifactContent(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := h.ParseDocument(ctx, "报价.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"parser": doc.Meta.Parser,
|
||||
"warnings": len(doc.Warnings),
|
||||
"blocks": len(doc.Blocks),
|
||||
}, nil
|
||||
})
|
||||
},
|
||||
})
|
||||
bus, st := newInprocTestApp(t, nil)
|
||||
|
||||
art := core.Artifact{ProjectID: "p1", Owner: "alice", Name: "报价.xlsx", Kind: "document"}
|
||||
if err := st.CreateArtifact(&art, testXLSXBytes(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
artID := art.ID
|
||||
|
||||
resp := bus.Dispatch(context.Background(), cmdbus.Request{
|
||||
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "alice", Entrance: "chat",
|
||||
})
|
||||
if resp.Receipt.State != agentcommand.StateCommitted {
|
||||
t.Fatalf("alice dispatch: %+v %s", resp.Receipt, resp.Receipt.Reason)
|
||||
}
|
||||
m, _ := resp.Payload.(map[string]any)
|
||||
if m["parser"] != "builtin:xlsx" || m["warnings"] != 0 || m["blocks"] == 0 {
|
||||
t.Fatalf("alice payload: %+v", m)
|
||||
}
|
||||
|
||||
// bob 同一工件 ID:取原件即被拒(跨 owner 不可见)。
|
||||
resp = bus.Dispatch(context.Background(), cmdbus.Request{
|
||||
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "bob", Entrance: "chat",
|
||||
})
|
||||
if resp.Receipt.State == agentcommand.StateCommitted {
|
||||
t.Fatalf("bob must not read alice's artifact: %+v", resp.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestInprocHostParseDocumentLimits:仅有的两条 error 路径——ctx 取消与输入
|
||||
// 超过 100MiB 上限;不可解析格式仍是带 Warnings 的显式失败。
|
||||
func TestInprocHostParseDocumentLimits(t *testing.T) {
|
||||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
h := inprocHost{caps: hostcaps{st: st}, pluginID: "tst", owner: "alice"}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := h.ParseDocument(ctx, "a.xlsx", "", []byte("x")); err == nil {
|
||||
t.Fatal("canceled ctx must error")
|
||||
}
|
||||
if _, err := h.ParseDocument(context.Background(), "a.xlsx", "", make([]byte, maxParseInput+1)); err == nil {
|
||||
t.Fatal("oversized input must error")
|
||||
}
|
||||
doc, err := h.ParseDocument(context.Background(), "旧.xls", "", []byte{0xD0, 0xCF})
|
||||
if err != nil {
|
||||
t.Fatalf("legacy format is a visible failure, not an error: %v", err)
|
||||
}
|
||||
if doc.Meta.Parser != "builtin:legacy-office" || len(doc.Warnings) != 1 {
|
||||
t.Fatalf("legacy doc: %+v %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// testXLSXBytes 构造最小可用 xlsx(单表两行),供句柄测试解析。
|
||||
func testXLSXBytes(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
files := map[string]string{
|
||||
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="报价" sheetId="1" r:id="rId1"/></sheets></workbook>`,
|
||||
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>`,
|
||||
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>
|
||||
<row r="1"><c r="A1" t="inlineStr"><is><t>物料</t></is></c><c r="B1" t="inlineStr"><is><t>单价</t></is></c></row>
|
||||
<row r="2"><c r="A2" t="inlineStr"><is><t>支架</t></is></c><c r="B2"><v>12.5</v></c></row>
|
||||
</sheetData></worksheet>`,
|
||||
}
|
||||
for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write([]byte(files[name]))
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
+49
-17
@@ -4,7 +4,6 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
@@ -52,6 +51,35 @@ func Parse(filename, mime string, content []byte) core.ParsedDocument {
|
||||
doc.Blocks = blocks
|
||||
doc.Meta.Confidence = 0.85
|
||||
}
|
||||
case "xlsx", "xlsm":
|
||||
doc.Meta.Parser = "builtin:xlsx"
|
||||
blocks, warns := parseXlsx(content)
|
||||
doc.Blocks = blocks
|
||||
doc.Warnings = append(doc.Warnings, warns...)
|
||||
switch {
|
||||
case len(blocks) == 0:
|
||||
doc.Meta.Confidence = 0.3
|
||||
case len(warns) > 0:
|
||||
doc.Meta.Confidence = 0.6 // 超限截断:内容可用但注明不完整
|
||||
default:
|
||||
doc.Meta.Confidence = 0.9
|
||||
}
|
||||
case "pptx", "pptm", "ppsx":
|
||||
doc.Meta.Parser = "builtin:pptx"
|
||||
blocks, warns := parsePptx(content)
|
||||
doc.Blocks = blocks
|
||||
doc.Warnings = append(doc.Warnings, warns...)
|
||||
switch {
|
||||
case len(blocks) == 0:
|
||||
doc.Meta.Confidence = 0.3
|
||||
case len(warns) > 0:
|
||||
doc.Meta.Confidence = 0.6
|
||||
default:
|
||||
doc.Meta.Confidence = 0.9
|
||||
}
|
||||
case "doc", "xls", "ppt":
|
||||
doc.Meta.Parser = "builtin:legacy-office"
|
||||
doc.Warnings = append(doc.Warnings, "旧版二进制格式 ."+ext+" 不解析:请转存为 ."+legacyOfficeTarget(ext)+" 后重传")
|
||||
default:
|
||||
if isImage(ext) {
|
||||
doc.Warnings = append(doc.Warnings, "图片不走文本解析(OCR 为 MinerU 深度档,远期)")
|
||||
@@ -164,24 +192,13 @@ type xtc struct {
|
||||
}
|
||||
|
||||
func parseDocx(content []byte) ([]core.DocBlock, error) {
|
||||
zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
||||
zr, err := openZip(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open zip: %w", err)
|
||||
}
|
||||
var docXML []byte
|
||||
for _, f := range zr.File {
|
||||
if f.Name == "word/document.xml" {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docXML, _ = io.ReadAll(rc)
|
||||
rc.Close()
|
||||
break
|
||||
}
|
||||
}
|
||||
if docXML == nil {
|
||||
return nil, fmt.Errorf("word/document.xml not found")
|
||||
docXML, err := zipPart(zr, "word/document.xml")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseDocxBody(docXML)
|
||||
}
|
||||
@@ -322,13 +339,28 @@ func isMostlyPrintable(s string) bool {
|
||||
return nonPrint*10 < len(s)
|
||||
}
|
||||
|
||||
// legacyOfficeTarget 旧版二进制 Office 格式的现代等价格式(提示转存用)。
|
||||
func legacyOfficeTarget(ext string) string {
|
||||
switch ext {
|
||||
case "doc":
|
||||
return "docx"
|
||||
case "xls":
|
||||
return "xlsx"
|
||||
case "ppt":
|
||||
return "pptx"
|
||||
}
|
||||
return ext
|
||||
}
|
||||
|
||||
// ParsersDescription 返回解析器注册表描述(来源标记;设置页/文档用)。
|
||||
func ParsersDescription() []core.ParserDesc {
|
||||
return []core.ParserDesc{
|
||||
{ID: "builtin:md", Formats: []string{"md", "markdown"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:txt", Formats: []string{"txt", "csv", "log"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:txt", Formats: []string{"txt", "text", "csv", "log"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:docx", Formats: []string{"docx"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层;扫描件为远期 MinerU 深度档"},
|
||||
{ID: "builtin:xlsx", Formats: []string{"xlsx", "xlsm"}, Source: core.SourceBuiltin},
|
||||
{ID: "builtin:pptx", Formats: []string{"pptx", "pptm", "ppsx"}, Source: core.SourceBuiltin},
|
||||
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选:深度解析与交叉验证(默认不部署)"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package ingest
|
||||
|
||||
// OOXML 公共件:zip 部件读取(解压上限防 zip 炸弹)、关系部件解析、
|
||||
// r:id 属性与相对路径归一。docx/xlsx/pptx 解析器共用;纯标准库。
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ooxmlPartLimit 单个 zip 部件的解压上限(测试可覆盖)。
|
||||
var ooxmlPartLimit int64 = 64 << 20
|
||||
|
||||
func openZip(content []byte) (*zip.Reader, error) {
|
||||
return zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
||||
}
|
||||
|
||||
// zipPart 按名称读取部件;解压超过 ooxmlPartLimit 视为异常拒绝。
|
||||
func zipPart(zr *zip.Reader, name string) ([]byte, error) {
|
||||
for _, f := range zr.File {
|
||||
if f.Name != name {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(rc, ooxmlPartLimit+1))
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(b)) > ooxmlPartLimit {
|
||||
return nil, fmt.Errorf("part %s exceeds %d bytes decompression limit", name, ooxmlPartLimit)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
return nil, fmt.Errorf("part %s not found", name)
|
||||
}
|
||||
|
||||
// relsPath 计算部件对应的关系部件名(xl/workbook.xml → xl/_rels/workbook.xml.rels)。
|
||||
func relsPath(part string) string {
|
||||
dir, base := path.Split(part)
|
||||
return dir + "_rels/" + base + ".rels"
|
||||
}
|
||||
|
||||
type ooxmlRel struct {
|
||||
ID, Target, Type string
|
||||
}
|
||||
|
||||
// parseRels 解析部件的关系表;无关系部件返回空表(不是错误)。
|
||||
func parseRels(zr *zip.Reader, part string) (map[string]ooxmlRel, error) {
|
||||
data, err := zipPart(zr, relsPath(part))
|
||||
if err != nil {
|
||||
return map[string]ooxmlRel{}, nil
|
||||
}
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
out := map[string]ooxmlRel{}
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok || st.Name.Local != "Relationship" {
|
||||
continue
|
||||
}
|
||||
r := ooxmlRel{}
|
||||
for _, a := range st.Attr {
|
||||
switch a.Name.Local {
|
||||
case "Id":
|
||||
r.ID = a.Value
|
||||
case "Target":
|
||||
r.Target = a.Value
|
||||
case "Type":
|
||||
r.Type = a.Value
|
||||
}
|
||||
}
|
||||
if r.ID != "" {
|
||||
out[r.ID] = r
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolveRelTarget 把关系 target 解析为 zip 部件名:绝对 target(/xl/…)去
|
||||
// 前导斜杠;相对 target 相对 partDir;归一化 ../ 与 ./ 段。
|
||||
func resolveRelTarget(partDir, target string) string {
|
||||
t := target
|
||||
if !strings.HasPrefix(target, "/") {
|
||||
t = partDir + "/" + target
|
||||
}
|
||||
var seg []string
|
||||
for _, s := range strings.Split(t, "/") {
|
||||
switch s {
|
||||
case "", ".":
|
||||
case "..":
|
||||
if len(seg) > 0 {
|
||||
seg = seg[:len(seg)-1]
|
||||
}
|
||||
default:
|
||||
seg = append(seg, s)
|
||||
}
|
||||
}
|
||||
return strings.Join(seg, "/")
|
||||
}
|
||||
|
||||
// ridAttr 取 r:id:局部名 id 且带命名空间前缀(Transitional 与 Strict OOXML
|
||||
// 的关系引用都满足该形态;无前缀的 id 是别的编号,不误取)。
|
||||
func ridAttr(st xml.StartElement) string {
|
||||
for _, a := range st.Attr {
|
||||
if a.Name.Local == "id" && a.Name.Space != "" {
|
||||
return a.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// attrLocal 取无命名空间前缀的属性值。
|
||||
func attrLocal(st xml.StartElement, name string) string {
|
||||
for _, a := range st.Attr {
|
||||
if a.Name.Local == name && a.Name.Space == "" {
|
||||
return a.Value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// relIsType 判断关系类型是否以标准关系类型后缀结尾(/worksheet、/slide 等)。
|
||||
func relIsType(rel ooxmlRel, suffix string) bool {
|
||||
return strings.HasSuffix(rel.Type, suffix)
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package ingest
|
||||
|
||||
// 真实生成器轮测:openpyxl / python-pptx 现场生成 xlsx/pptx 再解析,防止
|
||||
// 手工夹具只验证"自己写的 XML"。生成器缺失时 t.Skip(诚实出口);CI 已装
|
||||
// 这两个包(.gitea/workflows/ci.yaml 安装 python-docx python-pptx openpyxl)。
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func pythonModuleAvailable(t *testing.T, module string) bool {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("python3"); err != nil {
|
||||
return false
|
||||
}
|
||||
return exec.Command("python3", "-c", "import "+module).Run() == nil
|
||||
}
|
||||
|
||||
// runPythonScript 把脚本写到临时目录执行;失败即 t.Fatal(脚本属测试自身)。
|
||||
func runPythonScript(t *testing.T, script string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "gen.py")
|
||||
if err := os.WriteFile(path, []byte(script), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, err := exec.Command("python3", path).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("python generator failed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxFromOpenpyxl(t *testing.T) {
|
||||
if !pythonModuleAvailable(t, "openpyxl") {
|
||||
t.Skip("python3 openpyxl not available")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "报价单.xlsx")
|
||||
runPythonScript(t, `
|
||||
import openpyxl
|
||||
from datetime import datetime
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "报价"
|
||||
ws.append(["物料", "单价", "日期"])
|
||||
ws.append(["支架", 12.5, datetime(2024, 1, 1)])
|
||||
ws["C2"].number_format = "yyyy-mm-dd"
|
||||
ws["D2"] = "=A2&\"-x\""
|
||||
ws["D3"] = True
|
||||
wb.create_sheet("说明").append(["备注行"])
|
||||
wb.save(r"`+file+`")
|
||||
`)
|
||||
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := Parse(file, "", data)
|
||||
if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 {
|
||||
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
var heads []string
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads = append(heads, b.Text)
|
||||
}
|
||||
}
|
||||
if len(heads) != 2 || heads[0] != "报价" || heads[1] != "说明" {
|
||||
t.Fatalf("headings: %v", heads)
|
||||
}
|
||||
plain := doc.PlainText()
|
||||
for _, want := range []string{"物料", "支架", "12.5", "2024-01-01"} {
|
||||
if !strings.Contains(plain, want) {
|
||||
t.Errorf("missing %q in: %s", want, plain)
|
||||
}
|
||||
}
|
||||
// openpyxl 公式不带缓存值:回显公式本身(可检索、不冒充结果)。
|
||||
if !strings.Contains(plain, "=A2") {
|
||||
t.Errorf("formula without cache should echo: %s", plain)
|
||||
}
|
||||
if !strings.Contains(plain, "TRUE") {
|
||||
t.Errorf("boolean cell should render TRUE: %s", plain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePptxFromPythonPptx(t *testing.T) {
|
||||
if !pythonModuleAvailable(t, "pptx") {
|
||||
t.Skip("python3 pptx not available")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "方案评审.pptx")
|
||||
runPythonScript(t, `
|
||||
from pptx import Presentation
|
||||
from pptx.util import Inches
|
||||
prs = Presentation()
|
||||
s1 = prs.slides.add_slide(prs.slide_layouts[0])
|
||||
s1.shapes.title.text = "评审结论"
|
||||
s1.placeholders[1].text = "通过"
|
||||
s2 = prs.slides.add_slide(prs.slide_layouts[5])
|
||||
s2.shapes.title.text = "报价明细"
|
||||
tbl = s2.shapes.add_table(2, 2, Inches(1), Inches(2), Inches(6), Inches(1)).table
|
||||
tbl.cell(0, 0).text = "物料"
|
||||
tbl.cell(0, 1).text = "单价"
|
||||
tbl.cell(1, 0).text = "支架"
|
||||
tbl.cell(1, 1).text = "12.5"
|
||||
notes = s1.notes_slide.notes_text_frame
|
||||
notes.text = "需要补充法规依据"
|
||||
prs.save(r"`+file+`")
|
||||
`)
|
||||
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
doc := Parse(file, "", data)
|
||||
if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 {
|
||||
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
var heads []string
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads = append(heads, b.Text)
|
||||
}
|
||||
}
|
||||
if len(heads) != 2 || !strings.HasPrefix(heads[0], "幻灯片 1:评审结论") || !strings.HasPrefix(heads[1], "幻灯片 2:报价明细") {
|
||||
t.Fatalf("headings: %v", heads)
|
||||
}
|
||||
plain := doc.PlainText()
|
||||
for _, want := range []string{"通过", "物料", "支架", "12.5", "备注:需要补充法规依据"} {
|
||||
if !strings.Contains(plain, want) {
|
||||
t.Errorf("missing %q in: %s", want, plain)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
package ingest
|
||||
|
||||
// OOXML(xlsx/pptx)解析器测试:手工构造最小 zip 部件,覆盖顺序解析、富文本、
|
||||
// 公式、日期、稀疏列、隐藏表、组合形状、AlternateContent、表格合并、备注、
|
||||
// 上限与失败路径。真实生成器的轮测见 ooxml_python_test.go。
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
// zipBytes 按名称构造 zip(键排序保证确定性)。
|
||||
func zipBytes(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
names := make([]string, 0, len(files))
|
||||
for n := range files {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
w, err := zw.Create(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(files[n])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
const relTypeNS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"
|
||||
|
||||
func xlsxFixture() map[string]string {
|
||||
return map[string]string{
|
||||
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="` + relTypeNS + `">
|
||||
<sheets>
|
||||
<sheet name="报价" sheetId="1" r:id="rId1"/>
|
||||
<sheet name="草稿" sheetId="2" state="hidden" r:id="rId2"/>
|
||||
</sheets></workbook>`,
|
||||
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="` + relTypeNS + `/worksheet" Target="worksheets/sheet1.xml"/>
|
||||
<Relationship Id="rId2" Type="` + relTypeNS + `/worksheet" Target="/xl/worksheets/sheet2.xml"/>
|
||||
</Relationships>`,
|
||||
// shared[0]=物料;shared[1]=富文本"单价"(rPh 注音必须排除)。
|
||||
"xl/sharedStrings.xml": `<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<si><t>物料</t></si>
|
||||
<si><rPh sb="0" eb="2"><t>たんか</t></rPh><r><rPr lang="ja"/><t>单价</t></r></si>
|
||||
</sst>`,
|
||||
"xl/styles.xml": `<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<numFmts><numFmt numFmtId="164" formatCode="yyyy"年"m"月"d"日""/></numFmts>
|
||||
<cellXfs count="3">
|
||||
<xf numFmtId="0"/>
|
||||
<xf numFmtId="14" applyNumberFormat="1"/>
|
||||
<xf numFmtId="164" applyNumberFormat="1"/>
|
||||
</cellXfs></styleSheet>`,
|
||||
// A1=物料 B1=单价(富文本) A2=公式缓存 10 B2=无缓存公式 C2=日期(内置14)=2024-01-01
|
||||
// D2=TRUE E2=#DIV/0! F2=内联 G2=自定义日期格式(2024-01-02) 稀疏行 C4=物料。
|
||||
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>
|
||||
<row r="2"><c r="A2"><f>B9*2</f><v>10</v></c><c r="B2"><f>B9*3</f></c><c r="C2" s="1"><v>45292</v></c><c r="D2" t="b"><v>1</v></c><c r="E2" t="e"><v>#DIV/0!</v></c><c r="F2" t="inlineStr"><is><t>内联</t></is></c><c r="G2" s="2"><v>45293</v></c></row>
|
||||
<row r="4"><c r="C4" t="s"><v>0</v></c></row>
|
||||
</sheetData></worksheet>`,
|
||||
"xl/worksheets/sheet2.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData><row r="1"><c r="A1"><v>7</v></c></row></sheetData></worksheet>`,
|
||||
}
|
||||
}
|
||||
|
||||
func findTable(doc core.ParsedDocument, after int) *core.DocBlock {
|
||||
for i := after; i < len(doc.Blocks); i++ {
|
||||
if doc.Blocks[i].Type == core.BlockTable {
|
||||
return &doc.Blocks[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestParseXlsxBuiltFromParts(t *testing.T) {
|
||||
doc := Parse("报价单.xlsx", "", zipBytes(t, xlsxFixture()))
|
||||
if doc.Meta.Parser != "builtin:xlsx" || len(doc.Warnings) != 0 {
|
||||
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
if doc.Meta.Confidence != 0.9 {
|
||||
t.Fatalf("confidence: %v", doc.Meta.Confidence)
|
||||
}
|
||||
// 工作表顺序取 workbook.xml(报价在前),隐藏表标注;绝对 target /xl/… 也能定位。
|
||||
if doc.Blocks[0].Type != core.BlockHeading || doc.Blocks[0].Text != "报价" {
|
||||
t.Fatalf("first block: %+v", doc.Blocks[0])
|
||||
}
|
||||
heads := []string{}
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads = append(heads, b.Text)
|
||||
}
|
||||
}
|
||||
if strings.Join(heads, "|") != "报价|草稿(隐藏)" {
|
||||
t.Fatalf("headings: %v", heads)
|
||||
}
|
||||
tbl := findTable(doc, 0)
|
||||
if tbl == nil || len(tbl.Rows) != 3 {
|
||||
t.Fatalf("table: %+v", tbl)
|
||||
}
|
||||
row1, row2, row4 := tbl.Rows[0], tbl.Rows[1], tbl.Rows[2]
|
||||
if row1[0] != "物料" || row1[1] != "单价" {
|
||||
t.Errorf("header row (phonetic must be excluded): %v", row1)
|
||||
}
|
||||
want := []string{"10", "=B9*3", "2024-01-01", "TRUE", "#DIV/0!", "内联", "2024-01-02"}
|
||||
if len(row2) != len(want) {
|
||||
t.Fatalf("row2: %v", row2)
|
||||
}
|
||||
for i, w := range want {
|
||||
if row2[i] != w {
|
||||
t.Errorf("row2[%d]: got %q want %q", i, row2[i], w)
|
||||
}
|
||||
}
|
||||
// 稀疏单元格按引用就位:C4 → 第三列。
|
||||
if len(row4) != 3 || row4[0] != "" || row4[1] != "" || row4[2] != "物料" {
|
||||
t.Errorf("sparse row: %v", row4)
|
||||
}
|
||||
if !strings.Contains(doc.PlainText(), "2024-01-01") {
|
||||
t.Error("PlainText missing date text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxEmptySheetSkipped(t *testing.T) {
|
||||
files := xlsxFixture()
|
||||
files["xl/worksheets/sheet2.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData><row r="1"/></sheetData></worksheet>`
|
||||
doc := Parse("x.xlsx", "", zipBytes(t, files))
|
||||
heads := 0
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads++
|
||||
}
|
||||
}
|
||||
if heads != 1 || len(doc.Warnings) != 0 {
|
||||
t.Fatalf("empty sheet must be skipped: blocks=%+v warnings=%v", doc.Blocks, doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxInvalid(t *testing.T) {
|
||||
doc := Parse("坏.xlsx", "", []byte("not a zip"))
|
||||
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
|
||||
t.Fatalf("invalid xlsx should fail visibly: %+v %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
files := xlsxFixture()
|
||||
delete(files, "xl/workbook.xml")
|
||||
doc = Parse("无工作簿.xlsx", "", zipBytes(t, files))
|
||||
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
|
||||
t.Fatalf("missing workbook.xml should fail visibly: %v", doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxCellCap(t *testing.T) {
|
||||
old := xlsxMaxCells
|
||||
xlsxMaxCells = 3
|
||||
defer func() { xlsxMaxCells = old }()
|
||||
doc := Parse("超限.xlsx", "", zipBytes(t, xlsxFixture()))
|
||||
if len(doc.Blocks) == 0 {
|
||||
t.Fatal("truncated parse should keep partial blocks")
|
||||
}
|
||||
if len(doc.Warnings) == 0 || doc.Meta.Confidence != 0.6 {
|
||||
t.Fatalf("truncation must warn and lower confidence: %v %v", doc.Meta.Confidence, doc.Warnings)
|
||||
}
|
||||
if !strings.Contains(strings.Join(doc.Warnings, ";"), "上限") {
|
||||
t.Errorf("warnings: %v", doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxPartSizeCap(t *testing.T) {
|
||||
old := ooxmlPartLimit
|
||||
ooxmlPartLimit = 16
|
||||
defer func() { ooxmlPartLimit = old }()
|
||||
doc := Parse("炸弹.xlsx", "", zipBytes(t, xlsxFixture()))
|
||||
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
|
||||
t.Fatalf("oversized part must fail visibly: %+v %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseXlsxDate1904(t *testing.T) {
|
||||
files := xlsxFixture()
|
||||
files["xl/workbook.xml"] = strings.Replace(files["xl/workbook.xml"],
|
||||
"<sheets>", `<workbookPr date1904="1"/><sheets>`, 1)
|
||||
files["xl/worksheets/sheet1.xml"] = `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData><row r="1"><c r="A1" s="1"><v>1</v></c><c r="B1" s="1"><v>0.520833333333333</v></c></row></sheetData></worksheet>`
|
||||
doc := Parse("mac.xlsx", "", zipBytes(t, files))
|
||||
tbl := findTable(doc, 0)
|
||||
// 1904 历法:serial 0 = 1904-01-01,serial 1 = 1904-01-02。
|
||||
if tbl == nil || tbl.Rows[0][0] != "1904-01-02" {
|
||||
t.Fatalf("1904 epoch: %+v", tbl)
|
||||
}
|
||||
if !strings.Contains(tbl.Rows[0][1], "1904-01-01 12:30") {
|
||||
t.Errorf("time fraction: %q", tbl.Rows[0][1])
|
||||
}
|
||||
}
|
||||
|
||||
const pNS = "http://schemas.openxmlformats.org/presentationml/2006/main"
|
||||
const aNS = "http://schemas.openxmlformats.org/drawingml/2006/main"
|
||||
|
||||
func sp(phType, text string) string {
|
||||
ph := ""
|
||||
if phType != "" {
|
||||
ph = `<p:nvPr><p:ph type="` + phType + `"/></p:nvPr>`
|
||||
} else {
|
||||
ph = `<p:nvPr/>`
|
||||
}
|
||||
return `<p:sp><p:nvSpPr><p:cNvPr id="2" name="S"/><p:cNvSpPr/></p:nvSpPr>` + ph + `<p:spPr/>` +
|
||||
`<p:txBody><a:bodyPr/><a:p><a:r><a:t>` + text + `</a:t></a:r></a:p></p:txBody></p:sp>`
|
||||
}
|
||||
|
||||
func pptxFixture() map[string]string {
|
||||
presRel := func(id, target string) string {
|
||||
return `<Relationship Id="` + id + `" Type="` + relTypeNS + `/slide" Target="` + target + `"/>`
|
||||
}
|
||||
slide1 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `" xmlns:r="` + relTypeNS + `"><p:cSld><p:spTree>
|
||||
<p:nvGrpSpPr/><p:grpSpPr/>
|
||||
` + sp("title", "封面标题") + `
|
||||
` + sp("body", "") + `
|
||||
<p:sp><p:nvSpPr><p:cNvPr id="4" name="B"/><p:cNvSpPr/></p:nvSpPr><p:nvPr/><p:spPr/><p:txBody><a:bodyPr/>
|
||||
<a:p><a:r><a:t>第一行</a:t></a:r><a:br/><a:r><a:t>第二行</a:t></a:r></a:p></p:txBody></p:sp>
|
||||
<p:grpSp><p:nvGrpSpPr/><p:grpSpPr/>` + sp("", "组合内文本") + `</p:grpSp>
|
||||
<mc:AlternateContent xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006">
|
||||
<mc:Choice Requires="p14">` + sp("", "选内文本") + `</mc:Choice>
|
||||
<mc:Fallback>` + sp("", "弃内文本") + `</mc:Fallback>
|
||||
</mc:AlternateContent>
|
||||
<p:sp><p:nvSpPr><p:cNvPr id="6" name="N"/><p:cNvSpPr/></p:nvSpPr><p:nvPr><p:ph type="sldNum"/></p:nvPr><p:spPr/><p:txBody><a:p><a:fld id="{F1}"><a:t>3</a:t></a:fld></a:p></p:txBody></p:sp>
|
||||
<p:graphicFrame><p:nvGraphicFramePr/><p:xfrm/><a:graphic><a:graphicData uri="` + aNS + `/table"><a:tbl>
|
||||
<a:tr h="1"><a:tc gridSpan="2"><a:txBody><a:p><a:r><a:t>合并头</a:t></a:r></a:p></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>列三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
|
||||
<a:tr h="1"><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc hMerge="1"><a:txBody><a:p/></a:txBody></a:tc><a:tc><a:txBody><a:p><a:r><a:t>值三</a:t></a:r></a:p></a:txBody></a:tc></a:tr>
|
||||
</a:tbl></a:graphicData></a:graphic></p:graphicFrame>
|
||||
</p:spTree></p:cSld></p:sld>`
|
||||
slide2 := `<p:sld xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
|
||||
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "第二页要点") + `
|
||||
</p:spTree></p:cSld></p:sld>`
|
||||
return map[string]string{
|
||||
"ppt/presentation.xml": `<p:presentation xmlns:p="` + pNS + `" xmlns:r="` + relTypeNS + `"><p:sldIdLst>
|
||||
<p:sldId id="256" r:id="rIdA"/><p:sldId id="257" r:id="rIdB"/>
|
||||
</p:sldIdLst></p:presentation>`,
|
||||
"ppt/_rels/presentation.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">` +
|
||||
presRel("rIdA", "slides/slide1.xml") + presRel("rIdB", "slides/slide2.xml") + `</Relationships>`,
|
||||
"ppt/slides/slide1.xml": slide1,
|
||||
"ppt/slides/slide2.xml": slide2,
|
||||
"ppt/slides/_rels/slide1.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rIdN" Type="` + relTypeNS + `/notesSlide" Target="../notesSlides/notesSlide1.xml"/></Relationships>`,
|
||||
// slide2 的备注故意缺失(无 rels 文件)。
|
||||
"ppt/notesSlides/notesSlide1.xml": `<p:notes xmlns:p="` + pNS + `" xmlns:a="` + aNS + `"><p:cSld><p:spTree>
|
||||
<p:nvGrpSpPr/><p:grpSpPr/>` + sp("body", "补充说明") + `
|
||||
</p:spTree></p:cSld></p:notes>`,
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePptxBuiltFromParts(t *testing.T) {
|
||||
doc := Parse("演示.pptx", "", zipBytes(t, pptxFixture()))
|
||||
if doc.Meta.Parser != "builtin:pptx" || len(doc.Warnings) != 0 || doc.Meta.Confidence != 0.9 {
|
||||
t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
var heads []string
|
||||
for _, b := range doc.Blocks {
|
||||
if b.Type == core.BlockHeading {
|
||||
heads = append(heads, b.Text)
|
||||
}
|
||||
}
|
||||
// sldIdLst 顺序:slide1(有标题)在前;slide2 无标题占位符 → 无标题兜底。
|
||||
if len(heads) != 2 || heads[0] != "幻灯片 1:封面标题" || heads[1] != "幻灯片 2" {
|
||||
t.Fatalf("headings: %v", heads)
|
||||
}
|
||||
plain := doc.PlainText()
|
||||
for _, want := range []string{"第一行\n第二行", "组合内文本", "选内文本", "备注:补充说明"} {
|
||||
if !strings.Contains(plain, want) {
|
||||
t.Errorf("missing %q in plain text", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(plain, "弃内文本") {
|
||||
t.Error("mc:Fallback content must be skipped")
|
||||
}
|
||||
// 页码占位符与字段文本不出现。
|
||||
if strings.Contains(plain, "\n3\n") {
|
||||
t.Error("slide number placeholder text leaked")
|
||||
}
|
||||
// 表格:gridSpan 占位 + hMerge 续格留空,保持三列对齐。
|
||||
tbl := findTable(doc, 0)
|
||||
if tbl == nil || len(tbl.Rows) != 2 {
|
||||
t.Fatalf("table: %+v", tbl)
|
||||
}
|
||||
if strings.Join(tbl.Rows[0], "|") != "合并头||列三" {
|
||||
t.Errorf("row1: %v", tbl.Rows[0])
|
||||
}
|
||||
if strings.Join(tbl.Rows[1], "|") != "||值三" {
|
||||
t.Errorf("row2: %v", tbl.Rows[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePptxInvalidAndEmpty(t *testing.T) {
|
||||
doc := Parse("坏.pptx", "", []byte("garbage"))
|
||||
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 || doc.Meta.Confidence != 0.3 {
|
||||
t.Fatalf("invalid pptx: %+v %v", doc.Meta, doc.Warnings)
|
||||
}
|
||||
files := pptxFixture()
|
||||
files["ppt/presentation.xml"] = `<p:presentation xmlns:p="` + pNS + `"></p:presentation>`
|
||||
doc = Parse("空演示.pptx", "", zipBytes(t, files))
|
||||
if len(doc.Warnings) == 0 || len(doc.Blocks) != 0 {
|
||||
t.Fatalf("no slides: %v", doc.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLegacyOfficeHint(t *testing.T) {
|
||||
for ext, want := range map[string]string{
|
||||
"doc": "docx", "xls": "xlsx", "ppt": "pptx",
|
||||
} {
|
||||
doc := Parse("旧档."+ext, "", []byte{0xD0, 0xCF})
|
||||
if doc.Meta.Parser != "builtin:legacy-office" {
|
||||
t.Fatalf("%s parser: %+v", ext, doc.Meta)
|
||||
}
|
||||
if len(doc.Blocks) != 0 || len(doc.Warnings) != 1 {
|
||||
t.Fatalf("%s blocks/warnings: %+v %v", ext, doc.Blocks, doc.Warnings)
|
||||
}
|
||||
if !strings.Contains(doc.Warnings[0], want) {
|
||||
t.Errorf("%s hint should mention %s: %v", ext, want, doc.Warnings[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 注册表与分派行为一致:每个已注册格式的解析器 ID 与 Parse 分派结果一致,
|
||||
// 且 builtin 解析器覆盖的扩展名都出现在注册表里。
|
||||
func TestParsersDescriptionMatchesDispatch(t *testing.T) {
|
||||
byFormat := map[string]string{}
|
||||
for _, d := range ParsersDescription() {
|
||||
if d.Source != core.SourceBuiltin {
|
||||
continue
|
||||
}
|
||||
for _, f := range d.Formats {
|
||||
byFormat[f] = d.ID
|
||||
}
|
||||
}
|
||||
check := map[string]string{
|
||||
"md": "builtin:md", "markdown": "builtin:md",
|
||||
"txt": "builtin:txt", "text": "builtin:txt", "csv": "builtin:txt", "log": "builtin:txt",
|
||||
"docx": "builtin:docx", "pdf": "builtin:pdf",
|
||||
"xlsx": "builtin:xlsx", "xlsm": "builtin:xlsx",
|
||||
"pptx": "builtin:pptx", "pptm": "builtin:pptx", "ppsx": "builtin:pptx",
|
||||
}
|
||||
for ext, wantID := range check {
|
||||
got, ok := byFormat[ext]
|
||||
if !ok {
|
||||
t.Errorf("format .%s missing from registry", ext)
|
||||
continue
|
||||
}
|
||||
doc := Parse("a."+ext, "", []byte("x"))
|
||||
if doc.Meta.Parser != wantID || got != wantID {
|
||||
t.Errorf(".%s: registry=%s dispatch=%s want=%s", ext, got, doc.Meta.Parser, wantID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package ingest
|
||||
|
||||
// pptx/pptm/ppsx(PresentationML)内置解析器。幻灯片顺序取 presentation.xml
|
||||
// 的 sldIdLst;标题占位符(title/ctrTitle)产标题块,正文形状按段落输出
|
||||
//(a:br 换行),组合形状经同一 token 走查天然递归;mc:AlternateContent 只取
|
||||
// Choice 分支;graphicFrame 内的表格按 tr/tc 输出(gridSpan/hMerge 占位保列
|
||||
// 对齐);备注页正文输出「备注:…」;页码/日期/页脚占位符跳过。图片与
|
||||
// SmartArt 不解析。纯标准库。
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
func parsePptx(content []byte) ([]core.DocBlock, []string) {
|
||||
zr, err := openZip(content)
|
||||
if err != nil {
|
||||
return nil, []string{"pptx 解析失败:" + err.Error()}
|
||||
}
|
||||
slides, warns := pptxSlideOrder(zr)
|
||||
if slides == nil && len(warns) > 0 {
|
||||
return nil, warns
|
||||
}
|
||||
var blocks []core.DocBlock
|
||||
for i, sp := range slides {
|
||||
n := i + 1
|
||||
data, err := zipPart(zr, sp)
|
||||
if err != nil {
|
||||
warns = append(warns, fmt.Sprintf("幻灯片 %d 读取失败:%v", n, err))
|
||||
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: fmt.Sprintf("幻灯片 %d", n)})
|
||||
continue
|
||||
}
|
||||
title, paras, tables := pptxSlideBlocks(data)
|
||||
heading := fmt.Sprintf("幻灯片 %d", n)
|
||||
if title != "" {
|
||||
heading += ":" + title
|
||||
}
|
||||
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: 2, Text: heading})
|
||||
blocks = append(blocks, paras...)
|
||||
blocks = append(blocks, tables...)
|
||||
if note := pptxSlideNotes(zr, sp); note != "" {
|
||||
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: "备注:" + note})
|
||||
}
|
||||
}
|
||||
if len(blocks) == 0 && len(warns) == 0 {
|
||||
warns = append(warns, "pptx 无可解析的幻灯片内容")
|
||||
}
|
||||
return blocks, warns
|
||||
}
|
||||
|
||||
// pptxSlideOrder 读 presentation.xml 的 sldIdLst → 幻灯片部件名(按顺序)。
|
||||
func pptxSlideOrder(zr *zip.Reader) ([]string, []string) {
|
||||
data, err := zipPart(zr, "ppt/presentation.xml")
|
||||
if err != nil {
|
||||
return nil, []string{"pptx 解析失败:" + err.Error()}
|
||||
}
|
||||
rels, err := parseRels(zr, "ppt/presentation.xml")
|
||||
if err != nil {
|
||||
return nil, []string{"pptx 解析失败:" + err.Error()}
|
||||
}
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
var out, warns []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return out, append(warns, "presentation.xml 解码失败:"+err.Error())
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok || st.Name.Local != "sldId" {
|
||||
continue
|
||||
}
|
||||
rel, ok := rels[ridAttr(st)]
|
||||
if !ok || !relIsType(rel, "/slide") {
|
||||
warns = append(warns, "幻灯片关系缺失或类型不符")
|
||||
continue
|
||||
}
|
||||
out = append(out, resolveRelTarget("ppt", rel.Target))
|
||||
}
|
||||
return out, warns
|
||||
}
|
||||
|
||||
// pptxSlideBlocks 解析一张幻灯片:标题、正文段落、表格块。
|
||||
// 表格块统一附在段落后(图形框与正文的精确穿插对检索无价值)。
|
||||
func pptxSlideBlocks(data []byte) (title string, paras []core.DocBlock, tables []core.DocBlock) {
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
var stack []string
|
||||
has := func(name string) bool {
|
||||
for _, s := range stack {
|
||||
if s == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
skip := 0 // mc:Fallback 子树深度
|
||||
// 形状状态
|
||||
spDepth := -1
|
||||
phSkip := false
|
||||
isTitle := false
|
||||
var spParas []string
|
||||
var para strings.Builder
|
||||
paraOpen := false
|
||||
// 文本收集目标
|
||||
textInPara := false
|
||||
textInCell := false
|
||||
// 表格状态
|
||||
var tblRows [][]string
|
||||
var tblRow []string
|
||||
var cellBuf strings.Builder
|
||||
cellOpen := false
|
||||
gridSpan := 1
|
||||
cellHM := false
|
||||
|
||||
finishPara := func() {
|
||||
if !paraOpen {
|
||||
return
|
||||
}
|
||||
paraOpen = false
|
||||
if s := strings.TrimRight(para.String(), "\n"); strings.TrimSpace(s) != "" {
|
||||
spParas = append(spParas, strings.TrimSpace(s))
|
||||
}
|
||||
para.Reset()
|
||||
}
|
||||
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", paras, tables
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
if skip > 0 {
|
||||
skip++
|
||||
continue
|
||||
}
|
||||
local := t.Name.Local
|
||||
switch local {
|
||||
case "Fallback":
|
||||
if has("AlternateContent") {
|
||||
skip = 1
|
||||
continue
|
||||
}
|
||||
case "sp":
|
||||
spDepth = len(stack)
|
||||
phSkip, isTitle, spParas = false, false, nil
|
||||
case "ph":
|
||||
if spDepth >= 0 && len(stack) > spDepth {
|
||||
switch attrLocal(t, "type") {
|
||||
case "title", "ctrTitle":
|
||||
isTitle = true
|
||||
case "dt", "ftr", "sldNum":
|
||||
phSkip = true
|
||||
}
|
||||
}
|
||||
case "p":
|
||||
if spDepth >= 0 && has("txBody") && !has("tbl") {
|
||||
finishPara()
|
||||
paraOpen = true
|
||||
para.Reset()
|
||||
} else if cellOpen {
|
||||
if cellBuf.Len() > 0 {
|
||||
cellBuf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
case "br":
|
||||
if paraOpen {
|
||||
para.WriteByte('\n')
|
||||
}
|
||||
case "t":
|
||||
if has("fld") {
|
||||
break // 字段(页码/日期)不取
|
||||
}
|
||||
textInPara = paraOpen
|
||||
textInCell = cellOpen && !paraOpen
|
||||
case "tc":
|
||||
if has("tbl") {
|
||||
cellBuf.Reset()
|
||||
cellOpen = true
|
||||
gridSpan, _ = strconv.Atoi(attrLocal(t, "gridSpan"))
|
||||
if gridSpan < 1 {
|
||||
gridSpan = 1
|
||||
}
|
||||
hm := attrLocal(t, "hMerge")
|
||||
cellHM = hm == "1" || strings.EqualFold(hm, "true")
|
||||
}
|
||||
}
|
||||
stack = append(stack, local)
|
||||
case xml.CharData:
|
||||
if textInPara {
|
||||
para.Write(t)
|
||||
} else if textInCell {
|
||||
cellBuf.Write(t)
|
||||
}
|
||||
case xml.EndElement:
|
||||
if skip > 0 {
|
||||
skip--
|
||||
continue
|
||||
}
|
||||
if len(stack) > 0 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
switch t.Name.Local {
|
||||
case "t":
|
||||
textInPara, textInCell = false, false
|
||||
case "p":
|
||||
finishPara()
|
||||
case "sp":
|
||||
if spDepth >= 0 && len(stack) == spDepth {
|
||||
if !phSkip && len(spParas) > 0 {
|
||||
if isTitle && title == "" {
|
||||
title = strings.Join(spParas, " ")
|
||||
} else {
|
||||
for _, p := range spParas {
|
||||
paras = append(paras, core.DocBlock{Type: core.BlockPara, Text: p})
|
||||
}
|
||||
}
|
||||
}
|
||||
spDepth = -1
|
||||
phSkip, isTitle, spParas = false, false, nil
|
||||
}
|
||||
case "tc":
|
||||
if cellOpen {
|
||||
text := strings.TrimSpace(cellBuf.String())
|
||||
if cellHM {
|
||||
text = "" // 横向合并的续格:占位保列对齐
|
||||
}
|
||||
tblRow = append(tblRow, text)
|
||||
for i := 1; i < gridSpan; i++ {
|
||||
tblRow = append(tblRow, "") // 合并占位保列对齐
|
||||
}
|
||||
cellOpen = false
|
||||
}
|
||||
case "tr":
|
||||
if len(tblRow) > 0 {
|
||||
for len(tblRow) > 0 && tblRow[len(tblRow)-1] == "" {
|
||||
tblRow = tblRow[:len(tblRow)-1]
|
||||
}
|
||||
if len(tblRow) > 0 {
|
||||
tblRows = append(tblRows, tblRow)
|
||||
}
|
||||
tblRow = nil
|
||||
}
|
||||
case "tbl":
|
||||
if len(tblRows) > 0 {
|
||||
tables = append(tables, core.DocBlock{Type: core.BlockTable, Rows: tblRows})
|
||||
}
|
||||
tblRows = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
finishPara()
|
||||
return title, paras, tables
|
||||
}
|
||||
|
||||
// pptxSlideNotes 读幻灯片的备注页正文;无备注返回空串。
|
||||
func pptxSlideNotes(zr *zip.Reader, slidePart string) string {
|
||||
rels, err := parseRels(zr, slidePart)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, rel := range rels {
|
||||
if !relIsType(rel, "/notesSlide") {
|
||||
continue
|
||||
}
|
||||
data, err := zipPart(zr, resolveRelTarget(path.Dir(slidePart), rel.Target))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
_, paras, _ := pptxSlideBlocks(data)
|
||||
var lines []string
|
||||
for _, p := range paras {
|
||||
lines = append(lines, p.Text)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
package ingest
|
||||
|
||||
// xlsx/xlsm(SpreadsheetML)内置解析器。工作表顺序取 workbook.xml 的 sheets
|
||||
// 列表(target 经关系部件解析,支持相对/绝对路径);共享字符串含富文本 run
|
||||
//(注音 rPh 无对应解码字段,天然排除);单元格覆盖 inlineStr/str/b/e/n,
|
||||
// 公式取缓存值、无缓存回显公式;日期按数字格式(内置 ID 与自定义记号)转
|
||||
// ISO,1900/1904 双日期系统;稀疏单元格按引用就位。纯标准库。
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
core "agent-system-core"
|
||||
)
|
||||
|
||||
// 解析上限(包级变量,测试可覆盖)。
|
||||
var (
|
||||
xlsxMaxCells = 200000
|
||||
xlsxMaxCols = 1024
|
||||
)
|
||||
|
||||
type xlsxSheetRef struct {
|
||||
Name string
|
||||
State string
|
||||
Path string
|
||||
IsSheet bool
|
||||
}
|
||||
|
||||
func parseXlsx(content []byte) ([]core.DocBlock, []string) {
|
||||
zr, err := openZip(content)
|
||||
if err != nil {
|
||||
return nil, []string{"xlsx 解析失败:" + err.Error()}
|
||||
}
|
||||
sheets, date1904, warns := xlsxWorkbookSheets(zr)
|
||||
if len(warns) > 0 && sheets == nil {
|
||||
return nil, warns
|
||||
}
|
||||
shared := xlsxSharedStrings(zr)
|
||||
dateStyles := xlsxDateStyles(zr)
|
||||
var blocks []core.DocBlock
|
||||
cells := 0
|
||||
for _, sh := range sheets {
|
||||
if !sh.IsSheet || sh.Path == "" {
|
||||
continue // chartsheet 等非工作表部件
|
||||
}
|
||||
data, err := zipPart(zr, sh.Path)
|
||||
if err != nil {
|
||||
warns = append(warns, fmt.Sprintf("工作表 %s 读取失败:%v", sh.Name, err))
|
||||
continue
|
||||
}
|
||||
table, used, trunc := xlsxSheetBlocks(data, shared, dateStyles, date1904, cells)
|
||||
cells = used
|
||||
if table == nil {
|
||||
continue // 空表
|
||||
}
|
||||
title := sh.Name
|
||||
if sh.State == "hidden" || sh.State == "veryHidden" {
|
||||
title += "(隐藏)"
|
||||
}
|
||||
blocks = append(blocks,
|
||||
core.DocBlock{Type: core.BlockHeading, Level: 2, Text: title},
|
||||
*table,
|
||||
)
|
||||
if trunc {
|
||||
warns = append(warns, fmt.Sprintf("单元格总数超过 %d 上限,工作表 %s 之后未解析", xlsxMaxCells, sh.Name))
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(blocks) == 0 && len(warns) == 0 {
|
||||
warns = append(warns, "xlsx 无可解析的工作表内容")
|
||||
}
|
||||
return blocks, warns
|
||||
}
|
||||
|
||||
// xlsxWorkbookSheets 读 workbook.xml:工作表顺序、隐藏状态、1904 日期系统。
|
||||
func xlsxWorkbookSheets(zr *zip.Reader) (sheets []xlsxSheetRef, date1904 bool, warns []string) {
|
||||
data, err := zipPart(zr, "xl/workbook.xml")
|
||||
if err != nil {
|
||||
return nil, false, []string{"xlsx 解析失败:" + err.Error()}
|
||||
}
|
||||
rels, err := parseRels(zr, "xl/workbook.xml")
|
||||
if err != nil {
|
||||
return nil, false, []string{"xlsx 解析失败:" + err.Error()}
|
||||
}
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return sheets, date1904, append(warns, "workbook.xml 解码失败:"+err.Error())
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch st.Name.Local {
|
||||
case "workbookPr":
|
||||
if v := attrLocal(st, "date1904"); v == "1" || strings.EqualFold(v, "true") {
|
||||
date1904 = true
|
||||
}
|
||||
case "sheet":
|
||||
ref := xlsxSheetRef{Name: attrLocal(st, "name"), State: attrLocal(st, "state")}
|
||||
if rel, ok := rels[ridAttr(st)]; ok {
|
||||
ref.Path = resolveRelTarget("xl", rel.Target)
|
||||
ref.IsSheet = relIsType(rel, "/worksheet")
|
||||
} else {
|
||||
warns = append(warns, fmt.Sprintf("工作表 %s 的关系缺失", ref.Name))
|
||||
}
|
||||
sheets = append(sheets, ref)
|
||||
}
|
||||
}
|
||||
return sheets, date1904, warns
|
||||
}
|
||||
|
||||
// xlsxSharedStrings 读共享字符串表;富文本按 run 拼接,rPh 注音不进结果。
|
||||
func xlsxSharedStrings(zr *zip.Reader) []string {
|
||||
data, err := zipPart(zr, "xl/sharedStrings.xml")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
var out []string
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok || st.Name.Local != "si" {
|
||||
continue
|
||||
}
|
||||
var si struct {
|
||||
T string `xml:"t"`
|
||||
Runs []struct {
|
||||
T string `xml:"t"`
|
||||
} `xml:"r"`
|
||||
}
|
||||
if err := dec.DecodeElement(&si, &st); err != nil {
|
||||
continue
|
||||
}
|
||||
if si.T != "" || len(si.Runs) == 0 {
|
||||
out = append(out, si.T)
|
||||
continue
|
||||
}
|
||||
parts := make([]string, 0, len(si.Runs))
|
||||
for _, r := range si.Runs {
|
||||
parts = append(parts, r.T)
|
||||
}
|
||||
out = append(out, strings.Join(parts, ""))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// xlsxDateStyles 返回按日期呈现的样式索引集合(cellXfs 下标)。
|
||||
func xlsxDateStyles(zr *zip.Reader) map[int]bool {
|
||||
data, err := zipPart(zr, "xl/styles.xml")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
custom := map[int]string{}
|
||||
date := map[int]bool{}
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
inCellXfs := false
|
||||
xfIdx := 0
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return date
|
||||
}
|
||||
switch t := tok.(type) {
|
||||
case xml.StartElement:
|
||||
switch t.Name.Local {
|
||||
case "numFmt":
|
||||
id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
|
||||
custom[id] = attrLocal(t, "formatCode")
|
||||
case "cellXfs":
|
||||
inCellXfs = true
|
||||
xfIdx = 0
|
||||
case "xf":
|
||||
if !inCellXfs {
|
||||
continue
|
||||
}
|
||||
id, _ := strconv.Atoi(attrLocal(t, "numFmtId"))
|
||||
if xlsxFmtIsDate(id, custom) {
|
||||
date[xfIdx] = true
|
||||
}
|
||||
xfIdx++
|
||||
}
|
||||
case xml.EndElement:
|
||||
if t.Name.Local == "cellXfs" {
|
||||
inCellXfs = false
|
||||
}
|
||||
}
|
||||
}
|
||||
return date
|
||||
}
|
||||
|
||||
// xlsxFmtIsDate:自定义格式按记号判断;内置 ID 覆盖常见日期/时间区间。
|
||||
func xlsxFmtIsDate(id int, custom map[int]string) bool {
|
||||
if code, ok := custom[id]; ok {
|
||||
return formatCodeIsDate(code)
|
||||
}
|
||||
switch {
|
||||
case id >= 14 && id <= 22, id >= 27 && id <= 36,
|
||||
id >= 45 && id <= 47, id >= 50 && id <= 58:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// formatCodeIsDate 剥掉引号字面量与方括号节([Red]、[DBNum1]、[$-409] 等)后,
|
||||
// 剩余字母含 y/d/h/s/m 记号即按日期处理。
|
||||
func formatCodeIsDate(code string) bool {
|
||||
var b strings.Builder
|
||||
inQuote, inBracket := false, false
|
||||
runes := []rune(code)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
switch {
|
||||
case r == '"':
|
||||
inQuote = !inQuote
|
||||
case inQuote:
|
||||
case r == '[':
|
||||
inBracket = true
|
||||
case r == ']':
|
||||
inBracket = false
|
||||
case inBracket:
|
||||
case r == '\\':
|
||||
i++ // 转义字符:连同下一字符跳过
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.ContainsAny(strings.ToLower(b.String()), "ydhsm")
|
||||
}
|
||||
|
||||
type xlsxCellEl struct {
|
||||
Ref string `xml:"r,attr"`
|
||||
T string `xml:"t,attr"`
|
||||
S string `xml:"s,attr"`
|
||||
IS *struct {
|
||||
T string `xml:"t"`
|
||||
} `xml:"is"`
|
||||
F string `xml:"f"`
|
||||
V string `xml:"v"`
|
||||
}
|
||||
|
||||
type xlsxRowEl struct {
|
||||
Cells []xlsxCellEl `xml:"c"`
|
||||
}
|
||||
|
||||
// xlsxSheetBlocks 解析单张工作表;返回表格块、累计单元格数、是否触达上限。
|
||||
// 无内容行返回 nil 表。
|
||||
func xlsxSheetBlocks(data []byte, shared []string, dateStyles map[int]bool, date1904 bool, cellsUsed int) (*core.DocBlock, int, bool) {
|
||||
dec := xml.NewDecoder(bytes.NewReader(data))
|
||||
var rows [][]string
|
||||
count := cellsUsed
|
||||
truncated := false
|
||||
for {
|
||||
tok, err := dec.Token()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, count, false
|
||||
}
|
||||
st, ok := tok.(xml.StartElement)
|
||||
if !ok || st.Name.Local != "row" {
|
||||
continue
|
||||
}
|
||||
var re xlsxRowEl
|
||||
if err := dec.DecodeElement(&re, &st); err != nil {
|
||||
continue
|
||||
}
|
||||
cells := map[int]string{}
|
||||
maxCol := -1
|
||||
for _, c := range re.Cells {
|
||||
if count >= xlsxMaxCells {
|
||||
truncated = true
|
||||
break
|
||||
}
|
||||
count++
|
||||
col := xlsxColIndex(c.Ref)
|
||||
if col < 0 {
|
||||
col = maxCol + 1 // 无引用:按出现顺序接续
|
||||
}
|
||||
if col >= xlsxMaxCols {
|
||||
truncated = true
|
||||
continue
|
||||
}
|
||||
cells[col] = xlsxCellText(c, shared, dateStyles, date1904)
|
||||
if col > maxCol {
|
||||
maxCol = col
|
||||
}
|
||||
}
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
full := make([]string, maxCol+1)
|
||||
for i, v := range cells {
|
||||
full[i] = v
|
||||
}
|
||||
for len(full) > 0 && strings.TrimSpace(full[len(full)-1]) == "" {
|
||||
full = full[:len(full)-1]
|
||||
}
|
||||
if len(full) > 0 {
|
||||
rows = append(rows, full)
|
||||
}
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, count, truncated
|
||||
}
|
||||
return &core.DocBlock{Type: core.BlockTable, Rows: rows}, count, truncated
|
||||
}
|
||||
|
||||
// xlsxCellText 把单元格转成展示文本。
|
||||
func xlsxCellText(c xlsxCellEl, shared []string, dateStyles map[int]bool, date1904 bool) string {
|
||||
text := ""
|
||||
switch c.T {
|
||||
case "s":
|
||||
if idx, err := strconv.Atoi(c.V); err == nil && idx >= 0 && idx < len(shared) {
|
||||
text = shared[idx]
|
||||
}
|
||||
case "inlineStr":
|
||||
if c.IS != nil {
|
||||
text = c.IS.T
|
||||
}
|
||||
case "b":
|
||||
if c.V == "1" || strings.EqualFold(c.V, "true") {
|
||||
text = "TRUE"
|
||||
} else {
|
||||
text = "FALSE"
|
||||
}
|
||||
case "e":
|
||||
text = c.V
|
||||
default: // n 或空:数字 / 公式缓存值
|
||||
text = c.V
|
||||
if sIdx, err := strconv.Atoi(c.S); err == nil && dateStyles[sIdx] && c.V != "" {
|
||||
if f, err := strconv.ParseFloat(c.V, 64); err == nil {
|
||||
text = xlsxSerialString(f, date1904)
|
||||
}
|
||||
}
|
||||
}
|
||||
if text == "" && c.F != "" {
|
||||
return "=" + c.F // 无缓存值的公式:回显公式本身
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
// xlsxSerialString Excel 日期序列值 → ISO 字符串;带小数部分时输出到秒。
|
||||
func xlsxSerialString(serial float64, date1904 bool) string {
|
||||
days := math.Floor(serial)
|
||||
frac := serial - days
|
||||
var t time.Time
|
||||
switch {
|
||||
case date1904:
|
||||
t = time.Date(1904, 1, 1, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
|
||||
case days >= 61:
|
||||
t = time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
|
||||
default: // 1–60:兼容 Excel 的 1900 闰年 bug 区段
|
||||
t = time.Date(1899, 12, 31, 0, 0, 0, 0, time.UTC).AddDate(0, 0, int(days))
|
||||
}
|
||||
if frac > 0 {
|
||||
return t.Add(time.Duration(frac * 86400 * float64(time.Second))).Round(time.Second).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
return t.Format("2006-01-02")
|
||||
}
|
||||
|
||||
// xlsxColIndex "BC12" → 0 起列号;无引用或非法返回 -1。
|
||||
func xlsxColIndex(ref string) int {
|
||||
if ref == "" {
|
||||
return -1
|
||||
}
|
||||
i := 0
|
||||
for i < len(ref) {
|
||||
ch := ref[i]
|
||||
if (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if i == 0 || i == len(ref) {
|
||||
return -1
|
||||
}
|
||||
for _, ch := range ref[i:] {
|
||||
if ch < '0' || ch > '9' {
|
||||
return -1
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
for _, ch := range ref[:i] {
|
||||
c := ch
|
||||
if c >= 'a' && c <= 'z' {
|
||||
c = c - 'a' + 'A'
|
||||
}
|
||||
n = n*26 + int(c-'A') + 1
|
||||
}
|
||||
return n - 1
|
||||
}
|
||||
@@ -1199,3 +1199,76 @@ func TestSettingsDottedKeyAndSecretMask(t *testing.T) {
|
||||
t.Fatalf("apiKey mask = %v", chat["apiKey"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestUploadXlsxParsedAndIndexed:xlsx 上传走内置解析器(builtin:xlsx、无警告),
|
||||
// 单元格文本进入知识检索(EXT-5 验收路径)。
|
||||
func TestUploadXlsxParsedAndIndexed(t *testing.T) {
|
||||
srv, _ := newTestServer(t)
|
||||
h := srv.Handler()
|
||||
projectID, _ := setupConversation(t, h)
|
||||
|
||||
xlsx := buildTestXLSX(t)
|
||||
body := &bytes.Buffer{}
|
||||
mw := multipart.NewWriter(body)
|
||||
fw, _ := mw.CreateFormFile("file", "报价单.xlsx")
|
||||
fw.Write(xlsx)
|
||||
mw.Close()
|
||||
req := httptest.NewRequest("POST", "/api/projects/"+projectID+"/uploads", body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("upload status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
up := map[string]any{}
|
||||
json.Unmarshal(rec.Body.Bytes(), &up)
|
||||
parsed, _ := up["parsed"].(map[string]any)
|
||||
if parsed == nil {
|
||||
t.Fatalf("upload response missing parsed: %s", rec.Body.String())
|
||||
}
|
||||
meta, _ := parsed["meta"].(map[string]any)
|
||||
if meta["parser"] != "builtin:xlsx" {
|
||||
t.Fatalf("parser: %v", meta["parser"])
|
||||
}
|
||||
if w, ok := parsed["warnings"].([]any); ok && len(w) > 0 {
|
||||
t.Fatalf("xlsx upload should parse without warnings: %v", w)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, raw := doAny(t, h, "GET", "/api/knowledge/search?q=%E6%94%AF%E6%9E%B6", nil)
|
||||
if code == 200 {
|
||||
if hits, ok := raw.([]any); ok && len(hits) > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("knowledge search found nothing for xlsx cell text")
|
||||
}
|
||||
|
||||
// buildTestXLSX 构造最小可用 xlsx(inlineStr 单元格)。
|
||||
func buildTestXLSX(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
files := map[string]string{
|
||||
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="报价" sheetId="1" r:id="rId1"/></sheets></workbook>`,
|
||||
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>`,
|
||||
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>
|
||||
<row r="1"><c r="A1" t="inlineStr"><is><t>物料</t></is></c></row>
|
||||
<row r="2"><c r="A2" t="inlineStr"><is><t>支架</t></is></c></row>
|
||||
</sheetData></worksheet>`,
|
||||
}
|
||||
for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write([]byte(files[name]))
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
@@ -20,9 +20,9 @@ type Request struct {
|
||||
// Host 句柄传入,处理器经句柄访问基座能力,不得自行越权持有 owner 凭证。
|
||||
type Handler func(ctx context.Context, h Host, req Request) (any, error)
|
||||
|
||||
// Host 是基座在分发时构造的 owner 作用域宿主句柄(EXT-1 最小集)。
|
||||
// Host 是基座在分发时构造的 owner 作用域宿主句柄(EXT-1 最小集 + EXT-5 解析)。
|
||||
// 所有能力的可见范围都以本次分发的 Owner 为准;artifacts 只读
|
||||
//(写入属完整集,另行评估)。实现由基座 internal 提供。
|
||||
// (写入属完整集,另行评估)。实现由基座 internal 提供。
|
||||
type Host interface {
|
||||
// PluginID 返回注册处理器的插件标识(形如 rdm)。
|
||||
PluginID() string
|
||||
@@ -52,4 +52,10 @@ type Host interface {
|
||||
// ArtifactGet/ArtifactContent 工件只读(R1-3 ingest 读原件)。
|
||||
ArtifactGet(artifactID string) (core.Artifact, error)
|
||||
ArtifactContent(artifactID string) ([]byte, error)
|
||||
|
||||
// ParseDocument 用基座内置解析器把文档字节解析为 ParsedDocument(EXT-5)。
|
||||
// 纯计算,不读写 owner 数据——要解析工件原件先经 ArtifactContent 取字节,
|
||||
// owner 作用域由它保证。按扩展名分派;不可解析返回带 Warnings 的文档
|
||||
//(显式失败),只有 ctx 取消与输入超过 100MiB 两种情况返回 error。
|
||||
ParseDocument(ctx context.Context, filename, mime string, content []byte) (core.ParsedDocument, error)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,19 @@ Compiled
|
||||
ErrNotLinked
|
||||
Handler
|
||||
Host
|
||||
Host.ArtifactContent
|
||||
Host.ArtifactGet
|
||||
Host.Audit
|
||||
Host.Complete
|
||||
Host.KVGet
|
||||
Host.KVPut
|
||||
Host.KnowledgeSearch
|
||||
Host.KnowledgeWrite
|
||||
Host.Owner
|
||||
Host.ParseDocument
|
||||
Host.PluginID
|
||||
Host.TaskCreate
|
||||
Host.TaskUpdate
|
||||
HostClient
|
||||
InProc
|
||||
Main
|
||||
@@ -9,6 +22,10 @@ NewHostClient
|
||||
Options
|
||||
RegisterCompiled
|
||||
Registry
|
||||
Registry.AddCommand
|
||||
Registry.AddCommandHandler
|
||||
Registry.AddSkill
|
||||
Registry.Note
|
||||
Request
|
||||
ResetCompiled
|
||||
SetRunner
|
||||
|
||||
@@ -37,6 +37,17 @@ func TestSurfaceSnapshot(t *testing.T) {
|
||||
case *ast.TypeSpec:
|
||||
if s.Name.IsExported() {
|
||||
names = append(names, s.Name.Name)
|
||||
// 导出接口同时记录其方法:只加方法、不改顶层名的
|
||||
// 变更同样必须更新 surface.txt(EXT-5 起执行)。
|
||||
if it, ok := s.Type.(*ast.InterfaceType); ok {
|
||||
for _, m := range it.Methods.List {
|
||||
for _, id := range m.Names {
|
||||
if id.IsExported() {
|
||||
names = append(names, s.Name.Name+"."+id.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
case *ast.ValueSpec:
|
||||
for _, name := range s.Names {
|
||||
|
||||
Reference in New Issue
Block a user