weichen
727f3702d0
ci / vet-and-test (pull_request) Successful in 46s
Why: downstream parsers are unreachable via extension.Host and the base lacked xlsx/pptx parsers; MS1 needs Excel and PPT ingest (EXT-5 request, documents proposal 2026-10-06-ext-5-host-parse-document). Evidence: local GOWORK=off go test ./... all 29 packages pass incl. new ooxml_test (hand-built parts), ooxml_python_test (openpyxl/python-pptx round-trip), inproc owner-scope and limit tests, xlsx upload e2e; go vet clean; recorded as SELF on the Dev task card, PR CI to confirm. Impact: Host gains ParseDocument (pure compute; errors only on ctx cancel or >100MiB input); builtin:xlsx (sheet order via workbook.xml rels, rich-text shared strings, formula cache, 1900/1904 dates, sparse cells) and builtin:pptx (sldIdLst order, titles, group shapes, AlternateContent Choice, tables, notes) added with 200k-cell/64MiB-part caps; legacy doc/xls/ppt get an actionable hint; ParsersDescription gains xlsx/pptx and the missing text ext; surface snapshot now records exported interface methods; docx part reads share the 64MiB cap. Refs: documents 2026-10-06-ext-5-host-parse-document.md; EXT-1 host.go precedent.
307 lines
11 KiB
Go
307 lines
11 KiB
Go
package app
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"path/filepath"
|
||
"testing"
|
||
"time"
|
||
|
||
agentcommand "agent-command"
|
||
"agent-system-backend/internal/cmdbus"
|
||
"agent-system-backend/internal/rag"
|
||
"agent-system-backend/internal/settings"
|
||
"agent-system-backend/internal/store"
|
||
"agent-system-backend/pkg/extension"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
// newInprocTestApp 组合根测试版：走真实的 applyCompiled → cmdbus.Dispatch →
|
||
// inprocHost 链路（卡片端口 nil，仅普通危险级命令；embedder 非 nil 时为 hash 档）。
|
||
func newInprocTestApp(t *testing.T, embedder func() string) (*cmdbus.Bus, *store.Store) {
|
||
t.Helper()
|
||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "test.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { st.Close() })
|
||
set := settings.NewService(st)
|
||
pipe := rag.NewPipeline(st, set, nil, nil)
|
||
if embedder != nil {
|
||
pipe.SetEmbedder(rag.SwitchingEmbedder{Provider: embedder})
|
||
}
|
||
bus := cmdbus.New(st, nil, func() time.Time { return time.Unix(1700000000, 0).UTC() })
|
||
caps := hostcaps{st: st, pipe: pipe, set: set}
|
||
applyCompiled(filepath.Join(t.TempDir(), "skills"), bus, caps)
|
||
return bus, st
|
||
}
|
||
|
||
// TestInprocHostOwnerScope 是 D8/H4 烟测的单元级前置：两个 owner 各分发一次，
|
||
// 审计、知识、KV、任务台账必须各落自己 owner 名下（D8 烟测的 PG 版另行）。
|
||
func TestInprocHostOwnerScope(t *testing.T) {
|
||
extension.ResetCompiled()
|
||
t.Cleanup(extension.ResetCompiled)
|
||
extension.RegisterCompiled(extension.InProc{
|
||
ID: "tst",
|
||
Register: func(reg extension.Registry) error {
|
||
return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.record","title":"记录"}`),
|
||
func(ctx context.Context, h extension.Host, req extension.Request) (any, error) {
|
||
ref, _ := req.Params["ref"].(string)
|
||
if err := h.Audit("tst.record", ref, "{}"); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := h.KnowledgeWrite(ctx, []core.KnowledgeDoc{
|
||
{Title: "分块", Ref: ref, Chunk: string(h.Owner()) + " 的入库内容"},
|
||
}, true); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := h.KVPut("last", string(h.Owner())); err != nil {
|
||
return nil, err
|
||
}
|
||
taskID, err := h.TaskCreate("tst.record", "记录 "+ref, req.ConversationID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if err := h.TaskUpdate(taskID, "succeeded", 100, ""); err != nil {
|
||
return nil, err
|
||
}
|
||
hits, err := h.KnowledgeSearch(ctx, "", string(h.Owner())+" 的入库内容", 5)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return map[string]any{"owner": h.Owner(), "plugin": h.PluginID(), "hits": len(hits)}, nil
|
||
})
|
||
},
|
||
})
|
||
bus, st := newInprocTestApp(t, nil) // embedder nil：embed=true 仍只落全文（IndexChunks 降级路径）
|
||
|
||
for _, owner := range []string{"alice", "bob"} {
|
||
resp := bus.Dispatch(context.Background(), cmdbus.Request{
|
||
ID: "tst.record", Params: map[string]any{"ref": owner + "-ref"}, Owner: owner, Actor: "ui", Entrance: "chat",
|
||
})
|
||
if resp.Receipt.State != agentcommand.StateCommitted {
|
||
t.Fatalf("%s dispatch: %+v", owner, resp.Receipt)
|
||
}
|
||
m, _ := resp.Payload.(map[string]any)
|
||
if m["owner"] != owner || m["plugin"] != "tst" || m["hits"] != 1 {
|
||
t.Fatalf("%s payload: %+v", owner, resp.Payload)
|
||
}
|
||
}
|
||
|
||
// 审计各归其主，Actor 标注插件身份。
|
||
for _, owner := range []string{"alice", "bob"} {
|
||
evs, err := st.ListAuditEvents(owner, 10)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(evs) != 1 || evs[0].Owner != owner || evs[0].Actor != "plugin:tst" || evs[0].Action != "tst.record" {
|
||
t.Fatalf("%s audit: %+v", owner, evs)
|
||
}
|
||
}
|
||
|
||
// 知识各归其主：本人可见，他人检索不可见（visibility 过滤）。
|
||
aHits, err := st.SearchKnowledge("alice", core.RetrievalQuery{Text: "alice 的入库内容"})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(aHits) != 1 || string(aHits[0].Doc.Owner) != "alice" {
|
||
t.Fatalf("alice knowledge: %+v", aHits)
|
||
}
|
||
cross, err := st.SearchKnowledge("bob", core.RetrievalQuery{Text: "alice 的入库内容"})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(cross) != 0 {
|
||
t.Fatalf("owner leak in knowledge: %+v", cross)
|
||
}
|
||
|
||
// KV 按 pluginID+owner 隔离。
|
||
for _, owner := range []string{"alice", "bob"} {
|
||
v, ok, err := st.GetPluginKV("tst", owner, "last")
|
||
if err != nil || !ok || v != owner {
|
||
t.Fatalf("%s kv: %q %v %v", owner, v, ok, err)
|
||
}
|
||
if _, ok, _ := st.GetPluginKV("other-plugin", owner, "last"); ok {
|
||
t.Fatalf("%s kv leaked across plugins", owner)
|
||
}
|
||
}
|
||
|
||
// 任务台账各归其主且状态已更新。
|
||
for _, owner := range []string{"alice", "bob"} {
|
||
tasks, err := st.ListBackgroundTasks(owner)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(tasks) != 1 || tasks[0].Kind != "tst.record" || tasks[0].Status != "succeeded" || tasks[0].Owner != owner {
|
||
t.Fatalf("%s tasks: %+v", owner, tasks)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestInprocHostKnowledgeWriteEmbedHash：provider=hash 档下 KnowledgeWrite(embed=true)
|
||
// 写出向量行，且 Owner/Source 由基座强制改写。
|
||
func TestInprocHostKnowledgeWriteEmbedHash(t *testing.T) {
|
||
extension.ResetCompiled()
|
||
t.Cleanup(extension.ResetCompiled)
|
||
extension.RegisterCompiled(extension.InProc{
|
||
ID: "tst",
|
||
Register: func(reg extension.Registry) error {
|
||
return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.embed","title":"向量"}`),
|
||
func(ctx context.Context, h extension.Host, req extension.Request) (any, error) {
|
||
text, _ := req.Params["text"].(string)
|
||
// 故意不带 Owner/Source：验证基座强制改写。
|
||
return "ok", h.KnowledgeWrite(ctx, []core.KnowledgeDoc{{Title: "向量", Ref: "vec-1", Chunk: text}}, true)
|
||
})
|
||
},
|
||
})
|
||
bus, st := newInprocTestApp(t, func() string { return "hash" })
|
||
|
||
resp := bus.Dispatch(context.Background(), cmdbus.Request{
|
||
ID: "tst.embed", Params: map[string]any{"text": "Hash 向量内容"}, Owner: "alice", Entrance: "api",
|
||
})
|
||
if resp.Receipt.State != agentcommand.StateCommitted {
|
||
t.Fatalf("dispatch: %+v", resp.Receipt)
|
||
}
|
||
|
||
// Owner 强制 alice、Source 强制补 plugin:tst。
|
||
hits, err := st.SearchKnowledge("alice", core.RetrievalQuery{Text: "向量内容"})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(hits) != 1 || string(hits[0].Doc.Owner) != "alice" || hits[0].Doc.Source != "plugin:tst" {
|
||
t.Fatalf("doc attribution: %+v", hits)
|
||
}
|
||
|
||
// hash 档向量行存在且可被同文本向量召回。
|
||
want, err := rag.HashEmbedder{}.Embed(context.Background(), []string{"Hash 向量内容"})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
vhits, err := st.SearchKnowledgeVectors(context.Background(), "alice", want[0], 5, core.RetrievalQuery{})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(vhits) != 1 {
|
||
t.Fatalf("vector rows: %+v", vhits)
|
||
}
|
||
// bob 用同一向量检索不可见。
|
||
bhits, err := st.SearchKnowledgeVectors(context.Background(), "bob", want[0], 5, core.RetrievalQuery{})
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(bhits) != 0 {
|
||
t.Fatalf("vector owner leak: %+v", bhits)
|
||
}
|
||
}
|
||
|
||
// TestInprocHostParseDocumentOwnerScope：EXT-5 解析方法的 owner 作用域——
|
||
// alice 经句柄读自己的工件并解析；bob 用同一工件 ID 在 ArtifactContent 处
|
||
// 即被拒（解析是纯计算，作用域边界在取原件一环）。
|
||
func TestInprocHostParseDocumentOwnerScope(t *testing.T) {
|
||
extension.ResetCompiled()
|
||
t.Cleanup(extension.ResetCompiled)
|
||
extension.RegisterCompiled(extension.InProc{
|
||
ID: "tst",
|
||
Register: func(reg extension.Registry) error {
|
||
return reg.AddCommandHandler(json.RawMessage(`{"id":"tst.parse","title":"解析"}`),
|
||
func(ctx context.Context, h extension.Host, req extension.Request) (any, error) {
|
||
id, _ := req.Params["artifactId"].(string)
|
||
content, err := h.ArtifactContent(id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
doc, err := h.ParseDocument(ctx, "报价.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", content)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return map[string]any{
|
||
"parser": doc.Meta.Parser,
|
||
"warnings": len(doc.Warnings),
|
||
"blocks": len(doc.Blocks),
|
||
}, nil
|
||
})
|
||
},
|
||
})
|
||
bus, st := newInprocTestApp(t, nil)
|
||
|
||
art := core.Artifact{ProjectID: "p1", Owner: "alice", Name: "报价.xlsx", Kind: "document"}
|
||
if err := st.CreateArtifact(&art, testXLSXBytes(t)); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
artID := art.ID
|
||
|
||
resp := bus.Dispatch(context.Background(), cmdbus.Request{
|
||
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "alice", Entrance: "chat",
|
||
})
|
||
if resp.Receipt.State != agentcommand.StateCommitted {
|
||
t.Fatalf("alice dispatch: %+v %s", resp.Receipt, resp.Receipt.Reason)
|
||
}
|
||
m, _ := resp.Payload.(map[string]any)
|
||
if m["parser"] != "builtin:xlsx" || m["warnings"] != 0 || m["blocks"] == 0 {
|
||
t.Fatalf("alice payload: %+v", m)
|
||
}
|
||
|
||
// bob 同一工件 ID：取原件即被拒（跨 owner 不可见）。
|
||
resp = bus.Dispatch(context.Background(), cmdbus.Request{
|
||
ID: "tst.parse", Params: map[string]any{"artifactId": artID}, Owner: "bob", Entrance: "chat",
|
||
})
|
||
if resp.Receipt.State == agentcommand.StateCommitted {
|
||
t.Fatalf("bob must not read alice's artifact: %+v", resp.Payload)
|
||
}
|
||
}
|
||
|
||
// TestInprocHostParseDocumentLimits：仅有的两条 error 路径——ctx 取消与输入
|
||
// 超过 100MiB 上限；不可解析格式仍是带 Warnings 的显式失败。
|
||
func TestInprocHostParseDocumentLimits(t *testing.T) {
|
||
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db"))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { st.Close() })
|
||
h := inprocHost{caps: hostcaps{st: st}, pluginID: "tst", owner: "alice"}
|
||
|
||
ctx, cancel := context.WithCancel(context.Background())
|
||
cancel()
|
||
if _, err := h.ParseDocument(ctx, "a.xlsx", "", []byte("x")); err == nil {
|
||
t.Fatal("canceled ctx must error")
|
||
}
|
||
if _, err := h.ParseDocument(context.Background(), "a.xlsx", "", make([]byte, maxParseInput+1)); err == nil {
|
||
t.Fatal("oversized input must error")
|
||
}
|
||
doc, err := h.ParseDocument(context.Background(), "旧.xls", "", []byte{0xD0, 0xCF})
|
||
if err != nil {
|
||
t.Fatalf("legacy format is a visible failure, not an error: %v", err)
|
||
}
|
||
if doc.Meta.Parser != "builtin:legacy-office" || len(doc.Warnings) != 1 {
|
||
t.Fatalf("legacy doc: %+v %v", doc.Meta, doc.Warnings)
|
||
}
|
||
}
|
||
|
||
// testXLSXBytes 构造最小可用 xlsx（单表两行），供句柄测试解析。
|
||
func testXLSXBytes(t *testing.T) []byte {
|
||
t.Helper()
|
||
var buf bytes.Buffer
|
||
zw := zip.NewWriter(&buf)
|
||
files := map[string]string{
|
||
"xl/workbook.xml": `<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="报价" sheetId="1" r:id="rId1"/></sheets></workbook>`,
|
||
"xl/_rels/workbook.xml.rels": `<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>`,
|
||
"xl/worksheets/sheet1.xml": `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>
|
||
<row r="1"><c r="A1" t="inlineStr"><is><t>物料</t></is></c><c r="B1" t="inlineStr"><is><t>单价</t></is></c></row>
|
||
<row r="2"><c r="A2" t="inlineStr"><is><t>支架</t></is></c><c r="B2"><v>12.5</v></c></row>
|
||
</sheetData></worksheet>`,
|
||
}
|
||
for _, name := range []string{"xl/workbook.xml", "xl/_rels/workbook.xml.rels", "xl/worksheets/sheet1.xml"} {
|
||
w, err := zw.Create(name)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
w.Write([]byte(files[name]))
|
||
}
|
||
if err := zw.Close(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return buf.Bytes()
|
||
}
|