Reject pseudo-binary Office writes and index extracted text #34
@@ -12,6 +12,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"agent-system-backend/internal/bus"
|
||||
"agent-system-backend/internal/docgen"
|
||||
"agent-system-backend/internal/settings"
|
||||
"agent-system-backend/internal/store"
|
||||
"agent-system-backend/internal/tools"
|
||||
@@ -113,3 +114,33 @@ func TestExecuteToolNoArtifactByDefault(t *testing.T) {
|
||||
t.Errorf("无产物工具不应携带 Artifact: %+v", res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeneratedDocxIsSearchable(t *testing.T) {
|
||||
c, st := newExecEnv(t)
|
||||
const marker = "zebraindexword"
|
||||
docx, err := docgen.Build("纪要", "结论:"+marker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mime := "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
if _, err := c.WriteArtifact("note.docx", "document", mime, docx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hits, err := st.SearchKnowledge("alice", core.RetrievalQuery{Text: marker, ProjectID: c.run.ProjectID, TopK: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hits) == 0 {
|
||||
t.Fatal("generated docx text was not indexed")
|
||||
}
|
||||
if _, err := c.WriteArtifact("bad.docx", "document", mime, []byte("notazipmarker")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hits, err = st.SearchKnowledge("alice", core.RetrievalQuery{Text: "notazipmarker", ProjectID: c.run.ProjectID, TopK: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(hits) != 0 {
|
||||
t.Fatalf("unparseable docx bytes should not be indexed, got %d hits", len(hits))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"agent-system-backend/internal/bus"
|
||||
"agent-system-backend/internal/cardsvc"
|
||||
"agent-system-backend/internal/ingest"
|
||||
"agent-system-backend/internal/profiles"
|
||||
"agent-system-backend/internal/runtime"
|
||||
"agent-system-backend/internal/settings"
|
||||
@@ -352,14 +353,15 @@ func (c *runCtx) WriteArtifact(name, kind, mime string, content []byte) (core.Ar
|
||||
if _, err := c.mgr.emit(c.run.ConversationID, c.run.ID, core.EvArtifactCreated, core.ArtifactPayload{Artifact: art}); err != nil {
|
||||
return art, err
|
||||
}
|
||||
// 工件文本进检索索引(管线分块+全文+可选向量;未注入回落整文档单块)。
|
||||
// 二进制 Office/PDF 不进字符串索引(zip 字节当文本是垃圾数据);
|
||||
// docx 的可检索文本由 ingest 管线在上传路径解析,生成路径暂不索引。
|
||||
if kind == "document" && !binaryDocMime(mime) && len(content) < 200000 {
|
||||
if c.mgr.indexer != nil {
|
||||
_ = c.mgr.indexer.IndexDocument(c.run.Owner, c.run.ProjectID, "artifact", name, art.ID, string(content))
|
||||
} else {
|
||||
_ = c.mgr.st.AddKnowledge(core.KnowledgeDoc{Owner: c.run.Owner, ProjectID: c.run.ProjectID, Source: "artifact", Title: name, Ref: art.ID, Chunk: string(content)})
|
||||
// 工件文本进检索索引。二进制 Office/PDF 先经 ingest 抽出纯文本,抽到才入库;
|
||||
// 抽不到则跳过,避免把 ZIP 字节当文本索引。
|
||||
if kind == "document" && len(content) < 200000 {
|
||||
if text := indexTextOf(name, mime, content); text != "" {
|
||||
if c.mgr.indexer != nil {
|
||||
_ = c.mgr.indexer.IndexDocument(c.run.Owner, c.run.ProjectID, "artifact", name, art.ID, text)
|
||||
} else {
|
||||
_ = c.mgr.st.AddKnowledge(core.KnowledgeDoc{Owner: c.run.Owner, ProjectID: c.run.ProjectID, Source: "artifact", Title: name, Ref: art.ID, Chunk: text})
|
||||
}
|
||||
}
|
||||
}
|
||||
return art, nil
|
||||
@@ -381,6 +383,19 @@ func binaryDocMime(mime string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// indexTextOf 决定工件进检索索引的文本。普通文本原样返回;二进制文档 mime
|
||||
// 先走 ingest 抽文本,解析失败或为空则返回空串(跳过索引)。
|
||||
func indexTextOf(name, mime string, content []byte) string {
|
||||
if !binaryDocMime(mime) {
|
||||
return string(content)
|
||||
}
|
||||
doc := ingest.Parse(name, mime, content)
|
||||
if len(doc.Warnings) > 0 || len(doc.Blocks) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(doc.PlainText())
|
||||
}
|
||||
|
||||
func (c *runCtx) Retrieve(q core.RetrievalQuery) ([]core.RetrievalHit, error) {
|
||||
if q.TopK <= 0 {
|
||||
q.TopK = int(c.eff.GetNumber("retrieval.topK", 5))
|
||||
|
||||
@@ -113,6 +113,9 @@ func execFSWrite(rc ToolExecCtx, args json.RawMessage) (string, error) {
|
||||
if a.Path == "" {
|
||||
return "", fmt.Errorf("fs_write_file: path required")
|
||||
}
|
||||
if err := rejectBinaryOfficeWrite(a.Path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(a.Content) > fsMaxWrite {
|
||||
return "", fmt.Errorf("fs_write_file: 内容超过 2MB 上限")
|
||||
}
|
||||
@@ -218,3 +221,14 @@ func mimeOfExt(name string) string {
|
||||
return "text/plain; charset=utf-8"
|
||||
}
|
||||
}
|
||||
|
||||
// rejectBinaryOfficeWrite 拒绝把字符串直接写成伪二进制 Office/PDF。
|
||||
// 带排版的文件由 run_python 生成后再 register_artifact 登记。
|
||||
func rejectBinaryOfficeWrite(name string) error {
|
||||
switch strings.ToLower(filepath.Ext(name)) {
|
||||
case ".doc", ".docx", ".xls", ".xlsx", ".xlsm", ".ppt", ".pptx", ".pdf":
|
||||
return fmt.Errorf("%s 是二进制文档扩展名,不能把字符串直接写成该文件。请用 run_python 生成到工作区后,用 register_artifact 登记", filepath.Ext(name))
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,3 +161,21 @@ func TestFSUnconfiguredRoot(t *testing.T) {
|
||||
t.Errorf("unconfigured root should fail: %s", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBinaryOfficeWriteRejected(t *testing.T) {
|
||||
r := newFSRegistry(t)
|
||||
rc := &fakeFSCtx{root: t.TempDir(), owner: "o1", conv: "c1", fsToolsOn: true}
|
||||
for _, args := range []struct{ name, body string }{
|
||||
{"fs_write_file", `{"path":"a.docx","content":"plain"}`},
|
||||
{"fs_write_file", `{"path":"a.pdf","content":"plain"}`},
|
||||
{"write_artifact", `{"name":"a.xlsx","content":"plain"}`},
|
||||
} {
|
||||
res := call(t, r, rc, args.name, args.body)
|
||||
if res.Status != "error" || !strings.Contains(res.Output, "run_python") {
|
||||
t.Errorf("%s %s: %s %s", args.name, args.body, res.Status, res.Output)
|
||||
}
|
||||
}
|
||||
if res := call(t, r, rc, "write_artifact", `{"name":"a.md","content":"plain"}`); res.Status != "ok" {
|
||||
t.Fatalf("markdown write should stay allowed: %s", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,6 +164,9 @@ func (r *Registry) RegisterBuiltin() error {
|
||||
if a.Name == "" {
|
||||
return "", fmt.Errorf("write_artifact: name required")
|
||||
}
|
||||
if err := rejectBinaryOfficeWrite(a.Name); err != nil {
|
||||
return "", err
|
||||
}
|
||||
kind := a.Kind
|
||||
if kind == "" {
|
||||
kind = "document"
|
||||
|
||||
Reference in New Issue
Block a user