Agent-System-Backend/internal/ingest/ingest_test.go

package ingest

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"strings"
	"testing"

	core "git-ssh.u1680832.nyat.app/weichen/Agent-System-Core.git"
)

func TestParseMarkdown(t *testing.T) {
	doc := Parse("笔记.md", "text/markdown", []byte("# 标题一\n\n段落甲。\n\n- 要点 1\n- 要点 2\n\n## 标题二\n\n段落乙。\n"))
	if doc.Meta.Parser != "builtin:md" || doc.Meta.Confidence != 1.0 {
		t.Fatalf("meta: %+v", doc.Meta)
	}
	var heads, paras, lists int
	for _, b := range doc.Blocks {
		switch b.Type {
		case core.BlockHeading:
			heads++
		case core.BlockPara:
			paras++
		case core.BlockList:
			lists++
			if len(b.Items) != 2 {
				t.Errorf("list items: %v", b.Items)
			}
		}
	}
	if heads != 2 || paras != 2 || lists != 1 {
		t.Errorf("blocks: heads=%d paras=%d lists=%d", heads, paras, lists)
	}
	if !strings.Contains(doc.PlainText(), "# 标题一") {
		t.Error("PlainText missing heading")
	}
}

func TestParseUnknownBinary(t *testing.T) {
	doc := Parse("blob.bin", "application/octet-stream", []byte{0x00, 0x01, 0x02, 0xFF})
	if len(doc.Blocks) != 0 {
		t.Error("binary should not produce blocks")
	}
	if len(doc.Warnings) == 0 {
		t.Error("binary should carry explicit warning (fail visible, not silent)")
	}
}

// 构造最小 docx（zip + word/document.xml）验证 OOXML 解析。
func TestParseDocx(t *testing.T) {
	var buf bytes.Buffer
	zw := zip.NewWriter(&buf)
	docXML := `<?xml version="1.0"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
 <w:body>
  <w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>总则</w:t></w:r></w:p>
  <w:p><w:r><w:t>本文件说明医疗器械文档要求。</w:t></w:r></w:p>
  <w:tbl>
   <w:tr><w:tc><w:p><w:r><w:t>项</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>值</w:t></w:r></w:p></w:tc></w:tr>
   <w:tr><w:tc><w:p><w:r><w:t>版本</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>1.0</w:t></w:r></w:p></w:tc></w:tr>
  </w:tbl>
  <w:p><w:r><w:t>表格之后的段落。</w:t></w:r></w:p>
 </w:body>
</w:document>`
	if f, err := zw.Create("word/document.xml"); err == nil {
		f.Write([]byte(docXML))
	} else {
		t.Fatal(err)
	}
	zw.Close()

	doc := Parse("报告.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", buf.Bytes())
	if doc.Meta.Parser != "builtin:docx" || len(doc.Warnings) != 0 {
		t.Fatalf("meta: %+v warnings: %v", doc.Meta, doc.Warnings)
	}
	var heading, para, table, afterTable bool
	for _, b := range doc.Blocks {
		switch b.Type {
		case core.BlockHeading:
			heading = b.Level == 1 && b.Text == "总则"
		case core.BlockPara:
			para = para || strings.Contains(b.Text, "医疗器械")
			afterTable = afterTable || strings.Contains(b.Text, "表格之后的段落")
		case core.BlockTable:
			table = len(b.Rows) == 2 && b.Rows[1][1] == "1.0"
		}
	}
	if !afterTable {
		t.Error("paragraph after table missing (table must not reorder content)")
	}
	// 文档顺序：标题 → 段落 → 表格 → 表后段落（表格原位，不后置）。
	var seq []core.DocBlockType
	for _, b := range doc.Blocks {
		seq = append(seq, b.Type)
	}
	want := []core.DocBlockType{core.BlockHeading, core.BlockPara, core.BlockTable, core.BlockPara}
	if len(seq) != len(want) {
		t.Fatalf("block count: got %v want %v", seq, want)
	}
	for i := range want {
		if seq[i] != want[i] {
			t.Errorf("block order: got %v want %v", seq, want)
			break
		}
	}
	if !heading || !para || !table {
		t.Errorf("docx blocks wrong: heading=%v para=%v table=%v blocks=%+v", heading, para, table, doc.Blocks)
	}
	if !strings.Contains(doc.PlainText(), "医疗器械") {
		t.Error("PlainText missing docx text")
	}
}

// docxWith 把 body 片段包成最小 docx（zip 内仅 word/document.xml）。
func docxWith(t *testing.T, body string) []byte {
	t.Helper()
	var buf bytes.Buffer
	zw := zip.NewWriter(&buf)
	f, err := zw.Create("word/document.xml")
	if err != nil {
		t.Fatal(err)
	}
	f.Write([]byte(`<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><w:body>` + body + `</w:body></w:document>`))
	zw.Close()
	return buf.Bytes()
}

// 修订与超链接：插入/超链接/域结果内的文字必须保留；删除文字不进正文；
// 含修订时显式告警（SR-A5：禁止静默降级）。
func TestParseDocxTrackedChangesAndHyperlinks(t *testing.T) {
	body := `<w:p><w:r><w:t>周期</w:t></w:r><w:ins w:id="1" w:author="a"><w:r><w:t>18</w:t></w:r></w:ins>` +
		`<w:del w:id="2" w:author="a"><w:r><w:delText>12</w:delText></w:r></w:del><w:r><w:t>个月</w:t></w:r></w:p>` +
		`<w:p><w:r><w:t>见</w:t></w:r><w:hyperlink r:id="rId5"><w:r><w:t>附件A</w:t></w:r></w:hyperlink></w:p>` +
		`<w:p><w:fldSimple w:instr="PAGE"><w:r><w:t>3</w:t></w:r></w:fldSimple><w:r><w:t>页</w:t></w:r></w:p>` +
		`<w:tbl><w:tr><w:tc><w:p><w:ins w:id="3" w:author="a"><w:r><w:t>新增项</w:t></w:r></w:ins></w:p></w:tc></w:tr></w:tbl>`
	doc := Parse("修订稿.docx", "", docxWith(t, body))
	var paras []string
	var cell string
	for _, b := range doc.Blocks {
		switch b.Type {
		case core.BlockPara:
			paras = append(paras, b.Text)
		case core.BlockTable:
			if len(b.Rows) > 0 && len(b.Rows[0]) > 0 {
				cell = b.Rows[0][0]
			}
		}
	}
	want := []string{"周期18个月", "见附件A", "3页"}
	if strings.Join(paras, "|") != strings.Join(want, "|") {
		t.Errorf("paragraphs: got %q want %q", paras, want)
	}
	if cell != "新增项" {
		t.Errorf("table cell inside w:ins: got %q", cell)
	}
	if strings.Contains(doc.PlainText(), "12个月") {
		t.Errorf("deleted text leaked into body: %q", doc.PlainText())
	}
	warned := false
	for _, w := range doc.Warnings {
		warned = warned || strings.Contains(w, "修订")
	}
	if !warned {
		t.Errorf("tracked changes must be reported in warnings, got %v", doc.Warnings)
	}
}

// 无修订的普通文档不产生修订告警（避免噪音）。
func TestParseDocxNoRevisionNoWarning(t *testing.T) {
	doc := Parse("干净.docx", "", docxWith(t, `<w:p><w:r><w:t>正文</w:t></w:r></w:p>`))
	if len(doc.Warnings) != 0 {
		t.Errorf("clean docx warnings: %v", doc.Warnings)
	}
}

func TestParsePDFInvalid(t *testing.T) {
	doc := Parse("scan.pdf", "application/pdf", []byte("not a pdf"))
	if len(doc.Warnings) == 0 || doc.Meta.Confidence > 0.5 {
		t.Errorf("invalid pdf should warn with low confidence: %+v", doc.Meta)
	}
}

var _ = xml.Marshal