weichen
0d74ec94dd
Details (original message): Implement the backend: store/bus/runmgr core, mock and LLM runners (agent-native client), HITL three-branch routing, subagent runs, builtin and declarative HTTP tools, ingest parsers (md/txt/docx/pdf-text), knowledge fulltext, layered settings, profiles, plugin host, REST+SSE API, single-binary entrypoint; e2e and unit tests.
314 lines
8.2 KiB
Go
314 lines
8.2 KiB
Go
// Package ingest 实现文档摄取与解析（4.11）：上传/工件 → ParsedDocument IR。
|
||
// 主路线为内置代码解析器（确定性、纯 Go、可独立测试）；MinerU 深度解析与
|
||
// agent 驱动解析为远期档位（ParserProvider 注册位已留）。
|
||
package ingest
|
||
|
||
import (
|
||
"archive/zip"
|
||
"bytes"
|
||
"encoding/xml"
|
||
"fmt"
|
||
"io"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
pdflib "github.com/ledongthuc/pdf"
|
||
core "agent-system-core"
|
||
)
|
||
|
||
// Parse 按扩展名分派内置解析器；无法解析时返回带 warning 的空文档（显式失败，不静默）。
|
||
func Parse(filename, mime string, content []byte) core.ParsedDocument {
|
||
doc := core.ParsedDocument{Meta: core.DocMeta{
|
||
Filename: filename, MIME: mime, Parser: "builtin", ParsedAt: time.Now().UTC(),
|
||
}}
|
||
ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(filename), "."))
|
||
switch ext {
|
||
case "md", "markdown":
|
||
doc.Meta.Parser = "builtin:md"
|
||
doc.Blocks = parseMarkdown(string(content))
|
||
doc.Meta.Confidence = 1.0
|
||
case "txt", "text", "log", "csv":
|
||
doc.Meta.Parser = "builtin:txt"
|
||
doc.Blocks = parsePlain(string(content))
|
||
doc.Meta.Confidence = 1.0
|
||
case "docx":
|
||
doc.Meta.Parser = "builtin:docx"
|
||
blocks, err := parseDocx(content)
|
||
if err != nil {
|
||
doc.Warnings = append(doc.Warnings, "docx 解析失败："+err.Error())
|
||
doc.Meta.Confidence = 0.3
|
||
} else {
|
||
doc.Blocks = blocks
|
||
doc.Meta.Confidence = 0.92
|
||
}
|
||
case "pdf":
|
||
doc.Meta.Parser = "builtin:pdf"
|
||
blocks, err := parsePDF(content)
|
||
if err != nil {
|
||
doc.Warnings = append(doc.Warnings, "pdf 文本层解析失败："+err.Error()+"（扫描件需远期 MinerU 档）")
|
||
doc.Meta.Confidence = 0.2
|
||
} else {
|
||
doc.Blocks = blocks
|
||
doc.Meta.Confidence = 0.85
|
||
}
|
||
default:
|
||
if isImage(ext) {
|
||
doc.Warnings = append(doc.Warnings, "图片不走文本解析（OCR 为 MinerU 深度档，远期）")
|
||
} else {
|
||
// 未知类型按纯文本尝试（二进制内容会产生乱码 warning）。
|
||
text := string(content)
|
||
if !isMostlyPrintable(text) {
|
||
doc.Warnings = append(doc.Warnings, fmt.Sprintf("未知类型 .%s 且内容非文本，未解析", ext))
|
||
} else {
|
||
doc.Blocks = parsePlain(text)
|
||
doc.Meta.Confidence = 0.6
|
||
doc.Warnings = append(doc.Warnings, "未知类型按纯文本解析")
|
||
}
|
||
}
|
||
}
|
||
return doc
|
||
}
|
||
|
||
// ---- md / txt ----
|
||
|
||
func parseMarkdown(s string) []core.DocBlock {
|
||
var blocks []core.DocBlock
|
||
var para []string
|
||
flush := func() {
|
||
if len(para) > 0 {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: strings.Join(para, "\n")})
|
||
para = nil
|
||
}
|
||
}
|
||
for _, line := range strings.Split(s, "\n") {
|
||
t := strings.TrimSpace(line)
|
||
switch {
|
||
case t == "":
|
||
flush()
|
||
case strings.HasPrefix(t, "#"):
|
||
flush()
|
||
level := 0
|
||
for level < len(t) && t[level] == '#' {
|
||
level++
|
||
}
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: level, Text: strings.TrimSpace(t[level:])})
|
||
case strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* "):
|
||
flush()
|
||
if n := len(blocks); n > 0 && blocks[n-1].Type == core.BlockList {
|
||
blocks[n-1].Items = append(blocks[n-1].Items, strings.TrimSpace(t[2:]))
|
||
} else {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockList, Items: []string{strings.TrimSpace(t[2:])}})
|
||
}
|
||
case strings.HasPrefix(t, "```"):
|
||
flush()
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockCode, Text: ""}) // 简化：代码块内容并入口令块
|
||
default:
|
||
para = append(para, t)
|
||
}
|
||
}
|
||
flush()
|
||
return blocks
|
||
}
|
||
|
||
func parsePlain(s string) []core.DocBlock {
|
||
var blocks []core.DocBlock
|
||
var para []string
|
||
flush := func() {
|
||
if len(para) > 0 {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: strings.Join(para, "\n")})
|
||
para = nil
|
||
}
|
||
}
|
||
for _, line := range strings.Split(s, "\n") {
|
||
if strings.TrimSpace(line) == "" {
|
||
flush()
|
||
continue
|
||
}
|
||
para = append(para, strings.TrimRight(line, "\r"))
|
||
}
|
||
flush()
|
||
return blocks
|
||
}
|
||
|
||
// ---- docx（OOXML：zip 内 word/document.xml）----
|
||
|
||
type xdoc struct {
|
||
Body struct {
|
||
Paragraphs []xpar `xml:"p"`
|
||
Tables []xtbl `xml:"tbl"`
|
||
} `xml:"body"`
|
||
}
|
||
|
||
type xpar struct {
|
||
Props struct {
|
||
Style struct {
|
||
Val string `xml:"val,attr"`
|
||
} `xml:"pStyle"`
|
||
} `xml:"pPr"`
|
||
Runs []xrun `xml:"r"`
|
||
}
|
||
|
||
type xrun struct {
|
||
Text []struct {
|
||
Content string `xml:",chardata"`
|
||
} `xml:"t"`
|
||
Break struct {
|
||
Val string `xml:"val,attr"`
|
||
} `xml:"br"`
|
||
}
|
||
|
||
type xtbl struct {
|
||
Rows []xtr `xml:"tr"`
|
||
}
|
||
|
||
type xtr struct {
|
||
Cells []xtc `xml:"tc"`
|
||
}
|
||
|
||
type xtc struct {
|
||
Paragraphs []xpar `xml:"p"`
|
||
}
|
||
|
||
func parseDocx(content []byte) ([]core.DocBlock, error) {
|
||
zr, err := zip.NewReader(bytes.NewReader(content), int64(len(content)))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("open zip: %w", err)
|
||
}
|
||
var docXML []byte
|
||
for _, f := range zr.File {
|
||
if f.Name == "word/document.xml" {
|
||
rc, err := f.Open()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
docXML, _ = io.ReadAll(rc)
|
||
rc.Close()
|
||
break
|
||
}
|
||
}
|
||
if docXML == nil {
|
||
return nil, fmt.Errorf("word/document.xml not found")
|
||
}
|
||
var doc xdoc
|
||
if err := xml.Unmarshal(docXML, &doc); err != nil {
|
||
return nil, fmt.Errorf("unmarshal document.xml: %w", err)
|
||
}
|
||
var blocks []core.DocBlock
|
||
for _, p := range doc.Body.Paragraphs {
|
||
text := paragraphText(p)
|
||
if strings.TrimSpace(text) == "" {
|
||
continue
|
||
}
|
||
if lvl, ok := headingLevel(p.Props.Style.Val); ok {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockHeading, Level: lvl, Text: text})
|
||
} else {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: text})
|
||
}
|
||
}
|
||
for _, tbl := range doc.Body.Tables {
|
||
var rows [][]string
|
||
for _, tr := range tbl.Rows {
|
||
row := []string{}
|
||
for _, tc := range tr.Cells {
|
||
cell := []string{}
|
||
for _, p := range tc.Paragraphs {
|
||
cell = append(cell, paragraphText(p))
|
||
}
|
||
row = append(row, strings.Join(cell, " "))
|
||
}
|
||
rows = append(rows, row)
|
||
}
|
||
if len(rows) > 0 {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockTable, Rows: rows})
|
||
}
|
||
}
|
||
return blocks, nil
|
||
}
|
||
|
||
func paragraphText(p xpar) string {
|
||
var sb strings.Builder
|
||
for _, r := range p.Runs {
|
||
for _, t := range r.Text {
|
||
sb.WriteString(t.Content)
|
||
}
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
func headingLevel(style string) (int, bool) {
|
||
s := strings.ToLower(style)
|
||
switch {
|
||
case strings.Contains(s, "heading1") || s == "1":
|
||
return 1, true
|
||
case strings.Contains(s, "heading2") || s == "2":
|
||
return 2, true
|
||
case strings.Contains(s, "heading3") || s == "3" || strings.Contains(s, "title"):
|
||
return 3, true
|
||
}
|
||
return 0, false
|
||
}
|
||
|
||
// ---- pdf（文本层提取）----
|
||
|
||
func parsePDF(content []byte) ([]core.DocBlock, error) {
|
||
r, err := pdflib.NewReader(bytes.NewReader(content), int64(len(content)))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var blocks []core.DocBlock
|
||
for i := 1; i <= r.NumPage(); i++ {
|
||
page := r.Page(i)
|
||
if page.V.IsNull() {
|
||
continue
|
||
}
|
||
text, err := page.GetPlainText(nil)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
for _, para := range strings.Split(text, "\n\n") {
|
||
if t := strings.TrimSpace(para); t != "" {
|
||
blocks = append(blocks, core.DocBlock{Type: core.BlockPara, Text: t})
|
||
}
|
||
}
|
||
}
|
||
if len(blocks) == 0 {
|
||
return nil, fmt.Errorf("无文本层（可能是扫描件）")
|
||
}
|
||
return blocks, nil
|
||
}
|
||
|
||
// ---- 辅助 ----
|
||
|
||
func isImage(ext string) bool {
|
||
switch ext {
|
||
case "png", "jpg", "jpeg", "gif", "webp", "bmp", "svg":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isMostlyPrintable(s string) bool {
|
||
if len(s) == 0 {
|
||
return true
|
||
}
|
||
nonPrint := 0
|
||
for _, r := range s {
|
||
if r == 0xFFFD || (r < 0x20 && r != '\n' && r != '\r' && r != '\t') {
|
||
nonPrint++
|
||
}
|
||
}
|
||
return nonPrint*10 < len(s)
|
||
}
|
||
|
||
// ParsersDescription 返回解析器注册表描述（来源标记；设置页/文档用）。
|
||
func ParsersDescription() []core.ParserDesc {
|
||
return []core.ParserDesc{
|
||
{ID: "builtin:md", Formats: []string{"md", "markdown"}, Source: core.SourceBuiltin},
|
||
{ID: "builtin:txt", Formats: []string{"txt", "csv", "log"}, Source: core.SourceBuiltin},
|
||
{ID: "builtin:docx", Formats: []string{"docx"}, Source: core.SourceBuiltin},
|
||
{ID: "builtin:pdf", Formats: []string{"pdf"}, Source: core.SourceBuiltin, Note: "文本层；扫描件为远期 MinerU 深度档"},
|
||
{ID: "mineru", Formats: []string{"pdf", "png", "jpg"}, Source: core.SourcePlugin, Note: "远期可选：深度解析与交叉验证（默认不部署）"},
|
||
}
|
||
}
|