149 lines
4.9 KiB
Go
149 lines
4.9 KiB
Go
package core
|
||
|
||
import "time"
|
||
|
||
// ---- 文档摄取与解析（4.11）----
|
||
|
||
// ParsedDocument 是解析中间表示：所有下游（附件上下文/工件预览/检索索引/全文检索）
|
||
// 只消费 IR，不关心解析器——解析器可替换、可组合、可交叉验证。
|
||
type ParsedDocument struct {
|
||
Meta DocMeta `json:"meta"`
|
||
Blocks []DocBlock `json:"blocks"`
|
||
Warnings []string `json:"warnings,omitempty"`
|
||
}
|
||
|
||
type DocMeta struct {
|
||
Origin *DocumentOrigin `json:"origin,omitempty"`
|
||
Filename string `json:"filename"`
|
||
MIME string `json:"mime,omitempty"`
|
||
Parser string `json:"parser"` // builtin:md / builtin:docx / mineru / agent
|
||
Confidence float64 `json:"confidence,omitempty"` // 0–1；交叉验证差异块拉低
|
||
CrossCheck *CrossCheck `json:"crossCheck,omitempty"`
|
||
ParsedAt time.Time `json:"parsedAt"`
|
||
}
|
||
|
||
type CrossCheck struct {
|
||
With string `json:"with"` // 交叉验证的另一解析器
|
||
DiffBlocks int `json:"diffBlocks"`
|
||
Verdict string `json:"verdict"` // agree | differ
|
||
}
|
||
|
||
type DocBlockType string
|
||
|
||
const (
|
||
BlockHeading DocBlockType = "heading"
|
||
BlockPara DocBlockType = "para"
|
||
BlockList DocBlockType = "list"
|
||
BlockTable DocBlockType = "table"
|
||
BlockCode DocBlockType = "code"
|
||
BlockImageRef DocBlockType = "image-ref"
|
||
)
|
||
|
||
type DocBlock struct {
|
||
Type DocBlockType `json:"type"`
|
||
Level int `json:"level,omitempty"` // heading
|
||
Text string `json:"text,omitempty"` // para/heading/list 项拼接/code
|
||
Items []string `json:"items,omitempty"` // list
|
||
Rows [][]string `json:"rows,omitempty"` // table（首行为表头）
|
||
Ref string `json:"ref,omitempty"` // image-ref 的对象键
|
||
Confidence float64 `json:"confidence,omitempty"`
|
||
}
|
||
|
||
// PlainText 把文档压成纯文本（附件注入与全文索引用）；表格按行拼接。
|
||
func (d ParsedDocument) PlainText() string {
|
||
out := make([]byte, 0, 1024)
|
||
for _, b := range d.Blocks {
|
||
switch b.Type {
|
||
case BlockHeading:
|
||
out = append(out, '\n')
|
||
for i := 0; i < b.Level; i++ {
|
||
out = append(out, '#')
|
||
}
|
||
out = append(out, ' ')
|
||
out = append(out, b.Text...)
|
||
out = append(out, '\n')
|
||
case BlockPara, BlockCode:
|
||
out = append(out, b.Text...)
|
||
out = append(out, '\n')
|
||
case BlockList:
|
||
for _, it := range b.Items {
|
||
out = append(out, "- "...)
|
||
out = append(out, it...)
|
||
out = append(out, '\n')
|
||
}
|
||
case BlockTable:
|
||
for _, row := range b.Rows {
|
||
for i, cell := range row {
|
||
if i > 0 {
|
||
out = append(out, '\t')
|
||
}
|
||
out = append(out, cell...)
|
||
}
|
||
out = append(out, '\n')
|
||
}
|
||
}
|
||
out = append(out, '\n')
|
||
}
|
||
return string(out)
|
||
}
|
||
|
||
// ParserDesc 注册表条目（来源标记：builtin/plugin/...；一切可禁用）。
|
||
type ParserDesc struct {
|
||
ID string `json:"id"`
|
||
Formats []string `json:"formats"` // 支持的扩展名
|
||
Source SourceKind `json:"source"`
|
||
Note string `json:"note,omitempty"`
|
||
}
|
||
|
||
// ---- 检索（4.12）----
|
||
|
||
// KnowledgeDoc 是检索语料的一条文档（来自 ingest 或插件知识库，带来源标记）。
|
||
type KnowledgeDoc struct {
|
||
ID ID `json:"id"`
|
||
Owner ID `json:"owner,omitempty"` // 空 = 共享（插件级）知识
|
||
ProjectID ID `json:"projectId,omitempty"` // 空 = 插件级知识库
|
||
Source string `json:"source"` // upload | artifact | plugin:<id>
|
||
Title string `json:"title"`
|
||
Ref string `json:"ref,omitempty"` // 工件/文件引用
|
||
Chunk string `json:"chunk"`
|
||
CreatedAt time.Time `json:"createdAt"`
|
||
}
|
||
|
||
type RetrievalHit struct {
|
||
Doc KnowledgeDoc `json:"doc"`
|
||
Score float64 `json:"score"`
|
||
Kind string `json:"kind"` // fulltext | vector | plugin
|
||
}
|
||
|
||
type RetrievalQuery struct {
|
||
ProjectID ID `json:"projectId,omitempty"`
|
||
Text string `json:"text"`
|
||
TopK int `json:"topK,omitempty"`
|
||
Sources []string `json:"sources,omitempty"` // 过滤来源
|
||
}
|
||
|
||
// ---- 记忆（4.13 契约；实现远期）----
|
||
|
||
type MemoryEntry struct {
|
||
ID string `json:"id"`
|
||
Scope string `json:"scope"` // user | project
|
||
ProjectID ID `json:"projectId,omitempty"`
|
||
Type string `json:"type"` // fact | preference | decision | episode | procedure
|
||
Content string `json:"content"`
|
||
Source string `json:"source"` // run:<id> | manual
|
||
Confidence float64 `json:"confidence,omitempty"`
|
||
CreatedAt time.Time `json:"createdAt"`
|
||
LastAccessed time.Time `json:"lastAccessed"`
|
||
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
|
||
}
|
||
|
||
// ---- 上传文件 ----
|
||
|
||
type UploadInfo struct {
|
||
ArtifactID ID `json:"artifactId"`
|
||
Filename string `json:"filename"`
|
||
MIME string `json:"mime,omitempty"`
|
||
Size int64 `json:"size"`
|
||
Parsed *ParsedDocument `json:"parsed,omitempty"`
|
||
}
|