Merge pull request 'U4 第三步:textextract 抽包,消除 materialindex→squad 反向依赖' (#7) from feature/textextract-package-2026-09-29 into dev
CI / go-vet (push) Successful in 10s
CI / short-tests-report (push) Failing after 36s
CI / backup-restore-drill (push) Successful in 27s
CI / web (push) Successful in 32s

This commit was merged in pull request #7.
This commit is contained in:
2026-09-29 06:21:45 +00:00
14 changed files with 309 additions and 267 deletions
+7 -4
View File
@@ -27,8 +27,8 @@ imports:
- docduty
bizdev:
- agent
- authz
- app/uow
- authz
- config
- conflictresolve
- contracts/vocabulary
@@ -50,6 +50,7 @@ imports:
- squad
- storage
- stream
- textextract
- workflowdef
- workspace
chat:
@@ -214,6 +215,7 @@ imports:
- kb
- mineru
- squad
- textextract
maturity: []
mineru: []
notification: []
@@ -272,22 +274,22 @@ imports:
- storage
- stream
- stream/transport
- supplementimpact
- swboot
- sysadmin
- taskrun
- textextract
- traceimpact
- upload
- workflowdef
- workflowscope
- workspace
- supplementimpact
squad:
- agent
- config
- quote
- stream
- template
- workspace
- textextract
- workflowdef
storage:
- config
@@ -306,6 +308,7 @@ imports:
- projectworkflow
- stream
- workspace
textextract: []
template:
- workflowdef
upload: []
-16
View File
@@ -1,25 +1,9 @@
package bizdev
import (
"chat-agent-system/internal/quote"
"chat-agent-system/internal/squad"
)
// FormatMoney formats a currency amount for display in markdown (no grouping separators).
func FormatMoney(v float64) string {
return squad.FormatMoney(v)
}
// RenderCDOQuoteTable builds the pricing table from quote summary lines and total.
func RenderCDOQuoteTable(summary *quote.Summary) string {
return squad.RenderCDOQuoteTable(summary)
}
// BackfillCDOQuoteMarkdown replaces placeholder sections with computed pricing from summary.
func BackfillCDOQuoteMarkdown(body string, summary *quote.Summary) string {
return squad.BackfillCDOQuoteMarkdown(body, summary)
}
// WriteMaterials persists uploaded material text into workspace materials_text/.
func WriteMaterials(workDir string, materials []MaterialInput) error {
var items []squad.MaterialItem
+2 -1
View File
@@ -12,6 +12,7 @@ import (
"chat-agent-system/internal/materialindex"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"chat-agent-system/internal/storage"
)
@@ -154,7 +155,7 @@ func (s *Service) materialToItems(
meta["parse_engine"] = "mineru"
}
} else {
text, err = squad.ExtractTextWithAnchors(fileName, data)
text, err = textextract.ExtractTextWithAnchors(fileName, data)
if err != nil {
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
}
@@ -1,4 +1,4 @@
package squad
package bizdev
import (
"fmt"
+74
View File
@@ -0,0 +1,74 @@
package bizdev
import (
"os"
"path/filepath"
"chat-agent-system/internal/quote"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/workflowdef"
)
// handleCDOQuotePreparation 为 CDO 报价任务执行 Excel 核算并生成回填内容
// (KTD1:报价领域规则、核算与 xlsx 资产归 Core;只计算,不写 squad 工作区,
// 文件由 squad 经 TaskPreparation.Backfills 统一写入)。
func (s *Service) handleCDOQuotePreparation(workDir string, req squad.SquadRunRequest) (*squad.TaskPreparation, error) {
in := quote.Input{
CustomerName: req.CustomerName,
ProductName: req.ProjectName,
Hours: map[string]float64{
"结构工程师": 320,
"硬件工程师": 320,
"软件工程师": 480,
"项目经理": 160,
},
}
// 核算写入临时文件后读回字节;工作区文件由 squad 写入。
tmp, err := os.CreateTemp("", "cdo-quote-*.xlsx")
if err != nil {
return nil, err
}
tmpPath := tmp.Name()
tmp.Close()
defer os.Remove(tmpPath)
summary, err := quote.CalculateAndFill(in, "templates/xlsx/cdo_quote.xlsx", tmpPath)
if err != nil {
return nil, err
}
data, err := os.ReadFile(tmpPath)
if err != nil {
return nil, err
}
backfills := map[string][]byte{
"CDO报价核算-V01.xlsx": data,
"artifacts/CDO报价核算-V01.xlsx": data,
}
// 回填 template.md 报价占位(模板存在时);output.md 镜像由 squad 决定
tmplBytes, tmplErr := os.ReadFile(filepath.Join(workDir, "template.md"))
mirror := false
if tmplErr == nil {
filled := BackfillCDOQuoteMarkdown(string(tmplBytes), summary)
backfills["template.md"] = []byte(filled)
mirror = true
}
return &squad.TaskPreparation{
Attachment: &squad.Attachment{
Filename: "CDO报价核算-V01.xlsx",
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
Data: data,
},
Backfills: backfills,
MirrorTemplateToOutput: mirror,
}, nil
}
// squadTaskPreparer 仅对 CDO 报价任务触发领域准备,其余任务快速跳过。
func (s *Service) squadTaskPreparer(workDir string, req squad.SquadRunRequest) (*squad.TaskPreparation, error) {
if !workflowdef.SameTaskKey(req.TaskKey, "cdo_quote") {
return nil, nil
}
return s.handleCDOQuotePreparation(workDir, req)
}
+9 -8
View File
@@ -193,14 +193,6 @@ func NewService(cfg *config.Config) (*Service, error) {
svc := &Service{
cfg: cfg,
kb: kb.NewClient(cfg),
squad: squad.NewService(squad.Options{
BackendName: cfg.AgentBackend,
BackendTimeout: cfg.AgentTimeout,
DataDir: cfg.DataDir,
NewBackend: func(name string) agent.Backend {
return agent.NewBackend(name, cfg)
},
}),
ctxMgr: ctxMgr,
cas: casStore,
materialIndex: matIndex,
@@ -211,6 +203,15 @@ func NewService(cfg *config.Config) (*Service, error) {
pool: pool,
schemaVersion: schemaVersion,
}
svc.squad = squad.NewService(squad.Options{
BackendName: cfg.AgentBackend,
BackendTimeout: cfg.AgentTimeout,
DataDir: cfg.DataDir,
NewBackend: func(name string) agent.Backend {
return agent.NewBackend(name, cfg)
},
TaskPreparer: svc.squadTaskPreparer,
})
svc.parts = Parts{
Pool: pool,
Gov: docGov,
+2 -2
View File
@@ -18,7 +18,7 @@ import (
"chat-agent-system/internal/config"
"chat-agent-system/internal/kb"
"chat-agent-system/internal/mineru"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -233,7 +233,7 @@ func (p *Pipeline) ResetForReparse(ctx context.Context, blobSHA string) bool {
// fallback text from squad.ExtractTextWithAnchors (may be empty).
func (p *Pipeline) EnsureParsed(ctx context.Context, fileName string, blobSHA string, data []byte) (markdown string, fromMinerU bool, err error) {
fallback := func() (string, bool, error) {
txt, e := squad.ExtractTextWithAnchors(fileName, data)
txt, e := textextract.ExtractTextWithAnchors(fileName, data)
return strings.TrimSpace(txt), false, e
}
if p == nil {
+2 -2
View File
@@ -15,7 +15,7 @@ import (
"chat-agent-system/internal/docgov"
"chat-agent-system/internal/intent"
"chat-agent-system/internal/projectinstance"
"chat-agent-system/internal/squad"
"chat-agent-system/internal/textextract"
"chat-agent-system/internal/storage"
"chat-agent-system/internal/stream"
"chat-agent-system/internal/workflowscope"
@@ -179,7 +179,7 @@ func (s *Server) resolveLatestUploadDocument(ctx context.Context, projectID int6
}
markdown = strings.TrimSpace(md)
} else {
md, extractErr := squad.ExtractTextWithAnchors(ref.FileName, data)
md, extractErr := textextract.ExtractTextWithAnchors(ref.FileName, data)
if extractErr != nil {
return nil, extractErr
}
+2 -1
View File
@@ -1,6 +1,7 @@
package squad
import (
"chat-agent-system/internal/textextract"
"encoding/json"
"fmt"
"os"
@@ -145,7 +146,7 @@ func (cm *ContextManager) InjectMaterials(wsDir string, materials []MaterialItem
name = "material.txt"
}
textContent, err := ExtractTextWithAnchors(name, m.Content)
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
if err != nil || strings.TrimSpace(textContent) == "" {
textContent = string(m.Content)
}
-171
View File
@@ -1,183 +1,12 @@
package squad
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path/filepath"
"strings"
"unicode/utf8"
"chat-agent-system/internal/workflowdef"
)
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
if len(data) == 0 {
return "", nil
}
ext := strings.ToLower(filepath.Ext(filename))
switch ext {
case ".txt", ".md", ".json", ".yaml", ".yml":
return string(data), nil
case ".docx":
txt, err := extractTextFromDOCX(data)
if err == nil && len(strings.TrimSpace(txt)) > 0 {
return injectPageAnchorsForText(txt), nil
}
// 降级:从二进制中提取可见字符
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
default:
// 通用降级
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
}
}
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
func injectPageAnchorsForText(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
// 若原始文本中包含换页符 \f
if strings.Contains(raw, "\f") {
pages := strings.Split(raw, "\f")
var sb strings.Builder
for i, page := range pages {
pTrim := strings.TrimSpace(page)
if pTrim == "" {
continue
}
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
}
return strings.TrimSpace(sb.String())
}
// 否则按段落累加约 1200 字符为一页
paragraphs := strings.Split(raw, "\n")
var sb strings.Builder
currentPage := 1
currentLen := 0
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
for _, p := range paragraphs {
pTrim := strings.TrimRight(p, "\r\t ")
pLen := utf8.RuneCountInString(pTrim)
if currentLen > 0 && currentLen+pLen > 1200 {
currentPage++
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
currentLen = 0
}
sb.WriteString(pTrim)
sb.WriteString("\n")
currentLen += pLen + 1
}
return strings.TrimSpace(sb.String())
}
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
func extractTextFromDOCX(data []byte) (string, error) {
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return "", err
}
var docFile *zip.File
for _, f := range r.File {
if f.Name == "word/document.xml" {
docFile = f
break
}
}
if docFile == nil {
return "", fmt.Errorf("word/document.xml not found in docx")
}
rc, err := docFile.Open()
if err != nil {
return "", err
}
defer rc.Close()
xmlBytes, err := io.ReadAll(rc)
if err != nil {
return "", err
}
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
var sb strings.Builder
inParagraph := false
for {
tok, err := decoder.Token()
if err != nil {
break
}
switch elem := tok.(type) {
case xml.StartElement:
if elem.Name.Local == "p" {
inParagraph = true
} else if elem.Name.Local == "br" {
for _, attr := range elem.Attr {
if attr.Name.Local == "type" && attr.Value == "page" {
sb.WriteString("\f")
}
}
} else if elem.Name.Local == "lastRenderedPageBreak" {
sb.WriteString("\f")
}
case xml.EndElement:
if elem.Name.Local == "p" {
if inParagraph {
sb.WriteString("\n")
inParagraph = false
}
}
case xml.CharData:
sb.Write(elem)
}
}
return sb.String(), nil
}
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
func extractAsciiUtf8Strings(data []byte) string {
var sb strings.Builder
var cur bytes.Buffer
flush := func() {
if cur.Len() >= 4 {
s := cur.String()
s = strings.TrimSpace(s)
if s != "" {
sb.WriteString(s)
sb.WriteString("\n")
}
}
cur.Reset()
}
for i := 0; i < len(data); {
r, size := utf8.DecodeRune(data[i:])
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
cur.WriteRune(r)
i += size
} else {
flush()
i++
}
}
flush()
return sb.String()
}
// ExtractAdvisorBriefing 从 output.md 与黑板事实中提取 B 轨业务参谋决策锦囊
func ExtractAdvisorBriefing(taskKey string, markdownBody string, fields map[string]string) *AdvisorBriefing {
if fields == nil {
+30 -45
View File
@@ -10,7 +10,6 @@ import (
"time"
"chat-agent-system/internal/agent"
"chat-agent-system/internal/quote"
"chat-agent-system/internal/workflowdef"
)
@@ -119,6 +118,20 @@ type Options struct {
// NewBackend 由调用方注入的后端工厂(Core 装配:凭证、环境、并发上限
// 留在产品侧)。nil 时返回错误,不静默降级 Stub。
NewBackend func(name string) agent.Backend
// TaskPreparer 为可选的任务准备钩子(KTD1:报价领域规则、核算与 xlsx
// 资产归 Core;Core 只做计算并返回产物,工作区文件由 squad 统一写入)。
// nil 或返回 nil 表示无预处理。
TaskPreparer func(workDir string, req SquadRunRequest) (*TaskPreparation, error)
}
// TaskPreparation 是领域准备钩子的产物:附件与工作区内回写内容。
// 路径均为工作区相对路径;物理写入由 squad 执行(工作区单写者)。
type TaskPreparation struct {
Attachment *Attachment
Backfills map[string][]byte
// MirrorTemplateToOutput 为 true 时,template.md 回填内容同时镜像到
// output.md(工作区产物镜像语义归 squad,Core 不直接命名 output.md)。
MirrorTemplateToOutput bool
}
func (o Options) withDefaults() Options {
@@ -265,12 +278,23 @@ func (s *Service) ExecuteTask(ctx context.Context, req SquadRunRequest) (*SquadR
return nil, fmt.Errorf("prepare workspace: %w", err)
}
// 2. 若为 CDO 报价任务,先执行 Excel 报价核算引擎并回填
// 2. 任务领域预处理(如 CDO 报价核算+xlsx 回填):Core 计算,squad 写入
var attachments []Attachment
if workflowdef.SameTaskKey(req.TaskKey, "cdo_quote") {
att, err := s.handleCDOQuotePreparation(workDir, req)
if err == nil && att != nil {
attachments = append(attachments, *att)
if s.opts.TaskPreparer != nil {
prep, err := s.opts.TaskPreparer(workDir, req)
if err == nil && prep != nil {
for rel, content := range prep.Backfills {
_ = os.MkdirAll(filepath.Dir(filepath.Join(workDir, rel)), 0o755)
_ = os.WriteFile(filepath.Join(workDir, rel), content, 0o644)
}
if prep.MirrorTemplateToOutput {
if filled, ok := prep.Backfills["template.md"]; ok {
_ = os.WriteFile(filepath.Join(workDir, "output.md"), filled, 0o644)
}
}
if prep.Attachment != nil {
attachments = append(attachments, *prep.Attachment)
}
}
}
@@ -535,45 +559,6 @@ func (s *Service) ResumeOrModifyWorkspace(ctx context.Context, req ModifyWorkflo
return s.ExecuteWorkflow(ctx, req.WorkDir, backendName, opts)
}
func (s *Service) handleCDOQuotePreparation(workDir string, req SquadRunRequest) (*Attachment, error) {
xlsxPath := filepath.Join(workDir, "CDO报价核算-V01.xlsx")
artifactXlsxPath := filepath.Join(workDir, "artifacts", "CDO报价核算-V01.xlsx")
in := quote.Input{
CustomerName: req.CustomerName,
ProductName: req.ProjectName,
Hours: map[string]float64{
"结构工程师": 320,
"硬件工程师": 320,
"软件工程师": 480,
"项目经理": 160,
},
}
summary, err := quote.CalculateAndFill(in, "templates/xlsx/cdo_quote.xlsx", xlsxPath)
if err != nil {
return nil, err
}
data, err := os.ReadFile(xlsxPath)
if err != nil {
return nil, err
}
_ = os.WriteFile(artifactXlsxPath, data, 0o644)
// 回填 template.md 中的报价数据
tmplPath := filepath.Join(workDir, "template.md")
if tmplBytes, err := os.ReadFile(tmplPath); err == nil {
filled := BackfillCDOQuoteMarkdown(string(tmplBytes), summary)
_ = os.WriteFile(tmplPath, []byte(filled), 0o644)
_ = os.WriteFile(filepath.Join(workDir, "output.md"), []byte(filled), 0o644)
}
return &Attachment{
Filename: "CDO报价核算-V01.xlsx",
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
Data: data,
}, nil
}
func (s *Service) loadWorkspaceMeta(wsDir string) (WorkspaceMetadata, error) {
p := filepath.Join(wsDir, "workspace.json")
-15
View File
@@ -1,15 +0,0 @@
package squad
import "chat-agent-system/internal/workspace"
// WorkspaceRefForRun is the only squad→workspace DTO. Physical paths stay inside workspace.
func WorkspaceRefForRun(runID string, projectID int64, env, token string) workspace.WorkspaceRef {
return workspace.WorkspaceRef{
ID: "run-" + runID,
SchemaVersion: workspace.SchemaVersion,
FencingToken: token,
ProjectScope: projectID,
RunID: runID,
Environment: env,
}
}
+2 -1
View File
@@ -1,6 +1,7 @@
package squad
import (
"chat-agent-system/internal/textextract"
"fmt"
"os"
"path/filepath"
@@ -68,7 +69,7 @@ func (e *WorkspaceEngine) buildContextSystem(workDir string, req WorkspacePrepar
}
// 自动提取纯文本并注入页码锚点 <!-- page N -->
textContent, err := ExtractTextWithAnchors(name, m.Content)
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
if err != nil || strings.TrimSpace(textContent) == "" {
textContent = string(m.Content)
}
+178
View File
@@ -0,0 +1,178 @@
package textextract
import (
"archive/zip"
"bytes"
"encoding/xml"
"fmt"
"io"
"path/filepath"
"strings"
"unicode/utf8"
)
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
if len(data) == 0 {
return "", nil
}
ext := strings.ToLower(filepath.Ext(filename))
switch ext {
case ".txt", ".md", ".json", ".yaml", ".yml":
return string(data), nil
case ".docx":
txt, err := extractTextFromDOCX(data)
if err == nil && len(strings.TrimSpace(txt)) > 0 {
return injectPageAnchorsForText(txt), nil
}
// 降级:从二进制中提取可见字符
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
default:
// 通用降级
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
}
}
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
func injectPageAnchorsForText(raw string) string {
raw = strings.TrimSpace(raw)
if raw == "" {
return ""
}
// 若原始文本中包含换页符 \f
if strings.Contains(raw, "\f") {
pages := strings.Split(raw, "\f")
var sb strings.Builder
for i, page := range pages {
pTrim := strings.TrimSpace(page)
if pTrim == "" {
continue
}
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
}
return strings.TrimSpace(sb.String())
}
// 否则按段落累加约 1200 字符为一页
paragraphs := strings.Split(raw, "\n")
var sb strings.Builder
currentPage := 1
currentLen := 0
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
for _, p := range paragraphs {
pTrim := strings.TrimRight(p, "\r\t ")
pLen := utf8.RuneCountInString(pTrim)
if currentLen > 0 && currentLen+pLen > 1200 {
currentPage++
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
currentLen = 0
}
sb.WriteString(pTrim)
sb.WriteString("\n")
currentLen += pLen + 1
}
return strings.TrimSpace(sb.String())
}
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
func extractTextFromDOCX(data []byte) (string, error) {
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return "", err
}
var docFile *zip.File
for _, f := range r.File {
if f.Name == "word/document.xml" {
docFile = f
break
}
}
if docFile == nil {
return "", fmt.Errorf("word/document.xml not found in docx")
}
rc, err := docFile.Open()
if err != nil {
return "", err
}
defer rc.Close()
xmlBytes, err := io.ReadAll(rc)
if err != nil {
return "", err
}
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
var sb strings.Builder
inParagraph := false
for {
tok, err := decoder.Token()
if err != nil {
break
}
switch elem := tok.(type) {
case xml.StartElement:
if elem.Name.Local == "p" {
inParagraph = true
} else if elem.Name.Local == "br" {
for _, attr := range elem.Attr {
if attr.Name.Local == "type" && attr.Value == "page" {
sb.WriteString("\f")
}
}
} else if elem.Name.Local == "lastRenderedPageBreak" {
sb.WriteString("\f")
}
case xml.EndElement:
if elem.Name.Local == "p" {
if inParagraph {
sb.WriteString("\n")
inParagraph = false
}
}
case xml.CharData:
sb.Write(elem)
}
}
return sb.String(), nil
}
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
func extractAsciiUtf8Strings(data []byte) string {
var sb strings.Builder
var cur bytes.Buffer
flush := func() {
if cur.Len() >= 4 {
s := cur.String()
s = strings.TrimSpace(s)
if s != "" {
sb.WriteString(s)
sb.WriteString("\n")
}
}
cur.Reset()
}
for i := 0; i < len(data); {
r, size := utf8.DecodeRune(data[i:])
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
cur.WriteRune(r)
i += size
} else {
flush()
i++
}
}
flush()
return sb.String()
}