Merge pull request 'U4 第三步:textextract 抽包,消除 materialindex→squad 反向依赖' (#7) from feature/textextract-package-2026-09-29 into dev
This commit was merged in pull request #7.
This commit is contained in:
@@ -27,8 +27,8 @@ imports:
|
||||
- docduty
|
||||
bizdev:
|
||||
- agent
|
||||
- authz
|
||||
- app/uow
|
||||
- authz
|
||||
- config
|
||||
- conflictresolve
|
||||
- contracts/vocabulary
|
||||
@@ -50,6 +50,7 @@ imports:
|
||||
- squad
|
||||
- storage
|
||||
- stream
|
||||
- textextract
|
||||
- workflowdef
|
||||
- workspace
|
||||
chat:
|
||||
@@ -214,6 +215,7 @@ imports:
|
||||
- kb
|
||||
- mineru
|
||||
- squad
|
||||
- textextract
|
||||
maturity: []
|
||||
mineru: []
|
||||
notification: []
|
||||
@@ -272,22 +274,22 @@ imports:
|
||||
- storage
|
||||
- stream
|
||||
- stream/transport
|
||||
- supplementimpact
|
||||
- swboot
|
||||
- sysadmin
|
||||
- taskrun
|
||||
- textextract
|
||||
- traceimpact
|
||||
- upload
|
||||
- workflowdef
|
||||
- workflowscope
|
||||
- workspace
|
||||
- supplementimpact
|
||||
squad:
|
||||
- agent
|
||||
- config
|
||||
- quote
|
||||
- stream
|
||||
- template
|
||||
- workspace
|
||||
- textextract
|
||||
- workflowdef
|
||||
storage:
|
||||
- config
|
||||
@@ -306,6 +308,7 @@ imports:
|
||||
- projectworkflow
|
||||
- stream
|
||||
- workspace
|
||||
textextract: []
|
||||
template:
|
||||
- workflowdef
|
||||
upload: []
|
||||
|
||||
@@ -1,25 +1,9 @@
|
||||
package bizdev
|
||||
|
||||
import (
|
||||
"chat-agent-system/internal/quote"
|
||||
"chat-agent-system/internal/squad"
|
||||
)
|
||||
|
||||
// FormatMoney formats a currency amount for display in markdown (no grouping separators).
|
||||
func FormatMoney(v float64) string {
|
||||
return squad.FormatMoney(v)
|
||||
}
|
||||
|
||||
// RenderCDOQuoteTable builds the pricing table from quote summary lines and total.
|
||||
func RenderCDOQuoteTable(summary *quote.Summary) string {
|
||||
return squad.RenderCDOQuoteTable(summary)
|
||||
}
|
||||
|
||||
// BackfillCDOQuoteMarkdown replaces placeholder sections with computed pricing from summary.
|
||||
func BackfillCDOQuoteMarkdown(body string, summary *quote.Summary) string {
|
||||
return squad.BackfillCDOQuoteMarkdown(body, summary)
|
||||
}
|
||||
|
||||
// WriteMaterials persists uploaded material text into workspace materials_text/.
|
||||
func WriteMaterials(workDir string, materials []MaterialInput) error {
|
||||
var items []squad.MaterialItem
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"chat-agent-system/internal/materialindex"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
"chat-agent-system/internal/storage"
|
||||
)
|
||||
|
||||
@@ -154,7 +155,7 @@ func (s *Service) materialToItems(
|
||||
meta["parse_engine"] = "mineru"
|
||||
}
|
||||
} else {
|
||||
text, err = squad.ExtractTextWithAnchors(fileName, data)
|
||||
text, err = textextract.ExtractTextWithAnchors(fileName, data)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("extract material %q: %w", fileName, err)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package squad
|
||||
package bizdev
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -0,0 +1,74 @@
|
||||
package bizdev
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"chat-agent-system/internal/quote"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/workflowdef"
|
||||
)
|
||||
|
||||
// handleCDOQuotePreparation 为 CDO 报价任务执行 Excel 核算并生成回填内容
|
||||
// (KTD1:报价领域规则、核算与 xlsx 资产归 Core;只计算,不写 squad 工作区,
|
||||
// 文件由 squad 经 TaskPreparation.Backfills 统一写入)。
|
||||
func (s *Service) handleCDOQuotePreparation(workDir string, req squad.SquadRunRequest) (*squad.TaskPreparation, error) {
|
||||
in := quote.Input{
|
||||
CustomerName: req.CustomerName,
|
||||
ProductName: req.ProjectName,
|
||||
Hours: map[string]float64{
|
||||
"结构工程师": 320,
|
||||
"硬件工程师": 320,
|
||||
"软件工程师": 480,
|
||||
"项目经理": 160,
|
||||
},
|
||||
}
|
||||
// 核算写入临时文件后读回字节;工作区文件由 squad 写入。
|
||||
tmp, err := os.CreateTemp("", "cdo-quote-*.xlsx")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmpPath := tmp.Name()
|
||||
tmp.Close()
|
||||
defer os.Remove(tmpPath)
|
||||
|
||||
summary, err := quote.CalculateAndFill(in, "templates/xlsx/cdo_quote.xlsx", tmpPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := os.ReadFile(tmpPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
backfills := map[string][]byte{
|
||||
"CDO报价核算-V01.xlsx": data,
|
||||
"artifacts/CDO报价核算-V01.xlsx": data,
|
||||
}
|
||||
// 回填 template.md 报价占位(模板存在时);output.md 镜像由 squad 决定
|
||||
tmplBytes, tmplErr := os.ReadFile(filepath.Join(workDir, "template.md"))
|
||||
mirror := false
|
||||
if tmplErr == nil {
|
||||
filled := BackfillCDOQuoteMarkdown(string(tmplBytes), summary)
|
||||
backfills["template.md"] = []byte(filled)
|
||||
mirror = true
|
||||
}
|
||||
|
||||
return &squad.TaskPreparation{
|
||||
Attachment: &squad.Attachment{
|
||||
Filename: "CDO报价核算-V01.xlsx",
|
||||
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
Data: data,
|
||||
},
|
||||
Backfills: backfills,
|
||||
MirrorTemplateToOutput: mirror,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// squadTaskPreparer 仅对 CDO 报价任务触发领域准备,其余任务快速跳过。
|
||||
func (s *Service) squadTaskPreparer(workDir string, req squad.SquadRunRequest) (*squad.TaskPreparation, error) {
|
||||
if !workflowdef.SameTaskKey(req.TaskKey, "cdo_quote") {
|
||||
return nil, nil
|
||||
}
|
||||
return s.handleCDOQuotePreparation(workDir, req)
|
||||
}
|
||||
@@ -193,14 +193,6 @@ func NewService(cfg *config.Config) (*Service, error) {
|
||||
svc := &Service{
|
||||
cfg: cfg,
|
||||
kb: kb.NewClient(cfg),
|
||||
squad: squad.NewService(squad.Options{
|
||||
BackendName: cfg.AgentBackend,
|
||||
BackendTimeout: cfg.AgentTimeout,
|
||||
DataDir: cfg.DataDir,
|
||||
NewBackend: func(name string) agent.Backend {
|
||||
return agent.NewBackend(name, cfg)
|
||||
},
|
||||
}),
|
||||
ctxMgr: ctxMgr,
|
||||
cas: casStore,
|
||||
materialIndex: matIndex,
|
||||
@@ -211,6 +203,15 @@ func NewService(cfg *config.Config) (*Service, error) {
|
||||
pool: pool,
|
||||
schemaVersion: schemaVersion,
|
||||
}
|
||||
svc.squad = squad.NewService(squad.Options{
|
||||
BackendName: cfg.AgentBackend,
|
||||
BackendTimeout: cfg.AgentTimeout,
|
||||
DataDir: cfg.DataDir,
|
||||
NewBackend: func(name string) agent.Backend {
|
||||
return agent.NewBackend(name, cfg)
|
||||
},
|
||||
TaskPreparer: svc.squadTaskPreparer,
|
||||
})
|
||||
svc.parts = Parts{
|
||||
Pool: pool,
|
||||
Gov: docGov,
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"chat-agent-system/internal/config"
|
||||
"chat-agent-system/internal/kb"
|
||||
"chat-agent-system/internal/mineru"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -233,7 +233,7 @@ func (p *Pipeline) ResetForReparse(ctx context.Context, blobSHA string) bool {
|
||||
// fallback text from squad.ExtractTextWithAnchors (may be empty).
|
||||
func (p *Pipeline) EnsureParsed(ctx context.Context, fileName string, blobSHA string, data []byte) (markdown string, fromMinerU bool, err error) {
|
||||
fallback := func() (string, bool, error) {
|
||||
txt, e := squad.ExtractTextWithAnchors(fileName, data)
|
||||
txt, e := textextract.ExtractTextWithAnchors(fileName, data)
|
||||
return strings.TrimSpace(txt), false, e
|
||||
}
|
||||
if p == nil {
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"chat-agent-system/internal/docgov"
|
||||
"chat-agent-system/internal/intent"
|
||||
"chat-agent-system/internal/projectinstance"
|
||||
"chat-agent-system/internal/squad"
|
||||
"chat-agent-system/internal/textextract"
|
||||
"chat-agent-system/internal/storage"
|
||||
"chat-agent-system/internal/stream"
|
||||
"chat-agent-system/internal/workflowscope"
|
||||
@@ -179,7 +179,7 @@ func (s *Server) resolveLatestUploadDocument(ctx context.Context, projectID int6
|
||||
}
|
||||
markdown = strings.TrimSpace(md)
|
||||
} else {
|
||||
md, extractErr := squad.ExtractTextWithAnchors(ref.FileName, data)
|
||||
md, extractErr := textextract.ExtractTextWithAnchors(ref.FileName, data)
|
||||
if extractErr != nil {
|
||||
return nil, extractErr
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"chat-agent-system/internal/textextract"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -145,7 +146,7 @@ func (cm *ContextManager) InjectMaterials(wsDir string, materials []MaterialItem
|
||||
name = "material.txt"
|
||||
}
|
||||
|
||||
textContent, err := ExtractTextWithAnchors(name, m.Content)
|
||||
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
|
||||
if err != nil || strings.TrimSpace(textContent) == "" {
|
||||
textContent = string(m.Content)
|
||||
}
|
||||
|
||||
@@ -1,183 +1,12 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"chat-agent-system/internal/workflowdef"
|
||||
)
|
||||
|
||||
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
|
||||
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
case ".txt", ".md", ".json", ".yaml", ".yml":
|
||||
return string(data), nil
|
||||
case ".docx":
|
||||
txt, err := extractTextFromDOCX(data)
|
||||
if err == nil && len(strings.TrimSpace(txt)) > 0 {
|
||||
return injectPageAnchorsForText(txt), nil
|
||||
}
|
||||
// 降级:从二进制中提取可见字符
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
default:
|
||||
// 通用降级
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
|
||||
func injectPageAnchorsForText(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 若原始文本中包含换页符 \f
|
||||
if strings.Contains(raw, "\f") {
|
||||
pages := strings.Split(raw, "\f")
|
||||
var sb strings.Builder
|
||||
for i, page := range pages {
|
||||
pTrim := strings.TrimSpace(page)
|
||||
if pTrim == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// 否则按段落累加约 1200 字符为一页
|
||||
paragraphs := strings.Split(raw, "\n")
|
||||
var sb strings.Builder
|
||||
currentPage := 1
|
||||
currentLen := 0
|
||||
|
||||
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
|
||||
for _, p := range paragraphs {
|
||||
pTrim := strings.TrimRight(p, "\r\t ")
|
||||
pLen := utf8.RuneCountInString(pTrim)
|
||||
if currentLen > 0 && currentLen+pLen > 1200 {
|
||||
currentPage++
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
|
||||
currentLen = 0
|
||||
}
|
||||
sb.WriteString(pTrim)
|
||||
sb.WriteString("\n")
|
||||
currentLen += pLen + 1
|
||||
}
|
||||
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
|
||||
func extractTextFromDOCX(data []byte) (string, error) {
|
||||
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var docFile *zip.File
|
||||
for _, f := range r.File {
|
||||
if f.Name == "word/document.xml" {
|
||||
docFile = f
|
||||
break
|
||||
}
|
||||
}
|
||||
if docFile == nil {
|
||||
return "", fmt.Errorf("word/document.xml not found in docx")
|
||||
}
|
||||
|
||||
rc, err := docFile.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
xmlBytes, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
|
||||
var sb strings.Builder
|
||||
inParagraph := false
|
||||
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch elem := tok.(type) {
|
||||
case xml.StartElement:
|
||||
if elem.Name.Local == "p" {
|
||||
inParagraph = true
|
||||
} else if elem.Name.Local == "br" {
|
||||
for _, attr := range elem.Attr {
|
||||
if attr.Name.Local == "type" && attr.Value == "page" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
}
|
||||
} else if elem.Name.Local == "lastRenderedPageBreak" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem.Name.Local == "p" {
|
||||
if inParagraph {
|
||||
sb.WriteString("\n")
|
||||
inParagraph = false
|
||||
}
|
||||
}
|
||||
case xml.CharData:
|
||||
sb.Write(elem)
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
|
||||
func extractAsciiUtf8Strings(data []byte) string {
|
||||
var sb strings.Builder
|
||||
var cur bytes.Buffer
|
||||
|
||||
flush := func() {
|
||||
if cur.Len() >= 4 {
|
||||
s := cur.String()
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" {
|
||||
sb.WriteString(s)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
cur.Reset()
|
||||
}
|
||||
|
||||
for i := 0; i < len(data); {
|
||||
r, size := utf8.DecodeRune(data[i:])
|
||||
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
|
||||
cur.WriteRune(r)
|
||||
i += size
|
||||
} else {
|
||||
flush()
|
||||
i++
|
||||
}
|
||||
}
|
||||
flush()
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// ExtractAdvisorBriefing 从 output.md 与黑板事实中提取 B 轨业务参谋决策锦囊
|
||||
func ExtractAdvisorBriefing(taskKey string, markdownBody string, fields map[string]string) *AdvisorBriefing {
|
||||
if fields == nil {
|
||||
|
||||
+30
-45
@@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"chat-agent-system/internal/agent"
|
||||
"chat-agent-system/internal/quote"
|
||||
"chat-agent-system/internal/workflowdef"
|
||||
)
|
||||
|
||||
@@ -119,6 +118,20 @@ type Options struct {
|
||||
// NewBackend 由调用方注入的后端工厂(Core 装配:凭证、环境、并发上限
|
||||
// 留在产品侧)。nil 时返回错误,不静默降级 Stub。
|
||||
NewBackend func(name string) agent.Backend
|
||||
// TaskPreparer 为可选的任务准备钩子(KTD1:报价领域规则、核算与 xlsx
|
||||
// 资产归 Core;Core 只做计算并返回产物,工作区文件由 squad 统一写入)。
|
||||
// nil 或返回 nil 表示无预处理。
|
||||
TaskPreparer func(workDir string, req SquadRunRequest) (*TaskPreparation, error)
|
||||
}
|
||||
|
||||
// TaskPreparation 是领域准备钩子的产物:附件与工作区内回写内容。
|
||||
// 路径均为工作区相对路径;物理写入由 squad 执行(工作区单写者)。
|
||||
type TaskPreparation struct {
|
||||
Attachment *Attachment
|
||||
Backfills map[string][]byte
|
||||
// MirrorTemplateToOutput 为 true 时,template.md 回填内容同时镜像到
|
||||
// output.md(工作区产物镜像语义归 squad,Core 不直接命名 output.md)。
|
||||
MirrorTemplateToOutput bool
|
||||
}
|
||||
|
||||
func (o Options) withDefaults() Options {
|
||||
@@ -265,12 +278,23 @@ func (s *Service) ExecuteTask(ctx context.Context, req SquadRunRequest) (*SquadR
|
||||
return nil, fmt.Errorf("prepare workspace: %w", err)
|
||||
}
|
||||
|
||||
// 2. 若为 CDO 报价任务,先执行 Excel 报价核算引擎并回填
|
||||
// 2. 任务领域预处理(如 CDO 报价核算+xlsx 回填):Core 计算,squad 写入
|
||||
var attachments []Attachment
|
||||
if workflowdef.SameTaskKey(req.TaskKey, "cdo_quote") {
|
||||
att, err := s.handleCDOQuotePreparation(workDir, req)
|
||||
if err == nil && att != nil {
|
||||
attachments = append(attachments, *att)
|
||||
if s.opts.TaskPreparer != nil {
|
||||
prep, err := s.opts.TaskPreparer(workDir, req)
|
||||
if err == nil && prep != nil {
|
||||
for rel, content := range prep.Backfills {
|
||||
_ = os.MkdirAll(filepath.Dir(filepath.Join(workDir, rel)), 0o755)
|
||||
_ = os.WriteFile(filepath.Join(workDir, rel), content, 0o644)
|
||||
}
|
||||
if prep.MirrorTemplateToOutput {
|
||||
if filled, ok := prep.Backfills["template.md"]; ok {
|
||||
_ = os.WriteFile(filepath.Join(workDir, "output.md"), filled, 0o644)
|
||||
}
|
||||
}
|
||||
if prep.Attachment != nil {
|
||||
attachments = append(attachments, *prep.Attachment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -535,45 +559,6 @@ func (s *Service) ResumeOrModifyWorkspace(ctx context.Context, req ModifyWorkflo
|
||||
return s.ExecuteWorkflow(ctx, req.WorkDir, backendName, opts)
|
||||
}
|
||||
|
||||
func (s *Service) handleCDOQuotePreparation(workDir string, req SquadRunRequest) (*Attachment, error) {
|
||||
xlsxPath := filepath.Join(workDir, "CDO报价核算-V01.xlsx")
|
||||
artifactXlsxPath := filepath.Join(workDir, "artifacts", "CDO报价核算-V01.xlsx")
|
||||
|
||||
in := quote.Input{
|
||||
CustomerName: req.CustomerName,
|
||||
ProductName: req.ProjectName,
|
||||
Hours: map[string]float64{
|
||||
"结构工程师": 320,
|
||||
"硬件工程师": 320,
|
||||
"软件工程师": 480,
|
||||
"项目经理": 160,
|
||||
},
|
||||
}
|
||||
summary, err := quote.CalculateAndFill(in, "templates/xlsx/cdo_quote.xlsx", xlsxPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(xlsxPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = os.WriteFile(artifactXlsxPath, data, 0o644)
|
||||
|
||||
// 回填 template.md 中的报价数据
|
||||
tmplPath := filepath.Join(workDir, "template.md")
|
||||
if tmplBytes, err := os.ReadFile(tmplPath); err == nil {
|
||||
filled := BackfillCDOQuoteMarkdown(string(tmplBytes), summary)
|
||||
_ = os.WriteFile(tmplPath, []byte(filled), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(workDir, "output.md"), []byte(filled), 0o644)
|
||||
}
|
||||
|
||||
return &Attachment{
|
||||
Filename: "CDO报价核算-V01.xlsx",
|
||||
ContentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
Data: data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) loadWorkspaceMeta(wsDir string) (WorkspaceMetadata, error) {
|
||||
p := filepath.Join(wsDir, "workspace.json")
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
package squad
|
||||
|
||||
import "chat-agent-system/internal/workspace"
|
||||
|
||||
// WorkspaceRefForRun is the only squad→workspace DTO. Physical paths stay inside workspace.
|
||||
func WorkspaceRefForRun(runID string, projectID int64, env, token string) workspace.WorkspaceRef {
|
||||
return workspace.WorkspaceRef{
|
||||
ID: "run-" + runID,
|
||||
SchemaVersion: workspace.SchemaVersion,
|
||||
FencingToken: token,
|
||||
ProjectScope: projectID,
|
||||
RunID: runID,
|
||||
Environment: env,
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package squad
|
||||
|
||||
import (
|
||||
"chat-agent-system/internal/textextract"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -68,7 +69,7 @@ func (e *WorkspaceEngine) buildContextSystem(workDir string, req WorkspacePrepar
|
||||
}
|
||||
|
||||
// 自动提取纯文本并注入页码锚点 <!-- page N -->
|
||||
textContent, err := ExtractTextWithAnchors(name, m.Content)
|
||||
textContent, err := textextract.ExtractTextWithAnchors(name, m.Content)
|
||||
if err != nil || strings.TrimSpace(textContent) == "" {
|
||||
textContent = string(m.Content)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
package textextract
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
)
|
||||
|
||||
// ExtractTextWithAnchors 从不同格式材料中提取纯文本,并自动注入物理页码锚点 <!-- page N -->
|
||||
func ExtractTextWithAnchors(filename string, data []byte) (string, error) {
|
||||
if len(data) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
case ".txt", ".md", ".json", ".yaml", ".yml":
|
||||
return string(data), nil
|
||||
case ".docx":
|
||||
txt, err := extractTextFromDOCX(data)
|
||||
if err == nil && len(strings.TrimSpace(txt)) > 0 {
|
||||
return injectPageAnchorsForText(txt), nil
|
||||
}
|
||||
// 降级:从二进制中提取可见字符
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
default:
|
||||
// 通用降级
|
||||
return injectPageAnchorsForText(extractAsciiUtf8Strings(data)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// injectPageAnchorsForText 将连续文本按约 1200 字符切分,并注入 <!-- page N --> 锚点
|
||||
func injectPageAnchorsForText(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 若原始文本中包含换页符 \f
|
||||
if strings.Contains(raw, "\f") {
|
||||
pages := strings.Split(raw, "\f")
|
||||
var sb strings.Builder
|
||||
for i, page := range pages {
|
||||
pTrim := strings.TrimSpace(page)
|
||||
if pTrim == "" {
|
||||
continue
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n%s\n", i+1, pTrim))
|
||||
}
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// 否则按段落累加约 1200 字符为一页
|
||||
paragraphs := strings.Split(raw, "\n")
|
||||
var sb strings.Builder
|
||||
currentPage := 1
|
||||
currentLen := 0
|
||||
|
||||
sb.WriteString(fmt.Sprintf("<!-- page %d -->\n", currentPage))
|
||||
for _, p := range paragraphs {
|
||||
pTrim := strings.TrimRight(p, "\r\t ")
|
||||
pLen := utf8.RuneCountInString(pTrim)
|
||||
if currentLen > 0 && currentLen+pLen > 1200 {
|
||||
currentPage++
|
||||
sb.WriteString(fmt.Sprintf("\n<!-- page %d -->\n", currentPage))
|
||||
currentLen = 0
|
||||
}
|
||||
sb.WriteString(pTrim)
|
||||
sb.WriteString("\n")
|
||||
currentLen += pLen + 1
|
||||
}
|
||||
|
||||
return strings.TrimSpace(sb.String())
|
||||
}
|
||||
|
||||
// extractTextFromDOCX 解析 docx 的 word/document.xml 提取段落文本
|
||||
func extractTextFromDOCX(data []byte) (string, error) {
|
||||
r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var docFile *zip.File
|
||||
for _, f := range r.File {
|
||||
if f.Name == "word/document.xml" {
|
||||
docFile = f
|
||||
break
|
||||
}
|
||||
}
|
||||
if docFile == nil {
|
||||
return "", fmt.Errorf("word/document.xml not found in docx")
|
||||
}
|
||||
|
||||
rc, err := docFile.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
xmlBytes, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decoder := xml.NewDecoder(bytes.NewReader(xmlBytes))
|
||||
var sb strings.Builder
|
||||
inParagraph := false
|
||||
|
||||
for {
|
||||
tok, err := decoder.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch elem := tok.(type) {
|
||||
case xml.StartElement:
|
||||
if elem.Name.Local == "p" {
|
||||
inParagraph = true
|
||||
} else if elem.Name.Local == "br" {
|
||||
for _, attr := range elem.Attr {
|
||||
if attr.Name.Local == "type" && attr.Value == "page" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
}
|
||||
} else if elem.Name.Local == "lastRenderedPageBreak" {
|
||||
sb.WriteString("\f")
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem.Name.Local == "p" {
|
||||
if inParagraph {
|
||||
sb.WriteString("\n")
|
||||
inParagraph = false
|
||||
}
|
||||
}
|
||||
case xml.CharData:
|
||||
sb.Write(elem)
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// extractAsciiUtf8Strings 降级算法:从二进制流中提取连续的有效可读 UTF-8 文本
|
||||
func extractAsciiUtf8Strings(data []byte) string {
|
||||
var sb strings.Builder
|
||||
var cur bytes.Buffer
|
||||
|
||||
flush := func() {
|
||||
if cur.Len() >= 4 {
|
||||
s := cur.String()
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" {
|
||||
sb.WriteString(s)
|
||||
sb.WriteString("\n")
|
||||
}
|
||||
}
|
||||
cur.Reset()
|
||||
}
|
||||
|
||||
for i := 0; i < len(data); {
|
||||
r, size := utf8.DecodeRune(data[i:])
|
||||
if size > 0 && r != utf8.RuneError && (r >= 32 || r == '\n' || r == '\t' || r == '\r') {
|
||||
cur.WriteRune(r)
|
||||
i += size
|
||||
} else {
|
||||
flush()
|
||||
i++
|
||||
}
|
||||
}
|
||||
flush()
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
Reference in New Issue
Block a user