weichenandCursor
1fb59e12ef
大小阈值只决定全文注入还是目录加检索片段，不再决定是否切块。上传后的异步解析同样跳过 60KB 门槛。 Co-authored-by: Cursor <cursoragent@cursor.com>
46 lines
1.6 KiB
Go
46 lines
1.6 KiB
Go
package bizdev
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
"chat-agent-system/internal/config"
|
|
"chat-agent-system/internal/materialindex"
|
|
)
|
|
|
|
func TestMaterialToItems_SmallDocStillFulltextAfterIndex(t *testing.T) {
|
|
svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{}, nil)}
|
|
items, meta, err := svc.materialToItems(context.Background(), "note.txt", "abc123", []byte("能量参数是 2.5 bar"), 0, 0, 7, "能量")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if meta["inject_mode"] != "fulltext" {
|
|
t.Fatalf("small doc inject_mode=%v", meta["inject_mode"])
|
|
}
|
|
if meta["chunks_indexed"] != true {
|
|
t.Fatalf("small doc must attempt chunk index, meta=%v", meta)
|
|
}
|
|
if len(items) != 1 || !strings.Contains(string(items[0].Content), "能量参数是 2.5 bar") {
|
|
t.Fatalf("full text missing: %+v", items)
|
|
}
|
|
}
|
|
|
|
func TestMaterialToItems_LargeDocIndexesBeforeRAGInject(t *testing.T) {
|
|
svc := &Service{materialIndex: materialindex.NewPipeline(&config.Config{MaterialFulltextMaxBytes: 8}, nil)}
|
|
body := []byte("能量参数是 2.5 bar，并且还有更多正文用来越过全文阈值。")
|
|
items, meta, err := svc.materialToItems(context.Background(), "note.txt", "def456", body, 0, 0, 7, "能量")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if meta["chunks_indexed"] != true {
|
|
t.Fatalf("large doc must attempt chunk index, meta=%v", meta)
|
|
}
|
|
if meta["inject_mode"] != "rag" && meta["inject_mode"] != "rag_degraded" {
|
|
t.Fatalf("large doc must not use fulltext, inject_mode=%v", meta["inject_mode"])
|
|
}
|
|
if len(items) != 2 {
|
|
t.Fatalf("want toc+rag items, got %d", len(items))
|
|
}
|
|
}
|