feat(ctxmgr): 增加离线确定性高维嵌入生成器与向量字面量工具

Co-authored-by: Cursor <cursoragent@cursor.com>

internal/ctxmgr/embed.go

@@ -0,0 +1,113 @@
+package ctxmgr
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"strings"
+)
+
+// VectorEmbedder 统一向量化接口
+type VectorEmbedder interface {
+	Embed(ctx context.Context, texts []string) ([][]float32, error)
+}
+
+// DeterministicEmbedder 确定性特征高维投影嵌入器（无外部网络依赖，测试与离线兜底）
+type DeterministicEmbedder struct {
+	Dim int
+}
+
+// NewDeterministicEmbedder 创建确定性嵌入生成器
+func NewDeterministicEmbedder(dim int) *DeterministicEmbedder {
+	if dim <= 0 {
+		dim = 1024
+	}
+	return &DeterministicEmbedder{Dim: dim}
+}
+
+// Embed 为文本切片生成归一化的特征向量
+func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
+	if len(texts) == 0 {
+		return nil, nil
+	}
+
+	results := make([][]float32, len(texts))
+	for idx, text := range texts {
+		vec := make([]float32, e.Dim)
+		cleaned := strings.TrimSpace(strings.ToLower(text))
+		if cleaned == "" {
+			vec[0] = 1.0
+			results[idx] = vec
+			continue
+		}
+
+		// 分词提取（简单按空格或中文按单字/双字切分）
+		tokens := tokenizeForEmbedding(cleaned)
+		for _, tok := range tokens {
+			h := sha256.Sum256([]byte(tok))
+			for i := 0; i < e.Dim; i++ {
+				offset := (i * 4) % (len(h) - 4)
+				val := binary.BigEndian.Uint32(h[offset : offset+4])
+				// 映射到 [-1.0, 1.0]
+				fVal := float32((float64(val)/float64(math.MaxUint32))*2.0 - 1.0)
+				vec[i] += fVal
+			}
+		}
+
+		// L2 归一化 (Unit Normalization)
+		var sumSq float64
+		for _, v := range vec {
+			sumSq += float64(v * v)
+		}
+		norm := float32(math.Sqrt(sumSq))
+		if norm > 1e-9 {
+			for i := range vec {
+				vec[i] /= norm
+			}
+		} else {
+			vec[0] = 1.0
+		}
+
+		results[idx] = vec
+	}
+
+	return results, nil
+}
+
+// tokenizeForEmbedding 字符级与词级特征切分
+func tokenizeForEmbedding(s string) []string {
+	var tokens []string
+	words := strings.Fields(s)
+	if len(words) > 1 {
+		tokens = append(tokens, words...)
+	}
+
+	runes := []rune(s)
+	// 提取 2-gram 和单字符
+	for i := 0; i < len(runes); i++ {
+		tokens = append(tokens, string(runes[i]))
+		if i+1 < len(runes) {
+			tokens = append(tokens, string(runes[i:i+2]))
+		}
+	}
+	return tokens
+}
+
+// VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...]
+func VectorLiteral(vec []float32) string {
+	if len(vec) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	sb.WriteByte('[')
+	for i, f := range vec {
+		if i > 0 {
+			sb.WriteByte(',')
+		}
+		sb.WriteString(fmt.Sprintf("%g", f))
+	}
+	sb.WriteByte(']')
+	return sb.String()
+}

internal/ctxmgr/embed_test.go

@@ -0,0 +1,66 @@
+package ctxmgr_test
+
+import (
+	"context"
+	"math"
+	"testing"
+
+	"chat-agent-system/internal/ctxmgr"
+)
+
+func TestDeterministicEmbedder_EmbeddingProperties(t *testing.T) {
+	emb := ctxmgr.NewDeterministicEmbedder(1024)
+	ctx := context.Background()
+
+	texts := []string{
+		"超声软组织切割止血手术设备",
+		"微创外科硬管内窥镜摄像系统",
+		"超声软组织切割止血手术设备", // 重复文本验证确定性
+	}
+
+	vecs, err := emb.Embed(ctx, texts)
+	if err != nil {
+		t.Fatalf("embed failed: %v", err)
+	}
+	if len(vecs) != 3 {
+		t.Fatalf("expected 3 vectors, got %d", len(vecs))
+	}
+
+	// 1. 验证维度
+	for i, v := range vecs {
+		if len(v) != 1024 {
+			t.Fatalf("vector %d length expected 1024, got %d", i, len(v))
+		}
+		// 验证单位化模长 (L2 norm ≈ 1.0)
+		var sumSq float64
+		for _, val := range v {
+			sumSq += float64(val * val)
+		}
+		norm := math.Sqrt(sumSq)
+		if math.Abs(norm-1.0) > 1e-4 {
+			t.Fatalf("vector %d not normalized, norm=%f", i, norm)
+		}
+	}
+
+	// 2. 验证确定性：相同文本向量完全一致
+	for i := 0; i < 1024; i++ {
+		if vecs[0][i] != vecs[2][i] {
+			t.Fatalf("deterministic mismatch at index %d: %v vs %v", i, vecs[0][i], vecs[2][i])
+		}
+	}
+
+	// 3. 验证语义区别：不同文本余弦相似度应小于 0.95
+	var dotProduct float32
+	for i := 0; i < 1024; i++ {
+		dotProduct += vecs[0][i] * vecs[1][i]
+	}
+	if dotProduct > 0.95 {
+		t.Fatalf("distinct texts should not have near-identical embeddings: dotProduct=%f", dotProduct)
+	}
+
+	// 4. 验证 VectorLiteral
+	lit := ctxmgr.VectorLiteral(vecs[0][:3])
+	if lit == "" || lit[0] != '[' || lit[len(lit)-1] != ']' {
+		t.Fatalf("invalid vector literal: %s", lit)
+	}
+}