diff --git a/internal/ctxmgr/embed.go b/internal/ctxmgr/embed.go new file mode 100644 index 00000000..8b87d812 --- /dev/null +++ b/internal/ctxmgr/embed.go @@ -0,0 +1,113 @@ +package ctxmgr + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "math" + "strings" +) + +// VectorEmbedder 统一向量化接口 +type VectorEmbedder interface { + Embed(ctx context.Context, texts []string) ([][]float32, error) +} + +// DeterministicEmbedder 确定性特征高维投影嵌入器(无外部网络依赖,测试与离线兜底) +type DeterministicEmbedder struct { + Dim int +} + +// NewDeterministicEmbedder 创建确定性嵌入生成器 +func NewDeterministicEmbedder(dim int) *DeterministicEmbedder { + if dim <= 0 { + dim = 1024 + } + return &DeterministicEmbedder{Dim: dim} +} + +// Embed 为文本切片生成归一化的特征向量 +func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) { + if len(texts) == 0 { + return nil, nil + } + + results := make([][]float32, len(texts)) + for idx, text := range texts { + vec := make([]float32, e.Dim) + cleaned := strings.TrimSpace(strings.ToLower(text)) + if cleaned == "" { + vec[0] = 1.0 + results[idx] = vec + continue + } + + // 分词提取(简单按空格或中文按单字/双字切分) + tokens := tokenizeForEmbedding(cleaned) + for _, tok := range tokens { + h := sha256.Sum256([]byte(tok)) + for i := 0; i < e.Dim; i++ { + offset := (i * 4) % (len(h) - 4) + val := binary.BigEndian.Uint32(h[offset : offset+4]) + // 映射到 [-1.0, 1.0] + fVal := float32((float64(val)/float64(math.MaxUint32))*2.0 - 1.0) + vec[i] += fVal + } + } + + // L2 归一化 (Unit Normalization) + var sumSq float64 + for _, v := range vec { + sumSq += float64(v * v) + } + norm := float32(math.Sqrt(sumSq)) + if norm > 1e-9 { + for i := range vec { + vec[i] /= norm + } + } else { + vec[0] = 1.0 + } + + results[idx] = vec + } + + return results, nil +} + +// tokenizeForEmbedding 字符级与词级特征切分 +func tokenizeForEmbedding(s string) []string { + var tokens []string + words := strings.Fields(s) + if len(words) > 1 { + tokens = append(tokens, words...) + } + + runes := []rune(s) + // 提取 2-gram 和单字符 + for i := 0; i < len(runes); i++ { + tokens = append(tokens, string(runes[i])) + if i+1 < len(runes) { + tokens = append(tokens, string(runes[i:i+2])) + } + } + return tokens +} + +// VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...] +func VectorLiteral(vec []float32) string { + if len(vec) == 0 { + return "" + } + var sb strings.Builder + sb.WriteByte('[') + for i, f := range vec { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(fmt.Sprintf("%g", f)) + } + sb.WriteByte(']') + return sb.String() +} diff --git a/internal/ctxmgr/embed_test.go b/internal/ctxmgr/embed_test.go new file mode 100644 index 00000000..db690624 --- /dev/null +++ b/internal/ctxmgr/embed_test.go @@ -0,0 +1,66 @@ +package ctxmgr_test + +import ( + "context" + "math" + "testing" + + "chat-agent-system/internal/ctxmgr" +) + +func TestDeterministicEmbedder_EmbeddingProperties(t *testing.T) { + emb := ctxmgr.NewDeterministicEmbedder(1024) + ctx := context.Background() + + texts := []string{ + "超声软组织切割止血手术设备", + "微创外科硬管内窥镜摄像系统", + "超声软组织切割止血手术设备", // 重复文本验证确定性 + } + + vecs, err := emb.Embed(ctx, texts) + if err != nil { + t.Fatalf("embed failed: %v", err) + } + if len(vecs) != 3 { + t.Fatalf("expected 3 vectors, got %d", len(vecs)) + } + + // 1. 验证维度 + for i, v := range vecs { + if len(v) != 1024 { + t.Fatalf("vector %d length expected 1024, got %d", i, len(v)) + } + // 验证单位化模长 (L2 norm ≈ 1.0) + var sumSq float64 + for _, val := range v { + sumSq += float64(val * val) + } + norm := math.Sqrt(sumSq) + if math.Abs(norm-1.0) > 1e-4 { + t.Fatalf("vector %d not normalized, norm=%f", i, norm) + } + } + + // 2. 验证确定性:相同文本向量完全一致 + for i := 0; i < 1024; i++ { + if vecs[0][i] != vecs[2][i] { + t.Fatalf("deterministic mismatch at index %d: %v vs %v", i, vecs[0][i], vecs[2][i]) + } + } + + // 3. 验证语义区别:不同文本余弦相似度应小于 0.95 + var dotProduct float32 + for i := 0; i < 1024; i++ { + dotProduct += vecs[0][i] * vecs[1][i] + } + if dotProduct > 0.95 { + t.Fatalf("distinct texts should not have near-identical embeddings: dotProduct=%f", dotProduct) + } + + // 4. 验证 VectorLiteral + lit := ctxmgr.VectorLiteral(vecs[0][:3]) + if lit == "" || lit[0] != '[' || lit[len(lit)-1] != ']' { + t.Fatalf("invalid vector literal: %s", lit) + } +}