114 lines
2.5 KiB
Go
114 lines
2.5 KiB
Go
package ctxmgr
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"math"
|
|
"strings"
|
|
)
|
|
|
|
// VectorEmbedder 统一向量化接口
|
|
type VectorEmbedder interface {
|
|
Embed(ctx context.Context, texts []string) ([][]float32, error)
|
|
}
|
|
|
|
// DeterministicEmbedder 确定性特征高维投影嵌入器（无外部网络依赖，测试与离线兜底）
|
|
type DeterministicEmbedder struct {
|
|
Dim int
|
|
}
|
|
|
|
// NewDeterministicEmbedder 创建确定性嵌入生成器
|
|
func NewDeterministicEmbedder(dim int) *DeterministicEmbedder {
|
|
if dim <= 0 {
|
|
dim = 1024
|
|
}
|
|
return &DeterministicEmbedder{Dim: dim}
|
|
}
|
|
|
|
// Embed 为文本切片生成归一化的特征向量
|
|
func (e *DeterministicEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
|
if len(texts) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
results := make([][]float32, len(texts))
|
|
for idx, text := range texts {
|
|
vec := make([]float32, e.Dim)
|
|
cleaned := strings.TrimSpace(strings.ToLower(text))
|
|
if cleaned == "" {
|
|
vec[0] = 1.0
|
|
results[idx] = vec
|
|
continue
|
|
}
|
|
|
|
// 分词提取（简单按空格或中文按单字/双字切分）
|
|
tokens := tokenizeForEmbedding(cleaned)
|
|
for _, tok := range tokens {
|
|
h := sha256.Sum256([]byte(tok))
|
|
for i := 0; i < e.Dim; i++ {
|
|
offset := (i * 4) % (len(h) - 4)
|
|
val := binary.BigEndian.Uint32(h[offset : offset+4])
|
|
// 映射到 [-1.0, 1.0]
|
|
fVal := float32((float64(val)/float64(math.MaxUint32))*2.0 - 1.0)
|
|
vec[i] += fVal
|
|
}
|
|
}
|
|
|
|
// L2 归一化 (Unit Normalization)
|
|
var sumSq float64
|
|
for _, v := range vec {
|
|
sumSq += float64(v * v)
|
|
}
|
|
norm := float32(math.Sqrt(sumSq))
|
|
if norm > 1e-9 {
|
|
for i := range vec {
|
|
vec[i] /= norm
|
|
}
|
|
} else {
|
|
vec[0] = 1.0
|
|
}
|
|
|
|
results[idx] = vec
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
// tokenizeForEmbedding 字符级与词级特征切分
|
|
func tokenizeForEmbedding(s string) []string {
|
|
var tokens []string
|
|
words := strings.Fields(s)
|
|
if len(words) > 1 {
|
|
tokens = append(tokens, words...)
|
|
}
|
|
|
|
runes := []rune(s)
|
|
// 提取 2-gram 和单字符
|
|
for i := 0; i < len(runes); i++ {
|
|
tokens = append(tokens, string(runes[i]))
|
|
if i+1 < len(runes) {
|
|
tokens = append(tokens, string(runes[i:i+2]))
|
|
}
|
|
}
|
|
return tokens
|
|
}
|
|
|
|
// VectorLiteral 将 float32 向量序列化为 pgvector 接受的字面量字符串 [0.1,0.2,...]
|
|
func VectorLiteral(vec []float32) string {
|
|
if len(vec) == 0 {
|
|
return ""
|
|
}
|
|
var sb strings.Builder
|
|
sb.WriteByte('[')
|
|
for i, f := range vec {
|
|
if i > 0 {
|
|
sb.WriteByte(',')
|
|
}
|
|
sb.WriteString(fmt.Sprintf("%g", f))
|
|
}
|
|
sb.WriteByte(']')
|
|
return sb.String()
|
|
}
|