weichen
63f1fa5f87
- internal/controlcenter/embed: OpenAI-compatible /embeddings client
(batch<=16, 30s timeout, 5xx/network retried once, 4xx not), Embedder
interface, deterministic offline Stub, pgvector literal formatting
- internal/controlcenter/docs: .doc/**/*.md sync into cc_documents with
sha256 diffing and vanished-file removal; YAML front matter -> jsonb,
title fallback, taxonomy from first .doc dir; heading-based chunking
with paragraph bisection over ~1800 est. tokens; chunk_hash idempotent
inserts (unchanged chunks keep embeddings); embedding failures degrade
to NULL vectors without failing the sync; IndexArtifact for
cc_stage_artifacts; StartPeriodic ticker (CC_DOC_SYNC_SECONDS)
- server: GET /api/v1/docs (taxonomy/q filters), /api/v1/docs/{id},
/api/v1/docs/search (semantic|keyword|hybrid, ts_rank+ILIKE keyword
fallback, 0.6/0.4 max-normalized fusion, degraded flag when embedding
unavailable); async artifact indexing hooked into agent artifacts and
external-submit handlers
- archtest module-boundaries allowlist updated for new packages
59 lines
1.2 KiB
Go
59 lines
1.2 KiB
Go
package embed
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"math"
|
|
)
|
|
|
|
// Stub is a deterministic offline Embedder: each text maps to a fixed
|
|
// pseudo-random unit-ish vector derived from its sha256, so tests need no
|
|
// network and results are stable across runs.
|
|
type Stub struct {
|
|
dim int
|
|
}
|
|
|
|
func NewStub(dim int) *Stub {
|
|
if dim <= 0 {
|
|
dim = Dim
|
|
}
|
|
return &Stub{dim: dim}
|
|
}
|
|
|
|
func (s *Stub) Dim() int { return s.dim }
|
|
|
|
func (s *Stub) Embed(_ context.Context, texts []string) ([][]float32, error) {
|
|
out := make([][]float32, len(texts))
|
|
for i, t := range texts {
|
|
out[i] = s.vector(t)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Stub) vector(text string) []float32 {
|
|
seed := sha256.Sum256([]byte(text))
|
|
state := binary.LittleEndian.Uint64(seed[:8]) | 1
|
|
vec := make([]float32, s.dim)
|
|
for i := range vec {
|
|
// xorshift64*
|
|
state ^= state >> 12
|
|
state ^= state << 25
|
|
state ^= state >> 27
|
|
v := (state * 2685821657736338717) >> 11
|
|
vec[i] = float32(float64(v)/float64(uint64(1)<<53))*2 - 1
|
|
}
|
|
// normalize so cosine distances behave sanely
|
|
var norm float64
|
|
for _, f := range vec {
|
|
norm += float64(f * f)
|
|
}
|
|
norm = math.Sqrt(norm)
|
|
if norm > 0 {
|
|
for i := range vec {
|
|
vec[i] /= float32(norm)
|
|
}
|
|
}
|
|
return vec
|
|
}
|