weichen
63f1fa5f87
- internal/controlcenter/embed: OpenAI-compatible /embeddings client
(batch<=16, 30s timeout, 5xx/network retried once, 4xx not), Embedder
interface, deterministic offline Stub, pgvector literal formatting
- internal/controlcenter/docs: .doc/**/*.md sync into cc_documents with
sha256 diffing and vanished-file removal; YAML front matter -> jsonb,
title fallback, taxonomy from first .doc dir; heading-based chunking
with paragraph bisection over ~1800 est. tokens; chunk_hash idempotent
inserts (unchanged chunks keep embeddings); embedding failures degrade
to NULL vectors without failing the sync; IndexArtifact for
cc_stage_artifacts; StartPeriodic ticker (CC_DOC_SYNC_SECONDS)
- server: GET /api/v1/docs (taxonomy/q filters), /api/v1/docs/{id},
/api/v1/docs/search (semantic|keyword|hybrid, ts_rank+ILIKE keyword
fallback, 0.6/0.4 max-normalized fusion, degraded flag when embedding
unavailable); async artifact indexing hooked into agent artifacts and
external-submit handlers
- archtest module-boundaries allowlist updated for new packages
120 lines
3.4 KiB
Go
120 lines
3.4 KiB
Go
package embed
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestNormalizeEndpoint(t *testing.T) {
|
|
cases := map[string]string{
|
|
"http://x:8080": "http://x:8080/v1/embeddings",
|
|
"http://x:8080/": "http://x:8080/v1/embeddings",
|
|
"http://x:8080/v1": "http://x:8080/v1/embeddings",
|
|
"http://x:8080/v1/embeddings": "http://x:8080/v1/embeddings",
|
|
"": "",
|
|
}
|
|
for in, want := range cases {
|
|
assert.Equal(t, want, normalizeEndpoint(in), in)
|
|
}
|
|
}
|
|
|
|
func fakeServer(t *testing.T, dim int, status int, calls *atomic.Int32) *httptest.Server {
|
|
t.Helper()
|
|
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if calls != nil {
|
|
calls.Add(1)
|
|
}
|
|
assert.Equal(t, "/v1/embeddings", r.URL.Path)
|
|
assert.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
|
|
if status != 200 {
|
|
w.WriteHeader(status)
|
|
return
|
|
}
|
|
var req struct {
|
|
Model string `json:"model"`
|
|
Input []string `json:"input"`
|
|
}
|
|
require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
|
|
assert.Equal(t, "bge-m3", req.Model)
|
|
type item struct {
|
|
Embedding []float32 `json:"embedding"`
|
|
Index int `json:"index"`
|
|
}
|
|
data := make([]item, len(req.Input))
|
|
for i := range req.Input {
|
|
data[i] = item{Embedding: make([]float32, dim), Index: i}
|
|
}
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"data": data})
|
|
}))
|
|
}
|
|
|
|
func TestEmbedBatches(t *testing.T) {
|
|
var calls atomic.Int32
|
|
srv := fakeServer(t, 4, 200, &calls)
|
|
defer srv.Close()
|
|
|
|
c := NewHTTP(srv.URL, "test-key", "bge-m3", 4)
|
|
texts := make([]string, maxBatchItems+3)
|
|
vecs, err := c.Embed(context.Background(), texts)
|
|
require.NoError(t, err)
|
|
assert.Len(t, vecs, len(texts))
|
|
for _, v := range vecs {
|
|
assert.Len(t, v, 4)
|
|
}
|
|
assert.Equal(t, int32(2), calls.Load(), "17 texts must be split into 16+1")
|
|
}
|
|
|
|
func TestEmbedRetriesOn5xxOnce(t *testing.T) {
|
|
var calls atomic.Int32
|
|
srv := fakeServer(t, 4, 500, &calls)
|
|
defer srv.Close()
|
|
|
|
c := NewHTTP(srv.URL, "test-key", "bge-m3", 4)
|
|
_, err := c.Embed(context.Background(), []string{"x"})
|
|
require.Error(t, err)
|
|
assert.Equal(t, int32(2), calls.Load(), "5xx retried exactly once")
|
|
}
|
|
|
|
func TestEmbedNoRetryOn4xx(t *testing.T) {
|
|
var calls atomic.Int32
|
|
srv := fakeServer(t, 4, 400, &calls)
|
|
defer srv.Close()
|
|
|
|
c := NewHTTP(srv.URL, "test-key", "bge-m3", 4)
|
|
_, err := c.Embed(context.Background(), []string{"x"})
|
|
require.Error(t, err)
|
|
assert.Equal(t, int32(1), calls.Load(), "4xx must not be retried")
|
|
}
|
|
|
|
func TestEmbedEmptyEndpoint(t *testing.T) {
|
|
c := NewHTTP("", "", "bge-m3", 4)
|
|
_, err := c.Embed(context.Background(), []string{"x"})
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestStubDeterministic(t *testing.T) {
|
|
s := NewStub(8)
|
|
a, err := s.Embed(context.Background(), []string{"hello", "world"})
|
|
require.NoError(t, err)
|
|
b, err := s.Embed(context.Background(), []string{"hello"})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, a[0], b[0], "same text, same vector")
|
|
assert.NotEqual(t, a[0], a[1], "different texts differ")
|
|
assert.Equal(t, 8, s.Dim())
|
|
}
|
|
|
|
func TestVectorLiteral(t *testing.T) {
|
|
assert.Equal(t, "", VectorLiteral(nil))
|
|
lit := VectorLiteral([]float32{0.5, -1, 1.25})
|
|
assert.True(t, strings.HasPrefix(lit, "[") && strings.HasSuffix(lit, "]"))
|
|
assert.Equal(t, "[0.5,-1,1.25]", lit)
|
|
}
|