Expand dotted setting keys and mask secret pools #61

Merged
weichen merged 2 commits from fix/settings-dotted-keys into main 2026-10-04 13:43:05 +00:00
6 changed files with 514 additions and 25 deletions
+11
View File
@@ -112,6 +112,17 @@ func Run(opt Options) error {
}
setSvc := settings.NewService(st)
setSvc.SetSystemOverride(map[string]any{"storage": map[string]any{"dataDir": opt.DataDir}})
// 点路径键展开必须在读 tools.tavily.apiKeys 之前,否则启动仍注册旧池。
norm, err := setSvc.NormalizeStored()
if err != nil {
return fmt.Errorf("normalize settings: %w", err)
}
for _, p := range norm.MaskPaths {
log.Printf("settings: warning: stored value is the mask placeholder at %s", p)
}
if norm.DottedKeys > 0 {
log.Printf("settings: normalized %d dotted key(s) in %d scope(s)", norm.DottedKeys, norm.Scopes)
}
// 网络搜索(Tavily):配置密钥池才注册(对齐 agent-native 语义:未配置不静默降级)。
eff := setSvc.Effective(store.OwnerDefault, "")
if keys := splitKeyPool(eff.GetString("tools.tavily.apiKeys", "")); len(keys) > 0 {
+66
View File
@@ -1133,3 +1133,69 @@ func TestProjectInbox(t *testing.T) {
t.Errorf("unknown project subscribe should not leak, got %d", codeX)
}
}
// TestSettingsDottedKeyAndSecretMask:点路径写入生效、嵌套胜出、掩码不回写、GET 不回明文。
func TestSettingsDottedKeyAndSecretMask(t *testing.T) {
srv, st := newTestServer(t)
h := srv.Handler()
set := func(body map[string]any) {
t.Helper()
code, out := doJSON(t, h, "PUT", "/api/settings/user", body)
if code != 200 {
t.Fatalf("PUT: %d %v", code, out)
}
}
svc := settings.NewService(st)
set(map[string]any{"runtime.entry": "squad"})
if got := svc.Effective(store.OwnerDefault, "").GetString("runtime.entry", ""); got != "squad" {
t.Fatalf("dotted write did not take effect: %q", got)
}
raw, err := svc.UserScope(store.OwnerDefault)
if err != nil {
t.Fatal(err)
}
if _, ok := raw["runtime.entry"]; ok {
t.Fatalf("dotted key stored as-is: %#v", raw)
}
const pool = "tvly-should-not-leak"
const modelKey = "sk-should-not-leak"
set(map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": pool}}})
set(map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": modelKey}}})
set(map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": settings.MaskPlaceholder}}})
set(map[string]any{
"tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested-wins"}},
"tools.tavily.apiKeys": "flat-loses",
})
eff := svc.Effective(store.OwnerDefault, "")
if got := eff.GetString("models.chat.apiKey", ""); got != modelKey {
t.Fatalf("mask writeback overwrote key: %q", got)
}
if got := eff.GetString("tools.tavily.apiKeys", ""); got != "nested-wins" {
t.Fatalf("nested form should win, got %q", got)
}
code, out := doJSON(t, h, "GET", "/api/settings", nil)
if code != 200 {
t.Fatalf("GET settings: %d", code)
}
blob, err := json.Marshal(out)
if err != nil {
t.Fatal(err)
}
for _, secret := range []string{pool, modelKey, "nested-wins", "flat-loses"} {
if strings.Contains(string(blob), secret) {
t.Fatalf("GET leaked %q", secret)
}
}
user := out["user"].(map[string]any)
tavily := user["tools"].(map[string]any)["tavily"].(map[string]any)
if tavily["apiKeys"] != settings.MaskPlaceholder {
t.Fatalf("apiKeys mask = %v", tavily["apiKeys"])
}
chat := user["models"].(map[string]any)["chat"].(map[string]any)
if chat["apiKey"] != settings.MaskPlaceholder {
t.Fatalf("apiKey mask = %v", chat["apiKey"])
}
}
+3 -10
View File
@@ -1091,9 +1091,9 @@ func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) {
schema := append(settings.Schema(), s.plugins.SettingsSchema()...)
writeJSON(w, http.StatusOK, map[string]any{
"schema": schema,
"defaults": settings.Masked(core.EffectiveSettings(settings.SystemDefaults())),
"user": maskMap(mustUser(s.set, owner)),
"project": maskMap(mustScope(s.set, owner, projectID)),
"defaults": settings.Masked(settings.SystemDefaults(), schema),
"user": settings.Masked(mustUser(s.set, owner), schema),
"project": settings.Masked(mustScope(s.set, owner, projectID), schema),
})
}
@@ -1486,13 +1486,6 @@ func asciiFallbackName(name string) string {
return b.String()
}
func maskMap(m map[string]any) map[string]any {
if m == nil {
return map[string]any{}
}
return settings.Masked(m)
}
func orEmptySlice[T any](s []T) []T {
if s == nil {
return []T{}
+195 -10
View File
@@ -3,6 +3,9 @@
package settings
import (
"sort"
"strings"
"agent-system-backend/internal/store"
core "agent-system-core"
)
@@ -142,19 +145,77 @@ func (s *Service) ProjectScope(owner, projectID string) (map[string]any, error)
}
func (s *Service) MergeUser(owner string, values map[string]any) error {
return s.st.MergeSettingsScope(owner, ScopeUser, values)
return s.mergeScope(owner, ScopeUser, values)
}
func (s *Service) MergeProject(owner, projectID string, values map[string]any) error {
return s.st.MergeSettingsScope(owner, ScopeProjectPrefix+projectID, values)
return s.mergeScope(owner, ScopeProjectPrefix+projectID, values)
}
// Masked 返回脱敏视图(secret 类 key 掩码),用于 API 响应。Effective 每次
// 深合并产出新 map,就地掩码不影响后续计算。
func Masked(eff core.EffectiveSettings) map[string]any {
out := map[string]any(eff)
maskSecrets(out)
return out
// mergeScope 先把库里和本次写入的点路径键展开成嵌套,再深合并。
// 掩码占位符视为未修改,不覆盖已存密钥。
func (s *Service) mergeScope(owner, scope string, values map[string]any) error {
incoming := prepareWrite(values)
return s.st.UpdateSettingsScope(owner, scope, func(cur map[string]any) (map[string]any, error) {
return core.MergeSettings(normalizeKeys(cur), incoming), nil
})
}
// NormalizeStats 是启动迁移的结果。MaskPaths 只含 owner、scope 和键路径,不含值。
type NormalizeStats struct {
DottedKeys int
Scopes int
MaskPaths []string
}
// NormalizeStored 把已落库的点路径键展开成嵌套。嵌套对象胜出,已生效的值不变。
// 存量值恰好是掩码占位符时记入 MaskPaths,原值保留(密钥已无法恢复)。
func (s *Service) NormalizeStored() (NormalizeStats, error) {
var stats NormalizeStats
refs, err := s.st.ListSettingsScopes()
if err != nil {
return stats, err
}
for _, ref := range refs {
var dotted int
var masks []string
err := s.st.UpdateSettingsScope(ref.Owner, ref.Scope, func(cur map[string]any) (map[string]any, error) {
dotted = countDottedKeys(cur)
next := normalizeKeys(cur)
for _, p := range maskPlaceholderPaths(next, "") {
masks = append(masks, ref.Owner+" "+ref.Scope+" "+p)
}
return next, nil
})
if err != nil {
return stats, err
}
stats.MaskPaths = append(stats.MaskPaths, masks...)
if dotted > 0 {
stats.DottedKeys += dotted
stats.Scopes++
}
}
return stats, nil
}
// MaskPlaceholder 是 GET 对密钥的掩码。写回这个值视为未修改。
const MaskPlaceholder = "••••••"
// Masked 返回脱敏视图。schema 中 Type=secret 或 Secret=true 的路径一律掩码;
// 叶子键名(apiKey/apiKeys 等)再兜底一层,覆盖 schema 未声明的密钥。
// 就地掩码:调用方须传入可丢弃的 map(Effective 每次深合并产出新 map)。
func Masked(values map[string]any, schema []core.SettingDesc) map[string]any {
if values == nil {
return map[string]any{}
}
maskSecrets(values)
for _, d := range schema {
if d.Type == core.SettingSecret || d.Secret {
maskAtPath(values, d.Key)
}
}
return values
}
func maskSecrets(m map[string]any) {
@@ -165,12 +226,136 @@ func maskSecrets(m map[string]any) {
}
if isSecretKey(k) {
if s, ok := v.(string); ok && s != "" {
m[k] = "••••••"
m[k] = MaskPlaceholder
}
}
}
}
func isSecretKey(k string) bool {
return k == "apiKey" || k == "api_key" || k == "secret" || k == "token"
switch k {
case "apiKey", "api_key", "apiKeys", "secret", "token":
return true
default:
return false
}
}
// prepareWrite 丢弃掩码占位符,再把点路径键展开成嵌套。
func prepareWrite(values map[string]any) map[string]any {
return normalizeKeys(dropMaskPlaceholders(values))
}
// normalizeKeys 递归展开点路径键。同一层嵌套对象与点路径冲突时嵌套胜出;
// 点路径键按字典序依次合并,后写入的覆盖先写入的。不改入参。
func normalizeKeys(in map[string]any) map[string]any {
if len(in) == 0 {
return map[string]any{}
}
plain := make(map[string]any, len(in))
dotted := make([]string, 0)
for k, v := range in {
if strings.Contains(k, ".") {
dotted = append(dotted, k)
continue
}
plain[k] = normalizeValue(v)
}
sort.Strings(dotted)
expanded := map[string]any{}
for _, k := range dotted {
expanded = core.MergeSettings(expanded, expandDotted(k, normalizeValue(in[k])))
}
return core.MergeSettings(expanded, plain)
}
func normalizeValue(v any) any {
vm, ok := v.(map[string]any)
if !ok {
return v
}
return normalizeKeys(vm)
}
// expandDotted 把 "a.b.c" 收成 {a:{b:{c: val}}}。
func expandDotted(key string, val any) map[string]any {
parts := strings.Split(key, ".")
built := val
for i := len(parts) - 1; i >= 0; i-- {
built = map[string]any{parts[i]: built}
}
return built.(map[string]any)
}
func dropMaskPlaceholders(m map[string]any) map[string]any {
out := make(map[string]any, len(m))
for k, v := range m {
if vm, ok := v.(map[string]any); ok {
out[k] = dropMaskPlaceholders(vm)
continue
}
if s, ok := v.(string); ok && s == MaskPlaceholder {
continue
}
out[k] = v
}
return out
}
func countDottedKeys(m map[string]any) int {
n := 0
for k, v := range m {
if strings.Contains(k, ".") {
n++
}
if vm, ok := v.(map[string]any); ok {
n += countDottedKeys(vm)
}
}
return n
}
func maskPlaceholderPaths(m map[string]any, prefix string) []string {
var out []string
for k, v := range m {
path := k
if prefix != "" {
path = prefix + "." + k
}
if vm, ok := v.(map[string]any); ok {
out = append(out, maskPlaceholderPaths(vm, path)...)
continue
}
if s, ok := v.(string); ok && s == MaskPlaceholder {
out = append(out, path)
}
}
sort.Strings(out)
return out
}
// maskAtPath 掩码 schema 路径。路径作为单个点键存在时同样掩码。
func maskAtPath(m map[string]any, path string) {
if s, ok := m[path].(string); ok && s != "" {
m[path] = MaskPlaceholder
}
parts := strings.Split(path, ".")
cur := m
for i, p := range parts {
v, ok := cur[p]
if !ok {
return
}
if i == len(parts)-1 {
if s, ok := v.(string); ok && s != "" {
cur[p] = MaskPlaceholder
}
return
}
next, ok := v.(map[string]any)
if !ok {
return
}
cur = next
}
}
+191 -1
View File
@@ -1,6 +1,14 @@
package settings
import "testing"
import (
"path/filepath"
"reflect"
"strings"
"testing"
"agent-system-backend/internal/store"
core "agent-system-core"
)
func TestLLMRetriesDefault(t *testing.T) {
block := SystemDefaults()["runtime"].(map[string]any)
@@ -39,3 +47,185 @@ func TestDeliverableSettingsPresent(t *testing.T) {
t.Fatalf("schema missing deliverable keys: style=%v accent=%v template=%v", style, accent, template)
}
}
func TestNormalizeKeys(t *testing.T) {
cases := []struct {
name string
in map[string]any
want map[string]any
}{
{
name: "dotted only",
in: map[string]any{"runtime.entry": "squad"},
want: map[string]any{"runtime": map[string]any{"entry": "squad"}},
},
{
name: "nested wins over dotted",
in: map[string]any{
"tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}},
"tools.tavily.apiKeys": "flat",
},
want: map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}}},
},
{
name: "dotted key inside a nested object",
in: map[string]any{"tools": map[string]any{"tavily.apiKeys": "x"}},
want: map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": "x"}}},
},
{
name: "overlapping dotted keys merge in lexicographic order",
in: map[string]any{"a.b": 1, "a.b.c": 2, "a.c": 3},
want: map[string]any{"a": map[string]any{"b": map[string]any{"c": 2}, "c": 3}},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := normalizeKeys(tc.in)
if !reflect.DeepEqual(got, tc.want) {
t.Fatalf("got %#v\nwant %#v", got, tc.want)
}
})
}
in := map[string]any{"runtime.entry": "squad"}
_ = normalizeKeys(in)
if _, ok := in["runtime.entry"]; !ok {
t.Fatal("normalizeKeys mutated its input")
}
}
func TestPrepareWriteDropsMask(t *testing.T) {
got := prepareWrite(map[string]any{
"models": map[string]any{"chat": map[string]any{"apiKey": MaskPlaceholder, "model": "m"}},
"runtime.entry": "squad",
})
want := map[string]any{
"models": map[string]any{"chat": map[string]any{"model": "m"}},
"runtime": map[string]any{"entry": "squad"},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %#v", got)
}
}
func TestMaskedBySchemaAndLeafName(t *testing.T) {
values := map[string]any{
"tools": map[string]any{"tavily": map[string]any{"apiKeys": "tvly-secret"}},
"models": map[string]any{"chat": map[string]any{"apiKey": "sk-secret"}},
"plugins": map[string]any{"example": map[string]any{"token": "plugin-secret"}},
"custom": map[string]any{"apiKeys": "pool-secret"},
"tools.tavily.apiKeys": "flat-secret",
}
schema := []core.SettingDesc{
{Key: "tools.tavily.apiKeys", Type: core.SettingSecret},
{Key: "models.chat.apiKey", Type: core.SettingSecret},
{Key: "plugins.example.token", Type: core.SettingString, Secret: true},
}
out := Masked(values, schema)
blob := stringify(out)
for _, secret := range []string{"tvly-secret", "sk-secret", "plugin-secret", "pool-secret", "flat-secret"} {
if strings.Contains(blob, secret) {
t.Fatalf("secret %q leaked: %#v", secret, out)
}
}
if out["tools"].(map[string]any)["tavily"].(map[string]any)["apiKeys"] != MaskPlaceholder {
t.Fatal("tavily apiKeys not masked")
}
if out["plugins"].(map[string]any)["example"].(map[string]any)["token"] != MaskPlaceholder {
t.Fatal("plugin token not masked")
}
if out["tools.tavily.apiKeys"] != MaskPlaceholder {
t.Fatal("flat secret key not masked")
}
}
func TestMergeUserDottedKeyMaskAndNormalizeStored(t *testing.T) {
st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
svc := NewService(st)
if err := st.MergeSettingsScope("local", ScopeUser, map[string]any{
"runtime.entry": "squad",
"tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}},
"tools.tavily.apiKeys": "flat",
"models": map[string]any{"chat": map[string]any{"apiKey": MaskPlaceholder}},
}); err != nil {
t.Fatal(err)
}
if err := st.MergeSettingsScope("local", ScopeProjectPrefix+"p1", map[string]any{
"context.maxHistoryMessages": 30,
}); err != nil {
t.Fatal(err)
}
stats, err := svc.NormalizeStored()
if err != nil {
t.Fatal(err)
}
if stats.DottedKeys != 3 || stats.Scopes != 2 {
t.Fatalf("stats = %+v", stats)
}
foundMask := false
for _, p := range stats.MaskPaths {
if p == "local user models.chat.apiKey" {
foundMask = true
}
}
if !foundMask {
t.Fatalf("mask path missing: %v", stats.MaskPaths)
}
eff := svc.Effective("local", "p1")
if eff.GetString("runtime.entry", "") != "squad" {
t.Fatalf("entry = %q", eff.GetString("runtime.entry", ""))
}
if eff.GetString("tools.tavily.apiKeys", "") != "nested" {
t.Fatalf("nested key lost: %q", eff.GetString("tools.tavily.apiKeys", ""))
}
if eff.GetNumber("context.maxHistoryMessages", 0) != 30 {
t.Fatalf("history = %v", eff.GetNumber("context.maxHistoryMessages", 0))
}
again, err := svc.NormalizeStored()
if err != nil || again.DottedKeys != 0 {
t.Fatalf("second pass = %+v err=%v", again, err)
}
if err := svc.MergeUser("local", map[string]any{"runtime.entry": "direct"}); err != nil {
t.Fatal(err)
}
if svc.Effective("local", "").GetString("runtime.entry", "") != "direct" {
t.Fatal("dotted write did not take effect")
}
raw, err := svc.UserScope("local")
if err != nil {
t.Fatal(err)
}
if _, ok := raw["runtime.entry"]; ok {
t.Fatalf("dotted key persisted: %#v", raw)
}
if err := svc.MergeUser("local", map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": "sk-real"}}}); err != nil {
t.Fatal(err)
}
if err := svc.MergeUser("local", map[string]any{"models.chat.apiKey": MaskPlaceholder}); err != nil {
t.Fatal(err)
}
if got := svc.Effective("local", "").GetString("models.chat.apiKey", ""); got != "sk-real" {
t.Fatalf("mask writeback overwrote key: %q", got)
}
}
func stringify(v any) string {
switch x := v.(type) {
case string:
return x
case map[string]any:
out := ""
for k, val := range x {
out += k + "=" + stringify(val) + ";"
}
return out
default:
return ""
}
}
+48 -4
View File
@@ -84,8 +84,33 @@ func (s *Store) GetSettingsScope(owner, scope string) (map[string]any, error) {
return out, nil
}
// MergeSettingsScope 读改写走事务 + scope 锁:并发合并不再丢失更新。
func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) error {
// SettingsScopeRef 标识一行设置(owner + scope),不含值。
type SettingsScopeRef struct {
Owner string
Scope string
}
// ListSettingsScopes 返回全部已落库的 scope,供启动迁移遍历。
func (s *Store) ListSettingsScopes() ([]SettingsScopeRef, error) {
rows, err := s.db.Query(s.q(`SELECT owner, scope FROM settings ORDER BY owner, scope`))
if err != nil {
return nil, err
}
defer rows.Close()
out := []SettingsScopeRef{}
for rows.Next() {
var ref SettingsScopeRef
if err := rows.Scan(&ref.Owner, &ref.Scope); err != nil {
return nil, err
}
out = append(out, ref)
}
return out, rows.Err()
}
// UpdateSettingsScope 在事务和 scope 锁内读改写。fn 返回的 map 与当前值
// JSON 相同时不写库。并发合并不再丢失更新。
func (s *Store) UpdateSettingsScope(owner, scope string, fn func(cur map[string]any) (map[string]any, error)) error {
tx, err := s.db.Begin()
if err != nil {
return err
@@ -102,16 +127,35 @@ func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) e
cur := map[string]any{}
if payload != "" {
cur, _ = unmarshalJSON[map[string]any](payload)
if cur == nil {
cur = map[string]any{}
}
}
next, err := fn(cur)
if err != nil {
return err
}
if next == nil {
next = map[string]any{}
}
if marshalJSON(cur) == marshalJSON(next) {
return tx.Commit()
}
merged := core.MergeSettings(cur, values)
_, err = tx.Exec(s.q(`INSERT INTO settings(owner,scope,"values") VALUES(?,?,?)
ON CONFLICT(owner,scope) DO UPDATE SET "values"=excluded."values"`), owner, scope, marshalJSON(merged))
ON CONFLICT(owner,scope) DO UPDATE SET "values"=excluded."values"`), owner, scope, marshalJSON(next))
if err != nil {
return err
}
return tx.Commit()
}
// MergeSettingsScope 读改写走事务 + scope 锁:并发合并不再丢失更新。
func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) error {
return s.UpdateSettingsScope(owner, scope, func(cur map[string]any) (map[string]any, error) {
return core.MergeSettings(cur, values), nil
})
}
func (s *Store) DeleteSettingsScope(owner, scope string) error {
_, err := s.db.Exec(s.q(`DELETE FROM settings WHERE owner=? AND scope=?`), owner, scope)
return err