diff --git a/internal/app/app.go b/internal/app/app.go index cc6db91..9598eb9 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -112,6 +112,17 @@ func Run(opt Options) error { } setSvc := settings.NewService(st) setSvc.SetSystemOverride(map[string]any{"storage": map[string]any{"dataDir": opt.DataDir}}) + // 点路径键展开必须在读 tools.tavily.apiKeys 之前,否则启动仍注册旧池。 + norm, err := setSvc.NormalizeStored() + if err != nil { + return fmt.Errorf("normalize settings: %w", err) + } + for _, p := range norm.MaskPaths { + log.Printf("settings: warning: stored value is the mask placeholder at %s", p) + } + if norm.DottedKeys > 0 { + log.Printf("settings: normalized %d dotted key(s) in %d scope(s)", norm.DottedKeys, norm.Scopes) + } // 网络搜索(Tavily):配置密钥池才注册(对齐 agent-native 语义:未配置不静默降级)。 eff := setSvc.Effective(store.OwnerDefault, "") if keys := splitKeyPool(eff.GetString("tools.tavily.apiKeys", "")); len(keys) > 0 { diff --git a/internal/server/e2e_test.go b/internal/server/e2e_test.go index 0104371..2f3eb2e 100644 --- a/internal/server/e2e_test.go +++ b/internal/server/e2e_test.go @@ -1133,3 +1133,69 @@ func TestProjectInbox(t *testing.T) { t.Errorf("unknown project subscribe should not leak, got %d", codeX) } } + +// TestSettingsDottedKeyAndSecretMask:点路径写入生效、嵌套胜出、掩码不回写、GET 不回明文。 +func TestSettingsDottedKeyAndSecretMask(t *testing.T) { + srv, st := newTestServer(t) + h := srv.Handler() + set := func(body map[string]any) { + t.Helper() + code, out := doJSON(t, h, "PUT", "/api/settings/user", body) + if code != 200 { + t.Fatalf("PUT: %d %v", code, out) + } + } + svc := settings.NewService(st) + + set(map[string]any{"runtime.entry": "squad"}) + if got := svc.Effective(store.OwnerDefault, "").GetString("runtime.entry", ""); got != "squad" { + t.Fatalf("dotted write did not take effect: %q", got) + } + raw, err := svc.UserScope(store.OwnerDefault) + if err != nil { + t.Fatal(err) + } + if _, ok := raw["runtime.entry"]; ok { + t.Fatalf("dotted key stored as-is: %#v", raw) + } + + const pool = "tvly-should-not-leak" + const modelKey = "sk-should-not-leak" + set(map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": pool}}}) + set(map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": modelKey}}}) + set(map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": settings.MaskPlaceholder}}}) + set(map[string]any{ + "tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested-wins"}}, + "tools.tavily.apiKeys": "flat-loses", + }) + eff := svc.Effective(store.OwnerDefault, "") + if got := eff.GetString("models.chat.apiKey", ""); got != modelKey { + t.Fatalf("mask writeback overwrote key: %q", got) + } + if got := eff.GetString("tools.tavily.apiKeys", ""); got != "nested-wins" { + t.Fatalf("nested form should win, got %q", got) + } + + code, out := doJSON(t, h, "GET", "/api/settings", nil) + if code != 200 { + t.Fatalf("GET settings: %d", code) + } + blob, err := json.Marshal(out) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{pool, modelKey, "nested-wins", "flat-loses"} { + if strings.Contains(string(blob), secret) { + t.Fatalf("GET leaked %q", secret) + } + } + user := out["user"].(map[string]any) + tavily := user["tools"].(map[string]any)["tavily"].(map[string]any) + if tavily["apiKeys"] != settings.MaskPlaceholder { + t.Fatalf("apiKeys mask = %v", tavily["apiKeys"]) + } + chat := user["models"].(map[string]any)["chat"].(map[string]any) + if chat["apiKey"] != settings.MaskPlaceholder { + t.Fatalf("apiKey mask = %v", chat["apiKey"]) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index a06fb36..616fc90 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1091,9 +1091,9 @@ func (s *Server) getSettings(w http.ResponseWriter, r *http.Request) { schema := append(settings.Schema(), s.plugins.SettingsSchema()...) writeJSON(w, http.StatusOK, map[string]any{ "schema": schema, - "defaults": settings.Masked(core.EffectiveSettings(settings.SystemDefaults())), - "user": maskMap(mustUser(s.set, owner)), - "project": maskMap(mustScope(s.set, owner, projectID)), + "defaults": settings.Masked(settings.SystemDefaults(), schema), + "user": settings.Masked(mustUser(s.set, owner), schema), + "project": settings.Masked(mustScope(s.set, owner, projectID), schema), }) } @@ -1486,13 +1486,6 @@ func asciiFallbackName(name string) string { return b.String() } -func maskMap(m map[string]any) map[string]any { - if m == nil { - return map[string]any{} - } - return settings.Masked(m) -} - func orEmptySlice[T any](s []T) []T { if s == nil { return []T{} diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 6dfcb91..e53c128 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -3,6 +3,9 @@ package settings import ( + "sort" + "strings" + "agent-system-backend/internal/store" core "agent-system-core" ) @@ -142,19 +145,77 @@ func (s *Service) ProjectScope(owner, projectID string) (map[string]any, error) } func (s *Service) MergeUser(owner string, values map[string]any) error { - return s.st.MergeSettingsScope(owner, ScopeUser, values) + return s.mergeScope(owner, ScopeUser, values) } func (s *Service) MergeProject(owner, projectID string, values map[string]any) error { - return s.st.MergeSettingsScope(owner, ScopeProjectPrefix+projectID, values) + return s.mergeScope(owner, ScopeProjectPrefix+projectID, values) } -// Masked 返回脱敏视图(secret 类 key 掩码),用于 API 响应。Effective 每次 -// 深合并产出新 map,就地掩码不影响后续计算。 -func Masked(eff core.EffectiveSettings) map[string]any { - out := map[string]any(eff) - maskSecrets(out) - return out +// mergeScope 先把库里和本次写入的点路径键展开成嵌套,再深合并。 +// 掩码占位符视为未修改,不覆盖已存密钥。 +func (s *Service) mergeScope(owner, scope string, values map[string]any) error { + incoming := prepareWrite(values) + return s.st.UpdateSettingsScope(owner, scope, func(cur map[string]any) (map[string]any, error) { + return core.MergeSettings(normalizeKeys(cur), incoming), nil + }) +} + +// NormalizeStats 是启动迁移的结果。MaskPaths 只含 owner、scope 和键路径,不含值。 +type NormalizeStats struct { + DottedKeys int + Scopes int + MaskPaths []string +} + +// NormalizeStored 把已落库的点路径键展开成嵌套。嵌套对象胜出,已生效的值不变。 +// 存量值恰好是掩码占位符时记入 MaskPaths,原值保留(密钥已无法恢复)。 +func (s *Service) NormalizeStored() (NormalizeStats, error) { + var stats NormalizeStats + refs, err := s.st.ListSettingsScopes() + if err != nil { + return stats, err + } + for _, ref := range refs { + var dotted int + var masks []string + err := s.st.UpdateSettingsScope(ref.Owner, ref.Scope, func(cur map[string]any) (map[string]any, error) { + dotted = countDottedKeys(cur) + next := normalizeKeys(cur) + for _, p := range maskPlaceholderPaths(next, "") { + masks = append(masks, ref.Owner+" "+ref.Scope+" "+p) + } + return next, nil + }) + if err != nil { + return stats, err + } + stats.MaskPaths = append(stats.MaskPaths, masks...) + if dotted > 0 { + stats.DottedKeys += dotted + stats.Scopes++ + } + } + return stats, nil +} + +// MaskPlaceholder 是 GET 对密钥的掩码。写回这个值视为未修改。 +const MaskPlaceholder = "••••••" + +// Masked 返回脱敏视图。schema 中 Type=secret 或 Secret=true 的路径一律掩码; +// 叶子键名(apiKey/apiKeys 等)再兜底一层,覆盖 schema 未声明的密钥。 +// 就地掩码:调用方须传入可丢弃的 map(Effective 每次深合并产出新 map)。 +func Masked(values map[string]any, schema []core.SettingDesc) map[string]any { + if values == nil { + return map[string]any{} + } + maskSecrets(values) + for _, d := range schema { + if d.Type == core.SettingSecret || d.Secret { + maskAtPath(values, d.Key) + } + } + return values } func maskSecrets(m map[string]any) { @@ -165,12 +226,136 @@ func maskSecrets(m map[string]any) { } if isSecretKey(k) { if s, ok := v.(string); ok && s != "" { - m[k] = "••••••" + m[k] = MaskPlaceholder } } } } func isSecretKey(k string) bool { - return k == "apiKey" || k == "api_key" || k == "secret" || k == "token" + switch k { + case "apiKey", "api_key", "apiKeys", "secret", "token": + return true + default: + return false + } +} + +// prepareWrite 丢弃掩码占位符,再把点路径键展开成嵌套。 +func prepareWrite(values map[string]any) map[string]any { + return normalizeKeys(dropMaskPlaceholders(values)) +} + +// normalizeKeys 递归展开点路径键。同一层嵌套对象与点路径冲突时嵌套胜出; +// 点路径键按字典序依次合并,后写入的覆盖先写入的。不改入参。 +func normalizeKeys(in map[string]any) map[string]any { + if len(in) == 0 { + return map[string]any{} + } + plain := make(map[string]any, len(in)) + dotted := make([]string, 0) + for k, v := range in { + if strings.Contains(k, ".") { + dotted = append(dotted, k) + continue + } + plain[k] = normalizeValue(v) + } + sort.Strings(dotted) + expanded := map[string]any{} + for _, k := range dotted { + expanded = core.MergeSettings(expanded, expandDotted(k, normalizeValue(in[k]))) + } + return core.MergeSettings(expanded, plain) +} + +func normalizeValue(v any) any { + vm, ok := v.(map[string]any) + if !ok { + return v + } + return normalizeKeys(vm) +} + +// expandDotted 把 "a.b.c" 收成 {a:{b:{c: val}}}。 +func expandDotted(key string, val any) map[string]any { + parts := strings.Split(key, ".") + built := val + for i := len(parts) - 1; i >= 0; i-- { + built = map[string]any{parts[i]: built} + } + return built.(map[string]any) +} + +func dropMaskPlaceholders(m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + if vm, ok := v.(map[string]any); ok { + out[k] = dropMaskPlaceholders(vm) + continue + } + if s, ok := v.(string); ok && s == MaskPlaceholder { + continue + } + out[k] = v + } + return out +} + +func countDottedKeys(m map[string]any) int { + n := 0 + for k, v := range m { + if strings.Contains(k, ".") { + n++ + } + if vm, ok := v.(map[string]any); ok { + n += countDottedKeys(vm) + } + } + return n +} + +func maskPlaceholderPaths(m map[string]any, prefix string) []string { + var out []string + for k, v := range m { + path := k + if prefix != "" { + path = prefix + "." + k + } + if vm, ok := v.(map[string]any); ok { + out = append(out, maskPlaceholderPaths(vm, path)...) + continue + } + if s, ok := v.(string); ok && s == MaskPlaceholder { + out = append(out, path) + } + } + sort.Strings(out) + return out +} + +// maskAtPath 掩码 schema 路径。路径作为单个点键存在时同样掩码。 +func maskAtPath(m map[string]any, path string) { + if s, ok := m[path].(string); ok && s != "" { + m[path] = MaskPlaceholder + } + parts := strings.Split(path, ".") + cur := m + for i, p := range parts { + v, ok := cur[p] + if !ok { + return + } + if i == len(parts)-1 { + if s, ok := v.(string); ok && s != "" { + cur[p] = MaskPlaceholder + } + return + } + next, ok := v.(map[string]any) + if !ok { + return + } + cur = next + } } diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index 50a52a7..2fd2a20 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -1,6 +1,14 @@ package settings -import "testing" +import ( + "path/filepath" + "reflect" + "strings" + "testing" + + "agent-system-backend/internal/store" + core "agent-system-core" +) func TestLLMRetriesDefault(t *testing.T) { block := SystemDefaults()["runtime"].(map[string]any) @@ -39,3 +47,185 @@ func TestDeliverableSettingsPresent(t *testing.T) { t.Fatalf("schema missing deliverable keys: style=%v accent=%v template=%v", style, accent, template) } } + +func TestNormalizeKeys(t *testing.T) { + cases := []struct { + name string + in map[string]any + want map[string]any + }{ + { + name: "dotted only", + in: map[string]any{"runtime.entry": "squad"}, + want: map[string]any{"runtime": map[string]any{"entry": "squad"}}, + }, + { + name: "nested wins over dotted", + in: map[string]any{ + "tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}}, + "tools.tavily.apiKeys": "flat", + }, + want: map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}}}, + }, + { + name: "dotted key inside a nested object", + in: map[string]any{"tools": map[string]any{"tavily.apiKeys": "x"}}, + want: map[string]any{"tools": map[string]any{"tavily": map[string]any{"apiKeys": "x"}}}, + }, + { + name: "overlapping dotted keys merge in lexicographic order", + in: map[string]any{"a.b": 1, "a.b.c": 2, "a.c": 3}, + want: map[string]any{"a": map[string]any{"b": map[string]any{"c": 2}, "c": 3}}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := normalizeKeys(tc.in) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %#v\nwant %#v", got, tc.want) + } + }) + } + in := map[string]any{"runtime.entry": "squad"} + _ = normalizeKeys(in) + if _, ok := in["runtime.entry"]; !ok { + t.Fatal("normalizeKeys mutated its input") + } +} + +func TestPrepareWriteDropsMask(t *testing.T) { + got := prepareWrite(map[string]any{ + "models": map[string]any{"chat": map[string]any{"apiKey": MaskPlaceholder, "model": "m"}}, + "runtime.entry": "squad", + }) + want := map[string]any{ + "models": map[string]any{"chat": map[string]any{"model": "m"}}, + "runtime": map[string]any{"entry": "squad"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v", got) + } +} + +func TestMaskedBySchemaAndLeafName(t *testing.T) { + values := map[string]any{ + "tools": map[string]any{"tavily": map[string]any{"apiKeys": "tvly-secret"}}, + "models": map[string]any{"chat": map[string]any{"apiKey": "sk-secret"}}, + "plugins": map[string]any{"example": map[string]any{"token": "plugin-secret"}}, + "custom": map[string]any{"apiKeys": "pool-secret"}, + "tools.tavily.apiKeys": "flat-secret", + } + schema := []core.SettingDesc{ + {Key: "tools.tavily.apiKeys", Type: core.SettingSecret}, + {Key: "models.chat.apiKey", Type: core.SettingSecret}, + {Key: "plugins.example.token", Type: core.SettingString, Secret: true}, + } + out := Masked(values, schema) + blob := stringify(out) + for _, secret := range []string{"tvly-secret", "sk-secret", "plugin-secret", "pool-secret", "flat-secret"} { + if strings.Contains(blob, secret) { + t.Fatalf("secret %q leaked: %#v", secret, out) + } + } + if out["tools"].(map[string]any)["tavily"].(map[string]any)["apiKeys"] != MaskPlaceholder { + t.Fatal("tavily apiKeys not masked") + } + if out["plugins"].(map[string]any)["example"].(map[string]any)["token"] != MaskPlaceholder { + t.Fatal("plugin token not masked") + } + if out["tools.tavily.apiKeys"] != MaskPlaceholder { + t.Fatal("flat secret key not masked") + } +} + +func TestMergeUserDottedKeyMaskAndNormalizeStored(t *testing.T) { + st, err := store.OpenSQLite(filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + svc := NewService(st) + + if err := st.MergeSettingsScope("local", ScopeUser, map[string]any{ + "runtime.entry": "squad", + "tools": map[string]any{"tavily": map[string]any{"apiKeys": "nested"}}, + "tools.tavily.apiKeys": "flat", + "models": map[string]any{"chat": map[string]any{"apiKey": MaskPlaceholder}}, + }); err != nil { + t.Fatal(err) + } + if err := st.MergeSettingsScope("local", ScopeProjectPrefix+"p1", map[string]any{ + "context.maxHistoryMessages": 30, + }); err != nil { + t.Fatal(err) + } + + stats, err := svc.NormalizeStored() + if err != nil { + t.Fatal(err) + } + if stats.DottedKeys != 3 || stats.Scopes != 2 { + t.Fatalf("stats = %+v", stats) + } + foundMask := false + for _, p := range stats.MaskPaths { + if p == "local user models.chat.apiKey" { + foundMask = true + } + } + if !foundMask { + t.Fatalf("mask path missing: %v", stats.MaskPaths) + } + eff := svc.Effective("local", "p1") + if eff.GetString("runtime.entry", "") != "squad" { + t.Fatalf("entry = %q", eff.GetString("runtime.entry", "")) + } + if eff.GetString("tools.tavily.apiKeys", "") != "nested" { + t.Fatalf("nested key lost: %q", eff.GetString("tools.tavily.apiKeys", "")) + } + if eff.GetNumber("context.maxHistoryMessages", 0) != 30 { + t.Fatalf("history = %v", eff.GetNumber("context.maxHistoryMessages", 0)) + } + again, err := svc.NormalizeStored() + if err != nil || again.DottedKeys != 0 { + t.Fatalf("second pass = %+v err=%v", again, err) + } + + if err := svc.MergeUser("local", map[string]any{"runtime.entry": "direct"}); err != nil { + t.Fatal(err) + } + if svc.Effective("local", "").GetString("runtime.entry", "") != "direct" { + t.Fatal("dotted write did not take effect") + } + raw, err := svc.UserScope("local") + if err != nil { + t.Fatal(err) + } + if _, ok := raw["runtime.entry"]; ok { + t.Fatalf("dotted key persisted: %#v", raw) + } + if err := svc.MergeUser("local", map[string]any{"models": map[string]any{"chat": map[string]any{"apiKey": "sk-real"}}}); err != nil { + t.Fatal(err) + } + if err := svc.MergeUser("local", map[string]any{"models.chat.apiKey": MaskPlaceholder}); err != nil { + t.Fatal(err) + } + if got := svc.Effective("local", "").GetString("models.chat.apiKey", ""); got != "sk-real" { + t.Fatalf("mask writeback overwrote key: %q", got) + } +} + +func stringify(v any) string { + switch x := v.(type) { + case string: + return x + case map[string]any: + out := "" + for k, val := range x { + out += k + "=" + stringify(val) + ";" + } + return out + default: + return "" + } +} diff --git a/internal/store/config.go b/internal/store/config.go index 76a040a..47f6b19 100644 --- a/internal/store/config.go +++ b/internal/store/config.go @@ -84,8 +84,33 @@ func (s *Store) GetSettingsScope(owner, scope string) (map[string]any, error) { return out, nil } -// MergeSettingsScope 读改写走事务 + scope 锁:并发合并不再丢失更新。 -func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) error { +// SettingsScopeRef 标识一行设置(owner + scope),不含值。 +type SettingsScopeRef struct { + Owner string + Scope string +} + +// ListSettingsScopes 返回全部已落库的 scope,供启动迁移遍历。 +func (s *Store) ListSettingsScopes() ([]SettingsScopeRef, error) { + rows, err := s.db.Query(s.q(`SELECT owner, scope FROM settings ORDER BY owner, scope`)) + if err != nil { + return nil, err + } + defer rows.Close() + out := []SettingsScopeRef{} + for rows.Next() { + var ref SettingsScopeRef + if err := rows.Scan(&ref.Owner, &ref.Scope); err != nil { + return nil, err + } + out = append(out, ref) + } + return out, rows.Err() +} + +// UpdateSettingsScope 在事务和 scope 锁内读改写。fn 返回的 map 与当前值 +// JSON 相同时不写库。并发合并不再丢失更新。 +func (s *Store) UpdateSettingsScope(owner, scope string, fn func(cur map[string]any) (map[string]any, error)) error { tx, err := s.db.Begin() if err != nil { return err @@ -102,16 +127,35 @@ func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) e cur := map[string]any{} if payload != "" { cur, _ = unmarshalJSON[map[string]any](payload) + if cur == nil { + cur = map[string]any{} + } + } + next, err := fn(cur) + if err != nil { + return err + } + if next == nil { + next = map[string]any{} + } + if marshalJSON(cur) == marshalJSON(next) { + return tx.Commit() } - merged := core.MergeSettings(cur, values) _, err = tx.Exec(s.q(`INSERT INTO settings(owner,scope,"values") VALUES(?,?,?) - ON CONFLICT(owner,scope) DO UPDATE SET "values"=excluded."values"`), owner, scope, marshalJSON(merged)) + ON CONFLICT(owner,scope) DO UPDATE SET "values"=excluded."values"`), owner, scope, marshalJSON(next)) if err != nil { return err } return tx.Commit() } +// MergeSettingsScope 读改写走事务 + scope 锁:并发合并不再丢失更新。 +func (s *Store) MergeSettingsScope(owner, scope string, values map[string]any) error { + return s.UpdateSettingsScope(owner, scope, func(cur map[string]any) (map[string]any, error) { + return core.MergeSettings(cur, values), nil + }) +} + func (s *Store) DeleteSettingsScope(owner, scope string) error { _, err := s.db.Exec(s.q(`DELETE FROM settings WHERE owner=? AND scope=?`), owner, scope) return err