diff --git a/cmd/agent-system/env.go b/cmd/agent-system/env.go new file mode 100644 index 0000000..818d51d --- /dev/null +++ b/cmd/agent-system/env.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "time" + + "agent-system-backend/internal/execenv" +) + +func runEnv(args []string) int { + if len(args) == 0 { + fmt.Fprintln(os.Stderr, "用法：agent-system env --root ") + return 2 + } + sub := args[0] + fs := flag.NewFlagSet("env "+sub, flag.ExitOnError) + rootPath := fs.String("root", "", "环境根目录") + ttl := fs.Duration("ttl", 7*24*time.Hour, "gc 保留窗口") + bin := fs.String("bin", "", "python link 要指向的解释器") + _ = fs.Parse(args[1:]) + if *rootPath == "" { + fmt.Fprintln(os.Stderr, "env: --root 必填") + return 2 + } + root, err := execenv.Open(*rootPath) + if err != nil { + fmt.Fprintln(os.Stderr, "env:", err) + return 1 + } + ctx := context.Background() + switch sub { + case "ensure": + py, err := root.EnsurePython(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, "env ensure:", err) + return 1 + } + id, err := execenv.NewBroker(root, nil, execenv.Pip{Python: py}).EnsureBase(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, "env ensure:", err) + return 1 + } + fmt.Println(id) + return runDoctor(ctx, root, py, true) + case "lock": + id, err := root.CurrentBase() + if err != nil || id == "" { + fmt.Fprintln(os.Stderr, "env lock: 没有 current 基础层") + return 1 + } + lock, err := root.ReadLock(id) + if err != nil { + fmt.Fprintln(os.Stderr, "env lock:", err) + return 1 + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(lock); err != nil { + fmt.Fprintln(os.Stderr, "env lock:", err) + return 1 + } + return 0 + case "gc": + removed, err := execenv.NewBroker(root, nil, execenv.Pip{}).GC(*ttl) + if err != nil { + fmt.Fprintln(os.Stderr, "env gc:", err) + return 1 + } + for _, id := range removed { + fmt.Println(id) + } + return 0 + case "doctor": + py, ok := root.PythonBin() + if !ok { + py = "/usr/bin/python3" + } + return runDoctor(ctx, root, py, true) + case "rebase": + py, err := root.EnsurePython(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, "env rebase:", err) + return 1 + } + id, err := execenv.NewBroker(root, nil, execenv.Pip{Python: py}).Rebase(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, "env rebase:", err) + return 1 + } + fmt.Println(id) + return 0 + case "python": + action := "install" + if fs.NArg() > 0 { + action = fs.Arg(0) + } + switch action { + case "install": + py, err := root.EnsurePython(ctx) + if err != nil { + fmt.Fprintln(os.Stderr, "env python:", err) + return 1 + } + fmt.Println(py) + return 0 + case "link": + if *bin == "" { + fmt.Fprintln(os.Stderr, "env python link: --bin 必填") + return 2 + } + if err := root.LinkPython(*bin); err != nil { + fmt.Fprintln(os.Stderr, "env python link:", err) + return 1 + } + fmt.Println(*bin) + return 0 + default: + fmt.Fprintln(os.Stderr, "env python: install | link") + return 2 + } + default: + fmt.Fprintf(os.Stderr, "env: 未知动作 %s\n", sub) + return 2 + } +} + +func runDoctor(ctx context.Context, root *execenv.Root, py string, sandbox bool) int { + var pyPath []string + if id, err := root.CurrentBase(); err == nil && id != "" { + pyPath = append(pyPath, root.LayerDir(id)) + } + mounts := append([]string{}, pyPath...) + if prefix := root.PythonPrefix(); prefix != "" { + mounts = append(mounts, prefix) + } + if err := execenv.Doctor(ctx, py, pyPath, nil, sandbox, mounts); err != nil { + fmt.Fprintln(os.Stderr, "env doctor:", err) + return 1 + } + fmt.Println("doctor ok") + return 0 +} diff --git a/cmd/agent-system/main.go b/cmd/agent-system/main.go index 079d10e..52e9428 100644 --- a/cmd/agent-system/main.go +++ b/cmd/agent-system/main.go @@ -46,9 +46,11 @@ func main() { os.Exit(runIdentity(os.Args[2:])) case "capabilities": os.Exit(runCapabilities(os.Args[2:])) + case "env": + os.Exit(runEnv(os.Args[2:])) default: fmt.Fprintf(os.Stderr, "agent-system: 未知子命令 %s\n", os.Args[1]) - fmt.Fprintln(os.Stderr, "用法：agent-system [参数]") + fmt.Fprintln(os.Stderr, "用法：agent-system [参数]") os.Exit(2) } } @@ -68,6 +70,7 @@ func main() { execBackend := fs.String("exec-backend", "", "执行后端：direct（默认，现状直启）| sandbox（bwrap 隔离档：断网、只读系统根、仅会话目录可写、进程数限额）") execDepsDir := fs.String("exec-deps-dir", "", "sandbox 档只读依赖层目录（site-packages 等；空=不挂载）") execWorkspace := fs.String("exec-workspace", "", "sandbox 档工作区模式：空=直写会话目录 | overlay=冻结基底+私有 upper+CAS 回收（S0）") + execEnvRoot := fs.String("exec-env-root", "", "分层 Python 环境根（解释器、基础层、扩展层；空=不启用）") tier := fs.String("tier", "", "部署档：空=auto（DSN→PG，否则 SQLite）| local（SQLite，不允许 DSN）| team（PG fail-closed：缺 DSN/未认证拒绝启动，PG 故障拒绝就绪）") pgMaxOpen := fs.Int("pg-max-open", 0, "PG 最大打开连接（0=默认 16）") pgMaxIdle := fs.Int("pg-max-idle", 0, "PG 最大空闲连接（0=默认 8）") @@ -93,7 +96,7 @@ func main() { Addr: *addr, DataDir: *data, PluginsDir: *pluginsDir, Mock: *mock, StorageDSN: *dsn, ArtifactsDir: *artifacts, AuthMode: *auth, Instance: *instance, InsecureListen: *insecure, AllowedHostsRaw: *allowedHosts, DisableExecTools: *noExec, - ExecBackend: *execBackend, ExecDepsDir: *execDepsDir, ExecWorkspace: *execWorkspace, + ExecBackend: *execBackend, ExecDepsDir: *execDepsDir, ExecWorkspace: *execWorkspace, ExecEnvRoot: *execEnvRoot, Tier: *tier, PGMaxOpen: *pgMaxOpen, PGMaxIdle: *pgMaxIdle, PGConnMaxIdleSec: *pgIdleSec, PGConnMaxLifetimeSec: *pgLifetimeSec, PGQueryTimeoutSec: *pgQueryTimeoutSec, }, cliSet) @@ -137,8 +140,8 @@ func main() { DisableUploadIndexing: extOpt.DisableUploadIndexing, DisableConvCardMirror: extOpt.DisableConvCardMirror, DisableExecTools: extOpt.DisableExecTools, ExecBackend: extOpt.ExecBackend, ExecDepsDir: resolved.ExecDepsDir, - ExecWorkspace: resolved.ExecWorkspace, - Tier: resolved.Tier, + ExecWorkspace: resolved.ExecWorkspace, ExecEnvRoot: resolved.ExecEnvRoot, + Tier: resolved.Tier, PGPool: store.PoolOptions{ MaxOpenConn: resolved.PGMaxOpen, MaxIdleConn: resolved.PGMaxIdle, ConnMaxIdleTimeSec: resolved.PGConnMaxIdleSec, ConnMaxLifetimeSec: resolved.PGConnMaxLifetimeSec, @@ -162,6 +165,14 @@ func visited(fs *flag.FlagSet) map[string]bool { name = "allowedHosts" case "disable-exec-tools": name = "disableExecTools" + case "exec-backend": + name = "execBackend" + case "exec-deps-dir": + name = "execDepsDir" + case "exec-workspace": + name = "execWorkspace" + case "exec-env-root": + name = "execEnvRoot" case "pg-max-open": name = "pgMaxOpen" case "pg-max-idle": @@ -200,6 +211,7 @@ func runConfig(args []string) int { execBackend := fs.String("exec-backend", "", "") execDepsDir := fs.String("exec-deps-dir", "", "") execWorkspace := fs.String("exec-workspace", "", "") + execEnvRoot := fs.String("exec-env-root", "", "") tier := fs.String("tier", "", "") pgMaxOpen := fs.Int("pg-max-open", 0, "") pgMaxIdle := fs.Int("pg-max-idle", 0, "") @@ -211,7 +223,7 @@ func runConfig(args []string) int { Addr: *addr, DataDir: *data, PluginsDir: *pluginsDir, Mock: *mock, StorageDSN: *dsn, ArtifactsDir: *artifacts, AuthMode: *auth, Instance: *instance, InsecureListen: *insecure, AllowedHostsRaw: *allowedHosts, DisableExecTools: *noExec, - ExecBackend: *execBackend, ExecDepsDir: *execDepsDir, ExecWorkspace: *execWorkspace, + ExecBackend: *execBackend, ExecDepsDir: *execDepsDir, ExecWorkspace: *execWorkspace, ExecEnvRoot: *execEnvRoot, Tier: *tier, PGMaxOpen: *pgMaxOpen, PGMaxIdle: *pgMaxIdle, PGConnMaxIdleSec: *pgIdleSec, PGConnMaxLifetimeSec: *pgLifetimeSec, PGQueryTimeoutSec: *pgQueryTimeoutSec, }, visited(fs)) diff --git a/cmd/storage-migrate/main.go b/cmd/storage-migrate/main.go index 84543fc..d1a8521 100644 --- a/cmd/storage-migrate/main.go +++ b/cmd/storage-migrate/main.go @@ -298,6 +298,8 @@ var copyOrder = []string{ "account_events", "redeem_attempts", "recovery_proposals", "recovery_proposal_proofs", + // v35 分层 Python 环境（无外键；申请与会话绑定要随库迁走） + "exec_env_requests", "exec_env_bindings", } const controlDDL = ` diff --git a/internal/app/app.go b/internal/app/app.go index 7b85728..a6edf18 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -86,6 +86,9 @@ type Options struct { // ExecDepsDir sandbox 档只读依赖层（site-packages 等，空=不挂）：修复 // --tmpfs /home 遮蔽 ~/.local 导致交付物生成链路不可用（F3）。 ExecDepsDir string + // ExecEnvRoot 分层 Python 环境根（解释器、基础层、扩展层、白名单）。 + // 设置后，未显式给出 ExecDepsDir 时使用其中的 current 基础层。 + ExecEnvRoot string // ExecWorkspace = "overlay" 时 sandbox 档启用 S0 工作区（冻结基底+私有 // upper+CAS 回收）；空 = EXT-16 现状直写会话目录。 ExecWorkspace string @@ -249,9 +252,17 @@ func Run(opt Options) error { if tools.ExecBackendName() == "sandbox" { log.Printf("exec tools: backend=sandbox (bwrap: no network, ro root, session-only writes, proc limit)") } + broker, err := wireExecEnv(st, &opt) + if err != nil { + return fmt.Errorf("exec env: %w", err) + } if opt.ExecDepsDir != "" { tools.SetExecDepsDir(opt.ExecDepsDir) log.Printf("exec tools: sandbox deps layer=%s", opt.ExecDepsDir) + if err := tools.ExecDoctor(); err != nil { + return fmt.Errorf("exec doctor: %w", err) + } + log.Printf("exec tools: doctor ok (docx/openpyxl/pptx)") } if opt.ExecWorkspace == "overlay" { tools.SetWorkspaceOverlay(true) @@ -453,6 +464,15 @@ func Run(opt Options) error { subRunner := &runtime.SubLLMRunner{Prompts: promptSvc, Trace: traceFn} mockRunner := &runtime.MockRunner{} mgr := runmgr.New(st, b, reg, profSvc, setSvc, convRunner, subRunner, mockRunner, &runtime.MockSubagent{}) + if broker != nil { + mgr.SetExecEnvDecider(func(actor, id string, allow, isolate bool) (string, error) { + out, err := broker.Decide(context.Background(), id, actor, false, allow, isolate) + if err != nil { + return "", err + } + return out.Detail, nil + }) + } authOn := resolved.AuthEnabled() effectiveAccount := accountEffectiveAuthorization(st, authOn) mgr.SetAccountEffectiveAuthorization(effectiveAccount, !authOn) @@ -697,6 +717,7 @@ func Run(opt Options) error { } srv := server.New(st, mgr, reg, profSvc, setSvc, pluginHost) + srv.ExecEnv = broker projectSlot.wire(srv, caps) // EXT-8：装载期注册的资源授权适配器注入读取入口（未注册=默认拒绝，行为不变）。 srv.SetProjectAuthorizer(authzSlot.Authorize) diff --git a/internal/app/execenv_wire.go b/internal/app/execenv_wire.go new file mode 100644 index 0000000..d25cd24 --- /dev/null +++ b/internal/app/execenv_wire.go @@ -0,0 +1,132 @@ +package app + +import ( + "agent-system-backend/internal/execenv" + "agent-system-backend/internal/store" + "agent-system-backend/internal/tools" + core "agent-system-core" +) + +// envStore 把 store 的行类型适配成 execenv.Store。 +type envStore struct{ *store.Store } + +func (s envStore) InsertExecEnvRequest(r execenv.Request) error { + return s.Store.InsertExecEnvRequest(store.ExecEnvRequest{ + ID: r.ID, ConversationID: r.ConversationID, Owner: r.Owner, Spec: r.Spec, + Reason: r.Reason, Status: r.Status, LayerID: r.LayerID, Detail: r.Detail, + Approver: r.Approver, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, + }) +} + +func (s envStore) GetExecEnvRequest(id string) (execenv.Request, error) { + r, err := s.Store.GetExecEnvRequest(id) + if err != nil { + return execenv.Request{}, err + } + return execenv.Request{ + ID: r.ID, ConversationID: r.ConversationID, Owner: r.Owner, Spec: r.Spec, + Reason: r.Reason, Status: r.Status, LayerID: r.LayerID, Detail: r.Detail, + Approver: r.Approver, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, + }, nil +} + +func (s envStore) ListExecEnvRequests(conversationID string, limit int) ([]execenv.Request, error) { + rows, err := s.Store.ListExecEnvRequests(conversationID, limit) + if err != nil { + return nil, err + } + out := make([]execenv.Request, 0, len(rows)) + for _, r := range rows { + out = append(out, execenv.Request{ + ID: r.ID, ConversationID: r.ConversationID, Owner: r.Owner, Spec: r.Spec, + Reason: r.Reason, Status: r.Status, LayerID: r.LayerID, Detail: r.Detail, + Approver: r.Approver, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, + }) + } + return out, nil +} + +func (s envStore) UpdateExecEnvRequest(id, status, layerID, detail, approver string) error { + return s.Store.UpdateExecEnvRequest(id, status, layerID, detail, approver) +} + +func (s envStore) GetExecEnvBinding(conversationID string) (execenv.Binding, error) { + b, err := s.Store.GetExecEnvBinding(conversationID) + if err != nil { + return execenv.Binding{}, err + } + return execenv.Binding{ + ConversationID: b.ConversationID, BaseID: b.BaseID, Extensions: b.Layers, + Isolated: b.Isolated, UpdatedAt: b.UpdatedAt, + }, nil +} + +func (s envStore) PutExecEnvBinding(b execenv.Binding) error { + return s.Store.PutExecEnvBinding(store.ExecEnvBinding{ + ConversationID: b.ConversationID, BaseID: b.BaseID, Layers: b.Extensions, + Isolated: b.Isolated, UpdatedAt: b.UpdatedAt, + }) +} + +func (s envStore) ListExecEnvBindings() ([]execenv.Binding, error) { + rows, err := s.Store.ListExecEnvBindings() + if err != nil { + return nil, err + } + out := make([]execenv.Binding, 0, len(rows)) + for _, b := range rows { + out = append(out, execenv.Binding{ + ConversationID: b.ConversationID, BaseID: b.BaseID, Extensions: b.Layers, + Isolated: b.Isolated, UpdatedAt: b.UpdatedAt, + }) + } + return out, nil +} + +func (s envStore) InsertExecEnvAudit(owner, action, actor, target, outcome, entry string) error { + return s.Store.InsertExecEnvAudit(owner, action, actor, target, outcome, entry) +} + +// wireExecEnv 打开环境根，把基础层交给执行器，并接上申请代理。 +// 两者都空时不改变现状（解释器仍固定为 /usr/bin/python3，避免 linuxbrew 路径）。 +func wireExecEnv(st *store.Store, opt *Options) (*execenv.Broker, error) { + if opt.ExecEnvRoot == "" { + return nil, nil + } + root, err := execenv.Open(opt.ExecEnvRoot) + if err != nil { + return nil, err + } + if bin, ok := root.PythonBin(); ok { + tools.SetExecPython(bin, root.PythonPrefix()) + } + tools.SetExecShimDir(root.BinDir()) + if opt.ExecDepsDir == "" { + if id, err := root.CurrentBase(); err != nil { + return nil, err + } else if id != "" { + opt.ExecDepsDir = root.LayerDir(id) + } + } + py, _ := root.PythonBin() + if py == "" { + py = "/usr/bin/python3" + } + broker := execenv.NewBroker(root, envStore{st}, execenv.Pip{Python: py}) + tools.SetEnvBroker(broker) + tools.SetExecLayers(broker.LayersFor) + tools.SetEnvApproval(func(owner, conv, requestID, detail string) error { + card := &core.HITLCard{ + ID: core.ID("hitl_" + requestID), RunID: "execenv", ConversationID: core.ID(conv), + Question: "批准为这个会话安装额外的 Python 包？", Kind: core.HITLKindChoice, + Detail: "exec-env:" + requestID + "\n" + detail, + Choices: []core.HITLChoice{ + {ID: "approve", Label: "批准", Description: detail, Impact: "只挂到这个会话，不改变基础层", Recommended: true}, + {ID: "isolate", Label: "批准并使用隔离层", Description: "与基础层版本冲突时用这份独立闭包", Impact: "这个会话不再叠基础层"}, + {ID: "reject", Label: "拒绝", Description: "不安装", Impact: "会话保持现状"}, + }, + } + return st.SaveHITLCard(owner, card) + }) + return broker, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 45defa6..90d2538 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,6 +36,8 @@ type Config struct { ExecDepsDir string `json:"execDepsDir"` // ExecWorkspace sandbox 档工作区模式：空=直写；overlay=冻结基底+upper。 ExecWorkspace string `json:"execWorkspace"` + // ExecEnvRoot 分层 Python 环境根。空 = 不启用申请代理。 + ExecEnvRoot string `json:"execEnvRoot"` // Tier 部署档（协作存储方案 KTD1）：空=auto（DSN→PG，否则 SQLite）； // local=强制 SQLite（不允许 DSN）；team=强制 PG（缺 DSN/未开认证拒绝启动， // PG 故障拒绝就绪，绝不回落 SQLite）。 @@ -69,6 +71,7 @@ var envKey = map[string]string{ "execBackend": "AGENT_SYSTEM_EXEC_BACKEND", "execDepsDir": "AGENT_SYSTEM_EXEC_DEPS_DIR", "execWorkspace": "AGENT_SYSTEM_EXEC_WORKSPACE", + "execEnvRoot": "AGENT_SYSTEM_EXEC_ENV_ROOT", "insecureListen": "AGENT_SYSTEM_INSECURE_LISTEN", "allowedHosts": "AGENT_SYSTEM_ALLOWED_HOSTS", "disableExecTools": "AGENT_SYSTEM_DISABLE_EXEC_TOOLS", @@ -213,6 +216,8 @@ func loadFile(path string) (Config, map[string]bool, error) { dst = &c.ExecDepsDir case "execWorkspace": dst = &c.ExecWorkspace + case "execEnvRoot": + dst = &c.ExecEnvRoot case "tier": dst = &c.Tier case "pgMaxOpen": @@ -260,6 +265,8 @@ func stringField(c Config, f string) string { return c.ExecDepsDir case "execWorkspace": return c.ExecWorkspace + case "execEnvRoot": + return c.ExecEnvRoot case "tier": return c.Tier } @@ -355,6 +362,8 @@ func setString(c *Config, f, v string) { c.ExecDepsDir = v case "execWorkspace": c.ExecWorkspace = v + case "execEnvRoot": + c.ExecEnvRoot = v case "tier": c.Tier = v } @@ -386,7 +395,7 @@ func (r Resolved) Explain(w io.Writer, file string) { fmt.Fprintln(w, "config file: (none)") } fmt.Fprintln(w, "field value source") - for _, f := range []string{"addr", "data", "plugins", "dsn", "artifacts", "auth", "mock", "instance", "insecureListen", "allowedHosts", "disableExecTools", "execBackend", "tier", "pgMaxOpen", "pgMaxIdle", "pgConnMaxIdleSec", "pgConnMaxLifetimeSec", "pgQueryTimeoutSec"} { + for _, f := range []string{"addr", "data", "plugins", "dsn", "artifacts", "auth", "mock", "instance", "insecureListen", "allowedHosts", "disableExecTools", "execBackend", "execDepsDir", "execWorkspace", "execEnvRoot", "tier", "pgMaxOpen", "pgMaxIdle", "pgConnMaxIdleSec", "pgConnMaxLifetimeSec", "pgQueryTimeoutSec"} { var v string if isBoolField(f) { v = fmt.Sprintf("%t", boolField(r.Config, f)) diff --git a/internal/execenv/base-requirements.txt b/internal/execenv/base-requirements.txt new file mode 100644 index 0000000..b8f699c --- /dev/null +++ b/internal/execenv/base-requirements.txt @@ -0,0 +1,11 @@ +python-docx +openpyxl +python-pptx +lxml +pillow +numpy +matplotlib +pypdf +reportlab +markdown +pdfplumber diff --git a/internal/execenv/broker.go b/internal/execenv/broker.go new file mode 100644 index 0000000..53d99d7 --- /dev/null +++ b/internal/execenv/broker.go @@ -0,0 +1,349 @@ +package execenv + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +// Store 是申请与绑定的持久化。实现在 store 包，避免本包依赖数据库驱动。 +type Store interface { + InsertExecEnvRequest(Request) error + GetExecEnvRequest(id string) (Request, error) + ListExecEnvRequests(conversationID string, limit int) ([]Request, error) + UpdateExecEnvRequest(id, status, layerID, detail, approver string) error + GetExecEnvBinding(conversationID string) (Binding, error) + PutExecEnvBinding(Binding) error + ListExecEnvBindings() ([]Binding, error) + InsertExecEnvAudit(owner, action, actor, target, outcome, entry string) error +} + +// Request 是一次扩展包申请。 +type Request struct { + ID string `json:"id"` + ConversationID string `json:"conversationId"` + Owner string `json:"owner"` + Spec string `json:"spec"` + Reason string `json:"reason"` + Status string `json:"status"` + LayerID string `json:"layerId"` + Detail string `json:"detail"` + Approver string `json:"approver"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +// Binding 是会话当前挂载的层。Isolated 非空时只挂隔离层，不挂基础层。 +type Binding struct { + ConversationID string `json:"conversationId"` + BaseID string `json:"baseId"` + Extensions []string `json:"extensions"` + Isolated string `json:"isolated"` + UpdatedAt string `json:"updatedAt"` +} + +// Outcome 是工具直接回给模型的结果。 +type Outcome struct { + Status string `json:"status"` + RequestID string `json:"requestId,omitempty"` + LayerID string `json:"layerId,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// Resolver 在宿主侧解析并下载 wheel。沙箱不实现这个接口。 +type Resolver interface { + Resolve(ctx context.Context, reqs []string, base Lock) (Result, error) + Fetch(ctx context.Context, wheelhouse string, pkg LockedPkg) (string, error) +} + +// Broker 在宿主侧解析、分级、构建。沙箱不调用它的构建路径。 +type Broker struct { + Root *Root + Store Store + Pip Resolver + Now func() time.Time +} + +func NewBroker(root *Root, st Store, pip Resolver) *Broker { + return &Broker{Root: root, Store: st, Pip: pip, Now: time.Now} +} + +// StatusText 给模型看的紧凑状态。 +func (b *Broker) StatusText(conv string) (string, error) { + var buf strings.Builder + baseID, err := b.Root.CurrentBase() + if err != nil { + return "", err + } + if baseID == "" { + buf.WriteString("基础层：未建立（需要 agent-system env ensure）\n") + } else { + lock, err := b.Root.ReadLock(baseID) + if err != nil { + return "", err + } + buf.WriteString("基础层：" + baseID + "\n") + for _, p := range lock.Packages { + fmt.Fprintf(&buf, "- %s==%s\n", p.Name, p.Version) + } + } + if conv != "" && b.Store != nil { + bind, err := b.Store.GetExecEnvBinding(conv) + if err != nil { + return "", err + } + if bind.Isolated != "" { + fmt.Fprintf(&buf, "本会话隔离层：%s\n", bind.Isolated) + } else if len(bind.Extensions) > 0 { + fmt.Fprintf(&buf, "本会话扩展层：%s\n", strings.Join(bind.Extensions, ", ")) + } else { + buf.WriteString("本会话扩展层：无\n") + } + reqs, err := b.Store.ListExecEnvRequests(conv, 10) + if err != nil { + return "", err + } + for _, r := range reqs { + if r.Status == "ready" || r.Status == "rejected" || r.Status == "failed" { + continue + } + fmt.Fprintf(&buf, "申请 %s 状态 %s %s\n", r.ID, r.Status, r.Detail) + } + } + return buf.String(), nil +} + +// RequestPackages 解析并按分级处理。wheelPath 非空表示用户上传的 wheel，一律待审批。 +func (b *Broker) RequestPackages(ctx context.Context, conv, owner, reason string, packages []string, wheelPath string) (Outcome, error) { + if b.Store == nil { + return Outcome{}, errors.New("execenv: store is not configured") + } + baseID, err := b.Root.CurrentBase() + if err != nil { + return Outcome{}, err + } + var base Lock + if baseID != "" { + base, err = b.Root.ReadLock(baseID) + if err != nil { + return Outcome{}, err + } + } + cat, err := b.Root.LoadCatalog() + if err != nil { + return Outcome{}, err + } + var res Result + uploaded := wheelPath != "" + if uploaded { + pkg, err := b.stageUpload(wheelPath) + if err != nil { + return Outcome{}, err + } + res.Packages = []LockedPkg{pkg} + } else if len(packages) == 0 { + return Outcome{}, errors.New("execenv: packages or wheel required") + } else { + res, err = b.Pip.Resolve(ctx, packages, base) + if err != nil { + return Outcome{}, err + } + } + grade, detail := grade(cat, res, uploaded) + specRaw, _ := json.Marshal(map[string]any{"packages": packages, "wheel": wheelPath, "resolved": res.Packages}) + rec := Request{ + ID: newID("env"), ConversationID: conv, Owner: owner, + Spec: string(specRaw), Reason: reason, Status: grade, Detail: detail, + CreatedAt: b.now(), UpdatedAt: b.now(), + } + if err := b.Store.InsertExecEnvRequest(rec); err != nil { + return Outcome{}, err + } + _ = b.Store.InsertExecEnvAudit(owner, "execenv.request", owner, rec.ID, grade, detail) + switch grade { + case "rejected": + return Outcome{Status: "rejected", RequestID: rec.ID, Detail: detail}, nil + case "pending_approval": + return Outcome{Status: "pending_approval", RequestID: rec.ID, Detail: detail}, nil + default: + id, err := b.materialize(ctx, conv, owner, base, res, false) + if err != nil { + _ = b.Store.UpdateExecEnvRequest(rec.ID, "failed", "", err.Error(), "") + return Outcome{Status: "failed", RequestID: rec.ID, Detail: err.Error()}, nil + } + _ = b.Store.UpdateExecEnvRequest(rec.ID, "ready", id, "已绑定", "auto") + return Outcome{Status: "ready", RequestID: rec.ID, LayerID: id, Detail: "已绑定，下一次 run_python 即可导入"}, nil + } +} + +func grade(cat Catalog, res Result, uploaded bool) (status, detail string) { + if len(res.Denied) > 0 || hasDenied(cat, res) { + return "rejected", "命中黑名单：" + strings.Join(deniedNames(cat, res), ", ") + } + if len(res.SDist) > 0 { + return "rejected", "只有源码包，拒绝执行构建脚本：" + strings.Join(res.SDist, ", ") + } + if len(res.Missing) > 0 { + return "rejected", "解析失败：" + strings.Join(res.Missing, "; ") + } + if len(res.Packages) == 0 && len(res.Override) == 0 { + return "rejected", "没有需要安装的新包（基础层已满足，或规格为空）" + } + var why []string + if uploaded { + why = append(why, "用户上传的 wheel") + } + if len(res.Override) > 0 { + why = append(why, strings.Join(res.Override, "; ")) + } + for _, p := range res.Packages { + if !cat.allowed(p.Name) { + why = append(why, p.Name+" 不在白名单") + } + if cat.MaxWheelBytes > 0 && p.Size > cat.MaxWheelBytes { + why = append(why, fmt.Sprintf("%s 体积 %d 超过上限", p.Name, p.Size)) + } + if p.SHA256 == "" && !uploaded { + why = append(why, p.Name+" 缺少哈希") + } + } + if len(why) > 0 { + return "pending_approval", strings.Join(why, "；") + } + return "approved", "白名单内自动批准" +} + +func hasDenied(cat Catalog, res Result) bool { + return len(deniedNames(cat, res)) > 0 +} + +func deniedNames(cat Catalog, res Result) []string { + var out []string + for _, p := range res.Packages { + if cat.denied(p.Name) { + out = append(out, p.Name) + } + } + return out +} + +// Decide 处理审批。isolate 仅在申请确实与基础层冲突时建立隔离层。 +func (b *Broker) Decide(ctx context.Context, id, actor string, admin, allow, isolate bool) (Outcome, error) { + rec, err := b.Store.GetExecEnvRequest(id) + if err != nil { + return Outcome{}, err + } + if rec.Status != "pending_approval" { + return Outcome{}, fmt.Errorf("execenv: request %s is %s", id, rec.Status) + } + if !admin && actor != rec.Owner { + return Outcome{}, errors.New("execenv: only the session owner or an admin can decide") + } + if !allow { + if err := b.Store.UpdateExecEnvRequest(id, "rejected", "", "已拒绝", actor); err != nil { + return Outcome{}, err + } + _ = b.Store.InsertExecEnvAudit(rec.Owner, "execenv.decide", actor, id, "rejected", "") + return Outcome{Status: "rejected", RequestID: id, Detail: "已拒绝"}, nil + } + var spec struct { + Packages []string `json:"packages"` + Resolved []LockedPkg `json:"resolved"` + } + _ = json.Unmarshal([]byte(rec.Spec), &spec) + baseID, err := b.Root.CurrentBase() + if err != nil { + return Outcome{}, err + } + var base Lock + if baseID != "" { + base, err = b.Root.ReadLock(baseID) + if err != nil { + return Outcome{}, err + } + } + var res Result + if isolate { + // 隔离层不继承基础层约束，整份闭包单独成层。 + res, err = b.Pip.Resolve(ctx, spec.Packages, Lock{}) + if err != nil { + return Outcome{}, err + } + } else if len(spec.Resolved) > 0 { + res.Packages = spec.Resolved + } else if len(spec.Packages) > 0 { + res, err = b.Pip.Resolve(ctx, spec.Packages, base) + if err != nil { + return Outcome{}, err + } + } + if len(res.Override) > 0 && !isolate { + msg := "与基础层版本冲突，批准时需要选择隔离层：" + strings.Join(res.Override, "; ") + _ = b.Store.UpdateExecEnvRequest(id, "pending_approval", "", msg, "") + return Outcome{Status: "pending_approval", RequestID: id, Detail: msg}, nil + } + layer, err := b.materialize(ctx, rec.ConversationID, rec.Owner, base, res, isolate) + if err != nil { + _ = b.Store.UpdateExecEnvRequest(id, "failed", "", err.Error(), actor) + return Outcome{Status: "failed", RequestID: id, Detail: err.Error()}, nil + } + if err := b.Store.UpdateExecEnvRequest(id, "ready", layer, "已批准并绑定", actor); err != nil { + return Outcome{}, err + } + _ = b.Store.InsertExecEnvAudit(rec.Owner, "execenv.decide", actor, id, "ready", layer) + return Outcome{Status: "ready", RequestID: id, LayerID: layer, Detail: "已批准并绑定"}, nil +} + +func (b *Broker) now() string { return b.Now().UTC().Format(time.RFC3339Nano) } + +func newID(prefix string) string { + var buf [8]byte + _, _ = rand.Read(buf[:]) + return prefix + "_" + hex.EncodeToString(buf[:]) +} + +func (b *Broker) stageUpload(path string) (LockedPkg, error) { + if filepath.Ext(path) != ".whl" { + return LockedPkg{}, errors.New("execenv: upload must be a .whl") + } + f, err := os.Open(path) + if err != nil { + return LockedPkg{}, err + } + defer f.Close() + sum := sha256New() + n, err := io.Copy(sum, f) + if err != nil { + return LockedPkg{}, err + } + sha := sum.Sum() + dest := filepath.Join(b.Root.Wheelhouse(), sha+".whl") + if _, err := os.Stat(dest); err != nil { + in, err := os.Open(path) + if err != nil { + return LockedPkg{}, err + } + defer in.Close() + out, err := os.OpenFile(dest, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil && !errors.Is(err, os.ErrExist) { + return LockedPkg{}, err + } + if err == nil { + if _, err := io.Copy(out, in); err != nil { + out.Close() + return LockedPkg{}, err + } + out.Close() + } + } + name := strings.Split(filepath.Base(path), "-")[0] + return LockedPkg{Name: name, Version: "0", Filename: filepath.Base(path), SHA256: sha, Size: n}, nil +} diff --git a/internal/execenv/broker_test.go b/internal/execenv/broker_test.go new file mode 100644 index 0000000..b6b1f04 --- /dev/null +++ b/internal/execenv/broker_test.go @@ -0,0 +1,171 @@ +package execenv + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +type memStore struct { + reqs map[string]Request + bind map[string]Binding +} + +func newMem() *memStore { return &memStore{reqs: map[string]Request{}, bind: map[string]Binding{}} } + +func (m *memStore) InsertExecEnvRequest(r Request) error { + m.reqs[r.ID] = r + return nil +} +func (m *memStore) GetExecEnvRequest(id string) (Request, error) { return m.reqs[id], nil } +func (m *memStore) ListExecEnvRequests(conv string, limit int) ([]Request, error) { + var out []Request + for _, r := range m.reqs { + if conv == "" || r.ConversationID == conv { + out = append(out, r) + } + } + return out, nil +} +func (m *memStore) UpdateExecEnvRequest(id, status, layerID, detail, approver string) error { + r := m.reqs[id] + r.Status, r.LayerID, r.Detail, r.Approver = status, layerID, detail, approver + m.reqs[id] = r + return nil +} +func (m *memStore) GetExecEnvBinding(conv string) (Binding, error) { return m.bind[conv], nil } +func (m *memStore) PutExecEnvBinding(b Binding) error { + m.bind[b.ConversationID] = b + return nil +} +func (m *memStore) ListExecEnvBindings() ([]Binding, error) { + var out []Binding + for _, b := range m.bind { + out = append(out, b) + } + return out, nil +} +func (m *memStore) InsertExecEnvAudit(string, string, string, string, string, string) error { + return nil +} + +type scripted struct{ byReq map[string]Result } + +func (s scripted) Resolve(_ context.Context, reqs []string, _ Lock) (Result, error) { + if len(reqs) == 0 { + return Result{}, nil + } + return s.byReq[reqs[0]], nil +} +func (s scripted) Fetch(context.Context, string, LockedPkg) (string, error) { + return "", os.ErrNotExist +} + +func TestConversationsGetDistinctLayers(t *testing.T) { + root, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + cat := DefaultCatalog() + cat.Allow = append(cat.Allow, CatalogEntry{Name: "foo", Reason: "test"}) + raw, _ := json.Marshal(cat) + if err := os.WriteFile(root.CatalogPath(), raw, 0o644); err != nil { + t.Fatal(err) + } + baseSum := placeWheel(t, root, "bar.py") + baseID, err := root.BuildLayer(Lock{ + Schema: 1, Kind: "base", PythonTag: "cp314", Platform: "any", + Packages: []LockedPkg{{Name: "bar", Version: "1", Filename: baseSum + ".whl", SHA256: baseSum}}, + }) + if err != nil { + t.Fatal(err) + } + if err := root.SwitchCurrent(baseID); err != nil { + t.Fatal(err) + } + sum1 := placeWheel(t, root, "foo1.py") + sum2 := placeWheel(t, root, "foo2.py") + broker := NewBroker(root, newMem(), scripted{byReq: map[string]Result{ + "foo==1": {Packages: []LockedPkg{{Name: "foo", Version: "1", Filename: sum1 + ".whl", SHA256: sum1}}}, + "foo==2": {Packages: []LockedPkg{{Name: "foo", Version: "2", Filename: sum2 + ".whl", SHA256: sum2}}}, + }}) + a, err := broker.RequestPackages(context.Background(), "conv-a", "owner", "need 1", []string{"foo==1"}, "") + if err != nil || a.Status != "ready" { + t.Fatalf("A: %+v %v", a, err) + } + b, err := broker.RequestPackages(context.Background(), "conv-b", "owner", "need 2", []string{"foo==2"}, "") + if err != nil || b.Status != "ready" { + t.Fatalf("B: %+v %v", b, err) + } + if a.LayerID == b.LayerID { + t.Fatalf("layers should differ: %s", a.LayerID) + } + cur, err := root.CurrentBase() + if err != nil || cur != baseID { + t.Fatalf("base changed: %s -> %s (%v)", baseID, cur, err) + } + extraA, isoA := broker.LayersFor("conv-a") + extraB, isoB := broker.LayersFor("conv-b") + if isoA != "" || isoB != "" || len(extraA) != 1 || len(extraB) != 1 || extraA[0] == extraB[0] { + t.Fatalf("mounts A=%v %s B=%v %s", extraA, isoA, extraB, isoB) + } +} + +func TestRequestGrades(t *testing.T) { + root, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + broker := NewBroker(root, newMem(), scripted{byReq: map[string]Result{ + "onlysrc": {SDist: []string{"onlysrc"}}, + "secret": {Packages: []LockedPkg{{Name: "secret", Version: "1", SHA256: "abc"}}}, + }}) + sdist, err := broker.RequestPackages(context.Background(), "c", "o", "", []string{"onlysrc"}, "") + if err != nil || sdist.Status != "rejected" { + t.Fatalf("sdist: %+v %v", sdist, err) + } + outside, err := broker.RequestPackages(context.Background(), "c", "o", "", []string{"secret"}, "") + if err != nil || outside.Status != "pending_approval" { + t.Fatalf("allowlist: %+v %v", outside, err) + } +} + +func placeWheel(t *testing.T, root *Root, member string) string { + t.Helper() + buf := &bytes.Buffer{} + zw := zip.NewWriter(buf) + f, err := zw.Create(member) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write([]byte("x = 1\n")); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(buf.Bytes()) + hexSum := hex.EncodeToString(sum[:]) + if err := os.WriteFile(filepath.Join(root.Wheelhouse(), hexSum+".whl"), buf.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return hexSum +} + +func osWriteWheel(path, member string) error { + buf := &bytes.Buffer{} + zw := zip.NewWriter(buf) + if _, err := zw.Create(member); err != nil { + return err + } + if err := zw.Close(); err != nil { + return err + } + return os.WriteFile(path, buf.Bytes(), 0o644) +} diff --git a/internal/execenv/catalog.go b/internal/execenv/catalog.go new file mode 100644 index 0000000..f31ed3e --- /dev/null +++ b/internal/execenv/catalog.go @@ -0,0 +1,77 @@ +package execenv + +import ( + "encoding/json" + "os" +) + +// Catalog 是管理员可编辑的白名单。不在白名单里的包不会自动安装。 +type Catalog struct { + MaxWheelBytes int64 `json:"maxWheelBytes"` + Allow []CatalogEntry `json:"allow"` + Deny []string `json:"deny"` +} + +// CatalogEntry 的 Spec 为空表示任意版本；非空时是给人类看的范围说明， +// 自动批准只看包名是否在列，版本冲突另由基础层锁判定。 +type CatalogEntry struct { + Name string `json:"name"` + Spec string `json:"spec,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// DefaultCatalog 是第一次创建环境根时落下的名单。之后以磁盘上的 catalog.json 为准。 +func DefaultCatalog() Catalog { + return Catalog{ + MaxWheelBytes: 80 << 20, + Allow: []CatalogEntry{ + {Name: "python-docx", Reason: "Word 交付物"}, + {Name: "openpyxl", Reason: "Excel"}, + {Name: "python-pptx", Reason: "PPT"}, + {Name: "lxml", Reason: "OOXML"}, + {Name: "pillow", Reason: "图片"}, + {Name: "numpy", Reason: "数值"}, + {Name: "matplotlib", Reason: "图表"}, + {Name: "pypdf", Reason: "PDF 读取"}, + {Name: "reportlab", Reason: "PDF 生成"}, + {Name: "markdown", Reason: "Markdown"}, + {Name: "pdfplumber", Reason: "PDF 抽取"}, + }, + Deny: []string{"setuptools", "pip"}, + } +} + +func (r *Root) LoadCatalog() (Catalog, error) { + raw, err := os.ReadFile(r.CatalogPath()) + if err != nil { + return Catalog{}, err + } + var c Catalog + if err := json.Unmarshal(raw, &c); err != nil { + return Catalog{}, err + } + if c.MaxWheelBytes <= 0 { + c.MaxWheelBytes = 80 << 20 + } + return c, nil +} + +func (c Catalog) allowed(name string) bool { + n := normName(name) + for _, e := range c.Allow { + if normName(e.Name) == n { + return true + } + } + return false +} + +func (c Catalog) denied(name string) bool { + n := normName(name) + for _, d := range c.Deny { + if normName(d) == n { + return true + } + } + return false +} diff --git a/internal/execenv/doctor.go b/internal/execenv/doctor.go new file mode 100644 index 0000000..4d64ab3 --- /dev/null +++ b/internal/execenv/doctor.go @@ -0,0 +1,97 @@ +package execenv + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// Doctor 用给定解释器导入模块。sandbox 为真且系统有 bwrap 时，按与执行后端相同的隔离方式跑。 +func Doctor(ctx context.Context, python string, pyPath []string, modules []string, sandbox bool, mounts []string) error { + if python == "" { + return fmt.Errorf("execenv: doctor: python is empty") + } + if len(modules) == 0 { + modules = []string{"docx", "openpyxl", "pptx"} + } + script := "import " + strings.Join(modules, ", ") + if !sandbox { + cmd := exec.CommandContext(ctx, python, "-c", script) + cmd.Env = doctorEnv(pyPath) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("execenv: doctor: %v: %s", err, trim(out)) + } + return nil + } + bwrap, err := exec.LookPath("bwrap") + if err != nil { + return fmt.Errorf("execenv: doctor: sandbox requested but bwrap is missing: %w", err) + } + argv := []string{"--unshare-all", "--die-with-parent", "--ro-bind", "/", "/", "--tmpfs", "/home", "--tmpfs", "/tmp", "--proc", "/proc", "--dev", "/dev"} + for _, m := range mounts { + if m == "" { + continue + } + argv = append(argv, "--ro-bind", m, m) + } + argv = append(argv, "--clearenv") + for _, kv := range doctorEnv(pyPath) { + if i := strings.IndexByte(kv, '='); i > 0 { + argv = append(argv, "--setenv", kv[:i], kv[i+1:]) + } + } + argv = append(argv, python, "-c", script) + cmd := exec.CommandContext(ctx, bwrap, argv...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("execenv: doctor sandbox: %v: %s", err, trim(out)) + } + return nil +} + +func doctorEnv(pyPath []string) []string { + env := []string{ + "PATH=/usr/local/bin:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE=1", + "PIP_NO_INDEX=1", + } + if len(pyPath) > 0 { + env = append(env, "PYTHONPATH="+strings.Join(pyPath, string(os.PathListSeparator))) + } + return env +} + +func trim(b []byte) string { + s := string(bytes.TrimSpace(b)) + if len(s) > 500 { + s = s[:500] + } + return s +} + +// EnsurePython 优先下载独立 CPython；失败则用系统 python 建一份 venv。 +func (r *Root) EnsurePython(ctx context.Context) (string, error) { + if bin, ok := r.PythonBin(); ok { + return bin, nil + } + pctx, cancel := context.WithTimeout(ctx, 4*time.Minute) + defer cancel() + if err := r.InstallStandalone(pctx, ""); err == nil { + if bin, ok := r.PythonBin(); ok { + return bin, nil + } + } + if err := r.BootstrapVenv("/usr/bin/python3"); err != nil { + return "", err + } + bin, ok := r.PythonBin() + if !ok { + return "", fmt.Errorf("execenv: managed python missing after bootstrap") + } + return bin, nil +} diff --git a/internal/execenv/gc.go b/internal/execenv/gc.go new file mode 100644 index 0000000..9eb674b --- /dev/null +++ b/internal/execenv/gc.go @@ -0,0 +1,61 @@ +package execenv + +import ( + "os" + "path/filepath" + "time" +) + +// GC 删除未被 current、绑定或锁引用、且早于 ttl 的层目录。ttl<=0 时用 7 天。 +func (b *Broker) GC(ttl time.Duration) ([]string, error) { + if ttl <= 0 { + ttl = 7 * 24 * time.Hour + } + keep := map[string]bool{} + if id, err := b.Root.CurrentBase(); err != nil { + return nil, err + } else if id != "" { + keep[id] = true + } + if b.Store != nil { + binds, err := b.Store.ListExecEnvBindings() + if err != nil { + return nil, err + } + for _, bind := range binds { + if bind.BaseID != "" { + keep[bind.BaseID] = true + } + if bind.Isolated != "" { + keep[bind.Isolated] = true + } + for _, id := range bind.Extensions { + keep[id] = true + } + } + } + ents, err := os.ReadDir(b.Root.LayersDir()) + if err != nil { + return nil, err + } + cut := time.Now().Add(-ttl) + var removed []string + for _, ent := range ents { + if !ent.IsDir() || keep[ent.Name()] { + continue + } + info, err := ent.Info() + if err != nil { + return removed, err + } + if info.ModTime().After(cut) { + continue + } + if err := os.RemoveAll(filepath.Join(b.Root.LayersDir(), ent.Name())); err != nil { + return removed, err + } + _ = os.Remove(filepath.Join(b.Root.LocksDir(), ent.Name()+".json")) + removed = append(removed, ent.Name()) + } + return removed, nil +} diff --git a/internal/execenv/hashio.go b/internal/execenv/hashio.go new file mode 100644 index 0000000..8fce17e --- /dev/null +++ b/internal/execenv/hashio.go @@ -0,0 +1,20 @@ +package execenv + +import ( + "crypto/sha256" + "encoding/hex" + "hash" + "io" +) + +type shaWriter struct{ h hash.Hash } + +func sha256New() shaWriter { return shaWriter{h: sha256.New()} } + +func (s shaWriter) Sum() string { return hex.EncodeToString(s.h.Sum(nil)) } + +// Write 满足 io.Writer，便于 fileSHA256 之外的流式哈希。 +func (s shaWriter) Write(p []byte) (int, error) { return s.h.Write(p) } + +// 确认 shaWriter 可传给 io.Copy。 +var _ io.Writer = shaWriter{} diff --git a/internal/execenv/lock.go b/internal/execenv/lock.go new file mode 100644 index 0000000..31a127c --- /dev/null +++ b/internal/execenv/lock.go @@ -0,0 +1,62 @@ +package execenv + +import ( + "encoding/json" + "sort" + "strings" +) + +// Lock 是一层的不可变清单。ID 由规范 JSON 决定，与写盘时的空白无关。 +type Lock struct { + Schema int `json:"schema"` + Kind string `json:"kind"` // base | extension | isolated + PythonTag string `json:"pythonTag"` + Platform string `json:"platform"` + BaseID string `json:"baseId,omitempty"` + Packages []LockedPkg `json:"packages"` +} + +// LockedPkg 是一个 wheel。SHA256 是文件字节的哈希，不是文件名。 +type LockedPkg struct { + Name string `json:"name"` + Version string `json:"version"` + Filename string `json:"filename"` + SHA256 string `json:"sha256"` + Size int64 `json:"size,omitempty"` +} + +// ID 返回 base- / x- / iso-。 +func (l Lock) ID() string { + canon := l + canon.Packages = append([]LockedPkg(nil), l.Packages...) + sort.Slice(canon.Packages, func(i, j int) bool { + return normName(canon.Packages[i].Name) < normName(canon.Packages[j].Name) + }) + raw, _ := json.Marshal(canon) + sum := HashBytes(raw) + prefix := "x-" + switch l.Kind { + case "base": + prefix = "base-" + case "isolated": + prefix = "iso-" + } + return prefix + sum[:16] +} + +func normName(name string) string { + var b strings.Builder + prevDash := false + for _, r := range strings.ToLower(name) { + if r == '-' || r == '_' || r == '.' { + if !prevDash && b.Len() > 0 { + b.WriteByte('-') + prevDash = true + } + continue + } + prevDash = false + b.WriteRune(r) + } + return strings.Trim(b.String(), "-") +} diff --git a/internal/execenv/materialize.go b/internal/execenv/materialize.go new file mode 100644 index 0000000..bc2da95 --- /dev/null +++ b/internal/execenv/materialize.go @@ -0,0 +1,206 @@ +package execenv + +import ( + "context" + "errors" + "fmt" + "os" + "time" +) + +func (b *Broker) materialize(ctx context.Context, conv, owner string, base Lock, res Result, isolate bool) (string, error) { + pkgs := append([]LockedPkg(nil), res.Packages...) + for i := range pkgs { + if pkgs[i].SHA256 == "" || !wheelExists(b.Root, pkgs[i]) { + path, err := b.Pip.Fetch(ctx, b.Root.Wheelhouse(), pkgs[i]) + if err != nil { + return "", err + } + sum, err := fileSHA256(path) + if err != nil { + return "", err + } + if pkgs[i].SHA256 != "" && sum != pkgs[i].SHA256 { + return "", fmt.Errorf("execenv: hash mismatch for %s", pkgs[i].Name) + } + pkgs[i].SHA256 = sum + pkgs[i].Filename = pkgs[i].SHA256 + ".whl" + stable := b.Root.Wheelhouse() + "/" + pkgs[i].Filename + if path != stable { + if err := os.Rename(path, stable); err != nil && !errors.Is(err, os.ErrExist) { + // Fetch 可能已经改名。忽略已就位的文件。 + if _, statErr := os.Stat(stable); statErr != nil { + return "", err + } + } + } + } + } + kind := "extension" + baseID := "" + if id, _ := b.Root.CurrentBase(); id != "" { + baseID = id + } + lockPkgs := pkgs + if isolate { + kind = "isolated" + } else if baseID == "" { + kind = "base" + } + if !isolate && baseID != "" { + lockPkgs = res.Packages + } + lock := Lock{ + Schema: 1, Kind: kind, PythonTag: base.PythonTag, Platform: base.Platform, + BaseID: baseID, Packages: lockPkgs, + } + if lock.PythonTag == "" { + lock.PythonTag = "cp314" + } + if lock.Platform == "" { + lock.Platform = "any" + } + id, err := b.Root.BuildLayer(lock) + if err != nil { + return "", err + } + if kind == "base" { + if err := b.Root.SwitchCurrent(id); err != nil { + return "", err + } + return id, nil + } + bind, err := b.Store.GetExecEnvBinding(conv) + if err != nil { + return "", err + } + bind.ConversationID = conv + bind.BaseID = baseID + bind.UpdatedAt = b.now() + if isolate { + bind.Isolated = id + bind.Extensions = nil + } else { + bind.Isolated = "" + bind.Extensions = appendUnique(bind.Extensions, id) + } + if err := b.Store.PutExecEnvBinding(bind); err != nil { + return "", err + } + _ = owner + return id, nil +} + +func wheelExists(r *Root, pkg LockedPkg) bool { + if pkg.SHA256 == "" { + return false + } + _, err := os.Stat(r.Wheelhouse() + "/" + pkg.SHA256 + ".whl") + return err == nil +} + +func appendUnique(ss []string, id string) []string { + for _, s := range ss { + if s == id { + return ss + } + } + return append(ss, id) +} + +// LayersFor 返回本次执行要挂载的 site-packages。isolated 非空时调用方不要再挂基础层。 +func (b *Broker) LayersFor(conv string) (extra []string, isolated string) { + if b.Store == nil || conv == "" { + return nil, "" + } + bind, err := b.Store.GetExecEnvBinding(conv) + if err != nil { + return nil, "" + } + if bind.Isolated != "" { + return nil, b.Root.LayerDir(bind.Isolated) + } + for _, id := range bind.Extensions { + extra = append(extra, b.Root.LayerDir(id)) + } + return extra, "" +} + +// Wait 轮询申请直到终态或超时。 +func (b *Broker) Wait(ctx context.Context, id string, timeout time.Duration) (Outcome, error) { + deadline := time.Now().Add(timeout) + if timeout <= 0 { + deadline = time.Now().Add(2 * time.Minute) + } + for { + rec, err := b.Store.GetExecEnvRequest(id) + if err != nil { + return Outcome{}, err + } + switch rec.Status { + case "ready", "rejected", "failed": + return Outcome{Status: rec.Status, RequestID: id, LayerID: rec.LayerID, Detail: rec.Detail}, nil + } + if time.Now().After(deadline) || ctx.Err() != nil { + return Outcome{Status: rec.Status, RequestID: id, Detail: "仍在等待审批或构建"}, nil + } + select { + case <-ctx.Done(): + return Outcome{Status: rec.Status, RequestID: id, Detail: "等待被取消"}, nil + case <-time.After(200 * time.Millisecond): + } + } +} + +// EnsureBase 解析基础层需求并建立 current。已有 current 时直接返回其 ID。 +func (b *Broker) EnsureBase(ctx context.Context) (string, error) { + if id, err := b.Root.CurrentBase(); err != nil { + return "", err + } else if id != "" { + return id, nil + } + res, err := b.Pip.Resolve(ctx, BaseRequirements(), Lock{}) + if err != nil { + return "", err + } + if !res.OK() { + return "", fmt.Errorf("execenv: base resolve: override=%v sdist=%v missing=%v", res.Override, res.SDist, res.Missing) + } + return b.materialize(ctx, "", "", Lock{Kind: "base"}, res, false) +} + +// Rebase 强制按当前需求重建基础层并移动 current。已有会话绑定保持旧 ID。 +func (b *Broker) Rebase(ctx context.Context) (string, error) { + res, err := b.Pip.Resolve(ctx, BaseRequirements(), Lock{}) + if err != nil { + return "", err + } + if !res.OK() { + return "", fmt.Errorf("execenv: rebase: override=%v sdist=%v missing=%v", res.Override, res.SDist, res.Missing) + } + lock := Lock{Schema: 1, Kind: "base", PythonTag: "cp314", Platform: "any", Packages: res.Packages} + for i := range lock.Packages { + path, err := b.Pip.Fetch(ctx, b.Root.Wheelhouse(), lock.Packages[i]) + if err != nil { + return "", err + } + sum, err := fileSHA256(path) + if err != nil { + return "", err + } + lock.Packages[i].SHA256 = sum + lock.Packages[i].Filename = sum + ".whl" + stable := b.Root.Wheelhouse() + "/" + sum + ".whl" + if path != stable { + _ = os.Rename(path, stable) + } + } + id, err := b.Root.BuildLayer(lock) + if err != nil { + return "", err + } + if err := b.Root.SwitchCurrent(id); err != nil { + return "", err + } + return id, nil +} diff --git a/internal/execenv/pip.go b/internal/execenv/pip.go new file mode 100644 index 0000000..8125ce5 --- /dev/null +++ b/internal/execenv/pip.go @@ -0,0 +1,183 @@ +package execenv + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +//go:embed base-requirements.txt +var baseRequirements string + +// BaseRequirements 是基础层的包名清单（不带版本；版本由第一次解析锁定）。 +func BaseRequirements() []string { + var out []string + for _, line := range strings.Split(baseRequirements, "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "#") { + out = append(out, line) + } + } + return out +} + +// Pip 用宿主侧的 pip 下载 wheel。它不在沙箱里运行。 +type Pip struct { + Python string // 带 pip 的解释器 +} + +func (p Pip) python() string { + if p.Python != "" { + return p.Python + } + return "python3" +} + +// Resolve 以基础层锁为 constraints 做 dry-run。冲突记入 Override，不改锁。 +func (p Pip) Resolve(ctx context.Context, reqs []string, base Lock) (Result, error) { + if len(reqs) == 0 { + return Result{}, nil + } + dir, err := os.MkdirTemp("", "execenv-resolve-") + if err != nil { + return Result{}, err + } + defer os.RemoveAll(dir) + cons := filepath.Join(dir, "constraints.txt") + if err := os.WriteFile(cons, []byte(constraintText(base)), 0o644); err != nil { + return Result{}, err + } + args := []string{"-m", "pip", "install", "--dry-run", "--report", "-", "--only-binary=:all:", "--constraint", cons} + args = append(args, reqs...) + cmd := exec.CommandContext(ctx, p.python(), args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + text := stderr.String() + stdout.String() + if err != nil { + if strings.Contains(text, "ResolutionImpossible") || strings.Contains(text, "conflicting") { + return Result{Override: []string{firstLine(text)}}, nil + } + if strings.Contains(text, "only-binary") || strings.Contains(text, "No matching distribution") { + return Result{SDist: reqs, Missing: []string{firstLine(text)}}, nil + } + return Result{}, fmt.Errorf("execenv: pip resolve: %v: %s", err, firstLine(text)) + } + return parsePipReport(stdout.Bytes(), base) +} + +func parsePipReport(raw []byte, base Lock) (Result, error) { + var doc struct { + Install []struct { + Metadata struct { + Name string `json:"name"` + Version string `json:"version"` + } `json:"metadata"` + DownloadInfo struct { + ArchiveInfo struct { + Hash string `json:"hash"` + } `json:"archive_info"` + URL string `json:"url"` + } `json:"download_info"` + } `json:"install"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return Result{}, fmt.Errorf("execenv: pip report: %w", err) + } + pinned := map[string]LockedPkg{} + for _, c := range base.Packages { + pinned[normName(c.Name)] = c + } + var res Result + for _, item := range doc.Install { + name := item.Metadata.Name + if pin, ok := pinned[normName(name)]; ok { + if pin.Version != item.Metadata.Version { + res.Override = append(res.Override, fmt.Sprintf("%s 需要 %s，基础层固定为 %s", name, item.Metadata.Version, pin.Version)) + } + continue + } + hash := strings.TrimPrefix(item.DownloadInfo.ArchiveInfo.Hash, "sha256=") + filename := name + "-" + item.Metadata.Version + ".whl" + if item.DownloadInfo.URL != "" { + filename = filepath.Base(strings.Split(item.DownloadInfo.URL, "?")[0]) + } + res.Packages = append(res.Packages, LockedPkg{ + Name: name, Version: item.Metadata.Version, Filename: filename, SHA256: hash, + }) + } + return res, nil +} + +// Fetch 把一个包下载进 wheelhouse，并返回落地路径。哈希不符则删除并报错。 +func (p Pip) Fetch(ctx context.Context, wheelhouse string, pkg LockedPkg) (string, error) { + if err := os.MkdirAll(wheelhouse, 0o755); err != nil { + return "", err + } + dest := filepath.Join(wheelhouse, pkg.Filename) + if pkg.SHA256 != "" { + if sum, err := fileSHA256(dest); err == nil && strings.EqualFold(sum, pkg.SHA256) { + return dest, nil + } + } + cmd := exec.CommandContext(ctx, p.python(), "-m", "pip", "download", + "--only-binary=:all:", "--no-deps", "-d", wheelhouse, pkg.Name+"=="+pkg.Version) + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("execenv: pip download %s: %v: %s", pkg.Name, err, firstLine(string(out))) + } + found := dest + if _, err := os.Stat(found); err != nil { + matches, _ := filepath.Glob(filepath.Join(wheelhouse, normName(pkg.Name)+"*.whl")) + if len(matches) == 0 { + matches, _ = filepath.Glob(filepath.Join(wheelhouse, "*.whl")) + } + if len(matches) == 0 { + return "", fmt.Errorf("execenv: download produced no wheel for %s", pkg.Name) + } + found = matches[len(matches)-1] + } + sum, err := fileSHA256(found) + if err != nil { + return "", err + } + if pkg.SHA256 != "" && !strings.EqualFold(sum, pkg.SHA256) { + _ = os.Remove(found) + return "", fmt.Errorf("execenv: hash mismatch for %s", pkg.Name) + } + stable := filepath.Join(wheelhouse, sum+".whl") + if found != stable { + if err := os.Rename(found, stable); err != nil { + return found, nil + } + } + return stable, nil +} + +func constraintText(base Lock) string { + var b strings.Builder + for _, p := range base.Packages { + if p.Name != "" && p.Version != "" { + fmt.Fprintf(&b, "%s==%s\n", p.Name, p.Version) + } + } + return b.String() +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 400 { + s = s[:400] + } + return s +} diff --git a/internal/execenv/python.go b/internal/execenv/python.go new file mode 100644 index 0000000..b677195 --- /dev/null +++ b/internal/execenv/python.go @@ -0,0 +1,241 @@ +package execenv + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" +) + +// PythonBin 返回托管解释器。未安装时 ok=false。 +func (r *Root) PythonBin() (string, bool) { + bin := filepath.Join(r.PythonDir(), "current", "bin", "python3") + st, err := os.Stat(bin) + if err != nil || st.IsDir() { + return "", false + } + return bin, true +} + +// PythonPrefix 是需要在沙箱里重新挂载的前缀（current 的真实目录）。 +func (r *Root) PythonPrefix() string { + dest, err := filepath.EvalSymlinks(filepath.Join(r.PythonDir(), "current")) + if err != nil { + return "" + } + return dest +} + +// LinkPython 把已有解释器收成托管前缀：python/host/bin/python3 -> 绝对路径。 +// 这是下载独立 CPython 失败时的退化，仍然全系统只有一份，不是每个工作区一份。 +func (r *Root) LinkPython(bin string) error { + if !filepath.IsAbs(bin) { + return fmt.Errorf("execenv: python bin must be absolute: %s", bin) + } + if _, err := os.Stat(bin); err != nil { + return err + } + host := filepath.Join(r.PythonDir(), "host") + bindir := filepath.Join(host, "bin") + if err := os.MkdirAll(bindir, 0o755); err != nil { + return err + } + link := filepath.Join(bindir, "python3") + _ = os.Remove(link) + if err := os.Symlink(bin, link); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(r.PythonDir(), "SOURCE"), []byte("link "+bin+"\n"), 0o644); err != nil { + return err + } + return r.switchPython("host") +} + +func (r *Root) switchPython(name string) error { + link := filepath.Join(r.PythonDir(), "current") + tmp := link + ".tmp" + _ = os.Remove(tmp) + if err := os.Symlink(name, tmp); err != nil { + return err + } + return os.Rename(tmp, link) +} + +// InstallStandalone 下载 python-build-standalone 的 install_only 包并解到 python/。 +// url 为空时向 GitHub latest 要与本机架构匹配的 cpython 3.14 资产。 +func (r *Root) InstallStandalone(ctx context.Context, url string) error { + if url == "" { + var err error + url, err = latestStandaloneURL(ctx) + if err != nil { + return err + } + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + client := &http.Client{Timeout: 5 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return fmt.Errorf("execenv: download python: %s", resp.Status) + } + dest := filepath.Join(r.PythonDir(), "cpython-standalone") + if err := os.RemoveAll(dest); err != nil { + return err + } + if err := os.MkdirAll(dest, 0o755); err != nil { + return err + } + if err := extractTarGz(resp.Body, dest); err != nil { + return err + } + bin := filepath.Join(dest, "python", "bin", "python3") + if _, err := os.Stat(bin); err != nil { + return fmt.Errorf("execenv: standalone archive has no python/bin/python3") + } + // 让 current/bin/python3 成立：current -> cpython-standalone/python。 + inner := filepath.Join("cpython-standalone", "python") + if err := os.WriteFile(filepath.Join(r.PythonDir(), "SOURCE"), []byte("standalone "+url+"\n"), 0o644); err != nil { + return err + } + return r.switchPython(inner) +} + +func latestStandaloneURL(ctx context.Context) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + "https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest", nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("User-Agent", "agent-system-execenv") + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("execenv: github releases: %s", resp.Status) + } + var doc struct { + Assets []struct { + Name string `json:"name"` + URL string `json:"browser_download_url"` + } `json:"assets"` + } + if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { + return "", err + } + arch := "x86_64" + if runtime.GOARCH == "arm64" { + arch = "aarch64" + } + needle := arch + "-unknown-linux-gnu-install_only.tar.gz" + var fallback string + for _, a := range doc.Assets { + if !strings.Contains(a.Name, needle) || !strings.HasPrefix(a.Name, "cpython-3.14") { + continue + } + if strings.Contains(a.Name, "freethreaded") { + fallback = a.URL + continue + } + return a.URL, nil + } + if fallback != "" { + return fallback, nil + } + return "", errors.New("execenv: no cpython 3.14 install_only asset") +} + +func extractTarGz(rd io.Reader, dest string) error { + gz, err := gzip.NewReader(rd) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + name := filepath.Clean(hdr.Name) + if name == "." || strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + return fmt.Errorf("execenv: tar path escapes: %s", hdr.Name) + } + target := filepath.Join(dest, name) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) && target != filepath.Clean(dest) { + return fmt.Errorf("execenv: tar path escapes: %s", hdr.Name) + } + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + mode := os.FileMode(hdr.Mode) & 0o777 + if mode == 0 { + mode = 0o644 + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + _, cerr := io.Copy(f, tr) + f.Close() + if cerr != nil { + return cerr + } + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(hdr.Linkname, target); err != nil { + return err + } + } + } +} + +// BootstrapVenv 用系统 python3 做一份唯一的托管前缀。解释器仍指向系统 python， +// 但 pip 在这份前缀里，构建器可以下载 wheel。 +func (r *Root) BootstrapVenv(python string) error { + if python == "" { + python = "/usr/bin/python3" + } + dest := filepath.Join(r.PythonDir(), "venv") + if _, err := os.Stat(filepath.Join(dest, "bin", "python3")); err != nil { + cmd := exec.Command(python, "-m", "venv", dest) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("execenv: venv: %v: %s", err, out) + } + } + if err := os.WriteFile(filepath.Join(r.PythonDir(), "SOURCE"), []byte("venv "+python+"\n"), 0o644); err != nil { + return err + } + return r.switchPython("venv") +} diff --git a/internal/execenv/resolve.go b/internal/execenv/resolve.go new file mode 100644 index 0000000..d20f835 --- /dev/null +++ b/internal/execenv/resolve.go @@ -0,0 +1,221 @@ +package execenv + +import ( + "fmt" + "strconv" + "strings" +) + +// Wheel 是索引里的一个候选。Kind 为 wheel 或 sdist。 +type Wheel struct { + Name string + Version string + Filename string + SHA256 string + Kind string + Requires []string + Size int64 +} + +// Result 是一次解析。Packages 只含相对约束新增的包，不含基础层已有的。 +type Result struct { + Packages []LockedPkg + Override []string + SDist []string + Missing []string + Denied []string +} + +func (r Result) OK() bool { + return len(r.Override) == 0 && len(r.SDist) == 0 && len(r.Missing) == 0 && len(r.Denied) == 0 +} + +// ResolveMem 用内存索引解析。基础层里已有且版本满足的包不会进入 Packages。 +// 版本不满足时记入 Override，不改约束。 +func ResolveMem(index []Wheel, reqs []string, constraints []LockedPkg) Result { + byName := map[string][]Wheel{} + for _, w := range index { + byName[normName(w.Name)] = append(byName[normName(w.Name)], w) + } + pinned := map[string]LockedPkg{} + for _, c := range constraints { + pinned[normName(c.Name)] = c + } + chosen := map[string]LockedPkg{} + var res Result + queue := append([]string{}, reqs...) + seen := map[string]bool{} + for len(queue) > 0 { + req := strings.TrimSpace(queue[0]) + queue = queue[1:] + if req == "" { + continue + } + name, spec := splitReq(req) + key := normName(name) + mark := key + "\x00" + spec + if seen[mark] { + continue + } + seen[mark] = true + if pin, ok := pinned[key]; ok { + if spec != "" && !matchSpec(pin.Version, spec) { + res.Override = append(res.Override, fmt.Sprintf("%s 需要 %s，基础层固定为 %s", name, spec, pin.Version)) + } + continue + } + if prev, ok := chosen[key]; ok { + if spec != "" && !matchSpec(prev.Version, spec) { + res.Override = append(res.Override, fmt.Sprintf("%s 需要 %s，已选 %s", name, spec, prev.Version)) + } + continue + } + best, sdist := pickWheel(byName[key], spec) + if sdist && best == nil { + res.SDist = append(res.SDist, name) + continue + } + if best == nil { + res.Missing = append(res.Missing, name) + continue + } + pkg := LockedPkg{ + Name: best.Name, Version: best.Version, Filename: best.Filename, + SHA256: best.SHA256, Size: best.Size, + } + chosen[key] = pkg + for _, dep := range best.Requires { + queue = append(queue, dep) + } + } + names := make([]string, 0, len(chosen)) + for n := range chosen { + names = append(names, n) + } + sortStrings(names) + for _, n := range names { + res.Packages = append(res.Packages, chosen[n]) + } + return res +} + +func pickWheel(cands []Wheel, spec string) (best *Wheel, sawSDist bool) { + for i := range cands { + w := &cands[i] + if spec != "" && !matchSpec(w.Version, spec) { + continue + } + if w.Kind == "sdist" { + sawSDist = true + continue + } + if best == nil || cmpVer(w.Version, best.Version) > 0 { + best = w + } + } + return best, sawSDist +} + +func splitReq(req string) (name, spec string) { + for i, r := range req { + if r == '=' || r == '<' || r == '>' || r == '!' || r == '~' || r == ' ' { + return strings.TrimSpace(req[:i]), strings.TrimSpace(req[i:]) + } + } + return req, "" +} + +func matchSpec(version, spec string) bool { + for _, part := range strings.Split(spec, ",") { + part = strings.TrimSpace(part) + if part == "" || part == "*" { + continue + } + switch { + case strings.HasPrefix(part, "=="): + if version != strings.TrimPrefix(part, "==") { + return false + } + case strings.HasPrefix(part, "!="): + if version == strings.TrimPrefix(part, "!=") { + return false + } + case strings.HasPrefix(part, ">="): + if cmpVer(version, strings.TrimPrefix(part, ">=")) < 0 { + return false + } + case strings.HasPrefix(part, "<="): + if cmpVer(version, strings.TrimPrefix(part, "<=")) > 0 { + return false + } + case strings.HasPrefix(part, ">"): + if cmpVer(version, strings.TrimPrefix(part, ">")) <= 0 { + return false + } + case strings.HasPrefix(part, "<"): + if cmpVer(version, strings.TrimPrefix(part, "<")) >= 0 { + return false + } + default: + if version != part { + return false + } + } + } + return true +} + +func cmpVer(a, b string) int { + as := splitVer(a) + bs := splitVer(b) + n := len(as) + if len(bs) > n { + n = len(bs) + } + for i := 0; i < n; i++ { + ai, bi := 0, 0 + if i < len(as) { + ai = as[i] + } + if i < len(bs) { + bi = bs[i] + } + if ai != bi { + return ai - bi + } + } + return 0 +} + +func splitVer(v string) []int { + v = strings.TrimPrefix(v, "v") + if i := strings.IndexAny(v, "abrc"); i > 0 { + // 丢掉本地/预发布后缀，比较只看数字段。 + head := v[:i] + if _, err := strconv.Atoi(string(head[len(head)-1])); err == nil { + v = head + } + } + var out []int + for _, p := range strings.Split(v, ".") { + n := 0 + for _, r := range p { + if r < '0' || r > '9' { + break + } + n = n*10 + int(r-'0') + } + out = append(out, n) + } + return out +} + +func sortStrings(s []string) { + for i := 1; i < len(s); i++ { + j := i + for j > 0 && s[j] < s[j-1] { + s[j], s[j-1] = s[j-1], s[j] + j-- + } + } +} diff --git a/internal/execenv/resolve_test.go b/internal/execenv/resolve_test.go new file mode 100644 index 0000000..584c1dd --- /dev/null +++ b/internal/execenv/resolve_test.go @@ -0,0 +1,48 @@ +package execenv + +import ( + "strings" + "testing" +) + +func TestResolveMemConflictAllowlistAndSDist(t *testing.T) { + index := []Wheel{ + {Name: "Foo", Version: "1.0.0", Filename: "foo-1.whl", SHA256: "aaa", Kind: "wheel"}, + {Name: "Foo", Version: "2.0.0", Filename: "foo-2.whl", SHA256: "bbb", Kind: "wheel"}, + {Name: "OnlySrc", Version: "1", Filename: "onlysrc-1.tar.gz", Kind: "sdist"}, + } + got := ResolveMem(index, []string{"foo==1.0.0"}, []LockedPkg{{Name: "foo", Version: "2.0.0"}}) + if len(got.Override) != 1 || len(got.Packages) != 0 { + t.Fatalf("conflict should keep the pin and record an override: %+v", got) + } + fresh := ResolveMem(index, []string{"foo"}, nil) + if len(fresh.Packages) != 1 || fresh.Packages[0].Version != "2.0.0" { + t.Fatalf("unpinned foo should pick the newest wheel: %+v", fresh.Packages) + } + sdist := ResolveMem(index, []string{"onlysrc"}, nil) + if len(sdist.SDist) != 1 || len(sdist.Packages) != 0 { + t.Fatalf("sdist-only must be refused: %+v", sdist) + } + missing := ResolveMem(index, []string{"nope"}, nil) + if len(missing.Missing) != 1 { + t.Fatalf("missing: %+v", missing) + } +} + +func TestBuildLayerRejectsHashMismatch(t *testing.T) { + root, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + wheel := root.Wheelhouse() + "/foo.whl" + if err := osWriteWheel(wheel, "foo.py"); err != nil { + t.Fatal(err) + } + _, err = root.BuildLayer(Lock{ + Schema: 1, Kind: "base", PythonTag: "cp314", Platform: "any", + Packages: []LockedPkg{{Name: "foo", Version: "1", Filename: "foo.whl", SHA256: "deadbeef"}}, + }) + if err == nil || !strings.Contains(err.Error(), "hash mismatch") { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/execenv/root.go b/internal/execenv/root.go new file mode 100644 index 0000000..dfc89bb --- /dev/null +++ b/internal/execenv/root.go @@ -0,0 +1,133 @@ +// Package execenv 是系统托管的 Python 环境：一个解释器、一份不可变基础层、 +// 按内容哈希缓存的扩展层。工作区不各自建 venv，也不往一个可变全局环境里装包。 +package execenv + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Root 是环境根目录。多个服务实例可以共用同一份。 +type Root struct { + Path string +} + +// Open 创建环境根的目录骨架。已存在的层与锁不会被改写。 +func Open(path string) (*Root, error) { + if strings.TrimSpace(path) == "" { + return nil, errors.New("execenv: empty root") + } + r := &Root{Path: path} + for _, d := range []string{ + r.PythonDir(), r.LayersDir(), r.Wheelhouse(), r.LocksDir(), r.BinDir(), + } { + if err := os.MkdirAll(d, 0o755); err != nil { + return nil, err + } + } + if err := r.ensureCatalog(); err != nil { + return nil, err + } + if err := writePipShim(r.BinDir()); err != nil { + return nil, err + } + return r, nil +} + +func (r *Root) PythonDir() string { return filepath.Join(r.Path, "python") } +func (r *Root) LayersDir() string { return filepath.Join(r.Path, "layers") } +func (r *Root) Wheelhouse() string { return filepath.Join(r.Path, "wheelhouse") } +func (r *Root) LocksDir() string { return filepath.Join(r.Path, "locks") } +func (r *Root) BinDir() string { return filepath.Join(r.Path, "bin") } +func (r *Root) CatalogPath() string { return filepath.Join(r.Path, "catalog.json") } + +func (r *Root) ensureCatalog() error { + if _, err := os.Stat(r.CatalogPath()); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + raw, err := json.MarshalIndent(DefaultCatalog(), "", " ") + if err != nil { + return err + } + return os.WriteFile(r.CatalogPath(), append(raw, '\n'), 0o644) +} + +// LayerDir 返回某一层的 site-packages 目录（不一定已经存在）。 +func (r *Root) LayerDir(id string) string { + return filepath.Join(r.LayersDir(), id, "site-packages") +} + +// CurrentBase 读取 current 符号链接指向的基础层 ID。未建立时返回空串。 +func (r *Root) CurrentBase() (string, error) { + dest, err := os.Readlink(filepath.Join(r.Path, "current")) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + id := filepath.Base(dest) + if id == "" || id == "." { + return "", fmt.Errorf("execenv: current link %q is empty", dest) + } + return id, nil +} + +// SwitchCurrent 把 current 指到已存在的层。旧链接被替换，层目录本身不动。 +func (r *Root) SwitchCurrent(id string) error { + if _, err := os.Stat(r.LayerDir(id)); err != nil { + return fmt.Errorf("execenv: layer %s: %w", id, err) + } + link := filepath.Join(r.Path, "current") + tmp := link + ".tmp" + _ = os.Remove(tmp) + if err := os.Symlink(filepath.Join("layers", id), tmp); err != nil { + return err + } + return os.Rename(tmp, link) +} + +// ReadLock 读取某一层的锁。 +func (r *Root) ReadLock(id string) (Lock, error) { + raw, err := os.ReadFile(filepath.Join(r.LocksDir(), id+".json")) + if err != nil { + return Lock{}, err + } + var l Lock + if err := json.Unmarshal(raw, &l); err != nil { + return Lock{}, err + } + return l, nil +} + +// WriteLock 写入锁文件。内容按哈希寻址，已存在且字节相同则跳过。 +func (r *Root) WriteLock(l Lock) (string, error) { + id := l.ID() + path := filepath.Join(r.LocksDir(), id+".json") + raw, err := json.MarshalIndent(l, "", " ") + if err != nil { + return "", err + } + raw = append(raw, '\n') + if prev, err := os.ReadFile(path); err == nil { + if string(prev) != string(raw) { + return "", fmt.Errorf("execenv: lock %s already exists with different bytes", id) + } + return id, nil + } + return id, os.WriteFile(path, raw, 0o644) +} + +// HashBytes 是层键与 wheel 去重共用的 sha256。 +func HashBytes(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/execenv/sandbox_test.go b/internal/execenv/sandbox_test.go new file mode 100644 index 0000000..d27a096 --- /dev/null +++ b/internal/execenv/sandbox_test.go @@ -0,0 +1,44 @@ +package execenv + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestSandboxImportPipAndOffline(t *testing.T) { + if _, err := exec.LookPath("bwrap"); err != nil { + t.Skip("bwrap not available") + } + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not available") + } + root, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + site := t.TempDir() + if err := os.WriteFile(filepath.Join(site, "hello.py"), []byte("value = 1\n"), 0o644); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := Doctor(ctx, "/usr/bin/python3", []string{site}, []string{"hello"}, true, []string{site}); err != nil { + t.Fatal(err) + } + shim := filepath.Join(root.BinDir(), "pip") + out, err := exec.Command("bwrap", "--unshare-all", "--die-with-parent", "--ro-bind", "/", "/", + "--tmpfs", "/tmp", "--ro-bind", root.BinDir(), root.BinDir(), "--", shim).CombinedOutput() + if err == nil || !strings.Contains(string(out), "python_env") { + t.Fatalf("pip shim err=%v out=%s", err, out) + } + netOut, err := exec.Command("bwrap", "--unshare-net", "--die-with-parent", "--ro-bind", "/", "/", + "--", "/usr/bin/python3", "-c", "import socket; socket.create_connection(('1.1.1.1', 443), 2)").CombinedOutput() + if err == nil { + t.Fatalf("sandbox network should be unreachable, out=%s", netOut) + } +} diff --git a/internal/execenv/shim.go b/internal/execenv/shim.go new file mode 100644 index 0000000..081f08e --- /dev/null +++ b/internal/execenv/shim.go @@ -0,0 +1,22 @@ +package execenv + +import ( + "os" + "path/filepath" +) + +const pipShim = `#!/bin/sh +echo "沙箱不联网安装。请使用 python_env 工具申请扩展包：python_env request，packages 写包名，reason 写用途。" >&2 +exit 1 +` + +func writePipShim(dir string) error { + body := []byte(pipShim) + for _, name := range []string{"pip", "pip3"} { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, body, 0o755); err != nil { + return err + } + } + return nil +} diff --git a/internal/execenv/wheel.go b/internal/execenv/wheel.go new file mode 100644 index 0000000..417a722 --- /dev/null +++ b/internal/execenv/wheel.go @@ -0,0 +1,112 @@ +package execenv + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// UnpackWheel 把 wheel 解进 site-packages。wheel 是 zip；这里不执行任何入口脚本。 +func UnpackWheel(wheelPath, sitePackages string) error { + zr, err := zip.OpenReader(wheelPath) + if err != nil { + return fmt.Errorf("execenv: open wheel: %w", err) + } + defer zr.Close() + if err := os.MkdirAll(sitePackages, 0o755); err != nil { + return err + } + root := filepath.Clean(sitePackages) + for _, f := range zr.File { + name := filepath.Clean(f.Name) + if name == "." || strings.HasPrefix(name, "..") || filepath.IsAbs(name) { + return fmt.Errorf("execenv: wheel path escapes: %s", f.Name) + } + // 安装器脚本不落地，避免把「解包」变成「执行」。 + if strings.Contains(name, ".data/scripts/") { + continue + } + dest := filepath.Join(root, name) + if !strings.HasPrefix(dest, root+string(os.PathSeparator)) && dest != root { + return fmt.Errorf("execenv: wheel path escapes: %s", f.Name) + } + if f.FileInfo().IsDir() { + if err := os.MkdirAll(dest, 0o755); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return err + } + rc, err := f.Open() + if err != nil { + return err + } + out, err := os.OpenFile(dest, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + if err != nil { + rc.Close() + return err + } + _, cerr := io.Copy(out, rc) + out.Close() + rc.Close() + if cerr != nil { + return cerr + } + } + return nil +} + +// BuildLayer 把已经在 wheelhouse 里的 wheel 解进新层。目录已存在则直接复用。 +func (r *Root) BuildLayer(l Lock) (string, error) { + id, err := r.WriteLock(l) + if err != nil { + return "", err + } + dest := r.LayerDir(id) + if st, err := os.Stat(dest); err == nil && st.IsDir() { + return id, nil + } + for _, pkg := range l.Packages { + wheel := filepath.Join(r.Wheelhouse(), pkg.SHA256+".whl") + if _, err := os.Stat(wheel); err != nil { + alt := filepath.Join(r.Wheelhouse(), pkg.Filename) + if _, err2 := os.Stat(alt); err2 != nil { + return "", fmt.Errorf("execenv: wheel missing for %s", pkg.Name) + } + wheel = alt + } + sum, err := fileSHA256(wheel) + if err != nil { + return "", err + } + if pkg.SHA256 != "" && !strings.EqualFold(sum, pkg.SHA256) { + return "", fmt.Errorf("execenv: hash mismatch for %s: got %s want %s", pkg.Name, sum, pkg.SHA256) + } + if err := UnpackWheel(wheel, dest); err != nil { + _ = os.RemoveAll(filepath.Dir(dest)) + return "", err + } + } + if err := os.MkdirAll(dest, 0o755); err != nil { + return "", err + } + return id, nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return h.Sum(), nil +} diff --git a/internal/prompts/library/base-deliverables.md b/internal/prompts/library/base-deliverables.md index 7c9a0be..3203aaf 100644 --- a/internal/prompts/library/base-deliverables.md +++ b/internal/prompts/library/base-deliverables.md @@ -1,7 +1,7 @@ --- id: base.deliverables layer: base -version: 9 +version: 10 title: 交付物工作流 description: 生成文档、表格、演示稿的工作流、样式决策与技能入口（常驻基座） --- @@ -38,7 +38,7 @@ description: 生成文档、表格、演示稿的工作流、样式决策与技 - **docx-author / xlsx-author / pptx-author**：各类交付物的版面范式和最小示例。 - **frontend-design / canvas-design / copywriting / copy-editing**（第三方 vendor，溯源见各自 UPSTREAM.md）：视觉方向与排版审美、海报/信息图等 png·pdf 视觉产物（PIL 渲染，字体已随包）、营销/商务文案的撰写改写与编辑润色。交付物主题与配色仍以 house-style 为权威；需要超出主题规范的视觉判断或文案功力时 `load_skill` 对应技能。 - 小队角色用 `load_skill`；其他路径用 `run_shell` 查看技能文件，例如 `cat "$AGENT_SKILLS_DIR/house-style/SKILL.md"`。不要假设工作区里有 `skills/` 目录。 -- 缺库时：`pip install --user python-docx openpyxl python-pptx`。PDF 仿制可选 `pdfplumber`；没装就说明只能近似，并请用户确认主色和字体。 +- 缺库时用 `python_env`：先 `status` 看基础层是否已有，没有再 `request` 并写明理由。不要 `pip install`。PDF 仿制可用基础层里的 `pdfplumber`；基础层没有且申请未通过时说明只能近似，并请用户确认主色和字体。 - 缺库又装不上时，如实告知用户当前环境无法生成带排版的 Office 交付物并说明缺什么；禁止改用 zipfile/字符串手工拼 OOXML 充数——那样生成的包缺母版与主题，结构自检会拒绝登记。 要求与边界： diff --git a/internal/prompts/library/tool-run-python.md b/internal/prompts/library/tool-run-python.md index bbf465b..1a5cf53 100644 --- a/internal/prompts/library/tool-run-python.md +++ b/internal/prompts/library/tool-run-python.md @@ -1,8 +1,8 @@ --- id: tool.run_python layer: tool -version: 3 +version: 4 title: run_python 工具描述 description: Python 执行原语的模型侧说明文案 --- -执行完整 Python 脚本（python3，工作目录=会话工作区）。适用：数据处理、生成 Excel/PPT/图片（housestyle kit）、批量操作。Word 交付物优先写 .md 源后用 run_shell 跑 `python3 -m housestyle.build`。任何 Office 交付物（docx/xlsx/pptx）都禁止用 zipfile 或字符串手工拼 OOXML——手拼的包缺母版与主题，register_artifact 的结构自检会直接拒绝；一律走 housestyle kit 构件。缺库先 `pip install --user python-pptx python-docx openpyxl`，装不上就如实告知用户当前环境无法生成本类交付物，不要退回手拼。kit 路线执行失败（报错、超时、后台任务溢出）时，修复后仍走 kit 路线重试——任何失败都不构成退回手工拼 OOXML 的理由。报错先读退出码与 traceback 定位行号再改，同一错误不要原样重跑。生成带排版的文件（xlsx/pptx）先读 `$AGENT_SKILLS_DIR/house-style/SKILL.md` 的构件表，不确定签名先 `inspect.signature`。 +执行完整 Python 脚本（python3，工作目录=会话工作区）。适用：数据处理、生成 Excel/PPT/图片（housestyle kit）、批量操作。Word 交付物优先写 .md 源后用 run_shell 跑 `python3 -m housestyle.build`。任何 Office 交付物（docx/xlsx/pptx）都禁止用 zipfile 或字符串手工拼 OOXML——手拼的包缺母版与主题，register_artifact 的结构自检会直接拒绝；一律走 housestyle kit 构件。基础层已有 python-docx、openpyxl、python-pptx 等，直接 import。缺的包用 python_env 申请（status 先看清单，request 再等），不要 pip install：沙箱不联网。申请被拒绝就如实说明缺什么，不要退回手拼。kit 路线执行失败（报错、超时、后台任务溢出）时，修复后仍走 kit 路线重试——任何失败都不构成退回手工拼 OOXML 的理由。报错先读退出码与 traceback 定位行号再改，同一错误不要原样重跑。生成带排版的文件（xlsx/pptx）先读 `$AGENT_SKILLS_DIR/house-style/SKILL.md` 的构件表，不确定签名先 `inspect.signature`。 diff --git a/internal/prompts/library/tool-run-shell.md b/internal/prompts/library/tool-run-shell.md index ad989d8..2409e58 100644 --- a/internal/prompts/library/tool-run-shell.md +++ b/internal/prompts/library/tool-run-shell.md @@ -1,8 +1,8 @@ --- id: tool.run_shell layer: tool -version: 1 +version: 2 title: run_shell 工具描述 description: bash 执行原语的模型侧说明文案 --- -在会话工作区内执行 bash 脚本（环境最小化：PATH/HOME/LANG；AGENT_SKILLS_DIR 与 PYTHONPATH 已注入）。适用：housestyle build/lint、pip install --user、文件批操作。读文件用 read_file、找内容用 grep——不要用 cat/sed/grep 命令行代替专用工具。同一命令失败两次换方法。 +在会话工作区内执行 bash 脚本（环境最小化：PATH/HOME/LANG；AGENT_SKILLS_DIR 与 PYTHONPATH 已注入）。适用：housestyle build/lint、文件批操作。Python 缺包用 python_env，不要 pip install。读文件用 read_file、找内容用 grep——不要用 cat/sed/grep 命令行代替专用工具。同一命令失败两次换方法。 diff --git a/internal/runmgr/runmgr.go b/internal/runmgr/runmgr.go index 3b735f2..edc79f7 100644 --- a/internal/runmgr/runmgr.go +++ b/internal/runmgr/runmgr.go @@ -50,7 +50,9 @@ type Manager struct { wfRunner runtime.Runner // 意图或工具启动的工作流；nil 时用 runtime.CompiledWorkflowRunner workflows *workflow.Registry // run_workflow 按 id 查找已注册定义 gate IntentGate - cards *cardsvc.Service // 可选意图闸门（nil = 未启用，行为回退） + // execEnvDecide 处理不挂在活动 run 上的 Python 环境审批卡（Detail 前缀 exec-env:）。 + execEnvDecide func(actor, requestID string, allow, isolate bool) (string, error) + cards *cardsvc.Service // 可选意图闸门（nil = 未启用，行为回退） // OnUsage 在 run 结束时报告 token（日预算记账；nil = 不记）。 OnUsage func(owner string, tokens int) events *eventBuffer // 事件批处理（写放大治理；seq 权威仍在 store） @@ -191,6 +193,11 @@ func (m *Manager) EmitProjectSharedCardEvent(owner, producer, channel, runID str // SetGate 注入意图闸门（装配期一次；插件意图 overlay 经 pluginhost 回调重建）。 func (m *Manager) SetGate(g IntentGate) { m.gate = g } +// SetExecEnvDecider 接上分层环境的审批。卡片不要求 run 仍在本进程执行。 +func (m *Manager) SetExecEnvDecider(fn func(actor, requestID string, allow, isolate bool) (string, error)) { + m.execEnvDecide = fn +} + // SetCards 注入收件箱卡服务（nil = 不产卡；T0 退回基线仍成立）。 func (m *Manager) SetCards(s *cardsvc.Service) { m.cards = s } @@ -1960,6 +1967,9 @@ func (m *Manager) ResolveHITL(owner, cardID string, resp core.HITLResponse) (piv if card.Resolved { return "", false, fmt.Errorf("runmgr: card %s already resolved", cardID) } + if strings.HasPrefix(card.Detail, "exec-env:") { + return m.resolveExecEnvCard(owner, card, resp) + } run, err := m.st.GetRunByID(card.RunID) if err != nil { return "", false, err @@ -2061,6 +2071,28 @@ func (m *Manager) ResolveHITL(owner, cardID string, resp core.HITLResponse) (piv return "", false, nil } +func (m *Manager) resolveExecEnvCard(owner string, card core.HITLCard, resp core.HITLResponse) (string, bool, error) { + if m.execEnvDecide == nil { + return "", false, fmt.Errorf("runmgr: exec env decider is not configured") + } + id := strings.TrimPrefix(card.Detail, "exec-env:") + if i := strings.IndexByte(id, '\n'); i >= 0 { + id = id[:i] + } + choice := resp.ChoiceID + allow := choice == "approve" || choice == "isolate" + detail, err := m.execEnvDecide(owner, id, allow, choice == "isolate") + if err != nil { + return "", false, err + } + if err := m.resolveHITLCardForEffectiveAccount(owner, string(card.ID), core.HITLResolvedPayload{ + CardID: card.ID, Kind: "choice", Answer: choice, + }); err != nil { + return "", false, err + } + return detail, false, nil +} + // ---- 子代理（任务树）---- func (m *Manager) spawnSubagent(parentCtx context.Context, parent core.Run, agentID, name, task, hint string) (core.AgentResultPayload, error) { diff --git a/internal/server/admin_execenv.go b/internal/server/admin_execenv.go new file mode 100644 index 0000000..aa05d1e --- /dev/null +++ b/internal/server/admin_execenv.go @@ -0,0 +1,151 @@ +package server + +import ( + "io" + "net/http" + "os" + "path/filepath" + "time" +) + +func (s *Server) adminExecEnv(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireAdmin(w, r); !ok { + return + } + if s.ExecEnv == nil { + writeJSON(w, http.StatusOK, map[string]any{"configured": false}) + return + } + base, err := s.ExecEnv.Root.CurrentBase() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + reqs, err := s.ExecEnv.Store.ListExecEnvRequests("", 50) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + binds, err := s.ExecEnv.Store.ListExecEnvBindings() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + bin, _ := s.ExecEnv.Root.PythonBin() + ents, _ := os.ReadDir(s.ExecEnv.Root.LayersDir()) + layers := make([]string, 0, len(ents)) + for _, ent := range ents { + if ent.IsDir() { + layers = append(layers, ent.Name()) + } + } + writeJSON(w, http.StatusOK, map[string]any{ + "configured": true, + "root": s.ExecEnv.Root.Path, + "python": bin, + "base": base, + "layers": layers, + "requests": reqs, + "bindings": binds, + }) +} + +func (s *Server) adminExecEnvDecision(w http.ResponseWriter, r *http.Request) { + actor, ok := s.requireAdmin(w, r) + if !ok { + return + } + if s.ExecEnv == nil { + http.Error(w, "exec env is not configured", http.StatusConflict) + return + } + var body struct { + Allow bool `json:"allow"` + Isolate bool `json:"isolate"` + } + if err := decodeJSON(r, &body); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + out, err := s.ExecEnv.Decide(r.Context(), r.PathValue("id"), actor, true, body.Allow, body.Isolate) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, out) +} + +func (s *Server) adminExecEnvRebase(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireAdmin(w, r); !ok { + return + } + if s.ExecEnv == nil { + http.Error(w, "exec env is not configured", http.StatusConflict) + return + } + id, err := s.ExecEnv.Rebase(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + writeJSON(w, http.StatusOK, map[string]any{"base": id}) +} + +func (s *Server) adminExecEnvGC(w http.ResponseWriter, r *http.Request) { + if _, ok := s.requireAdmin(w, r); !ok { + return + } + if s.ExecEnv == nil { + http.Error(w, "exec env is not configured", http.StatusConflict) + return + } + removed, err := s.ExecEnv.GC(7 * 24 * time.Hour) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, map[string]any{"removed": removed}) +} + +func (s *Server) adminExecEnvWheel(w http.ResponseWriter, r *http.Request) { + actor, ok := s.requireAdmin(w, r) + if !ok { + return + } + if s.ExecEnv == nil { + http.Error(w, "exec env is not configured", http.StatusConflict) + return + } + if err := r.ParseMultipartForm(64 << 20); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + file, hdr, err := r.FormFile("wheel") + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer file.Close() + dir := filepath.Join(s.ExecEnv.Root.Path, "uploads") + if err := os.MkdirAll(dir, 0o755); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + dst, err := os.CreateTemp(dir, "upload-*.whl") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if _, err := io.Copy(dst, file); err != nil { + dst.Close() + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + dst.Close() + out, err := s.ExecEnv.RequestPackages(r.Context(), r.FormValue("conversationId"), actor, r.FormValue("reason"), nil, dst.Name()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, map[string]any{"file": hdr.Filename, "outcome": out}) +} diff --git a/internal/server/server.go b/internal/server/server.go index 79d0dd7..b0b02a8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,7 @@ import ( agentcommand "agent-command" "agent-system-backend/internal/cardsvc" "agent-system-backend/internal/cmdbus" + "agent-system-backend/internal/execenv" "agent-system-backend/internal/ingest" "agent-system-backend/internal/intentgate" "agent-system-backend/internal/plugins" @@ -53,17 +54,19 @@ type Server struct { Limiter ratelimit.Limiter // 租户限流（nil = 不限；组合根注入） LimitLookup func() int // 当前限流值（次/分；<=0 不限；组合根缓存注入） mgr *runmgr.Manager - Cards *cardsvc.Service // 收件箱任务卡（nil = 不启用；组合根注入） - Surfaces *surfacesvc.Service // Generative UI Surface（nil = 不启用；EXT-17） - Bus *cmdbus.Bus // 命令总线（nil = 不启用；组合根注入；四入口同源 D27） - Gate NLIntentGate // 意图闸门（chat 入口同源判定；nil = 不接） - Workflows *workflow.Registry // 已装载工作流（nil = 不接） - Prompts *prompts.Service // 提示词库（nil = 不启用 API 与技能命令；执行器另有回退） - reg *tools.Registry - prof *profiles.Service - set *settings.Service - plugins *plugins.Host - static fs.FS + // ExecEnv 分层 Python 环境代理（nil = 未配置）。管理区用它审批、回收和重建基础层。 + ExecEnv *execenv.Broker + Cards *cardsvc.Service // 收件箱任务卡（nil = 不启用；组合根注入） + Surfaces *surfacesvc.Service // Generative UI Surface（nil = 不启用；EXT-17） + Bus *cmdbus.Bus // 命令总线（nil = 不启用；组合根注入；四入口同源 D27） + Gate NLIntentGate // 意图闸门（chat 入口同源判定；nil = 不接） + Workflows *workflow.Registry // 已装载工作流（nil = 不接） + Prompts *prompts.Service // 提示词库（nil = 不启用 API 与技能命令；执行器另有回退） + reg *tools.Registry + prof *profiles.Service + set *settings.Service + plugins *plugins.Host + static fs.FS // Subscribe 注入 bus 订阅（事件在线转发；重放走 store 游标）。 Subscribe func(conversationID string) (<-chan core.StreamEvent, func()) // DisableUploadIndexing EXT-9：true 时上传端点不再自动写检索索引（原件与 @@ -290,6 +293,11 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /api/auth/logout", s.logout) mux.HandleFunc("POST /api/auth/password", s.changePassword) mux.HandleFunc("GET /api/admin/users", s.listAdminUsers) + mux.HandleFunc("GET /api/admin/exec-env", s.adminExecEnv) + mux.HandleFunc("POST /api/admin/exec-env/requests/{id}/decision", s.adminExecEnvDecision) + mux.HandleFunc("POST /api/admin/exec-env/rebase", s.adminExecEnvRebase) + mux.HandleFunc("POST /api/admin/exec-env/gc", s.adminExecEnvGC) + mux.HandleFunc("POST /api/admin/exec-env/wheels", s.adminExecEnvWheel) mux.HandleFunc("GET /api/admin/users/search", s.searchAdminUsers) mux.HandleFunc("POST /api/admin/users", s.createAdminUser) mux.HandleFunc("GET /api/me/profile", s.meProfile) diff --git a/internal/store/exec_env.go b/internal/store/exec_env.go new file mode 100644 index 0000000..535566e --- /dev/null +++ b/internal/store/exec_env.go @@ -0,0 +1,149 @@ +package store + +import ( + "database/sql" + "encoding/json" + "errors" +) + +// ExecEnvRequest 是一次扩展包申请（分层 Python 环境）。 +type ExecEnvRequest struct { + ID string + ConversationID string + Owner string + Spec string + Reason string + Status string + LayerID string + Detail string + Approver string + CreatedAt string + UpdatedAt string +} + +// ExecEnvBinding 是会话挂载的层。Layers 是扩展层 ID；Isolated 非空时替换基础层。 +type ExecEnvBinding struct { + ConversationID string + BaseID string + Layers []string + Isolated string + UpdatedAt string +} + +func (s *Store) InsertExecEnvRequest(r ExecEnvRequest) error { + _, err := s.db.Exec(s.q(`INSERT INTO exec_env_requests( + id,conversation_id,owner,spec,reason,status,layer_id,detail,approver,created_at,updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?)`), + r.ID, r.ConversationID, r.Owner, r.Spec, r.Reason, r.Status, r.LayerID, r.Detail, r.Approver, r.CreatedAt, r.UpdatedAt) + return err +} + +func (s *Store) GetExecEnvRequest(id string) (ExecEnvRequest, error) { + var r ExecEnvRequest + err := s.db.QueryRow(s.q(`SELECT id,conversation_id,owner,spec,reason,status,layer_id,detail,approver,created_at,updated_at + FROM exec_env_requests WHERE id=?`), id).Scan( + &r.ID, &r.ConversationID, &r.Owner, &r.Spec, &r.Reason, &r.Status, &r.LayerID, &r.Detail, &r.Approver, &r.CreatedAt, &r.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return ExecEnvRequest{}, ErrNotFound + } + return r, err +} + +func (s *Store) ListExecEnvRequests(conversationID string, limit int) ([]ExecEnvRequest, error) { + if limit <= 0 { + limit = 20 + } + q := `SELECT id,conversation_id,owner,spec,reason,status,layer_id,detail,approver,created_at,updated_at + FROM exec_env_requests` + var args []any + if conversationID != "" { + q += ` WHERE conversation_id=?` + args = append(args, conversationID) + } + q += ` ORDER BY created_at DESC LIMIT ?` + args = append(args, limit) + rows, err := s.db.Query(s.q(q), args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanExecEnvRequests(rows) +} + +func scanExecEnvRequests(rows *sql.Rows) ([]ExecEnvRequest, error) { + var out []ExecEnvRequest + for rows.Next() { + var r ExecEnvRequest + if err := rows.Scan(&r.ID, &r.ConversationID, &r.Owner, &r.Spec, &r.Reason, &r.Status, &r.LayerID, &r.Detail, &r.Approver, &r.CreatedAt, &r.UpdatedAt); err != nil { + return nil, err + } + out = append(out, r) + } + return out, rows.Err() +} + +func (s *Store) UpdateExecEnvRequest(id, status, layerID, detail, approver string) error { + res, err := s.db.Exec(s.q(`UPDATE exec_env_requests SET status=?, layer_id=?, detail=?, approver=?, updated_at=? WHERE id=?`), + status, layerID, detail, approver, nowISO(), id) + if err != nil { + return err + } + n, _ := res.RowsAffected() + if n == 0 { + return ErrNotFound + } + return nil +} + +func (s *Store) GetExecEnvBinding(conversationID string) (ExecEnvBinding, error) { + var b ExecEnvBinding + var layers string + err := s.db.QueryRow(s.q(`SELECT conversation_id,base_id,layers,isolated,updated_at FROM exec_env_bindings WHERE conversation_id=?`), conversationID). + Scan(&b.ConversationID, &b.BaseID, &layers, &b.Isolated, &b.UpdatedAt) + if errors.Is(err, sql.ErrNoRows) { + return ExecEnvBinding{}, nil + } + if err != nil { + return ExecEnvBinding{}, err + } + _ = json.Unmarshal([]byte(layers), &b.Layers) + return b, nil +} + +func (s *Store) PutExecEnvBinding(b ExecEnvBinding) error { + raw, err := json.Marshal(b.Layers) + if err != nil { + return err + } + _, err = s.db.Exec(s.q(`INSERT INTO exec_env_bindings(conversation_id,base_id,layers,isolated,updated_at) + VALUES(?,?,?,?,?) + ON CONFLICT(conversation_id) DO UPDATE SET base_id=excluded.base_id, layers=excluded.layers, isolated=excluded.isolated, updated_at=excluded.updated_at`), + b.ConversationID, b.BaseID, string(raw), b.Isolated, b.UpdatedAt) + return err +} + +func (s *Store) ListExecEnvBindings() ([]ExecEnvBinding, error) { + rows, err := s.db.Query(s.q(`SELECT conversation_id,base_id,layers,isolated,updated_at FROM exec_env_bindings`)) + if err != nil { + return nil, err + } + defer rows.Close() + var out []ExecEnvBinding + for rows.Next() { + var b ExecEnvBinding + var layers string + if err := rows.Scan(&b.ConversationID, &b.BaseID, &layers, &b.Isolated, &b.UpdatedAt); err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(layers), &b.Layers) + out = append(out, b) + } + return out, rows.Err() +} + +func (s *Store) InsertExecEnvAudit(owner, action, actor, target, outcome, entry string) error { + _, err := s.db.Exec(s.q(`INSERT INTO audit_events(time,owner,action,actor,target,evidence,trace_id,entry,instance_id,operation_id,outcome) + VALUES(?,?,?,?,?,?,?,?,?,?,?)`), + nowISO(), owner, action, actor, target, entry, "", "execenv", s.instanceID, "", outcome) + return err +} diff --git a/internal/store/migrate.go b/internal/store/migrate.go index 23ea6ce..a175ae4 100644 --- a/internal/store/migrate.go +++ b/internal/store/migrate.go @@ -897,9 +897,44 @@ func init() { `ALTER TABLE account_directory_revision DROP COLUMN cursor_secret`, `ALTER TABLE account_directory_revision DROP COLUMN cursor_key_version`, }}}, + migration{version: 35, name: "exec_env_layers", + up: map[string][]string{"sqlite": execEnvUp, "postgres": execEnvUp}, + down: map[string][]string{"sqlite": execEnvDown, "postgres": execEnvDown}, + }, ) } +// v35 分层 Python 环境：扩展包申请与会话绑定。层目录本身在环境根，不入库。 +var execEnvUp = []string{ + `CREATE TABLE IF NOT EXISTS exec_env_requests ( + id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', + spec TEXT NOT NULL, + reason TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, + layer_id TEXT NOT NULL DEFAULT '', + detail TEXT NOT NULL DEFAULT '', + approver TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_exec_env_requests_conv ON exec_env_requests(conversation_id, created_at)`, + `CREATE TABLE IF NOT EXISTS exec_env_bindings ( + conversation_id TEXT PRIMARY KEY, + base_id TEXT NOT NULL DEFAULT '', + layers TEXT NOT NULL DEFAULT '[]', + isolated TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL + )`, +} + +var execEnvDown = []string{ + `DROP TABLE IF EXISTS exec_env_bindings`, + `DROP INDEX IF EXISTS idx_exec_env_requests_conv`, + `DROP TABLE IF EXISTS exec_env_requests`, +} + var recoveryFoundationUp = []string{ `ALTER TABLE login_keys ADD COLUMN grant_origin TEXT NOT NULL DEFAULT 'legacy_unknown'`, `ALTER TABLE login_keys ADD COLUMN generation BIGINT NOT NULL DEFAULT 0`, diff --git a/internal/store/migrate_compat.go b/internal/store/migrate_compat.go index 066fcbd..fd96457 100644 --- a/internal/store/migrate_compat.go +++ b/internal/store/migrate_compat.go @@ -39,6 +39,7 @@ var migrationBreaksOldReaders = map[int]bool{ 32: true, // 账号有效态/凭据来源是安全协议边界；旧认证写者必须排除 33: true, // 本人证明与 grant 绑定协议；旧恢复写者和兑换者必须排除 34: false, // 游标签名密钥只新增内部列；旧读者按列名读取既有字段 + 35: false, // exec_env 新表；旧读者不读这些表 } // SchemaTarget 是本二进制会迁移到的 schema 版本。 diff --git a/internal/tools/docx_e2e_test.go b/internal/tools/docx_e2e_test.go new file mode 100644 index 0000000..62eed2b --- /dev/null +++ b/internal/tools/docx_e2e_test.go @@ -0,0 +1,47 @@ +package tools + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestHousestyleBuildPassesOOXMLGate(t *testing.T) { + py, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 not available") + } + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + skill := filepath.Join(wd, "..", "skills", "recipes", "house-style") + if _, err := os.Stat(filepath.Join(skill, "housestyle", "build.py")); err != nil { + t.Skip("house-style skill is not in this checkout") + } + probe := exec.Command(py, "-c", "import docx") + if out, err := probe.CombinedOutput(); err != nil { + t.Skipf("python-docx is not installed: %s", out) + } + dir := t.TempDir() + src := filepath.Join(dir, "summary.md") + body := "# 摘要\n\n## 结论\n\n- 沙箱里可以直接生成 docx\n" + if err := os.WriteFile(src, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(py, "-m", "housestyle.build", src) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "PYTHONPATH="+skill) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("housestyle.build: %v\n%s", err, out) + } + docx := filepath.Join(dir, "summary.docx") + content, err := os.ReadFile(docx) + if err != nil { + t.Fatal(err) + } + if err := ooxmlStructuralCheck("summary.docx", content); err != nil { + t.Fatal(err) + } +} diff --git a/internal/tools/exec_backend.go b/internal/tools/exec_backend.go index 379479c..3d11eaa 100644 --- a/internal/tools/exec_backend.go +++ b/internal/tools/exec_backend.go @@ -34,6 +34,12 @@ type ExecSpec struct { // ~/.local 下会被 --tmpfs /home 遮蔽，只读挂回其原路径并并入 PYTHONPATH， // 交付物生成链路（F3）在隔离档下才可用。 DepsDir string + // LayerDirs 是本次执行要只读挂载的 site-packages（基础层与扩展层，或单独的隔离层）。 + // 路径若位于 /home 或 /tmp，必须在对应 tmpfs 之后挂载。 + LayerDirs []string + // PythonPrefix / ShimDir 同样可能落在被 tmpfs 遮住的路径上，需要挂回。 + PythonPrefix string + ShimDir string // S0 工作区 overlay（sandbox 档；三者须同批非空）：冻结基底（--overlay-src）、 // 私有 upper（--overlay RWSRC）、overlayfs work 目录。空 = EXT-16 现状直写。 WorkspaceFrozen string @@ -199,6 +205,18 @@ func sandboxArgv(spec ExecSpec) []string { // 必须后于 --tmpfs /home：先遮蔽再只读挂回依赖层（挂载顺序敏感）。 argv = append(argv, "--ro-bind", spec.DepsDir, spec.DepsDir) } + for _, dir := range spec.LayerDirs { + if dir == "" || dir == spec.DepsDir { + continue + } + argv = append(argv, "--ro-bind", dir, dir) + } + if spec.PythonPrefix != "" { + argv = append(argv, "--ro-bind", spec.PythonPrefix, spec.PythonPrefix) + } + if spec.ShimDir != "" && spec.ShimDir != spec.PythonPrefix { + argv = append(argv, "--ro-bind", spec.ShimDir, spec.ShimDir) + } argv = append(argv, "--chdir", spec.Dir, "--clearenv") for _, kv := range spec.Env { if i := strings.IndexByte(kv, '='); i > 0 { diff --git a/internal/tools/exec_backend_test.go b/internal/tools/exec_backend_test.go index 41006f3..5b5953f 100644 --- a/internal/tools/exec_backend_test.go +++ b/internal/tools/exec_backend_test.go @@ -270,7 +270,7 @@ func TestDirectBackendRunInterpreterUnchanged(t *testing.T) { if err := os.WriteFile(p, []byte("echo hi; exit 7\n"), 0o600); err != nil { t.Fatal(err) } - got, err := runInterpreter(context.Background(), "bash", p, dir, 30, nil) + got, err := runInterpreter(context.Background(), "bash", p, dir, "", 30, nil) if err != nil || !strings.HasPrefix(got, "exit=7\n") || !strings.Contains(got, "hi") { t.Fatalf("direct regression: err=%v got=%q", err, got) } @@ -278,7 +278,7 @@ func TestDirectBackendRunInterpreterUnchanged(t *testing.T) { if err := os.WriteFile(p2, []byte("sleep 60\n"), 0o600); err != nil { t.Fatal(err) } - got, err = runInterpreter(context.Background(), "bash", p2, dir, 2, nil) + got, err = runInterpreter(context.Background(), "bash", p2, dir, "", 2, nil) if err != nil || !strings.HasPrefix(got, "exit=timeout(2s)") { t.Fatalf("direct timeout regression: err=%v got=%q", err, got) } @@ -303,6 +303,35 @@ func TestSandboxArgvDepsLayer(t *testing.T) { } } +func TestSandboxArgvLayersAndPath(t *testing.T) { + argv := sandboxArgv(ExecSpec{ + Bin: "/usr/bin/python3", Args: []string{"/data/s/x.py"}, + Dir: "/data/s", + LayerDirs: []string{"/opt/base/site-packages", "/opt/ext/site-packages"}, + PythonPrefix: "/opt/py", + ShimDir: "/opt/env/bin", + Env: []string{"PATH=/opt/py/bin:/opt/env/bin:/usr/bin:/bin", "PIP_NO_INDEX=1"}, + }) + joined := strings.Join(argv, "\x00") + for _, m := range []string{ + "--ro-bind\x00/opt/base/site-packages\x00/opt/base/site-packages", + "--ro-bind\x00/opt/ext/site-packages\x00/opt/ext/site-packages", + "--ro-bind\x00/opt/py\x00/opt/py", + "--ro-bind\x00/opt/env/bin\x00/opt/env/bin", + "--setenv\x00PATH\x00/opt/py/bin:/opt/env/bin:/usr/bin:/bin", + "--setenv\x00PIP_NO_INDEX\x001", + } { + if !strings.Contains(joined, m) { + t.Fatalf("sandbox argv missing %q: %v", m, argv) + } + } + baseAt := strings.Index(joined, "/opt/base/site-packages") + extAt := strings.Index(joined, "/opt/ext/site-packages") + if baseAt < 0 || extAt < baseAt { + t.Fatalf("extension layer must mount after the base layer: %v", argv) + } +} + // TestExecEnvJoinsPythonPath F3：依赖层与技能根合成一条 PYTHONPATH， // 不允许重复键互相覆盖（否则只挂载了依赖层却 import 不到）。 func TestExecEnvJoinsPythonPath(t *testing.T) { diff --git a/internal/tools/exec_env_tool.go b/internal/tools/exec_env_tool.go new file mode 100644 index 0000000..5db5ea2 --- /dev/null +++ b/internal/tools/exec_env_tool.go @@ -0,0 +1,124 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + "time" + + "agent-system-backend/internal/execenv" + core "agent-system-core" +) + +// envBroker 由组合根注入。未注入时 python_env 明确报错，不假装能 pip install。 +var envBroker *execenv.Broker + +// envApproval 在申请需要人批时写一张选择卡。空则只把 pending 文本回给模型。 +var envApproval func(owner, conversationID, requestID, detail string) error + +// SetEnvBroker 注入分层环境代理。 +func SetEnvBroker(b *execenv.Broker) { envBroker = b } + +// SetEnvApproval 注入审批卡写入。 +func SetEnvApproval(fn func(owner, conversationID, requestID, detail string) error) { + envApproval = fn +} + +func registerPythonEnvTool(r *Registry) error { + return r.Register(core.ToolDesc{ + Name: "python_env", Source: core.SourceBuiltin, Risk: "medium", + Description: "向系统申请 Python 包，或查看本会话已经能用的环境。沙箱不能联网安装。action=status 查看基础层与本会话的层；action=request 申请包（packages）或登记工作区内的 wheel；白名单内的包会自动装好并在下一次 run_python 生效，其余会进入审批。action=wait 等待 requestId 结束。不要用 pip install。", + Parameters: json.RawMessage(`{"type":"object","properties":{` + + `"action":{"type":"string","enum":["status","request","wait"],"description":"status | request | wait"},` + + `"packages":{"type":"array","items":{"type":"string"},"description":"包规格，如 pdfplumber 或 numpy==2.2.0"},` + + `"reason":{"type":"string","description":"为什么需要这些包"},` + + `"wheel":{"type":"string","description":"工作区内的 .whl 相对路径，上传的包需要审批"},` + + `"request_id":{"type":"string","description":"wait 时的申请 ID"},` + + `"timeout_sec":{"type":"integer","description":"wait 的秒数，默认 120"}}` + + `,"required":["action"]}`), + }, execPythonEnv) +} + +func execPythonEnv(rc ToolExecCtx, args json.RawMessage) (string, error) { + if err := execGate(rc); err != nil { + return "", err + } + if envBroker == nil { + return "", fmt.Errorf("python_env: 环境代理未配置（启动参数 execEnvRoot）") + } + var a struct { + Action string `json:"action"` + Packages []string `json:"packages"` + Reason string `json:"reason"` + Wheel string `json:"wheel"` + RequestID string `json:"request_id"` + TimeoutS int `json:"timeout_sec"` + } + if len(args) > 0 { + if err := json.Unmarshal(args, &a); err != nil { + return "", fmt.Errorf("python_env: bad args: %w", err) + } + } + switch a.Action { + case "status": + return envBroker.StatusText(rc.ConversationID()) + case "wait": + if strings.TrimSpace(a.RequestID) == "" { + return "", fmt.Errorf("python_env: request_id required") + } + timeout := time.Duration(a.TimeoutS) * time.Second + out, err := envBroker.Wait(execParentCtx(rc), a.RequestID, timeout) + if err != nil { + return "", err + } + raw, _ := json.Marshal(out) + return string(raw), nil + case "request": + wheel := "" + if strings.TrimSpace(a.Wheel) != "" { + sandbox, err := execSandbox(rc) + if err != nil { + return "", err + } + wheel, err = filepath.Abs(filepath.Join(sandbox, a.Wheel)) + if err != nil { + return "", err + } + if !strings.HasPrefix(wheel, sandbox+string(filepath.Separator)) { + return "", fmt.Errorf("python_env: wheel must stay inside the workspace") + } + } + out, err := envBroker.RequestPackages(execParentCtx(rc), rc.ConversationID(), rc.OwnerID(), a.Reason, a.Packages, wheel) + if err != nil { + return "", err + } + if out.Status == "pending_approval" && envApproval != nil { + if err := envApproval(rc.OwnerID(), rc.ConversationID(), out.RequestID, out.Detail); err != nil { + out.Detail += "；审批卡未写入：" + err.Error() + } + } + raw, _ := json.Marshal(out) + return string(raw), nil + default: + return "", fmt.Errorf("python_env: unknown action %q", a.Action) + } +} + +// ExecDoctor 用当前执行后端探针导入基础交付物库。依赖层已配置但导入失败时返回错误。 +func ExecDoctor() error { + bin, err := resolveInterpreter("python3") + if err != nil { + return err + } + dirs, _ := pythonPathFor("") + mounts := append([]string{}, dirs...) + if execPythonPrefix != "" { + mounts = append(mounts, execPythonPrefix) + } + if execShimDir != "" { + mounts = append(mounts, execShimDir) + } + return execenv.Doctor(context.Background(), bin, dirs, nil, ExecBackendName() == "sandbox", mounts) +} diff --git a/internal/tools/exec_interpreter.go b/internal/tools/exec_interpreter.go index 7030839..55f2917 100644 --- a/internal/tools/exec_interpreter.go +++ b/internal/tools/exec_interpreter.go @@ -31,7 +31,7 @@ import ( func RegisterInterpreterTools(r *Registry) error { if err := r.Register(core.ToolDesc{ Name: "run_shell", Source: core.SourceBuiltin, Risk: "medium", - Description: "在会话工作区目录内执行 bash 脚本：检查环境、安装依赖（pip install --user …）、处理文件等。脚本原样落盘执行；返回退出码与合并输出（stdout+stderr，超限头尾截断）。以当前用户身份运行。", + Description: "在会话工作区目录内执行 bash 脚本：检查环境、处理文件、调用 python3 -m housestyle.build。脚本原样落盘执行；返回退出码与合并输出（stdout+stderr，超限头尾截断）。沙箱断网且不继承宿主家目录，不要用 pip/apt 安装；缺的 Python 包用 python_env 向系统申请。", Parameters: json.RawMessage(`{"type":"object","properties":{` + `"script":{"type":"string","description":"bash 脚本（可多行）"},` + `"timeout_sec":{"type":"integer","description":"超时秒数（默认 600，可自行设更长）"}}` + @@ -39,9 +39,12 @@ func RegisterInterpreterTools(r *Registry) error { }, execShell); err != nil { return err } + if err := registerPythonEnvTool(r); err != nil { + return err + } return r.Register(core.ToolDesc{ Name: "run_python", Source: core.SourceBuiltin, Risk: "medium", - Description: "执行 Python 脚本（python3，工作目录为会话工作区）：生成文档/表格/演示稿/数据处理的首选路径。脚本原样落盘执行；stdout/stderr 合并返回，超限头尾截断。python-docx/openpyxl/python-pptx 缺失时可先 pip install --user 安装。交付物用 import housestyle（技能目录在环境变量 AGENT_SKILLS_DIR，house-style 已在 PYTHONPATH）。", + Description: "执行 Python 脚本（python3，工作目录为会话工作区）：生成文档/表格/演示稿/数据处理的首选路径。脚本原样落盘执行；stdout/stderr 合并返回，超限头尾截断。基础层已提供 python-docx、openpyxl、python-pptx 等，可直接 import。缺的包用 python_env 申请，不要 pip install。交付物用 import housestyle（技能目录在环境变量 AGENT_SKILLS_DIR，house-style 已在 PYTHONPATH）。", Parameters: json.RawMessage(`{"type":"object","properties":{` + `"script":{"type":"string","description":"完整 Python 脚本（utf-8）"},` + `"timeout_sec":{"type":"integer","description":"超时秒数（默认 600，可自行设更长）"}}` + @@ -55,39 +58,95 @@ var skillsRoot string // SetSkillsRoot 让 run_python / run_shell 能 import housestyle，并看到技能目录。 func SetSkillsRoot(dir string) { skillsRoot = dir } -// execDepsDir 是 sandbox 档的只读依赖层（site-packages 等）；组合根启动时注入。 +// execDepsDir 是 sandbox 档的只读基础依赖层（site-packages）；组合根启动时注入。 var execDepsDir string -// SetExecDepsDir 配置 sandbox 档依赖层目录（空 = 不挂载，现状）。 +// SetExecDepsDir 配置 sandbox 档基础依赖层目录（空 = 不挂载，现状）。 func SetExecDepsDir(dir string) { execDepsDir = dir } -// execEnv 最小环境：PATH/HOME/LANG/TZ/TERM 继承（pip --user、字体、中文 locale）， -// 其余变量不带入子进程。技能根配置后追加 AGENT_SKILLS_DIR 与 PYTHONPATH。 -func execEnv() []string { +// 托管解释器与 pip 垫片。空表示尚未配置，python3 回落到 /usr/bin/python3， +// 不再使用 PATH 里可能指向 /home 的解释器（沙箱会把 /home 遮掉）。 +var ( + execPythonBin string + execPythonPrefix string + execShimDir string + execLayers func(conversationID string) (extra []string, isolated string) +) + +// SetExecPython 配置托管解释器。prefix 是需要在沙箱里重新挂载的目录。 +func SetExecPython(bin, prefix string) { + execPythonBin = bin + execPythonPrefix = prefix +} + +// SetExecShimDir 把 pip 垫片目录放到 PATH 最前。 +func SetExecShimDir(dir string) { execShimDir = dir } + +// SetExecLayers 按会话返回扩展层或隔离层的 site-packages 目录。 +func SetExecLayers(fn func(conversationID string) (extra []string, isolated string)) { + execLayers = fn +} + +// execEnv 最小环境。PATH 固定，不继承宿主 PATH：宿主 PATH 里的 linuxbrew +// 位于 /home 下，沙箱 tmpfs 会把它遮掉，解析到那里的 python3 无法启动。 +func execEnv() []string { return execEnvFor("") } + +func execEnvFor(conv string) []string { var out []string for _, kv := range os.Environ() { - for _, keep := range []string{"PATH=", "HOME=", "LANG=", "LC_", "TZ=", "TERM="} { + for _, keep := range []string{"HOME=", "LANG=", "LC_", "TZ=", "TERM="} { if strings.HasPrefix(kv, keep) { out = append(out, kv) break } } } - out = append(out, "PYTHONDONTWRITEBYTECODE=1") - var pypaths []string - if execDepsDir != "" { - pypaths = append(pypaths, execDepsDir) - } + out = append(out, + "PATH="+execPATH(), + "PYTHONDONTWRITEBYTECODE=1", + "PIP_NO_INDEX=1", + ) + dirs, _ := pythonPathFor(conv) if skillsRoot != "" { out = append(out, "AGENT_SKILLS_DIR="+skillsRoot) - pypaths = append(pypaths, filepath.Join(skillsRoot, "house-style")) + dirs = append(dirs, filepath.Join(skillsRoot, "house-style")) } - if len(pypaths) > 0 { // 合成一条，避免重复键互相覆盖 - out = append(out, "PYTHONPATH="+strings.Join(pypaths, string(os.PathListSeparator))) + if len(dirs) > 0 { + out = append(out, "PYTHONPATH="+strings.Join(dirs, string(os.PathListSeparator))) } return out } +func execPATH() string { + var parts []string + if execPythonBin != "" { + parts = append(parts, filepath.Dir(execPythonBin)) + } + if execShimDir != "" { + parts = append(parts, execShimDir) + } + parts = append(parts, "/usr/local/bin", "/usr/bin", "/bin") + return strings.Join(parts, ":") +} + +// pythonPathFor 返回要放进 PYTHONPATH、并且在沙箱里只读挂载的目录。 +// 隔离层替换基础层，不与基础层叠放。 +func pythonPathFor(conv string) (dirs []string, isolated bool) { + var extra []string + var iso string + if execLayers != nil && conv != "" { + extra, iso = execLayers(conv) + } + if iso != "" { + return []string{iso}, true + } + if execDepsDir != "" { + dirs = append(dirs, execDepsDir) + } + dirs = append(dirs, extra...) + return dirs, false +} + // truncateOutput 头 2/3 + 尾 1/3 保留，中部标记省略。 func truncateOutput(b []byte) string { if len(b) <= execMaxOutput { @@ -130,10 +189,27 @@ func execParentCtx(rc ToolExecCtx) context.Context { return context.Background() } -// resolveInterpreter 把白名单解释器名钉成绝对路径，避免 PATH 在子进程里被换掉。 +// resolveInterpreter 把白名单解释器名钉成绝对路径。 +// python3 优先用托管解释器，其次 /usr/bin/python3。不用 LookPath： +// 服务进程的 PATH 常把 /home 下的 linuxbrew 放在前面，那个路径在沙箱里不存在。 func resolveInterpreter(name string) (string, error) { switch name { - case "bash", "python3": + case "bash": + for _, p := range []string{"/bin/bash", "/usr/bin/bash"} { + if st, err := os.Stat(p); err == nil && !st.IsDir() { + return p, nil + } + } + case "python3": + if execPythonBin != "" { + if st, err := os.Stat(execPythonBin); err == nil && !st.IsDir() { + return execPythonBin, nil + } + return "", fmt.Errorf("python3: configured interpreter missing: %s", execPythonBin) + } + if st, err := os.Stat("/usr/bin/python3"); err == nil && !st.IsDir() { + return "/usr/bin/python3", nil + } default: return "", fmt.Errorf("interpreter %s is not allowlisted", name) } @@ -144,13 +220,16 @@ func resolveInterpreter(name string) (string, error) { if !filepath.IsAbs(p) { return "", fmt.Errorf("%s: resolved path is not absolute: %s", name, p) } + if strings.HasPrefix(p, "/home/") || strings.HasPrefix(p, "/root/") { + return "", fmt.Errorf("%s: refusing %s because the sandbox hides that path", name, p) + } return p, nil } // runInterpreter 启动白名单解释器执行落盘脚本（脚本路径为唯一参数）。 // 进程启动经 EXT-16 执行后端（direct=现状直启 / sandbox=bwrap 隔离档）。 // 非零退出、超时与请求取消都是正常反馈，以文本返回；仅启动失败算错误。 -func runInterpreter(parent context.Context, interpreter, scriptPath, dir string, timeoutS int, sink func(stream, text string), checks ...func() error) (string, error) { +func runInterpreter(parent context.Context, interpreter, scriptPath, dir, convID string, timeoutS int, sink func(stream, text string), checks ...func() error) (string, error) { if parent == nil { parent = context.Background() } @@ -168,10 +247,18 @@ func runInterpreter(parent context.Context, interpreter, scriptPath, dir string, check = checks[0] } spec := ExecSpec{ - Bin: bin, Args: []string{scriptPath}, Dir: dir, Env: execEnv(), + Bin: bin, Args: []string{scriptPath}, Dir: dir, Env: execEnvFor(convID), SkillsDir: skillsRoot, DepsDir: execDepsDir, Timeout: time.Duration(timeoutS) * time.Second, + PythonPrefix: execPythonPrefix, ShimDir: execShimDir, CheckExternalEffect: check, } + if dirs, isolated := pythonPathFor(convID); isolated { + spec.DepsDir = "" + spec.LayerDirs = dirs + } else if len(dirs) > 1 || (len(dirs) == 1 && dirs[0] != execDepsDir) { + spec.LayerDirs = dirs + spec.DepsDir = "" + } pr, pw := io.Pipe() var buf bytes.Buffer readDone := make(chan struct{}) @@ -256,7 +343,7 @@ func execShell(rc ToolExecCtx, args json.RawMessage) (string, error) { if err != nil { return "", err } - return runInterpreter(execParentCtx(rc), "bash", p, sandbox, firstPos(a.TimeoutS, a.TimeoutSec), toolOutputSink(rc), func() error { return checkToolExternalEffect(rc) }) + return runInterpreter(execParentCtx(rc), "bash", p, sandbox, rc.ConversationID(), firstPos(a.TimeoutS, a.TimeoutSec), toolOutputSink(rc), func() error { return checkToolExternalEffect(rc) }) } // ScriptCommand 准备一条以当前用户身份、工作目录为会话沙箱的 python3 命令。 @@ -271,7 +358,7 @@ func ScriptCommand(ctx context.Context, sandbox, path string) (*exec.Cmd, error) } cmd := exec.CommandContext(ctx, bin, path) cmd.Dir = sandbox - cmd.Env = execEnv() + cmd.Env = execEnvFor("") cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.WaitDelay = 2 * time.Second cmd.Cancel = func() error { @@ -312,7 +399,7 @@ func execPython(rc ToolExecCtx, args json.RawMessage) (string, error) { if err != nil { return "", err } - return runInterpreter(execParentCtx(rc), "python3", p, sandbox, firstPos(a.TimeoutS, a.TimeoutSec), toolOutputSink(rc), func() error { return checkToolExternalEffect(rc) }) + return runInterpreter(execParentCtx(rc), "python3", p, sandbox, rc.ConversationID(), firstPos(a.TimeoutS, a.TimeoutSec), toolOutputSink(rc), func() error { return checkToolExternalEffect(rc) }) } // toolOutputSink 在执行上下文实现了 EmitToolOutput 时，把 stdout 增量交出去。 @@ -343,7 +430,7 @@ func InterpreterExecConfig() *agenttools.ExecConfig { if ExecDisabled() { return nil } - cfg := &agenttools.ExecConfig{Env: execEnv()} + cfg := &agenttools.ExecConfig{Env: execEnvFor("")} if p, err := exec.LookPath("bash"); err == nil { cfg.Bash = p } diff --git a/internal/tools/exec_interpreter_test.go b/internal/tools/exec_interpreter_test.go index 41d28ee..da0aa29 100644 --- a/internal/tools/exec_interpreter_test.go +++ b/internal/tools/exec_interpreter_test.go @@ -227,7 +227,7 @@ func TestInterpreterExecConfigForKernel(t *testing.T) { for _, kv := range cfg.Env { if strings.HasPrefix(kv, "PATH=") || strings.HasPrefix(kv, "HOME=") || strings.HasPrefix(kv, "LANG=") || strings.HasPrefix(kv, "LC_") || strings.HasPrefix(kv, "TZ=") || strings.HasPrefix(kv, "TERM=") || - kv == "PYTHONDONTWRITEBYTECODE=1" || strings.HasPrefix(kv, "AGENT_SKILLS_DIR=") || strings.HasPrefix(kv, "PYTHONPATH=") { + kv == "PYTHONDONTWRITEBYTECODE=1" || kv == "PIP_NO_INDEX=1" || strings.HasPrefix(kv, "AGENT_SKILLS_DIR=") || strings.HasPrefix(kv, "PYTHONPATH=") { continue } t.Fatalf("env whitelist violated: %q", kv) diff --git a/internal/tools/ooxmlgate.go b/internal/tools/ooxmlgate.go index 9bfd839..e0cf467 100644 --- a/internal/tools/ooxmlgate.go +++ b/internal/tools/ooxmlgate.go @@ -378,7 +378,8 @@ func ooxmlDeclaredNamespaces(f *zip.File) (map[string]bool, error) { return nil, fmt.Errorf("部件 %s 无法读取（%v）", f.Name, err) } defer rc.Close() - out := map[string]bool{"xml": true, "xmlns": true} + // xml 前缀由 XML 规范预绑定。Go 解码器把它展开成 URI，而不是留下前缀名。 + out := map[string]bool{"xml": true, "xmlns": true, "http://www.w3.org/XML/1998/namespace": true} dec := xml.NewDecoder(rc) for { tok, err := dec.Token()