ci / vet-and-test (pull_request) Successful in 40s
Why: User rulings 4/5 plus the P2-P4 live evidence. Keyword scoring dimensions (readability, actionability) kept failing valid documents - normal Chinese section headings flagged as column titles, loops of 4 corrections - because they grade writing preferences, not verifiable facts. Unverified numbers (~10-13 per run) remained the largest quality leak through every phase. And the verifier runs on the same model as the worker (lightweight falls back to chat), so "independent" verification was self-review. Separately, svc seeding after boot never registered the tavily tools until a restart because keys were snapshotted at startup. Changes: quality.HardFacts (lint actually ran + zero errors) replaces the full ScoreBytes projection in CheckArtifacts - keyword dimensions remain in ScoreBytes for eval visibility only. New fact-consistency check (runtime.factCheck, default on): extract percentages/dates/large numbers from deliverable text, trace each against the material corpus (attachment text + tool evidence); misses get exactly one correction round (fix, drop, or mark as target/estimate), never a loop. runVerifier skips same-source model verification unless runtime.verifierSameModel is set (hard facts + fact check still run). Search tools now register lazily, reading tavily keys per call (settings/seeding take effect immediately). Evidence: SELF - TestHardFactsIgnoresKeywordDimensions, TestHardFactsKeepsLintUnavailableBlocking, TestUnverifiedFactsAndNotice, TestFactCheckContinuationOnce (fact note fires, corrected run passes); go vet + go test ./... pass; live before/after on the golden task set recorded on the phase05 task card per the policy clause. Impact: Gate stops rejecting valid documents on stylistic grounds; invented figures get one forced correction round; same-model verification no longer pretends to be independent; tavily config is hot. No contract or storage changes.
286 lines
9.1 KiB
Go
286 lines
9.1 KiB
Go
package tools
|
|
|
|
// EXT-16 执行后端测试:direct 档回归(现状语义)+ sandbox 档 bwrap 参数组装与
|
|
// 逃逸负例(提案 2026-10-09-ext-g1-b1-b3 §4 验收:读 ~/.secrets、读其他会话目录、
|
|
// 访问外网、fork 炸弹、超时不可杀——须全部被拦;会话目录内正常任务成功)。
|
|
// bwrap 缺席的环境跳过 sandbox 集成项(CI 只兜 direct 回归;隔离证据在具备
|
|
// bwrap 的评审环境重放)。
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSetExecBackendValidation(t *testing.T) {
|
|
defer func() { _ = SetExecBackend("") }()
|
|
if err := SetExecBackend("jail"); err == nil {
|
|
t.Fatal("unknown backend must be rejected (fail-closed)")
|
|
}
|
|
for _, ok := range []string{"", "direct", "sandbox"} {
|
|
if err := SetExecBackend(ok); err != nil {
|
|
t.Fatalf("%q: %v", ok, err)
|
|
}
|
|
}
|
|
if err := SetExecBackend("sandbox"); err != nil || ExecBackendName() != "sandbox" {
|
|
t.Fatalf("sandbox not selected: %v %q", err, ExecBackendName())
|
|
}
|
|
}
|
|
|
|
func TestSandboxArgvShape(t *testing.T) {
|
|
argv := sandboxArgv(ExecSpec{
|
|
Bin: "/usr/bin/python3", Args: []string{"/data/s/x.py"},
|
|
Dir: "/data/s", SkillsDir: "/data/skills",
|
|
Env: []string{"PATH=/usr/bin:/bin", "HOME=/tmp/h"},
|
|
})
|
|
joined := strings.Join(argv, "\x00")
|
|
musts := []string{
|
|
"--unshare-all", "--die-with-parent",
|
|
"--ro-bind\x00/\x00/", // 只读系统根
|
|
"--tmpfs\x00/home", // 遮蔽真实家目录
|
|
"--tmpfs\x00/tmp",
|
|
"--bind\x00/data/s\x00/data/s", // 会话目录唯一可写挂载
|
|
"--ro-bind\x00/data/skills\x00/data/skills",
|
|
"--chdir\x00/data/s", "--clearenv",
|
|
"--setenv\x00PATH", "--setenv\x00HOME",
|
|
"/bin/bash", "-c",
|
|
}
|
|
for _, m := range musts {
|
|
if !strings.Contains(joined, m) {
|
|
t.Fatalf("sandbox argv missing %q: %v", m, argv)
|
|
}
|
|
}
|
|
wrapper := argv[len(argv)-1]
|
|
if !strings.Contains(wrapper, "ulimit -u "+itoa(execSandboxMaxProcs)) ||
|
|
!strings.Contains(wrapper, "exec '/usr/bin/python3' '/data/s/x.py'") {
|
|
t.Fatalf("wrapper = %q(须 ulimit -u 限进程数并 exec 白名单解释器)", wrapper)
|
|
}
|
|
}
|
|
|
|
func itoa(n int) string {
|
|
if n == 0 {
|
|
return "0"
|
|
}
|
|
var b [20]byte
|
|
i := len(b)
|
|
for n > 0 {
|
|
i--
|
|
b[i] = byte('0' + n%10)
|
|
n /= 10
|
|
}
|
|
return string(b[i:])
|
|
}
|
|
|
|
// ---- sandbox 集成(需 bwrap;缺席即跳过)----
|
|
|
|
func requireBwrap(t *testing.T) {
|
|
t.Helper()
|
|
if _, err := exec.LookPath("bwrap"); err != nil {
|
|
t.Skip("bwrap not available; sandbox integration replayed on capable hosts")
|
|
}
|
|
}
|
|
|
|
// sandboxRun 经 sandbox 后端执行一段 bash 脚本,返回合并输出与退出描述。
|
|
func sandboxRun(t *testing.T, script string, timeout time.Duration) (string, string) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "s.sh")
|
|
if err := os.WriteFile(p, []byte(script), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
defer cancel()
|
|
var buf bytes.Buffer
|
|
spec := ExecSpec{
|
|
Bin: "/bin/bash", Args: []string{p}, Dir: dir,
|
|
Env: []string{"PATH=/usr/bin:/bin", "HOME=/home/agent"},
|
|
}
|
|
proc, err := sandboxBackend{}.Start(ctx, spec, &buf)
|
|
if err != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
werr := proc.Wait()
|
|
out := buf.String()
|
|
state := "exit=0"
|
|
if werr != nil {
|
|
if ee, ok := werr.(*exec.ExitError); ok {
|
|
state = "exit=" + itoa(ee.ExitCode())
|
|
} else {
|
|
state = "err:" + werr.Error()
|
|
}
|
|
}
|
|
if ctx.Err() == context.DeadlineExceeded {
|
|
state = "exit=timeout"
|
|
}
|
|
return out, state
|
|
}
|
|
|
|
func TestSandboxHomeHidden(t *testing.T) {
|
|
requireBwrap(t)
|
|
home, err := os.UserHomeDir()
|
|
if err != nil || home == "" || !strings.HasPrefix(home, "/home/") {
|
|
t.Skipf("home %q not under /home; fixture would not be shadowed", home)
|
|
}
|
|
secret := filepath.Join(home, ".agent-ext16-marker")
|
|
if err := os.WriteFile(secret, []byte("topsecret"), 0o600); err != nil {
|
|
t.Skipf("cannot create home fixture: %v", err)
|
|
}
|
|
defer os.Remove(secret)
|
|
out, state := sandboxRun(t, "cat \"$HOME/.agent-ext16-marker\" 2>err.txt; cat err.txt; exit 7", 30*time.Second)
|
|
if strings.Contains(out, "topsecret") {
|
|
t.Fatalf("sandbox read real home secret (state=%s out=%q)", state, out)
|
|
}
|
|
if state == "exit=0" || state == "exit=7" && strings.Contains(out, "topsecret") {
|
|
t.Fatalf("unexpected success: state=%s out=%q", state, out)
|
|
}
|
|
}
|
|
|
|
func TestSandboxSiblingSessionUnreadable(t *testing.T) {
|
|
requireBwrap(t)
|
|
base := t.TempDir()
|
|
a, b := filepath.Join(base, "session-a"), filepath.Join(base, "session-b")
|
|
if err := os.MkdirAll(a, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.MkdirAll(b, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
marker := filepath.Join(b, "other-session.txt")
|
|
if err := os.WriteFile(marker, []byte("others-data"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
var buf bytes.Buffer
|
|
spec := ExecSpec{
|
|
Bin: "/bin/bash", Args: []string{"-c", "cat '" + marker + "'"}, Dir: a,
|
|
Env: []string{"PATH=/usr/bin:/bin", "HOME=/home/agent"},
|
|
}
|
|
proc, err := sandboxBackend{}.Start(ctx, spec, &buf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
werr := proc.Wait()
|
|
if werr == nil || buf.String() == "others-data" {
|
|
t.Fatalf("sibling session readable inside sandbox: wait=%v out=%q", werr, buf.String())
|
|
}
|
|
}
|
|
|
|
func TestSandboxNoNetwork(t *testing.T) {
|
|
requireBwrap(t)
|
|
out, state := sandboxRun(t,
|
|
"timeout 5 bash -c 'echo > /dev/tcp/1.1.1.1/80' 2>/dev/null && echo NET-OPEN || echo NET-BLOCKED",
|
|
30*time.Second)
|
|
if strings.Contains(out, "NET-OPEN") {
|
|
t.Fatalf("network reachable in sandbox (state=%s)", state)
|
|
}
|
|
if !strings.Contains(out, "NET-BLOCKED") {
|
|
t.Fatalf("probe did not run: state=%s out=%q", state, out)
|
|
}
|
|
}
|
|
|
|
func TestSandboxForkBombContained(t *testing.T) {
|
|
requireBwrap(t)
|
|
// 确定性进程数压测(替代经典 :(){ :|:& };:——其在沙箱 user namespace 内
|
|
// 递归行为依 bash 实现而变,不适合做门禁):400 个并发 sleep 撞 ulimit -u
|
|
// 256 的墙,fork: retry 即限额生效的直接证据。
|
|
start := time.Now()
|
|
out, state := sandboxRun(t,
|
|
"n=0; for i in $(seq 1 400); do sleep 5 & n=$((n+1)); done; echo spawned=$n; wait",
|
|
20*time.Second)
|
|
elapsed := time.Since(start)
|
|
if elapsed > 40*time.Second {
|
|
t.Fatalf("spawn storm stalled the runner: %s", elapsed)
|
|
}
|
|
if !strings.Contains(out, "fork: retry") {
|
|
t.Fatalf("process limit not enforced (state=%s out=%.200q)", state, out)
|
|
}
|
|
if state == "exit=0" && strings.Contains(out, "spawned=400") {
|
|
t.Fatalf("400 processes spawned unbounded: %q", out)
|
|
}
|
|
}
|
|
|
|
func TestSandboxOrphansDieWithNamespace(t *testing.T) {
|
|
requireBwrap(t)
|
|
out, state := sandboxRun(t, ":(){ :|:& };:; echo gone", 30*time.Second)
|
|
if state != "exit=0" || !strings.Contains(out, "gone") {
|
|
t.Fatalf("state=%s out=%q", state, out)
|
|
}
|
|
time.Sleep(500 * time.Millisecond) // 给孤儿进程留存活窗口
|
|
hb, err := exec.Command("bash", "-c", "ps -eo args | grep -c '[s]\\.sh' || true").Output()
|
|
if err != nil {
|
|
t.Skipf("host ps probe failed: %v", err)
|
|
}
|
|
if strings.TrimSpace(string(hb)) != "0" {
|
|
t.Fatalf("orphaned bomb survived outside the namespace: %s", hb)
|
|
}
|
|
}
|
|
|
|
func TestSandboxTimeoutKills(t *testing.T) {
|
|
requireBwrap(t)
|
|
start := time.Now()
|
|
_, state := sandboxRun(t, "sleep 60", 3*time.Second)
|
|
if d := time.Since(start); d > 20*time.Second {
|
|
t.Fatalf("timeout did not kill the sandbox: %s", d)
|
|
}
|
|
if state != "exit=timeout" {
|
|
t.Fatalf("state = %s, want exit=timeout", state)
|
|
}
|
|
}
|
|
|
|
func TestSandboxSessionWriteVisible(t *testing.T) {
|
|
requireBwrap(t)
|
|
dir := t.TempDir()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
script := filepath.Join(dir, "w.sh")
|
|
if err := os.WriteFile(script, []byte("printf 'docx-ish' > \"$(dirname \"$0\")/out.txt\"\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var buf bytes.Buffer
|
|
spec := ExecSpec{
|
|
Bin: "/bin/bash", Args: []string{script}, Dir: dir,
|
|
Env: []string{"PATH=/usr/bin:/bin", "HOME=/home/agent"},
|
|
}
|
|
proc, err := sandboxBackend{}.Start(ctx, spec, &buf)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := proc.Wait(); err != nil {
|
|
t.Fatalf("sandbox task failed: %v out=%q", err, buf.String())
|
|
}
|
|
got, err := os.ReadFile(filepath.Join(dir, "out.txt"))
|
|
if err != nil || string(got) != "docx-ish" {
|
|
t.Fatalf("product not visible on host: %v %q", err, got)
|
|
}
|
|
}
|
|
|
|
// direct 档回归:后端抽象后 runInterpreter 语义不变(退出码/超时文本口径)。
|
|
func TestDirectBackendRunInterpreterUnchanged(t *testing.T) {
|
|
if _, err := exec.LookPath("bash"); err != nil {
|
|
t.Skip("bash missing")
|
|
}
|
|
dir := t.TempDir()
|
|
p := filepath.Join(dir, "x.sh")
|
|
if err := os.WriteFile(p, []byte("echo hi; exit 7\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := runInterpreter(context.Background(), "bash", p, dir, 30, nil)
|
|
if err != nil || !strings.HasPrefix(got, "exit=7\n") || !strings.Contains(got, "hi") {
|
|
t.Fatalf("direct regression: err=%v got=%q", err, got)
|
|
}
|
|
p2 := filepath.Join(dir, "t.sh")
|
|
if err := os.WriteFile(p2, []byte("sleep 60\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err = runInterpreter(context.Background(), "bash", p2, dir, 2, nil)
|
|
if err != nil || !strings.HasPrefix(got, "exit=timeout(2s)") {
|
|
t.Fatalf("direct timeout regression: err=%v got=%q", err, got)
|
|
}
|
|
}
|