diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 33abf249..59cd840b 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -263,6 +263,16 @@ jobs: tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz" ln -sfn /usr/local/go/bin/go /usr/local/bin/go ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt + - name: Install boundary checker prerequisite + run: &install_boundary_prerequisite | + set -euo pipefail + if ! command -v rg >/dev/null 2>&1; then + SUDO="" + if command -v sudo >/dev/null 2>&1 && [ "$(id -u)" != 0 ]; then SUDO=sudo; fi + $SUDO apt-get update -y + $SUDO apt-get install -y ripgrep + fi + rg --version - name: Prefetch Go modules without executing candidate code env: SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }} @@ -769,6 +779,8 @@ jobs: tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz" ln -sfn /usr/local/go/bin/go /usr/local/bin/go ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt + - name: Install boundary checker prerequisite + run: *install_boundary_prerequisite - name: Install house-style Python packages run: python3 -m pip install --break-system-packages -i https://pypi.tuna.tsinghua.edu.cn/simple python-docx python-pptx openpyxl - name: Install PostgreSQL, pgvector and Redis @@ -995,6 +1007,7 @@ jobs: # Future lifecycle fetches use only local fixed-source mirrors. git(directory, "remote", "set-url", "origin", (mirrors / "Agent-System-RDM.git").as_uri()) git(directory, "update-ref", "refs/remotes/origin/main", pins["sha"]) + git(directory, "update-ref", "refs/heads/main", pins["sha"]) mirror_env() (root / ".ci-rdm-pins").write_text("\n".join(revisions) + "\n") except (ValueError, OSError, subprocess.CalledProcessError) as error: @@ -1082,6 +1095,8 @@ jobs: tar -C /usr/local -xzf "$RUNNER_TEMP/go.tgz" ln -sfn /usr/local/go/bin/go /usr/local/bin/go ln -sfn /usr/local/go/bin/gofmt /usr/local/bin/gofmt + - name: Install boundary checker prerequisite + run: *install_boundary_prerequisite - name: Clone Agent-System-RDM with nested pins (reviewed allowlist addition) env: SUBMODULE_TOKEN: ${{ secrets.SUBMODULE_TOKEN }} diff --git a/scripts/component-check.py b/scripts/component-check.py index b9737fa4..41d17209 100644 --- a/scripts/component-check.py +++ b/scripts/component-check.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 """Prepare selected fixed sources and run explicitly local component checks.""" import argparse +from contextlib import contextmanager import os from pathlib import Path import subprocess import sys +import tempfile from workspace.cli import Workspace, process_environment from workspace.common import WorkspaceError @@ -15,6 +17,8 @@ SINGLE = {"test-backend": "Agent-System-Backend", "test-backend-pg": "Agent-Syst "test-frontend": "Agent-System-Frontend", "test-ui": "Agent-System-UI", "test-multi-instance": "Agent-System-Backend", "eval-deliverables": "Agent-System-Backend", "dev-backend": "Agent-System-Backend", "dev-frontend": "Agent-System-Frontend"} +GO_CHECKS = {"test-backend", "test-backend-pg", "test-core", "test-agentcmd", + "test-libs", "test-multi-instance", "eval-deliverables"} def run(command, cwd, env): @@ -40,6 +44,34 @@ def prepare_local_source(workspace, component): workspace.prepare_one(component) +@contextmanager +def check_sources(workspace, selected, target, overlay): + paths = {component["id"]: workspace.root / component["path"] for component in selected} + fixed = [component for component in selected + if workspace.state.data["components"][component["id"]].get("mode") == "versioned"] + if overlay or target not in GO_CHECKS or not fixed: + yield paths + return + # Go tests can create ignored outputs (including Python bytecode). Keep + # strict source guards and execute fixed commits in owned disposable clones. + # Editable sources and explicit overlays still exercise the owner's code. + with tempfile.TemporaryDirectory(prefix="component-check-") as temporary: + area = Path(temporary) + for component in fixed: + name = component["id"] + source = paths[name] + actual = workspace.git.safe(source, component["repo"]) + commit = workspace.state.data["components"][name]["baseline"] + if actual["head"] != commit or actual["branch"]: + raise WorkspaceError(f"fixed source changed before local check: {name}") + destination = area / name + workspace.git.call(area, "clone", "--no-checkout", "--no-hardlinks", "--", str(source), str(destination)) + workspace.git.call(destination, "checkout", "--detach", commit) + paths[name] = destination + print(f"[component-check] fixed check snapshot {name} commit={commit}", flush=True) + yield paths + + def check(root, target, overlay): root = root.resolve() if target in ("test-backend-pg", "test-multi-instance") and not os.environ.get("AGENT_PG_TEST_DSN"): @@ -67,7 +99,11 @@ def check(root, target, overlay): else: env["GOWORK"] = "off" print(f"[component-check] local developer sources; GOWORK={env['GOWORK']}; registered edit branches and unsaved code retained", flush=True) - paths = {component["id"]: root / component["path"] for component in selected} + with check_sources(workspace, selected, target, overlay) as paths: + run_checks(root, target, selected, names, paths, env) + + +def run_checks(root, target, selected, names, paths, env): if target == "dev": print("Backend: make dev-backend [LOCAL_OVERLAY=1]\nFrontend: make dev-frontend\nPrivate module transport is scoped to those processes.") return diff --git a/scripts/lint-boundaries.sh b/scripts/lint-boundaries.sh index e9c6d682..5c34f72c 100755 --- a/scripts/lint-boundaries.sh +++ b/scripts/lint-boundaries.sh @@ -2,6 +2,7 @@ # Public SDK isolation and sibling-library ratchets use the fixed composition map # when supplied; a manual invocation uses sources prepared by the checkout owner. set -euo pipefail +command -v rg >/dev/null 2>&1 || { echo "boundary: requires ripgrep (rg)" >&2; exit 2; } root=$(cd "$(dirname "$0")/.." && pwd) fail=0 source_path() { python3 "$root/scripts/component-source.py" "$1"; } diff --git a/tests/test_ci_credentials.py b/tests/test_ci_credentials.py index 95222fd9..1c910bed 100644 --- a/tests/test_ci_credentials.py +++ b/tests/test_ci_credentials.py @@ -337,6 +337,11 @@ class CheckoutTests(unittest.TestCase): for sha in shas: self.assertEqual(self.git(work / "rdm", "cat-file", "-t", sha), "commit") self.assertEqual(self.git(work / "rdm", "rev-parse", "HEAD"), shas[2]) + # A lifecycle cache clones this isolated checkout. Local clones copy + # heads, so the fixed source must expose a deterministic main head. + lifecycle = self.base / "lifecycle-source" + self.git(self.base, "clone", "--no-hardlinks", str(work / "rdm"), str(lifecycle)) + self.assertEqual(self.git(lifecycle, "rev-parse", "origin/main"), shas[2]) # Candidate builds are a separate phase: the shell has no checkout token, # and lifecycle source fetches resolve through public local mirror paths. from tests.test_dependency_ci import phase diff --git a/tests/test_component_checks.py b/tests/test_component_checks.py index 77e4ef8e..ca9d4508 100644 --- a/tests/test_component_checks.py +++ b/tests/test_component_checks.py @@ -31,6 +31,7 @@ class ComponentCheckTests(unittest.TestCase): shutil.copyfile(SOURCE / "Makefile", self.root / "Makefile") (self.root / ".gitignore").write_text(".workspace/\ngo.work\ngo.work.sum\ntests/integration/multi-instance/.build/\n") app = self.base / "backend" + self.backend = app app.mkdir() self.call(app, "git", "init", "--initial-branch=main") (app / "go.mod").write_text("module example.com/backend\n\ngo 1.23\n") @@ -77,13 +78,98 @@ class ComponentCheckTests(unittest.TestCase): self.call(self.root, "make", "test-backend-pg", env=self.env | {"AGENT_PG_TEST_DSN": "fixture-dsn", "AGENT_REDIS_TEST_ADDR": "fixture-redis"}) def test_multi_instance_binary_is_built_inside_dev_root_and_reached_by_suite(self): + suite = self.add_multi_instance_suite() + self.call(self.root, "make", "test-multi-instance", env=self.env | {"AGENT_PG_TEST_DSN": "fixture-dsn"}) + self.assertTrue((suite / ".build/agent-system").is_file()) + self.assertFalse((self.root / ".workspace/tests").exists()) + + def add_multi_instance_suite(self): suite = self.root / "tests/integration/multi-instance" suite.mkdir(parents=True) (suite / "go.mod").write_text("module example.com/multi\n\ngo 1.23\n") (suite / "binary_test.go").write_text('package multi\nimport("os";"os/exec";"testing")\nfunc TestBinary(t *testing.T){b,e:=exec.Command(os.Getenv("AGENT_BIN")).Output();if e!=nil||string(b)!="fixed-binary"{t.Fatalf("%s %v",b,e)}}\n') - self.call(self.root, "make", "test-multi-instance", env=self.env | {"AGENT_PG_TEST_DSN": "fixture-dsn"}) + return suite + + def add_go_byproduct_fixture(self, fail=False): + (self.backend / ".gitignore").write_text("__pycache__/\n.fixture-output\n") + (self.backend / "fixture_module.py").write_text("VALUE = 'imported from Go'\n") + (self.backend / "byproduct_test.go").write_text('''package backend +import("os";"os/exec";"testing") +func TestByproducts(t *testing.T) { + if err := os.WriteFile(".fixture-output", []byte("generated by Go test"), 0600); err != nil {t.Fatal(err)} + cmd := exec.Command("python3", "-c", "import fixture_module") + cmd.Env = append(os.Environ(), "PYTHONDONTWRITEBYTECODE=", "PYTHONPYCACHEPREFIX=") + if out, err := cmd.CombinedOutput(); err != nil {t.Fatalf("%s %v", out, err)} +''' + (' t.Fatal("deliberate fixture failure")\n' if fail else '') + '}\n') + self.call(self.backend, "git", "add", ".") + self.call(self.backend, "git", "commit", "-m", "Add Go and Python byproducts fixture") + lock = json.loads((self.root / "components.lock.json").read_text()) + lock["components"]["Agent-System-Backend"]["commit"] = self.call(self.backend, "git", "rev-parse", "HEAD").stdout.strip() + (self.root / "components.lock.json").write_text(json.dumps(lock)) + self.call(self.root, "git", "add", "components.lock.json") + self.call(self.root, "git", "commit", "-m", "Pin byproducts fixture") + self.call(self.root, "python3", "scripts/workspace.py", "--root", str(self.root), "prepare") + return self.root / ".workspace/Agent-System-Backend" + + def source_snapshot(self, child): + return {"head": self.call(child, "git", "rev-parse", "HEAD").stdout, + "branch": self.call(child, "git", "branch", "--show-current").stdout, + "index": (child / ".git/index").read_bytes(), + "status": self.call(child, "git", "status", "--porcelain", "--untracked-files=all").stdout, + "ignored": self.call(child, "git", "ls-files", "--others", "--ignored", "--exclude-standard").stdout, + "files": {path.relative_to(child).as_posix(): path.read_bytes() for path in child.rglob("*") + if path.is_file() and ".git" not in path.relative_to(child).parts}} + + def test_fixed_go_checks_keep_source_clean_across_pg_and_multi_instance(self): + child = self.add_go_byproduct_fixture() + suite = self.add_multi_instance_suite() + before = self.source_snapshot(child) + env = self.env | {"AGENT_PG_TEST_DSN": "fixture-dsn", "AGENT_REDIS_TEST_ADDR": "fixture-redis"} + self.call(self.root, "make", "test-backend-pg", env=env) + generated = self.call(child, "git", "ls-files", "--others", "--ignored", "--exclude-standard").stdout + print("fixed checkout byproducts after PostgreSQL fixture: " + repr(generated), flush=True) + self.call(self.root, "make", "test-multi-instance", env=env) + self.assertEqual(self.source_snapshot(child), before) self.assertTrue((suite / ".build/agent-system").is_file()) - self.assertFalse((self.root / ".workspace/tests").exists()) + + def test_fixed_go_failure_is_reported_without_polluting_source(self): + child = self.add_go_byproduct_fixture(fail=True) + before = self.source_snapshot(child) + result = self.call(self.root, "make", "test-backend", check=False) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("deliberate fixture failure", result.stdout) + self.assertIn("local component command failed (1)", result.stdout) + self.assertEqual(self.source_snapshot(child), before) + + def test_fixed_go_checks_refuse_and_preserve_owner_changes_and_ignored_files(self): + child = self.add_go_byproduct_fixture() + for kind in ("unstaged", "staged", "ignored"): + with self.subTest(kind=kind): + if kind == "ignored": + (child / ".fixture-output").write_text("owner's unique ignored file") + else: + (child / "source.go").write_text('package backend\nfunc Value() string {return "owner-edit"}\n') + if kind == "staged": + self.call(child, "git", "add", "source.go") + before = self.source_snapshot(child) + result = self.call(self.root, "make", "test-backend", check=False) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("dirty or ignored unique files", result.stdout) + self.assertEqual(self.source_snapshot(child), before) + # This fixture owns these files and its index; reset only between cases. + if kind != "ignored": + self.call(child, "git", "restore", "--source=HEAD", "--staged", "--worktree", "source.go") + + def test_pg_and_multi_instance_prerequisites_still_refuse_before_preparing_sources(self): + for target, supplied in (("test-backend-pg", {}), + ("test-backend-pg", {"AGENT_PG_TEST_DSN": "fixture-dsn"}), + ("test-multi-instance", {})): + with self.subTest(target=target, supplied=supplied): + env = {key: value for key, value in self.env.items() + if key not in ("AGENT_PG_TEST_DSN", "AGENT_REDIS_TEST_ADDR")} + result = self.call(self.root, "make", target, check=False, env=env | supplied) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse((self.root / ".workspace").exists()) def test_new_pin_gate_validates_fixed_sources_and_rejects_local_replaces(self): self.call(self.root, "python3", "scripts/workspace.py", "--root", str(self.root), "prepare") diff --git a/tests/test_dependency_ci.py b/tests/test_dependency_ci.py index 52b69995..60313452 100644 --- a/tests/test_dependency_ci.py +++ b/tests/test_dependency_ci.py @@ -90,6 +90,13 @@ class ComponentWorkflowFixtureTests(unittest.TestCase): class DependencyCITests(unittest.TestCase): + def test_boundary_gate_refuses_missing_ripgrep_before_resolving_sources(self): + with tempfile.TemporaryDirectory(prefix="boundary-no-tools-") as directory: + result = subprocess.run([shutil.which("bash"), str(WORKFLOW.parents[2] / "scripts/lint-boundaries.sh")], + cwd=directory, env=os.environ | {"PATH": directory}, capture_output=True, text=True) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("requires ripgrep", result.stderr) + def test_privileged_acquisition_finishes_before_candidate_execution_in_each_job(self): content = WORKFLOW.read_text() jobs = re.split(r"^ ([A-Za-z0-9_-]+):\n", content, flags=re.M)