diff --git a/scripts/build-composition.py b/scripts/build-composition.py index 7c2bbb2b..118e418d 100644 --- a/scripts/build-composition.py +++ b/scripts/build-composition.py @@ -87,9 +87,9 @@ def validate_repo(repo): raise CompositionError("file repository must be local") -def tooling_identity(root): +def tooling_identity(root, environment=None): """Bind executed tools before importing acquisition configuration.""" - env = os.environ.copy() + env = (os.environ if environment is None else environment).copy() for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"): env.pop(key, None) commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip() @@ -317,15 +317,17 @@ def provider_revision(module, downloads): raise CompositionError(f"cannot prove source commit for selected Go module: {module['Path']}@{module['Version']}") -def provider_tests(root, root_commit, components, sources, manifest, temporary, - fetch_env, clean_env, rdm=None, source_map_path=None): - """Preserve the full test gates while taking provider identities from consumers.""" +def prepare_provider_tests(root, root_commit, components, sources, manifest, temporary, + fetch_env, rdm=None, rdm_admin=None, source_map_path=None): + """Acquire every provider and cache before constructing the execution plan.""" graph = manifest["go"]["backend"] source_records = {name: {"path": str(path), "commit": manifest["applications"][name]["commit"], "repo": components[name]["repo"]} for name, path in sources.items()} module_sources = {} manifest["test_sources"] = {} manifest["checks"] = [] + go_checks = [{"component": BACKEND, "path": sources[BACKEND], "mode": "readonly"}] + npm_checks = [{"component": FRONTEND, "path": sources[FRONTEND], "commands": [["npm", "test"]]}] for name, component in components.items(): if component.get("role") != "go-library": continue @@ -357,10 +359,7 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary, raise CompositionError(f"declared provider module is missing: {mapping['path']}") module_sources[mapping["path"]] = {"path": str(location), "commit": module_commit, "repo": component["repo"]} manifest["go"][mapping["path"]] = go_dependencies(location, fetch_env) - for action in ("test", "vet"): - command = ["go", action, "-mod=readonly", "./..."] - run(command, location, clean_env) - manifest["checks"].append({"component": name, "module": mapping["path"], "command": command, "result": "PASS"}) + go_checks.append({"component": name, "module": mapping["path"], "path": location, "mode": "readonly"}) # UI is a package provider; its source identity belongs to the npm lock. ui_component = components.get("Agent-System-UI") if ui_component: @@ -379,21 +378,14 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary, continue manifest.setdefault("provider_npm", {})[name] = npm_dependencies(source, components, fetch_env) scripts = json.loads((source / "package.json").read_text()).get("scripts", {}) - for action in ("typecheck", "test"): - if action in scripts: - command = ["npm", "run", action] - run(command, source, clean_env) - manifest["checks"].append({"component": name, "command": command, "result": "PASS"}) - run(["go", "vet", "-mod=readonly", "./..."], sources[BACKEND], clean_env) - manifest["checks"].extend([ - {"component": FRONTEND, "command": ["npm", "test"], "result": "PASS"}, - {"component": BACKEND, "command": ["go", "test", "-mod=readonly", "./..."], "result": "PASS"}, - {"component": BACKEND, "command": ["go", "vet", "-mod=readonly", "./..."], "result": "PASS"}, - ]) + commands = [["npm", "run", action] for action in ("typecheck", "test") if action in scripts] + npm_checks.append({"component": name, "path": source, "commands": commands}) if rdm: source_records["Agent-System-RDM"] = {"path": str(rdm), **{key: manifest["applications"]["Agent-System-RDM"][key] for key in ("commit", "repo")}} - run(["go", "vet", "-mod=vendor", "./..."], rdm, clean_env) - manifest["checks"].extend({"component": "Agent-System-RDM", "command": ["go", action, "-mod=vendor", "./..."], "result": "PASS"} for action in ("test", "vet")) + go_checks.extend([ + {"component": "Agent-System-RDM", "path": rdm, "mode": "vendor"}, + {"component": "RDM-Backend", "path": rdm_admin, "mode": "readonly", "commit": manifest["go"]["rdm"]["backend_source"]["commit"]}, + ]) formal_root = temporary / "Dev-test-root" archive = run(["git", "archive", "--format=tar", root_commit], root, fetch_env, binary=True, quiet=True) extract_archive(archive, formal_root) @@ -401,9 +393,6 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary, map_file = source_map_path or temporary / "composition-sources.json" map_file.parent.mkdir(parents=True, exist_ok=True) map_file.write_text(json.dumps(source_map, indent=2) + "\n") - gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(map_file)} - if rdm: - gate_env["COMPOSITION_RDM_SOURCE"] = str(rdm) gates = ((["bash", "scripts/lint-boundaries.sh"], "scripts/lint-boundaries.sh"), (["bash", "scripts/check-pins.sh"], "scripts/check-pins.sh"), (["make", "test-governance"], "Makefile")) @@ -411,9 +400,33 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary, for command, required_path in gates: if project_gates and not (formal_root / required_path).is_file(): raise CompositionError(f"fixed root is missing required test gate: {required_path}") - if (formal_root / required_path).is_file(): - run(command, formal_root, gate_env) - manifest["checks"].append({"component": "Agent-System-Dev", "commit": root_commit, "command": command, "result": "PASS"}) + return {"go": go_checks, "npm": npm_checks, "root": formal_root, "root_commit": root_commit, + "map_file": map_file, "rdm": rdm, + "gates": [command for command, required_path in gates if (formal_root / required_path).is_file()]} + + +def run_provider_tests(prepared, manifest, clean_env): + """Execute a completed plan; this phase has no acquisition environment.""" + for check in prepared["npm"][:1]: + for command in check["commands"]: + run(command, check["path"], clean_env) + manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"}) + for check in prepared["go"]: + for action in ("test", "vet"): + command = ["go", action, "-mod=" + check["mode"], "./..."] + run(command, check["path"], clean_env) + identity = {key: check[key] for key in ("component", "module", "commit") if key in check} + manifest["checks"].append({**identity, "command": command, "result": "PASS"}) + for check in prepared["npm"][1:]: + for command in check["commands"]: + run(command, check["path"], clean_env) + manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"}) + gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(prepared["map_file"])} + if prepared["rdm"]: + gate_env["COMPOSITION_RDM_SOURCE"] = str(prepared["rdm"]) + for command in prepared["gates"]: + run(command, prepared["root"], gate_env) + manifest["checks"].append({"component": "Agent-System-Dev", "commit": prepared["root_commit"], "command": command, "result": "PASS"}) def compose(root, target, output, rdm_source=None, source_map_path=None): @@ -442,27 +455,17 @@ def compose(root, target, output, rdm_source=None, source_map_path=None): sources[name] = source frontend = sources[FRONTEND] manifest["npm"] = npm_dependencies(frontend, components, fetch_env) - dist, manifest["frontend"] = frontend_assets(frontend, clean_env) - manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(), - "node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(), - "npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()} products = temporary / "products" products.mkdir() rdm = None + admin_source = None + vendored_backend = None + test_plan = None manifest["version"] = version(manifest, lock) if target != "frontend": backend = sources[BACKEND] manifest["go"]["backend"] = go_dependencies(backend, fetch_env) manifest["version"] = version(manifest, lock) - inject(dist, backend / "web/dist") - manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"] - if target == "test": - run(["npm", "test"], frontend, clean_env) - run(["go", "test", "-mod=readonly", "./..."], backend, clean_env) - else: - for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")): - flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "") - run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env) if rdm_source: rdm_commit = run(["git", "rev-parse", "HEAD"], rdm_source, fetch_env, quiet=True).strip() rdm = temporary / "Agent-System-RDM" @@ -484,39 +487,55 @@ def compose(root, target, output, rdm_source=None, source_map_path=None): temporary, fetch_env, abbreviated=abbreviated) graph["backend_source"] = {**backend_record, "module": backend_module, "version": effective["Version"]} manifest["go"]["rdm_admin"] = go_dependencies(admin_source, fetch_env) - inject(dist, admin_source / "web/dist") - manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"] - graph["admin_version"] = version(manifest, lock, backend_commit=backend_record["commit"], - resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]}) run(["go", "mod", "vendor"], rdm, fetch_env) vendored_backend = rdm / "vendor" / backend_module if not (vendored_backend / "web/embed.go").is_file(): raise CompositionError("RDM vendored Backend does not contain the embed package") graph["vendor_modules_sha256"] = digest(rdm / "vendor/modules.txt") graph["vendor_backend_source_sha256"] = tree_digest(vendored_backend)["sha256"] - inject(dist, vendored_backend / "web/dist") - graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"] - if target == "test": - run(["go", "test", "-mod=vendor", "./..."], rdm, clean_env) - run(["go", "test", "-mod=readonly", "./..."], admin_source, clean_env) - run(["go", "vet", "-mod=readonly", "./..."], admin_source, clean_env) - else: - run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env) - run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", - "-ldflags=-s -w -X main.version=" + graph["admin_version"], - "-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env) if target == "test": - provider_tests(root, root_commit, components, sources, manifest, temporary, - fetch_env, clean_env, rdm, source_map_path) + test_plan = prepare_provider_tests(root, root_commit, components, sources, manifest, temporary, + fetch_env, rdm, admin_source, source_map_path) if target == "build" and run(["git", "ls-tree", root_commit, "--", "plugins"], root, fetch_env, quiet=True).strip(): archive = run(["git", "archive", "--format=tar", root_commit, "plugins"], root, fetch_env, binary=True, quiet=True) root_assets = temporary / "root-assets" extract_archive(archive, root_assets) manifest["assets"]["plugins"] = tree_digest(root_assets / "plugins") manifest["assets"]["plugins"]["root_commit"] = root_commit + + # All fixed sources, package caches, vendor files and root assets are + # acquired above. Candidate commands below receive only clean_env; + # there is no acquisition environment in the execution plan. + del fetch_env + dist, manifest["frontend"] = frontend_assets(frontend, clean_env) + manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(), + "node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(), + "npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()} + if target != "frontend": + inject(dist, backend / "web/dist") + manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"] + if rdm: + graph = manifest["go"]["rdm"] + inject(dist, admin_source / "web/dist") + manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"] + graph["admin_version"] = version(manifest, lock, backend_commit=graph["backend_source"]["commit"], + resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]}) + inject(dist, vendored_backend / "web/dist") + graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"] + if target == "test": + run_provider_tests(test_plan, manifest, clean_env) + else: + for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")): + flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "") + run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env) + if rdm: + run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env) + run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", + "-ldflags=-s -w -X main.version=" + graph["admin_version"], + "-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env) for artifact in products.iterdir(): manifest["artifacts"][artifact.name] = {"sha256": digest(artifact), "size": artifact.stat().st_size} - if tooling_identity(root) != tooling: + if tooling_identity(root, clean_env) != tooling: raise CompositionError("executing tooling changed during composition") # Publish only after all required build/test commands succeeded. output.mkdir(parents=True, exist_ok=True) diff --git a/tests/test_composition.py b/tests/test_composition.py index 96fdfb62..c34b355b 100644 --- a/tests/test_composition.py +++ b/tests/test_composition.py @@ -105,6 +105,47 @@ if (fs.readFileSync('dist/index.html', 'utf8') !== 'immutable-page') process.exi def compose(self, target="build", extra=(), env=None): return self.run_command([sys.executable, "-B", str(SCRIPT), "--root", str(self.root), "--target", target, "--output", str(self.out), *extra], env=env, check=False) + def acquisition_guard(self, env, *, mark_frontend=True): + """Forward to real tools, refusing acquisition after candidate execution.""" + marker = self.base / "candidate-executed" + tools = self.base / "guarded-tools" + tools.mkdir() + for name in ("git", "go", "npm"): + executable = shutil.which(name) + wrapper = tools / name + wrapper.write_text("#!" + sys.executable + "\n" + '''import os, pathlib, sys +args = sys.argv[1:] +name = pathlib.Path(sys.argv[0]).name +acquisition = (name == 'git' and args and args[0] in {'clone', 'fetch', 'archive'} + or name == 'go' and args and (args[0] == 'mod' or args[0] == 'list') + or name == 'npm' and args and args[0] in {'ci', 'install'}) +if acquisition and pathlib.Path(os.environ['COMPOSITION_EXEC_MARKER']).exists(): + print('acquisition-after-execution: ' + name + ' ' + ' '.join(args[:2]), file=sys.stderr) + sys.exit(86) +''' + "os.execv(" + repr(executable) + ", [" + repr(executable) + ", *args])\n") + wrapper.chmod(0o755) + if mark_frontend: + self.write(self.frontend, "build.cjs", (self.frontend / "build.cjs").read_text() + + "\nrequire('node:fs').writeFileSync(process.env.COMPOSITION_EXEC_MARKER, 'frontend-executed')\n") + self.lock["components"]["Agent-System-Frontend"]["commit"] = self.commit(self.frontend) + self.save_inputs() + self.commit(self.root) + return env | {"PATH": str(tools) + os.pathsep + env["PATH"], + "COMPOSITION_EXEC_MARKER": str(marker), "GITEA_TOKEN": "synthetic-phase-token"} + + def test_all_product_acquisition_precedes_candidate_execution(self): + self.test_rdm_vendored_remote_backend_embeds_assets_without_mutating_module_cache() + self.write(self.root, "plugins/example/plugin.json", '{"name":"fixed-plugin"}') + self.commit(self.root) + env = self.acquisition_guard(self.env | {"GOPROXY": (self.base / "proxy").as_uri(), + "GOMODCACHE": str(self.base / "module-cache")}) + result = self.compose(extra=("--rdm-source", str(self.base / "rdm")), env=env) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertEqual((self.base / "candidate-executed").read_text(), "frontend-executed") + self.assertEqual(self.run_command([str(self.out / "agent-system-rdm-product")]).stdout, "immutable-page") + self.assertEqual(self.run_command([str(self.out / "agent-system-admin")]).stdout, "immutable-page") + self.assertEqual((self.out / "plugins/example/plugin.json").read_text(), '{"name":"fixed-plugin"}') + def test_fixed_commits_embed_assets_without_touching_developer_checkout(self): # Reading working trees or allowing go.work/npm overlays would produce DIRTY or fail. self.write(self.backend, "web/dist/index.html", "DIRTY") @@ -181,7 +222,10 @@ if (fs.readFileSync('dist/index.html', 'utf8') !== 'immutable-page') process.exi self.write(library, "probe.go", "package probe\nfunc Text() string { return \"\" }\n") self.write(library, "probe_test.go", '''package probe import ("testing"; "os") -func TestFixedLibrary(t *testing.T) { if err:=os.WriteFile(os.Getenv("COMPOSITION_GO_MARKER"), []byte("fixed-go-test"), 0600); err!=nil {t.Fatal(err)} } +func TestFixedLibrary(t *testing.T) { + if err:=os.WriteFile(os.Getenv("COMPOSITION_GO_MARKER"), []byte("fixed-go-test"), 0600); err!=nil {t.Fatal(err)} + if path:=os.Getenv("COMPOSITION_EXEC_MARKER"); path!="" { if err:=os.WriteFile(path, []byte("library-executed"), 0600); err!=nil {t.Fatal(err)} } +} ''') library_sha = self.commit(library) version = "v0.0.0-" + self.run_command(["git", "show", "-s", "--format=%cd", "--date=format-local:%Y%m%d%H%M%S", library_sha], library).stdout.strip() + "-" + library_sha[:12] @@ -221,6 +265,7 @@ func TestFixedLibrary(t *testing.T) { if err:=os.WriteFile(os.Getenv("COMPOSITIO ]) self.save_inputs() self.commit(self.root) + env = self.acquisition_guard(env, mark_frontend=False) result = self.compose("test", env=env) self.assertEqual(result.returncode, 0, result.stdout) self.assertTrue((self.base / "go-tested").is_file(), "resolved Go provider tests were not executed")