Finish composition acquisition before running candidate code (U5)
Why: Later authenticated source acquisition must never follow a frontend build or provider test. Evidence: SELF host replay passed all 23 real composition fixtures; both acquisition-order regressions failed before the repair. Impact: Prepare fixed sources and Go/npm caches before execution; retain committed tooling checks and RDM-selected Backend versions. Refs: R7, R8, R11, R14; dependency-workspace U5
This commit is contained in:
@@ -87,9 +87,9 @@ def validate_repo(repo):
|
||||
raise CompositionError("file repository must be local")
|
||||
|
||||
|
||||
def tooling_identity(root):
|
||||
def tooling_identity(root, environment=None):
|
||||
"""Bind executed tools before importing acquisition configuration."""
|
||||
env = os.environ.copy()
|
||||
env = (os.environ if environment is None else environment).copy()
|
||||
for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"):
|
||||
env.pop(key, None)
|
||||
commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip()
|
||||
@@ -317,15 +317,17 @@ def provider_revision(module, downloads):
|
||||
raise CompositionError(f"cannot prove source commit for selected Go module: {module['Path']}@{module['Version']}")
|
||||
|
||||
|
||||
def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
fetch_env, clean_env, rdm=None, source_map_path=None):
|
||||
"""Preserve the full test gates while taking provider identities from consumers."""
|
||||
def prepare_provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
fetch_env, rdm=None, rdm_admin=None, source_map_path=None):
|
||||
"""Acquire every provider and cache before constructing the execution plan."""
|
||||
graph = manifest["go"]["backend"]
|
||||
source_records = {name: {"path": str(path), "commit": manifest["applications"][name]["commit"], "repo": components[name]["repo"]}
|
||||
for name, path in sources.items()}
|
||||
module_sources = {}
|
||||
manifest["test_sources"] = {}
|
||||
manifest["checks"] = []
|
||||
go_checks = [{"component": BACKEND, "path": sources[BACKEND], "mode": "readonly"}]
|
||||
npm_checks = [{"component": FRONTEND, "path": sources[FRONTEND], "commands": [["npm", "test"]]}]
|
||||
for name, component in components.items():
|
||||
if component.get("role") != "go-library":
|
||||
continue
|
||||
@@ -357,10 +359,7 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
raise CompositionError(f"declared provider module is missing: {mapping['path']}")
|
||||
module_sources[mapping["path"]] = {"path": str(location), "commit": module_commit, "repo": component["repo"]}
|
||||
manifest["go"][mapping["path"]] = go_dependencies(location, fetch_env)
|
||||
for action in ("test", "vet"):
|
||||
command = ["go", action, "-mod=readonly", "./..."]
|
||||
run(command, location, clean_env)
|
||||
manifest["checks"].append({"component": name, "module": mapping["path"], "command": command, "result": "PASS"})
|
||||
go_checks.append({"component": name, "module": mapping["path"], "path": location, "mode": "readonly"})
|
||||
# UI is a package provider; its source identity belongs to the npm lock.
|
||||
ui_component = components.get("Agent-System-UI")
|
||||
if ui_component:
|
||||
@@ -379,21 +378,14 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
continue
|
||||
manifest.setdefault("provider_npm", {})[name] = npm_dependencies(source, components, fetch_env)
|
||||
scripts = json.loads((source / "package.json").read_text()).get("scripts", {})
|
||||
for action in ("typecheck", "test"):
|
||||
if action in scripts:
|
||||
command = ["npm", "run", action]
|
||||
run(command, source, clean_env)
|
||||
manifest["checks"].append({"component": name, "command": command, "result": "PASS"})
|
||||
run(["go", "vet", "-mod=readonly", "./..."], sources[BACKEND], clean_env)
|
||||
manifest["checks"].extend([
|
||||
{"component": FRONTEND, "command": ["npm", "test"], "result": "PASS"},
|
||||
{"component": BACKEND, "command": ["go", "test", "-mod=readonly", "./..."], "result": "PASS"},
|
||||
{"component": BACKEND, "command": ["go", "vet", "-mod=readonly", "./..."], "result": "PASS"},
|
||||
])
|
||||
commands = [["npm", "run", action] for action in ("typecheck", "test") if action in scripts]
|
||||
npm_checks.append({"component": name, "path": source, "commands": commands})
|
||||
if rdm:
|
||||
source_records["Agent-System-RDM"] = {"path": str(rdm), **{key: manifest["applications"]["Agent-System-RDM"][key] for key in ("commit", "repo")}}
|
||||
run(["go", "vet", "-mod=vendor", "./..."], rdm, clean_env)
|
||||
manifest["checks"].extend({"component": "Agent-System-RDM", "command": ["go", action, "-mod=vendor", "./..."], "result": "PASS"} for action in ("test", "vet"))
|
||||
go_checks.extend([
|
||||
{"component": "Agent-System-RDM", "path": rdm, "mode": "vendor"},
|
||||
{"component": "RDM-Backend", "path": rdm_admin, "mode": "readonly", "commit": manifest["go"]["rdm"]["backend_source"]["commit"]},
|
||||
])
|
||||
formal_root = temporary / "Dev-test-root"
|
||||
archive = run(["git", "archive", "--format=tar", root_commit], root, fetch_env, binary=True, quiet=True)
|
||||
extract_archive(archive, formal_root)
|
||||
@@ -401,9 +393,6 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
map_file = source_map_path or temporary / "composition-sources.json"
|
||||
map_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
map_file.write_text(json.dumps(source_map, indent=2) + "\n")
|
||||
gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(map_file)}
|
||||
if rdm:
|
||||
gate_env["COMPOSITION_RDM_SOURCE"] = str(rdm)
|
||||
gates = ((["bash", "scripts/lint-boundaries.sh"], "scripts/lint-boundaries.sh"),
|
||||
(["bash", "scripts/check-pins.sh"], "scripts/check-pins.sh"),
|
||||
(["make", "test-governance"], "Makefile"))
|
||||
@@ -411,9 +400,33 @@ def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
for command, required_path in gates:
|
||||
if project_gates and not (formal_root / required_path).is_file():
|
||||
raise CompositionError(f"fixed root is missing required test gate: {required_path}")
|
||||
if (formal_root / required_path).is_file():
|
||||
run(command, formal_root, gate_env)
|
||||
manifest["checks"].append({"component": "Agent-System-Dev", "commit": root_commit, "command": command, "result": "PASS"})
|
||||
return {"go": go_checks, "npm": npm_checks, "root": formal_root, "root_commit": root_commit,
|
||||
"map_file": map_file, "rdm": rdm,
|
||||
"gates": [command for command, required_path in gates if (formal_root / required_path).is_file()]}
|
||||
|
||||
|
||||
def run_provider_tests(prepared, manifest, clean_env):
|
||||
"""Execute a completed plan; this phase has no acquisition environment."""
|
||||
for check in prepared["npm"][:1]:
|
||||
for command in check["commands"]:
|
||||
run(command, check["path"], clean_env)
|
||||
manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"})
|
||||
for check in prepared["go"]:
|
||||
for action in ("test", "vet"):
|
||||
command = ["go", action, "-mod=" + check["mode"], "./..."]
|
||||
run(command, check["path"], clean_env)
|
||||
identity = {key: check[key] for key in ("component", "module", "commit") if key in check}
|
||||
manifest["checks"].append({**identity, "command": command, "result": "PASS"})
|
||||
for check in prepared["npm"][1:]:
|
||||
for command in check["commands"]:
|
||||
run(command, check["path"], clean_env)
|
||||
manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"})
|
||||
gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(prepared["map_file"])}
|
||||
if prepared["rdm"]:
|
||||
gate_env["COMPOSITION_RDM_SOURCE"] = str(prepared["rdm"])
|
||||
for command in prepared["gates"]:
|
||||
run(command, prepared["root"], gate_env)
|
||||
manifest["checks"].append({"component": "Agent-System-Dev", "commit": prepared["root_commit"], "command": command, "result": "PASS"})
|
||||
|
||||
|
||||
def compose(root, target, output, rdm_source=None, source_map_path=None):
|
||||
@@ -442,27 +455,17 @@ def compose(root, target, output, rdm_source=None, source_map_path=None):
|
||||
sources[name] = source
|
||||
frontend = sources[FRONTEND]
|
||||
manifest["npm"] = npm_dependencies(frontend, components, fetch_env)
|
||||
dist, manifest["frontend"] = frontend_assets(frontend, clean_env)
|
||||
manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(),
|
||||
"node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(),
|
||||
"npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()}
|
||||
products = temporary / "products"
|
||||
products.mkdir()
|
||||
rdm = None
|
||||
admin_source = None
|
||||
vendored_backend = None
|
||||
test_plan = None
|
||||
manifest["version"] = version(manifest, lock)
|
||||
if target != "frontend":
|
||||
backend = sources[BACKEND]
|
||||
manifest["go"]["backend"] = go_dependencies(backend, fetch_env)
|
||||
manifest["version"] = version(manifest, lock)
|
||||
inject(dist, backend / "web/dist")
|
||||
manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"]
|
||||
if target == "test":
|
||||
run(["npm", "test"], frontend, clean_env)
|
||||
run(["go", "test", "-mod=readonly", "./..."], backend, clean_env)
|
||||
else:
|
||||
for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")):
|
||||
flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "")
|
||||
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env)
|
||||
if rdm_source:
|
||||
rdm_commit = run(["git", "rev-parse", "HEAD"], rdm_source, fetch_env, quiet=True).strip()
|
||||
rdm = temporary / "Agent-System-RDM"
|
||||
@@ -484,39 +487,55 @@ def compose(root, target, output, rdm_source=None, source_map_path=None):
|
||||
temporary, fetch_env, abbreviated=abbreviated)
|
||||
graph["backend_source"] = {**backend_record, "module": backend_module, "version": effective["Version"]}
|
||||
manifest["go"]["rdm_admin"] = go_dependencies(admin_source, fetch_env)
|
||||
inject(dist, admin_source / "web/dist")
|
||||
manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"]
|
||||
graph["admin_version"] = version(manifest, lock, backend_commit=backend_record["commit"],
|
||||
resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]})
|
||||
run(["go", "mod", "vendor"], rdm, fetch_env)
|
||||
vendored_backend = rdm / "vendor" / backend_module
|
||||
if not (vendored_backend / "web/embed.go").is_file():
|
||||
raise CompositionError("RDM vendored Backend does not contain the embed package")
|
||||
graph["vendor_modules_sha256"] = digest(rdm / "vendor/modules.txt")
|
||||
graph["vendor_backend_source_sha256"] = tree_digest(vendored_backend)["sha256"]
|
||||
inject(dist, vendored_backend / "web/dist")
|
||||
graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"]
|
||||
if target == "test":
|
||||
run(["go", "test", "-mod=vendor", "./..."], rdm, clean_env)
|
||||
run(["go", "test", "-mod=readonly", "./..."], admin_source, clean_env)
|
||||
run(["go", "vet", "-mod=readonly", "./..."], admin_source, clean_env)
|
||||
else:
|
||||
run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env)
|
||||
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false",
|
||||
"-ldflags=-s -w -X main.version=" + graph["admin_version"],
|
||||
"-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env)
|
||||
if target == "test":
|
||||
provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
fetch_env, clean_env, rdm, source_map_path)
|
||||
test_plan = prepare_provider_tests(root, root_commit, components, sources, manifest, temporary,
|
||||
fetch_env, rdm, admin_source, source_map_path)
|
||||
if target == "build" and run(["git", "ls-tree", root_commit, "--", "plugins"], root, fetch_env, quiet=True).strip():
|
||||
archive = run(["git", "archive", "--format=tar", root_commit, "plugins"], root, fetch_env, binary=True, quiet=True)
|
||||
root_assets = temporary / "root-assets"
|
||||
extract_archive(archive, root_assets)
|
||||
manifest["assets"]["plugins"] = tree_digest(root_assets / "plugins")
|
||||
manifest["assets"]["plugins"]["root_commit"] = root_commit
|
||||
|
||||
# All fixed sources, package caches, vendor files and root assets are
|
||||
# acquired above. Candidate commands below receive only clean_env;
|
||||
# there is no acquisition environment in the execution plan.
|
||||
del fetch_env
|
||||
dist, manifest["frontend"] = frontend_assets(frontend, clean_env)
|
||||
manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(),
|
||||
"node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(),
|
||||
"npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()}
|
||||
if target != "frontend":
|
||||
inject(dist, backend / "web/dist")
|
||||
manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"]
|
||||
if rdm:
|
||||
graph = manifest["go"]["rdm"]
|
||||
inject(dist, admin_source / "web/dist")
|
||||
manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"]
|
||||
graph["admin_version"] = version(manifest, lock, backend_commit=graph["backend_source"]["commit"],
|
||||
resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]})
|
||||
inject(dist, vendored_backend / "web/dist")
|
||||
graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"]
|
||||
if target == "test":
|
||||
run_provider_tests(test_plan, manifest, clean_env)
|
||||
else:
|
||||
for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")):
|
||||
flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "")
|
||||
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env)
|
||||
if rdm:
|
||||
run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env)
|
||||
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false",
|
||||
"-ldflags=-s -w -X main.version=" + graph["admin_version"],
|
||||
"-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env)
|
||||
for artifact in products.iterdir():
|
||||
manifest["artifacts"][artifact.name] = {"sha256": digest(artifact), "size": artifact.stat().st_size}
|
||||
if tooling_identity(root) != tooling:
|
||||
if tooling_identity(root, clean_env) != tooling:
|
||||
raise CompositionError("executing tooling changed during composition")
|
||||
# Publish only after all required build/test commands succeeded.
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -105,6 +105,47 @@ if (fs.readFileSync('dist/index.html', 'utf8') !== 'immutable-page') process.exi
|
||||
def compose(self, target="build", extra=(), env=None):
|
||||
return self.run_command([sys.executable, "-B", str(SCRIPT), "--root", str(self.root), "--target", target, "--output", str(self.out), *extra], env=env, check=False)
|
||||
|
||||
def acquisition_guard(self, env, *, mark_frontend=True):
|
||||
"""Forward to real tools, refusing acquisition after candidate execution."""
|
||||
marker = self.base / "candidate-executed"
|
||||
tools = self.base / "guarded-tools"
|
||||
tools.mkdir()
|
||||
for name in ("git", "go", "npm"):
|
||||
executable = shutil.which(name)
|
||||
wrapper = tools / name
|
||||
wrapper.write_text("#!" + sys.executable + "\n" + '''import os, pathlib, sys
|
||||
args = sys.argv[1:]
|
||||
name = pathlib.Path(sys.argv[0]).name
|
||||
acquisition = (name == 'git' and args and args[0] in {'clone', 'fetch', 'archive'}
|
||||
or name == 'go' and args and (args[0] == 'mod' or args[0] == 'list')
|
||||
or name == 'npm' and args and args[0] in {'ci', 'install'})
|
||||
if acquisition and pathlib.Path(os.environ['COMPOSITION_EXEC_MARKER']).exists():
|
||||
print('acquisition-after-execution: ' + name + ' ' + ' '.join(args[:2]), file=sys.stderr)
|
||||
sys.exit(86)
|
||||
''' + "os.execv(" + repr(executable) + ", [" + repr(executable) + ", *args])\n")
|
||||
wrapper.chmod(0o755)
|
||||
if mark_frontend:
|
||||
self.write(self.frontend, "build.cjs", (self.frontend / "build.cjs").read_text()
|
||||
+ "\nrequire('node:fs').writeFileSync(process.env.COMPOSITION_EXEC_MARKER, 'frontend-executed')\n")
|
||||
self.lock["components"]["Agent-System-Frontend"]["commit"] = self.commit(self.frontend)
|
||||
self.save_inputs()
|
||||
self.commit(self.root)
|
||||
return env | {"PATH": str(tools) + os.pathsep + env["PATH"],
|
||||
"COMPOSITION_EXEC_MARKER": str(marker), "GITEA_TOKEN": "synthetic-phase-token"}
|
||||
|
||||
def test_all_product_acquisition_precedes_candidate_execution(self):
|
||||
self.test_rdm_vendored_remote_backend_embeds_assets_without_mutating_module_cache()
|
||||
self.write(self.root, "plugins/example/plugin.json", '{"name":"fixed-plugin"}')
|
||||
self.commit(self.root)
|
||||
env = self.acquisition_guard(self.env | {"GOPROXY": (self.base / "proxy").as_uri(),
|
||||
"GOMODCACHE": str(self.base / "module-cache")})
|
||||
result = self.compose(extra=("--rdm-source", str(self.base / "rdm")), env=env)
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertEqual((self.base / "candidate-executed").read_text(), "frontend-executed")
|
||||
self.assertEqual(self.run_command([str(self.out / "agent-system-rdm-product")]).stdout, "immutable-page")
|
||||
self.assertEqual(self.run_command([str(self.out / "agent-system-admin")]).stdout, "immutable-page")
|
||||
self.assertEqual((self.out / "plugins/example/plugin.json").read_text(), '{"name":"fixed-plugin"}')
|
||||
|
||||
def test_fixed_commits_embed_assets_without_touching_developer_checkout(self):
|
||||
# Reading working trees or allowing go.work/npm overlays would produce DIRTY or fail.
|
||||
self.write(self.backend, "web/dist/index.html", "DIRTY")
|
||||
@@ -181,7 +222,10 @@ if (fs.readFileSync('dist/index.html', 'utf8') !== 'immutable-page') process.exi
|
||||
self.write(library, "probe.go", "package probe\nfunc Text() string { return \"\" }\n")
|
||||
self.write(library, "probe_test.go", '''package probe
|
||||
import ("testing"; "os")
|
||||
func TestFixedLibrary(t *testing.T) { if err:=os.WriteFile(os.Getenv("COMPOSITION_GO_MARKER"), []byte("fixed-go-test"), 0600); err!=nil {t.Fatal(err)} }
|
||||
func TestFixedLibrary(t *testing.T) {
|
||||
if err:=os.WriteFile(os.Getenv("COMPOSITION_GO_MARKER"), []byte("fixed-go-test"), 0600); err!=nil {t.Fatal(err)}
|
||||
if path:=os.Getenv("COMPOSITION_EXEC_MARKER"); path!="" { if err:=os.WriteFile(path, []byte("library-executed"), 0600); err!=nil {t.Fatal(err)} }
|
||||
}
|
||||
''')
|
||||
library_sha = self.commit(library)
|
||||
version = "v0.0.0-" + self.run_command(["git", "show", "-s", "--format=%cd", "--date=format-local:%Y%m%d%H%M%S", library_sha], library).stdout.strip() + "-" + library_sha[:12]
|
||||
@@ -221,6 +265,7 @@ func TestFixedLibrary(t *testing.T) { if err:=os.WriteFile(os.Getenv("COMPOSITIO
|
||||
])
|
||||
self.save_inputs()
|
||||
self.commit(self.root)
|
||||
env = self.acquisition_guard(env, mark_frontend=False)
|
||||
result = self.compose("test", env=env)
|
||||
self.assertEqual(result.returncode, 0, result.stdout)
|
||||
self.assertTrue((self.base / "go-tested").is_file(), "resolved Go provider tests were not executed")
|
||||
|
||||
Reference in New Issue
Block a user