weichen
0d9f44a7e2
Why: A dirty composer could change artifacts under a committed Dev identity, and two CI jobs reacquired credentials after executing candidate build code. Evidence: SELF regressions reproduce both issues before the fixes; all22 real composition cases and21 dependency/CI cases pass after. CI syntax checked; full exact committed replay follows this checkpoint. Impact: Verify executing composer/transport bytes before import and publication, record explicit tooling provenance, order all privileged acquisition first, and model CGO correctly in the nested component race fixture. Existing product and required CI gates remain. Refs: Dependency-workspace early code review; correctness/standards/security P1 findings and R7-R8/R14.
561 lines
32 KiB
Python
561 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""Build immutable application commits without developer workspace overlays.
|
|
|
|
Go versions remain authoritative in go.mod; npm versions in package-lock.json.
|
|
The output manifest records the selected graph rather than creating another lock.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
|
|
SHA = re.compile(r"[0-9a-f]{40}\Z")
|
|
IDS = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z")
|
|
BACKEND = "Agent-System-Backend"
|
|
FRONTEND = "Agent-System-Frontend"
|
|
|
|
|
|
class CompositionError(Exception):
|
|
pass
|
|
|
|
|
|
def run(command, cwd, env, *, binary=False, quiet=False):
|
|
if not quiet:
|
|
print("[composition] " + " ".join(map(str, command)), flush=True)
|
|
result = subprocess.run(command, cwd=cwd, env=env, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=not binary)
|
|
if not quiet and not binary:
|
|
sys.stdout.write(result.stdout)
|
|
sys.stderr.write(result.stderr)
|
|
if result.returncode:
|
|
detail = result.stderr.decode(errors="replace") if binary else result.stderr
|
|
raise CompositionError(f"command failed ({result.returncode}): {command[0]} {detail.strip()}")
|
|
return result.stdout
|
|
|
|
|
|
def digest(path):
|
|
# The composer also supports standalone fixed-root wrapper fixtures.
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def tree_digest(root):
|
|
if root.is_symlink():
|
|
raise CompositionError(f"artifact root symlink is forbidden: {root.name}")
|
|
entries = []
|
|
for path in sorted(root.rglob("*")):
|
|
if path.is_symlink():
|
|
raise CompositionError(f"artifact symlink is forbidden: {path.name}")
|
|
if path.is_file():
|
|
entries.append({"path": path.relative_to(root).as_posix(), "sha256": digest(path), "size": path.stat().st_size})
|
|
encoded = json.dumps(entries, sort_keys=True, separators=(",", ":")).encode()
|
|
return {"sha256": hashlib.sha256(encoded).hexdigest(), "files": entries}
|
|
|
|
|
|
def safe_relative(root, value):
|
|
if not isinstance(value, str) or not value or "\\" in value:
|
|
raise CompositionError("invalid component path")
|
|
relative = PurePosixPath(value)
|
|
if relative.is_absolute() or ".." in relative.parts or (root / value).resolve().is_relative_to(root) is False:
|
|
raise CompositionError("component path escapes root")
|
|
|
|
|
|
def validate_repo(repo):
|
|
if not isinstance(repo, str) or not repo or repo.startswith("-") or any(c.isspace() for c in repo):
|
|
raise CompositionError("invalid repository source")
|
|
if "://" not in repo:
|
|
if not Path(repo).is_absolute():
|
|
raise CompositionError("repository must be an approved URL or absolute local path")
|
|
return
|
|
parsed = urlsplit(repo)
|
|
if parsed.scheme not in {"ssh", "https", "file"} or parsed.password or parsed.query or parsed.fragment:
|
|
raise CompositionError("repository URL contains credentials or unsupported transport")
|
|
if parsed.scheme == "https" and parsed.username:
|
|
raise CompositionError("repository credentials are forbidden")
|
|
if parsed.scheme == "file" and parsed.netloc not in {"", "localhost"}:
|
|
raise CompositionError("file repository must be local")
|
|
|
|
|
|
def tooling_identity(root):
|
|
"""Bind executed tools before importing acquisition configuration."""
|
|
env = os.environ.copy()
|
|
for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"):
|
|
env.pop(key, None)
|
|
commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip()
|
|
result = {"root_commit": commit, "files": {}}
|
|
for relative in ("scripts/build-composition.py", "scripts/workspace/transport.py"):
|
|
executing = Path(__file__).resolve().parent / relative.removeprefix("scripts/")
|
|
fixed = subprocess.run(["git", "show", f"{commit}:{relative}"], cwd=root, env=env,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if fixed.returncode == 0:
|
|
if executing.is_symlink() or executing.read_bytes() != fixed.stdout:
|
|
raise CompositionError(f"executing tooling differs from committed source: {relative}")
|
|
mode = "committed-root"
|
|
elif (root / relative).exists():
|
|
raise CompositionError(f"root tooling has no committed identity: {relative}")
|
|
else:
|
|
# A standalone consumer fixture can use an external composer. Its
|
|
# exact tool bytes are a separate identity, never the consumer SHA.
|
|
mode = "external-tool"
|
|
result["files"][relative] = {"sha256": digest(executing), "source": mode}
|
|
return result
|
|
|
|
|
|
def inputs(root, env):
|
|
commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip()
|
|
if not SHA.fullmatch(commit):
|
|
raise CompositionError("root must have a committed Git revision")
|
|
documents = {}
|
|
hashes = {}
|
|
for name in ("components.json", "components.lock.json"):
|
|
committed = run(["git", "show", f"{commit}:{name}"], root, env, binary=True, quiet=True)
|
|
path = root / name
|
|
if path.is_symlink() or path.read_bytes() != committed:
|
|
raise CompositionError(f"{name} differs from committed root {commit}")
|
|
documents[name] = json.loads(committed)
|
|
hashes[name] = hashlib.sha256(committed).hexdigest()
|
|
manifest, lock = documents["components.json"], documents["components.lock.json"]
|
|
if manifest.get("schema") != 1 or not isinstance(manifest.get("components"), list):
|
|
raise CompositionError("unsupported components schema")
|
|
if lock.get("schema") != 1 or not isinstance(lock.get("components"), dict):
|
|
raise CompositionError("unsupported component lock schema")
|
|
components = {}
|
|
for item in manifest["components"]:
|
|
name = item.get("id")
|
|
if not isinstance(name, str) or not IDS.fullmatch(name) or name in components:
|
|
raise CompositionError("invalid or duplicate component id")
|
|
if item.get("path") != f".workspace/{name}":
|
|
raise CompositionError(f"invalid component path: {name}")
|
|
safe_relative(root, item["path"])
|
|
if item.get("legacy_path"):
|
|
safe_relative(root, item["legacy_path"])
|
|
validate_repo(item.get("repo"))
|
|
for module in item.get("modules", []):
|
|
safe_relative(root, module.get("directory", "."))
|
|
components[name] = item
|
|
for name, value in lock["components"].items():
|
|
if name not in components or not isinstance(value, dict) or not SHA.fullmatch(value.get("commit", "")):
|
|
raise CompositionError(f"invalid locked component: {name}")
|
|
if components[name].get("role") in {"go-library", "npm-library"}:
|
|
raise CompositionError(f"language library cannot have a competing source lock: {name}")
|
|
for name in (BACKEND, FRONTEND):
|
|
if name not in components or components[name].get("role") != "application" or name not in lock["components"]:
|
|
raise CompositionError(f"missing fixed application source: {name}")
|
|
return commit, components, lock["components"], hashes
|
|
|
|
|
|
def execution_environment(env):
|
|
"""Acquisition-only authentication/transport never reaches npm/Go candidate code."""
|
|
clean = env.copy()
|
|
for key in list(clean):
|
|
upper = key.upper()
|
|
if (any(word in upper for word in ("TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "API_KEY", "ACCESS_KEY"))
|
|
or upper.startswith(("GIT_CONFIG_", "GIT_SSH", "NPM_CONFIG_", "BASH_FUNC_"))
|
|
or upper in {"SSH_AUTH_SOCK", "SSH_AGENT_PID", "NODE_OPTIONS", "BASH_ENV", "ENV", "GOFLAGS", "GIT_ASKPASS", "SSH_ASKPASS", "CONTRACTS_TARBALL", "UI_TARBALL"}):
|
|
del clean[key]
|
|
clean.update(GOWORK="off", GOTOOLCHAIN="local", GOFLAGS="", CGO_ENABLED="0")
|
|
return clean
|
|
|
|
|
|
def export_source(repo, commit, destination, temporary, env, *, abbreviated=False):
|
|
validate_repo(repo)
|
|
clone = temporary / (destination.name + "-git")
|
|
run(["git", "clone", "--no-checkout", "--no-hardlinks", "--", repo, str(clone)], temporary, env)
|
|
exists = subprocess.run(["git", "cat-file", "-e", commit + "^{commit}"], cwd=clone, env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
|
|
if not exists:
|
|
run(["git", "fetch", "--no-tags", "origin", commit], clone, env)
|
|
actual = run(["git", "rev-parse", commit + "^{commit}"], clone, env, quiet=True).strip()
|
|
if not SHA.fullmatch(actual) or (actual != commit and not (abbreviated and len(commit) == 12 and actual.startswith(commit))):
|
|
raise CompositionError("application revision is not an immutable full commit")
|
|
archive = run(["git", "archive", "--format=tar", actual], clone, env, binary=True, quiet=True)
|
|
extract_archive(archive, destination)
|
|
return {"repo": repo, "commit": actual, "archive_sha256": hashlib.sha256(archive).hexdigest()}
|
|
|
|
|
|
def extract_archive(archive, destination):
|
|
destination.mkdir()
|
|
with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
|
|
for entry in tar.getmembers():
|
|
safe_relative(destination, entry.name)
|
|
if not (entry.isfile() or entry.isdir()):
|
|
raise CompositionError(f"source archive contains an unsupported link: {entry.name}")
|
|
tar.extractall(destination, filter="data")
|
|
|
|
|
|
def version(manifest, lock, *, backend_commit=None, resolved_go=None):
|
|
resolved = json.dumps({"go": manifest["go"] if resolved_go is None else resolved_go,
|
|
"npm_lock": manifest["npm"]["lock_sha256"]},
|
|
sort_keys=True, separators=(",", ":")).encode()
|
|
return (manifest["root"]["commit"][:12] + "+backend=" + (backend_commit or lock[BACKEND]["commit"])[:12]
|
|
+ "+frontend=" + lock[FRONTEND]["commit"][:12]
|
|
+ "+deps=" + hashlib.sha256(resolved).hexdigest()[:12])
|
|
|
|
|
|
def json_stream(output):
|
|
decoder = json.JSONDecoder()
|
|
result = []
|
|
while output.strip():
|
|
value, end = decoder.raw_decode(output.lstrip())
|
|
result.append(value)
|
|
output = output.lstrip()[end:]
|
|
return result
|
|
|
|
|
|
def go_dependencies(source, fetch_env):
|
|
document = json.loads(run(["go", "mod", "edit", "-json"], source, fetch_env, quiet=True))
|
|
for replacement in document.get("Replace") or []:
|
|
if not replacement["New"].get("Version"):
|
|
raise CompositionError("formal Go build forbids local directory replace")
|
|
original = {name: (source / name).read_bytes() if (source / name).exists() else None for name in ("go.mod", "go.sum")}
|
|
# Tidy locks need sums for consumed packages, not unused transitive zip files.
|
|
# Load actual build/test metadata before ending acquisition; no tests execute.
|
|
# Download may normalize sum lines even for a valid readonly consumer. Keep
|
|
# that package-manager bookkeeping in an identical disposable manifest.
|
|
with tempfile.TemporaryDirectory(prefix="composition-go-prefetch-") as directory:
|
|
isolated = Path(directory)
|
|
for name, content in original.items():
|
|
if content is not None:
|
|
(isolated / name).write_bytes(content)
|
|
downloads = json_stream(run(["go", "mod", "download", "-json"], isolated, fetch_env, quiet=True))
|
|
run(["go", "list", "-mod=readonly", "-deps", "-test", "./..."], source,
|
|
fetch_env | {"CGO_ENABLED": "0"}, quiet=True)
|
|
run(["go", "mod", "verify"], source, fetch_env)
|
|
modules = json_stream(run(["go", "list", "-mod=readonly", "-m", "-json", "all"], source, fetch_env, quiet=True))
|
|
selected = []
|
|
for module in modules:
|
|
item = {key: module[key] for key in ("Path", "Version", "Main", "Sum", "GoModSum", "GoVersion") if key in module}
|
|
if "Replace" in module:
|
|
replacement = module["Replace"]
|
|
if not replacement.get("Version"):
|
|
raise CompositionError("resolved Go graph contains a local replace")
|
|
item["Replace"] = {key: replacement[key] for key in ("Path", "Version", "Sum", "GoModSum") if key in replacement}
|
|
selected.append(item)
|
|
for name, content in original.items():
|
|
actual = (source / name).read_bytes() if (source / name).exists() else None
|
|
if actual != content:
|
|
raise CompositionError(f"Go acquisition changed committed {name}; update the consumer first")
|
|
return {"modules": selected, "go_mod_sha256": digest(source / "go.mod"),
|
|
"go_sum_sha256": digest(source / "go.sum") if (source / "go.sum").exists() else None,
|
|
"downloads": [{key: value[key] for key in ("Path", "Version", "Sum", "GoModSum", "Origin") if key in value} for value in downloads]}
|
|
|
|
|
|
def npm_dependencies(source, components, fetch_env):
|
|
path = source / "package-lock.json"
|
|
lock = json.loads(path.read_text())
|
|
if lock.get("lockfileVersion", 0) < 2 or not isinstance(lock.get("packages"), dict):
|
|
raise CompositionError("frontend requires a complete npm lock")
|
|
approved = {component["repo"] for component in components.values()}
|
|
packages = {}
|
|
for name, package in lock["packages"].items():
|
|
if package.get("link"):
|
|
raise CompositionError("formal npm build forbids workspace links")
|
|
for value in [package.get("resolved", ""), *package.get("dependencies", {}).values()]:
|
|
if not isinstance(value, str):
|
|
raise CompositionError("invalid npm lock dependency")
|
|
if value.startswith(("file:", "link:", "workspace:")):
|
|
raise CompositionError("formal npm build forbids local dependency overlays")
|
|
if value.startswith(("git+", "git:", "ssh:")):
|
|
repo, sep, revision = value.removeprefix("git+").rpartition("#")
|
|
if not sep or not SHA.fullmatch(revision) or repo not in approved:
|
|
raise CompositionError("npm Git dependency must use an approved source and full commit")
|
|
packages[name] = {key: package[key] for key in ("version", "resolved", "integrity") if key in package}
|
|
before = digest(path)
|
|
run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], source, fetch_env)
|
|
if digest(path) != before:
|
|
raise CompositionError("npm acquisition changed committed lock")
|
|
result = {"lock_sha256": before, "packages": packages}
|
|
contract = source / "node_modules/@agent-system/contracts"
|
|
if contract.is_dir():
|
|
result["contracts"] = tree_digest(contract)
|
|
return result
|
|
|
|
|
|
def frontend_assets(source, clean_env):
|
|
dist = source / "dist"
|
|
if dist.exists():
|
|
shutil.rmtree(dist)
|
|
run(["npm", "run", "build"], source, clean_env)
|
|
if not (dist / "index.html").is_file() or not (dist / "index.html").stat().st_size:
|
|
raise CompositionError("frontend dist requires nonempty index.html")
|
|
if not (dist / "assets").is_dir() or not any(p.is_file() and p.stat().st_size for p in (dist / "assets").rglob("*")):
|
|
raise CompositionError("frontend dist requires nonempty assets/")
|
|
return dist, tree_digest(dist)
|
|
|
|
|
|
def inject(dist, destination):
|
|
if destination.is_symlink():
|
|
raise CompositionError("embed destination must not be a symlink")
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(dist, destination)
|
|
(destination / ".gitkeep").touch()
|
|
|
|
|
|
def provider_revision(module, downloads):
|
|
metadata = next((entry for entry in downloads if entry["Path"] == module["Path"] and entry["Version"] == module["Version"]), {})
|
|
origin = metadata.get("Origin", {})
|
|
commit = origin.get("Hash", "")
|
|
pseudo = re.search(r"-([0-9a-f]{12})\Z", module["Version"])
|
|
if SHA.fullmatch(commit):
|
|
if pseudo and not commit.startswith(pseudo.group(1)):
|
|
raise CompositionError(f"Go Origin.Hash differs from selected version: {module['Path']}")
|
|
return commit, False
|
|
if pseudo:
|
|
return pseudo.group(1), True
|
|
raise CompositionError(f"cannot prove source commit for selected Go module: {module['Path']}@{module['Version']}")
|
|
|
|
|
|
def provider_tests(root, root_commit, components, sources, manifest, temporary,
|
|
fetch_env, clean_env, rdm=None, source_map_path=None):
|
|
"""Preserve the full test gates while taking provider identities from consumers."""
|
|
graph = manifest["go"]["backend"]
|
|
source_records = {name: {"path": str(path), "commit": manifest["applications"][name]["commit"], "repo": components[name]["repo"]}
|
|
for name, path in sources.items()}
|
|
module_sources = {}
|
|
manifest["test_sources"] = {}
|
|
manifest["checks"] = []
|
|
for name, component in components.items():
|
|
if component.get("role") != "go-library":
|
|
continue
|
|
selected = [(mapping, next((m for m in graph["modules"] if m["Path"] == mapping["path"]), None))
|
|
for mapping in component.get("modules", [])]
|
|
selected = [(mapping, module) for mapping, module in selected if module is not None]
|
|
if not selected:
|
|
continue
|
|
primary = next((pair for pair in selected if pair[0].get("directory", ".") == "."), selected[0])
|
|
revision, abbreviated = provider_revision(primary[1], graph["downloads"])
|
|
source = temporary / name
|
|
record = export_source(component["repo"], revision, source, temporary, fetch_env, abbreviated=abbreviated)
|
|
manifest["test_sources"][name] = record
|
|
source_records[name] = {"path": str(source), "commit": record["commit"], "repo": component["repo"]}
|
|
sources[name] = source
|
|
for mapping in component.get("modules", []):
|
|
module = next((m for m in graph["modules"] if m["Path"] == mapping["path"]), None)
|
|
module_root = source
|
|
module_commit = record["commit"]
|
|
if module:
|
|
module_revision, module_abbreviated = provider_revision(module, graph["downloads"])
|
|
if not module_commit.startswith(module_revision):
|
|
module_root = temporary / (name + "-" + mapping["directory"].replace("/", "-"))
|
|
alternate = export_source(component["repo"], module_revision, module_root, temporary, fetch_env, abbreviated=module_abbreviated)
|
|
module_commit = alternate["commit"]
|
|
manifest["test_sources"][mapping["path"]] = alternate
|
|
location = module_root / mapping.get("directory", ".")
|
|
if not (location / "go.mod").is_file():
|
|
raise CompositionError(f"declared provider module is missing: {mapping['path']}")
|
|
module_sources[mapping["path"]] = {"path": str(location), "commit": module_commit, "repo": component["repo"]}
|
|
manifest["go"][mapping["path"]] = go_dependencies(location, fetch_env)
|
|
for action in ("test", "vet"):
|
|
command = ["go", action, "-mod=readonly", "./..."]
|
|
run(command, location, clean_env)
|
|
manifest["checks"].append({"component": name, "module": mapping["path"], "command": command, "result": "PASS"})
|
|
# UI is a package provider; its source identity belongs to the npm lock.
|
|
ui_component = components.get("Agent-System-UI")
|
|
if ui_component:
|
|
package_name = ui_component.get("package_name", "@agent-system/ui")
|
|
resolved = manifest["npm"]["packages"].get("node_modules/" + package_name, {}).get("resolved", "")
|
|
repo, _, commit = resolved.removeprefix("git+").rpartition("#")
|
|
if repo != ui_component["repo"] or not SHA.fullmatch(commit):
|
|
raise CompositionError("cannot prove UI source from the Frontend npm lock")
|
|
ui = temporary / "Agent-System-UI"
|
|
record = export_source(repo, commit, ui, temporary, fetch_env)
|
|
sources["Agent-System-UI"] = ui
|
|
manifest["test_sources"]["Agent-System-UI"] = record
|
|
source_records["Agent-System-UI"] = {"path": str(ui), "commit": commit, "repo": repo}
|
|
for name, source in sources.items():
|
|
if name == FRONTEND or not (source / "package.json").is_file():
|
|
continue
|
|
manifest.setdefault("provider_npm", {})[name] = npm_dependencies(source, components, fetch_env)
|
|
scripts = json.loads((source / "package.json").read_text()).get("scripts", {})
|
|
for action in ("typecheck", "test"):
|
|
if action in scripts:
|
|
command = ["npm", "run", action]
|
|
run(command, source, clean_env)
|
|
manifest["checks"].append({"component": name, "command": command, "result": "PASS"})
|
|
run(["go", "vet", "-mod=readonly", "./..."], sources[BACKEND], clean_env)
|
|
manifest["checks"].extend([
|
|
{"component": FRONTEND, "command": ["npm", "test"], "result": "PASS"},
|
|
{"component": BACKEND, "command": ["go", "test", "-mod=readonly", "./..."], "result": "PASS"},
|
|
{"component": BACKEND, "command": ["go", "vet", "-mod=readonly", "./..."], "result": "PASS"},
|
|
])
|
|
if rdm:
|
|
source_records["Agent-System-RDM"] = {"path": str(rdm), **{key: manifest["applications"]["Agent-System-RDM"][key] for key in ("commit", "repo")}}
|
|
run(["go", "vet", "-mod=vendor", "./..."], rdm, clean_env)
|
|
manifest["checks"].extend({"component": "Agent-System-RDM", "command": ["go", action, "-mod=vendor", "./..."], "result": "PASS"} for action in ("test", "vet"))
|
|
formal_root = temporary / "Dev-test-root"
|
|
archive = run(["git", "archive", "--format=tar", root_commit], root, fetch_env, binary=True, quiet=True)
|
|
extract_archive(archive, formal_root)
|
|
source_map = {"schema": 1, "root_commit": root_commit, "components": source_records, "modules": module_sources}
|
|
map_file = source_map_path or temporary / "composition-sources.json"
|
|
map_file.parent.mkdir(parents=True, exist_ok=True)
|
|
map_file.write_text(json.dumps(source_map, indent=2) + "\n")
|
|
gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(map_file)}
|
|
if rdm:
|
|
gate_env["COMPOSITION_RDM_SOURCE"] = str(rdm)
|
|
gates = ((["bash", "scripts/lint-boundaries.sh"], "scripts/lint-boundaries.sh"),
|
|
(["bash", "scripts/check-pins.sh"], "scripts/check-pins.sh"),
|
|
(["make", "test-governance"], "Makefile"))
|
|
project_gates = any((formal_root / required_path).is_file() for _, required_path in gates)
|
|
for command, required_path in gates:
|
|
if project_gates and not (formal_root / required_path).is_file():
|
|
raise CompositionError(f"fixed root is missing required test gate: {required_path}")
|
|
if (formal_root / required_path).is_file():
|
|
run(command, formal_root, gate_env)
|
|
manifest["checks"].append({"component": "Agent-System-Dev", "commit": root_commit, "command": command, "result": "PASS"})
|
|
|
|
|
|
def compose(root, target, output, rdm_source=None, source_map_path=None):
|
|
tooling = tooling_identity(root)
|
|
transport_path = Path(__file__).resolve().parent / "workspace/transport.py"
|
|
spec = importlib.util.spec_from_file_location("composition_transport", transport_path)
|
|
transport = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(transport)
|
|
fetch_env = transport.environment(root)
|
|
fetch_env.update(GOWORK="off", GOFLAGS="", GOTOOLCHAIN="local")
|
|
root_commit, components, lock, input_hashes = inputs(root, fetch_env)
|
|
if root_commit != tooling["root_commit"]:
|
|
raise CompositionError("root identity changed during tooling validation")
|
|
clean_env = execution_environment(fetch_env)
|
|
manifest = {"schema": 1, "kind": "agent-system-composition/1", "target": target,
|
|
"root": {"commit": root_commit, "inputs": input_hashes}, "tooling": tooling,
|
|
"applications": {}, "go": {}, "artifacts": {}, "assets": {}}
|
|
with tempfile.TemporaryDirectory(prefix="agent-system-composition-") as directory:
|
|
temporary = Path(directory)
|
|
sources = {}
|
|
for name in (FRONTEND, BACKEND):
|
|
if target == "frontend" and name == BACKEND:
|
|
continue
|
|
source = temporary / name
|
|
manifest["applications"][name] = export_source(components[name]["repo"], lock[name]["commit"], source, temporary, fetch_env)
|
|
sources[name] = source
|
|
frontend = sources[FRONTEND]
|
|
manifest["npm"] = npm_dependencies(frontend, components, fetch_env)
|
|
dist, manifest["frontend"] = frontend_assets(frontend, clean_env)
|
|
manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(),
|
|
"node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(),
|
|
"npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()}
|
|
products = temporary / "products"
|
|
products.mkdir()
|
|
rdm = None
|
|
manifest["version"] = version(manifest, lock)
|
|
if target != "frontend":
|
|
backend = sources[BACKEND]
|
|
manifest["go"]["backend"] = go_dependencies(backend, fetch_env)
|
|
manifest["version"] = version(manifest, lock)
|
|
inject(dist, backend / "web/dist")
|
|
manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"]
|
|
if target == "test":
|
|
run(["npm", "test"], frontend, clean_env)
|
|
run(["go", "test", "-mod=readonly", "./..."], backend, clean_env)
|
|
else:
|
|
for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")):
|
|
flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "")
|
|
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env)
|
|
if rdm_source:
|
|
rdm_commit = run(["git", "rev-parse", "HEAD"], rdm_source, fetch_env, quiet=True).strip()
|
|
rdm = temporary / "Agent-System-RDM"
|
|
manifest["applications"]["Agent-System-RDM"] = export_source(str(rdm_source), rdm_commit, rdm, temporary, fetch_env)
|
|
graph = go_dependencies(rdm, fetch_env)
|
|
manifest["go"]["rdm"] = graph
|
|
backend_module = next((m["path"] for m in components[BACKEND]["modules"] if m.get("directory", ".") == "."), None)
|
|
selected = next((m for m in graph["modules"] if m["Path"] == backend_module), None)
|
|
if not selected:
|
|
raise CompositionError("RDM must consume the fixed Backend as an ordinary versioned Go module")
|
|
effective = selected.get("Replace", selected)
|
|
if effective.get("Path") != backend_module or not effective.get("Version"):
|
|
raise CompositionError("RDM Backend must resolve to the approved versioned module")
|
|
# RDM's module graph is its dependency authority. The Dev app
|
|
# source lock fixes a separate application and may differ.
|
|
revision, abbreviated = provider_revision(effective, graph["downloads"])
|
|
admin_source = temporary / "RDM-Backend"
|
|
backend_record = export_source(components[BACKEND]["repo"], revision, admin_source,
|
|
temporary, fetch_env, abbreviated=abbreviated)
|
|
graph["backend_source"] = {**backend_record, "module": backend_module, "version": effective["Version"]}
|
|
manifest["go"]["rdm_admin"] = go_dependencies(admin_source, fetch_env)
|
|
inject(dist, admin_source / "web/dist")
|
|
manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"]
|
|
graph["admin_version"] = version(manifest, lock, backend_commit=backend_record["commit"],
|
|
resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]})
|
|
run(["go", "mod", "vendor"], rdm, fetch_env)
|
|
vendored_backend = rdm / "vendor" / backend_module
|
|
if not (vendored_backend / "web/embed.go").is_file():
|
|
raise CompositionError("RDM vendored Backend does not contain the embed package")
|
|
graph["vendor_modules_sha256"] = digest(rdm / "vendor/modules.txt")
|
|
graph["vendor_backend_source_sha256"] = tree_digest(vendored_backend)["sha256"]
|
|
inject(dist, vendored_backend / "web/dist")
|
|
graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"]
|
|
if target == "test":
|
|
run(["go", "test", "-mod=vendor", "./..."], rdm, clean_env)
|
|
run(["go", "test", "-mod=readonly", "./..."], admin_source, clean_env)
|
|
run(["go", "vet", "-mod=readonly", "./..."], admin_source, clean_env)
|
|
else:
|
|
run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env)
|
|
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false",
|
|
"-ldflags=-s -w -X main.version=" + graph["admin_version"],
|
|
"-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env)
|
|
if target == "test":
|
|
provider_tests(root, root_commit, components, sources, manifest, temporary,
|
|
fetch_env, clean_env, rdm, source_map_path)
|
|
if target == "build" and run(["git", "ls-tree", root_commit, "--", "plugins"], root, fetch_env, quiet=True).strip():
|
|
archive = run(["git", "archive", "--format=tar", root_commit, "plugins"], root, fetch_env, binary=True, quiet=True)
|
|
root_assets = temporary / "root-assets"
|
|
extract_archive(archive, root_assets)
|
|
manifest["assets"]["plugins"] = tree_digest(root_assets / "plugins")
|
|
manifest["assets"]["plugins"]["root_commit"] = root_commit
|
|
for artifact in products.iterdir():
|
|
manifest["artifacts"][artifact.name] = {"sha256": digest(artifact), "size": artifact.stat().st_size}
|
|
if tooling_identity(root) != tooling:
|
|
raise CompositionError("executing tooling changed during composition")
|
|
# Publish only after all required build/test commands succeeded.
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
for artifact in products.iterdir():
|
|
shutil.copy2(artifact, output / artifact.name)
|
|
if "plugins" in manifest["assets"]:
|
|
destination = output / "plugins"
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(root_assets / "plugins", destination)
|
|
if target == "frontend":
|
|
destination = output / "frontend-dist"
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(dist, destination)
|
|
pending = output / ".resolved-manifest.json.tmp"
|
|
pending.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
|
|
pending.replace(output / "resolved-manifest.json")
|
|
print(f"[composition] complete: {target}; manifest={output / 'resolved-manifest.json'}", flush=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--target", required=True, choices=("frontend", "build", "test"))
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--rdm-source", type=Path)
|
|
parser.add_argument("--source-map", type=Path, help="Write source identity map used by test gates; paths exist only during this run")
|
|
args = parser.parse_args()
|
|
root = args.root.resolve()
|
|
try:
|
|
compose(root, args.target, (args.output or root / "bin").resolve(), args.rdm_source.resolve() if args.rdm_source else None,
|
|
args.source_map.resolve() if args.source_map else None)
|
|
except (CompositionError, OSError, ValueError, KeyError, TypeError) as error:
|
|
print(f"composition: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|