weichen
01b7799663
Why: An immutable npm lock caches tarballs but packed consumers also require resolver metadata before candidate execution. Evidence: SELF host replay passed 57 composition and CI credential cases; real npm 10 and 11 cold-cache consumers passed without lock changes. Impact: Acquire only installed exact-version npmjs metadata with clean config, preserve cache paths for offline execution, adopt the repaired Frontend source and record immutable CI fixtures. Refs: R6-R8, R14; dependency-workspace U4/U5; UI 63d289e and Frontend 5467517
625 lines
35 KiB
Python
625 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
"""Build immutable application commits without developer workspace overlays.
|
|
|
|
Go versions remain authoritative in go.mod; npm versions in package-lock.json.
|
|
The output manifest records the selected graph rather than creating another lock.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
|
|
SHA = re.compile(r"[0-9a-f]{40}\Z")
|
|
IDS = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z")
|
|
BACKEND = "Agent-System-Backend"
|
|
FRONTEND = "Agent-System-Frontend"
|
|
NPM_NAME = re.compile(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*\Z")
|
|
NPM_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\Z")
|
|
|
|
|
|
class CompositionError(Exception):
|
|
pass
|
|
|
|
|
|
def run(command, cwd, env, *, binary=False, quiet=False):
|
|
if not quiet:
|
|
print("[composition] " + " ".join(map(str, command)), flush=True)
|
|
result = subprocess.run(command, cwd=cwd, env=env, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, text=not binary)
|
|
if not quiet and not binary:
|
|
sys.stdout.write(result.stdout)
|
|
sys.stderr.write(result.stderr)
|
|
if result.returncode:
|
|
detail = result.stderr.decode(errors="replace") if binary else result.stderr
|
|
raise CompositionError(f"command failed ({result.returncode}): {command[0]} {detail.strip()}")
|
|
return result.stdout
|
|
|
|
|
|
def digest(path):
|
|
# The composer also supports standalone fixed-root wrapper fixtures.
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def tree_digest(root):
|
|
if root.is_symlink():
|
|
raise CompositionError(f"artifact root symlink is forbidden: {root.name}")
|
|
entries = []
|
|
for path in sorted(root.rglob("*")):
|
|
if path.is_symlink():
|
|
raise CompositionError(f"artifact symlink is forbidden: {path.name}")
|
|
if path.is_file():
|
|
entries.append({"path": path.relative_to(root).as_posix(), "sha256": digest(path), "size": path.stat().st_size})
|
|
encoded = json.dumps(entries, sort_keys=True, separators=(",", ":")).encode()
|
|
return {"sha256": hashlib.sha256(encoded).hexdigest(), "files": entries}
|
|
|
|
|
|
def safe_relative(root, value):
|
|
if not isinstance(value, str) or not value or "\\" in value:
|
|
raise CompositionError("invalid component path")
|
|
relative = PurePosixPath(value)
|
|
if relative.is_absolute() or ".." in relative.parts or (root / value).resolve().is_relative_to(root) is False:
|
|
raise CompositionError("component path escapes root")
|
|
|
|
|
|
def validate_repo(repo):
|
|
if not isinstance(repo, str) or not repo or repo.startswith("-") or any(c.isspace() for c in repo):
|
|
raise CompositionError("invalid repository source")
|
|
if "://" not in repo:
|
|
if not Path(repo).is_absolute():
|
|
raise CompositionError("repository must be an approved URL or absolute local path")
|
|
return
|
|
parsed = urlsplit(repo)
|
|
if parsed.scheme not in {"ssh", "https", "file"} or parsed.password or parsed.query or parsed.fragment:
|
|
raise CompositionError("repository URL contains credentials or unsupported transport")
|
|
if parsed.scheme == "https" and parsed.username:
|
|
raise CompositionError("repository credentials are forbidden")
|
|
if parsed.scheme == "file" and parsed.netloc not in {"", "localhost"}:
|
|
raise CompositionError("file repository must be local")
|
|
|
|
|
|
def tooling_identity(root, environment=None):
|
|
"""Bind executed tools before importing acquisition configuration."""
|
|
env = (os.environ if environment is None else environment).copy()
|
|
for key in ("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR"):
|
|
env.pop(key, None)
|
|
commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip()
|
|
result = {"root_commit": commit, "files": {}}
|
|
for relative in ("scripts/build-composition.py", "scripts/workspace/transport.py"):
|
|
executing = Path(__file__).resolve().parent / relative.removeprefix("scripts/")
|
|
fixed = subprocess.run(["git", "show", f"{commit}:{relative}"], cwd=root, env=env,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if fixed.returncode == 0:
|
|
if executing.is_symlink() or executing.read_bytes() != fixed.stdout:
|
|
raise CompositionError(f"executing tooling differs from committed source: {relative}")
|
|
mode = "committed-root"
|
|
elif (root / relative).exists():
|
|
raise CompositionError(f"root tooling has no committed identity: {relative}")
|
|
else:
|
|
# A standalone consumer fixture can use an external composer. Its
|
|
# exact tool bytes are a separate identity, never the consumer SHA.
|
|
mode = "external-tool"
|
|
result["files"][relative] = {"sha256": digest(executing), "source": mode}
|
|
return result
|
|
|
|
|
|
def inputs(root, env):
|
|
commit = run(["git", "rev-parse", "HEAD"], root, env, quiet=True).strip()
|
|
if not SHA.fullmatch(commit):
|
|
raise CompositionError("root must have a committed Git revision")
|
|
documents = {}
|
|
hashes = {}
|
|
for name in ("components.json", "components.lock.json"):
|
|
committed = run(["git", "show", f"{commit}:{name}"], root, env, binary=True, quiet=True)
|
|
path = root / name
|
|
if path.is_symlink() or path.read_bytes() != committed:
|
|
raise CompositionError(f"{name} differs from committed root {commit}")
|
|
documents[name] = json.loads(committed)
|
|
hashes[name] = hashlib.sha256(committed).hexdigest()
|
|
manifest, lock = documents["components.json"], documents["components.lock.json"]
|
|
if manifest.get("schema") != 1 or not isinstance(manifest.get("components"), list):
|
|
raise CompositionError("unsupported components schema")
|
|
if lock.get("schema") != 1 or not isinstance(lock.get("components"), dict):
|
|
raise CompositionError("unsupported component lock schema")
|
|
components = {}
|
|
for item in manifest["components"]:
|
|
name = item.get("id")
|
|
if not isinstance(name, str) or not IDS.fullmatch(name) or name in components:
|
|
raise CompositionError("invalid or duplicate component id")
|
|
if item.get("path") != f".workspace/{name}":
|
|
raise CompositionError(f"invalid component path: {name}")
|
|
safe_relative(root, item["path"])
|
|
if item.get("legacy_path"):
|
|
safe_relative(root, item["legacy_path"])
|
|
validate_repo(item.get("repo"))
|
|
for module in item.get("modules", []):
|
|
safe_relative(root, module.get("directory", "."))
|
|
components[name] = item
|
|
for name, value in lock["components"].items():
|
|
if name not in components or not isinstance(value, dict) or not SHA.fullmatch(value.get("commit", "")):
|
|
raise CompositionError(f"invalid locked component: {name}")
|
|
if components[name].get("role") in {"go-library", "npm-library"}:
|
|
raise CompositionError(f"language library cannot have a competing source lock: {name}")
|
|
for name in (BACKEND, FRONTEND):
|
|
if name not in components or components[name].get("role") != "application" or name not in lock["components"]:
|
|
raise CompositionError(f"missing fixed application source: {name}")
|
|
return commit, components, lock["components"], hashes
|
|
|
|
|
|
def execution_environment(env):
|
|
"""Acquisition-only authentication/transport never reaches npm/Go candidate code."""
|
|
clean = env.copy()
|
|
for key in list(clean):
|
|
upper = key.upper()
|
|
if (any(word in upper for word in ("TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "API_KEY", "ACCESS_KEY"))
|
|
or upper.startswith(("GIT_CONFIG_", "GIT_SSH", "NPM_CONFIG_", "BASH_FUNC_"))
|
|
or upper in {"SSH_AUTH_SOCK", "SSH_AGENT_PID", "NODE_OPTIONS", "BASH_ENV", "ENV", "GOFLAGS", "GIT_ASKPASS", "SSH_ASKPASS", "CONTRACTS_TARBALL", "UI_TARBALL"}):
|
|
del clean[key]
|
|
clean.update(GOWORK="off", GOTOOLCHAIN="local", GOFLAGS="", CGO_ENABLED="0")
|
|
# Cache location is transport-independent state, shared by acquisition and
|
|
# offline consumers. Other npm settings can carry credentials or hooks.
|
|
for key in ("npm_config_cache", "NPM_CONFIG_CACHE"):
|
|
if env.get(key):
|
|
clean[key] = env[key]
|
|
return clean
|
|
|
|
|
|
def export_source(repo, commit, destination, temporary, env, *, abbreviated=False):
|
|
validate_repo(repo)
|
|
clone = temporary / (destination.name + "-git")
|
|
run(["git", "clone", "--no-checkout", "--no-hardlinks", "--", repo, str(clone)], temporary, env)
|
|
exists = subprocess.run(["git", "cat-file", "-e", commit + "^{commit}"], cwd=clone, env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
|
|
if not exists:
|
|
run(["git", "fetch", "--no-tags", "origin", commit], clone, env)
|
|
actual = run(["git", "rev-parse", commit + "^{commit}"], clone, env, quiet=True).strip()
|
|
if not SHA.fullmatch(actual) or (actual != commit and not (abbreviated and len(commit) == 12 and actual.startswith(commit))):
|
|
raise CompositionError("application revision is not an immutable full commit")
|
|
archive = run(["git", "archive", "--format=tar", actual], clone, env, binary=True, quiet=True)
|
|
extract_archive(archive, destination)
|
|
return {"repo": repo, "commit": actual, "archive_sha256": hashlib.sha256(archive).hexdigest()}
|
|
|
|
|
|
def extract_archive(archive, destination):
|
|
destination.mkdir()
|
|
with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
|
|
for entry in tar.getmembers():
|
|
safe_relative(destination, entry.name)
|
|
if not (entry.isfile() or entry.isdir()):
|
|
raise CompositionError(f"source archive contains an unsupported link: {entry.name}")
|
|
tar.extractall(destination, filter="data")
|
|
|
|
|
|
def version(manifest, lock, *, backend_commit=None, resolved_go=None):
|
|
resolved = json.dumps({"go": manifest["go"] if resolved_go is None else resolved_go,
|
|
"npm_lock": manifest["npm"]["lock_sha256"]},
|
|
sort_keys=True, separators=(",", ":")).encode()
|
|
return (manifest["root"]["commit"][:12] + "+backend=" + (backend_commit or lock[BACKEND]["commit"])[:12]
|
|
+ "+frontend=" + lock[FRONTEND]["commit"][:12]
|
|
+ "+deps=" + hashlib.sha256(resolved).hexdigest()[:12])
|
|
|
|
|
|
def json_stream(output):
|
|
decoder = json.JSONDecoder()
|
|
result = []
|
|
while output.strip():
|
|
value, end = decoder.raw_decode(output.lstrip())
|
|
result.append(value)
|
|
output = output.lstrip()[end:]
|
|
return result
|
|
|
|
|
|
def go_dependencies(source, fetch_env):
|
|
document = json.loads(run(["go", "mod", "edit", "-json"], source, fetch_env, quiet=True))
|
|
for replacement in document.get("Replace") or []:
|
|
if not replacement["New"].get("Version"):
|
|
raise CompositionError("formal Go build forbids local directory replace")
|
|
original = {name: (source / name).read_bytes() if (source / name).exists() else None for name in ("go.mod", "go.sum")}
|
|
# Tidy locks need sums for consumed packages, not unused transitive zip files.
|
|
# Load actual build/test metadata before ending acquisition; no tests execute.
|
|
# Download may normalize sum lines even for a valid readonly consumer. Keep
|
|
# that package-manager bookkeeping in an identical disposable manifest.
|
|
with tempfile.TemporaryDirectory(prefix="composition-go-prefetch-") as directory:
|
|
isolated = Path(directory)
|
|
for name, content in original.items():
|
|
if content is not None:
|
|
(isolated / name).write_bytes(content)
|
|
downloads = json_stream(run(["go", "mod", "download", "-json"], isolated, fetch_env, quiet=True))
|
|
run(["go", "list", "-mod=readonly", "-deps", "-test", "./..."], source,
|
|
fetch_env | {"CGO_ENABLED": "0"}, quiet=True)
|
|
run(["go", "mod", "verify"], source, fetch_env)
|
|
modules = json_stream(run(["go", "list", "-mod=readonly", "-m", "-json", "all"], source, fetch_env, quiet=True))
|
|
selected = []
|
|
for module in modules:
|
|
item = {key: module[key] for key in ("Path", "Version", "Main", "Sum", "GoModSum", "GoVersion") if key in module}
|
|
if "Replace" in module:
|
|
replacement = module["Replace"]
|
|
if not replacement.get("Version"):
|
|
raise CompositionError("resolved Go graph contains a local replace")
|
|
item["Replace"] = {key: replacement[key] for key in ("Path", "Version", "Sum", "GoModSum") if key in replacement}
|
|
selected.append(item)
|
|
for name, content in original.items():
|
|
actual = (source / name).read_bytes() if (source / name).exists() else None
|
|
if actual != content:
|
|
raise CompositionError(f"Go acquisition changed committed {name}; update the consumer first")
|
|
return {"modules": selected, "go_mod_sha256": digest(source / "go.mod"),
|
|
"go_sum_sha256": digest(source / "go.sum") if (source / "go.sum").exists() else None,
|
|
"downloads": [{key: value[key] for key in ("Path", "Version", "Sum", "GoModSum", "Origin") if key in value} for value in downloads]}
|
|
|
|
|
|
def npm_registry_package(location, package):
|
|
resolved = package.get("resolved", "")
|
|
if not isinstance(resolved, str):
|
|
raise CompositionError("invalid npm lock dependency")
|
|
if not resolved.startswith(("https://", "http://")):
|
|
return None
|
|
registry = urlsplit(resolved)
|
|
if (registry.scheme != "https" or registry.netloc != "registry.npmjs.org" or
|
|
registry.username or registry.password or registry.query or registry.fragment):
|
|
raise CompositionError("npm tarball source is outside the reviewed public registry")
|
|
name = package.get("name") or location.rsplit("node_modules/", 1)[-1]
|
|
version = package.get("version", "")
|
|
if (not isinstance(name, str) or not NPM_NAME.fullmatch(name) or
|
|
not isinstance(version, str) or not NPM_VERSION.fullmatch(version)):
|
|
raise CompositionError("invalid locked registry package name or version")
|
|
return name + "@" + version
|
|
|
|
|
|
def npm_dependencies(source, components, fetch_env):
|
|
path = source / "package-lock.json"
|
|
lock = json.loads(path.read_text())
|
|
if lock.get("lockfileVersion", 0) < 2 or not isinstance(lock.get("packages"), dict):
|
|
raise CompositionError("frontend requires a complete npm lock")
|
|
approved = {component["repo"] for component in components.values()}
|
|
packages = {}
|
|
registry_packages = []
|
|
for name, package in lock["packages"].items():
|
|
if name:
|
|
safe_relative(source, name)
|
|
registry_package = npm_registry_package(name, package)
|
|
if registry_package:
|
|
registry_packages.append((name, registry_package))
|
|
if package.get("link"):
|
|
raise CompositionError("formal npm build forbids workspace links")
|
|
for value in [package.get("resolved", ""), *package.get("dependencies", {}).values()]:
|
|
if not isinstance(value, str):
|
|
raise CompositionError("invalid npm lock dependency")
|
|
if value.startswith(("file:", "link:", "workspace:")):
|
|
raise CompositionError("formal npm build forbids local dependency overlays")
|
|
if value.startswith(("git+", "git:", "ssh:")):
|
|
repo, sep, revision = value.removeprefix("git+").rpartition("#")
|
|
if not sep or not SHA.fullmatch(revision) or repo not in approved:
|
|
raise CompositionError("npm Git dependency must use an approved source and full commit")
|
|
packages[name] = {key: package[key] for key in ("version", "resolved", "integrity") if key in package}
|
|
before = digest(path)
|
|
run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], source, fetch_env)
|
|
# npm ci normally stores tarballs without the packuments required when a
|
|
# disposable packed-package consumer re-resolves peers offline. Fetch the
|
|
# installed locked versions before candidate execution, from a fixed public
|
|
# registry with empty config and no acquisition credentials or candidate rc.
|
|
metadata_env = execution_environment(fetch_env)
|
|
with tempfile.TemporaryDirectory(prefix="composition-npm-metadata-") as directory:
|
|
isolated = Path(directory)
|
|
global_config = isolated / "empty-global.npmrc"
|
|
global_config.write_text("")
|
|
selected = {spec for location, spec in registry_packages if (source / location).is_dir()}
|
|
if selected:
|
|
run(["npm", "cache", "add", *sorted(selected), "--ignore-scripts",
|
|
"--registry=https://registry.npmjs.org", "--userconfig=/dev/null",
|
|
"--globalconfig=" + str(global_config)], isolated, metadata_env)
|
|
if digest(path) != before:
|
|
raise CompositionError("npm acquisition changed committed lock")
|
|
result = {"lock_sha256": before, "packages": packages}
|
|
contract = source / "node_modules/@agent-system/contracts"
|
|
if contract.is_dir():
|
|
result["contracts"] = tree_digest(contract)
|
|
return result
|
|
|
|
|
|
def frontend_assets(source, clean_env):
|
|
dist = source / "dist"
|
|
if dist.exists():
|
|
shutil.rmtree(dist)
|
|
run(["npm", "run", "build"], source, clean_env)
|
|
if not (dist / "index.html").is_file() or not (dist / "index.html").stat().st_size:
|
|
raise CompositionError("frontend dist requires nonempty index.html")
|
|
if not (dist / "assets").is_dir() or not any(p.is_file() and p.stat().st_size for p in (dist / "assets").rglob("*")):
|
|
raise CompositionError("frontend dist requires nonempty assets/")
|
|
return dist, tree_digest(dist)
|
|
|
|
|
|
def inject(dist, destination):
|
|
if destination.is_symlink():
|
|
raise CompositionError("embed destination must not be a symlink")
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(dist, destination)
|
|
(destination / ".gitkeep").touch()
|
|
|
|
|
|
def provider_revision(module, downloads):
|
|
metadata = next((entry for entry in downloads if entry["Path"] == module["Path"] and entry["Version"] == module["Version"]), {})
|
|
origin = metadata.get("Origin", {})
|
|
commit = origin.get("Hash", "")
|
|
pseudo = re.search(r"-([0-9a-f]{12})\Z", module["Version"])
|
|
if SHA.fullmatch(commit):
|
|
if pseudo and not commit.startswith(pseudo.group(1)):
|
|
raise CompositionError(f"Go Origin.Hash differs from selected version: {module['Path']}")
|
|
return commit, False
|
|
if pseudo:
|
|
return pseudo.group(1), True
|
|
raise CompositionError(f"cannot prove source commit for selected Go module: {module['Path']}@{module['Version']}")
|
|
|
|
|
|
def prepare_provider_tests(root, root_commit, components, sources, manifest, temporary,
|
|
fetch_env, rdm=None, rdm_admin=None, source_map_path=None):
|
|
"""Acquire every provider and cache before constructing the execution plan."""
|
|
graph = manifest["go"]["backend"]
|
|
source_records = {name: {"path": str(path), "commit": manifest["applications"][name]["commit"], "repo": components[name]["repo"]}
|
|
for name, path in sources.items()}
|
|
module_sources = {}
|
|
manifest["test_sources"] = {}
|
|
manifest["checks"] = []
|
|
go_checks = [{"component": BACKEND, "path": sources[BACKEND], "mode": "readonly"}]
|
|
npm_checks = [{"component": FRONTEND, "path": sources[FRONTEND], "commands": [["npm", "test"]]}]
|
|
for name, component in components.items():
|
|
if component.get("role") != "go-library":
|
|
continue
|
|
selected = [(mapping, next((m for m in graph["modules"] if m["Path"] == mapping["path"]), None))
|
|
for mapping in component.get("modules", [])]
|
|
selected = [(mapping, module) for mapping, module in selected if module is not None]
|
|
if not selected:
|
|
continue
|
|
primary = next((pair for pair in selected if pair[0].get("directory", ".") == "."), selected[0])
|
|
revision, abbreviated = provider_revision(primary[1], graph["downloads"])
|
|
source = temporary / name
|
|
record = export_source(component["repo"], revision, source, temporary, fetch_env, abbreviated=abbreviated)
|
|
manifest["test_sources"][name] = record
|
|
source_records[name] = {"path": str(source), "commit": record["commit"], "repo": component["repo"]}
|
|
sources[name] = source
|
|
for mapping in component.get("modules", []):
|
|
module = next((m for m in graph["modules"] if m["Path"] == mapping["path"]), None)
|
|
module_root = source
|
|
module_commit = record["commit"]
|
|
if module:
|
|
module_revision, module_abbreviated = provider_revision(module, graph["downloads"])
|
|
if not module_commit.startswith(module_revision):
|
|
module_root = temporary / (name + "-" + mapping["directory"].replace("/", "-"))
|
|
alternate = export_source(component["repo"], module_revision, module_root, temporary, fetch_env, abbreviated=module_abbreviated)
|
|
module_commit = alternate["commit"]
|
|
manifest["test_sources"][mapping["path"]] = alternate
|
|
location = module_root / mapping.get("directory", ".")
|
|
if not (location / "go.mod").is_file():
|
|
raise CompositionError(f"declared provider module is missing: {mapping['path']}")
|
|
module_sources[mapping["path"]] = {"path": str(location), "commit": module_commit, "repo": component["repo"]}
|
|
manifest["go"][mapping["path"]] = go_dependencies(location, fetch_env)
|
|
go_checks.append({"component": name, "module": mapping["path"], "path": location, "mode": "readonly"})
|
|
# UI is a package provider; its source identity belongs to the npm lock.
|
|
ui_component = components.get("Agent-System-UI")
|
|
if ui_component:
|
|
package_name = ui_component.get("package_name", "@agent-system/ui")
|
|
resolved = manifest["npm"]["packages"].get("node_modules/" + package_name, {}).get("resolved", "")
|
|
repo, _, commit = resolved.removeprefix("git+").rpartition("#")
|
|
if repo != ui_component["repo"] or not SHA.fullmatch(commit):
|
|
raise CompositionError("cannot prove UI source from the Frontend npm lock")
|
|
ui = temporary / "Agent-System-UI"
|
|
record = export_source(repo, commit, ui, temporary, fetch_env)
|
|
sources["Agent-System-UI"] = ui
|
|
manifest["test_sources"]["Agent-System-UI"] = record
|
|
source_records["Agent-System-UI"] = {"path": str(ui), "commit": commit, "repo": repo}
|
|
for name, source in sources.items():
|
|
if name == FRONTEND or not (source / "package.json").is_file():
|
|
continue
|
|
manifest.setdefault("provider_npm", {})[name] = npm_dependencies(source, components, fetch_env)
|
|
scripts = json.loads((source / "package.json").read_text()).get("scripts", {})
|
|
commands = [["npm", "run", action] for action in ("typecheck", "test") if action in scripts]
|
|
npm_checks.append({"component": name, "path": source, "commands": commands})
|
|
if rdm:
|
|
source_records["Agent-System-RDM"] = {"path": str(rdm), **{key: manifest["applications"]["Agent-System-RDM"][key] for key in ("commit", "repo")}}
|
|
go_checks.extend([
|
|
{"component": "Agent-System-RDM", "path": rdm, "mode": "vendor"},
|
|
{"component": "RDM-Backend", "path": rdm_admin, "mode": "readonly", "commit": manifest["go"]["rdm"]["backend_source"]["commit"]},
|
|
])
|
|
formal_root = temporary / "Dev-test-root"
|
|
archive = run(["git", "archive", "--format=tar", root_commit], root, fetch_env, binary=True, quiet=True)
|
|
extract_archive(archive, formal_root)
|
|
source_map = {"schema": 1, "root_commit": root_commit, "components": source_records, "modules": module_sources}
|
|
map_file = source_map_path or temporary / "composition-sources.json"
|
|
map_file.parent.mkdir(parents=True, exist_ok=True)
|
|
map_file.write_text(json.dumps(source_map, indent=2) + "\n")
|
|
gates = ((["bash", "scripts/lint-boundaries.sh"], "scripts/lint-boundaries.sh"),
|
|
(["bash", "scripts/check-pins.sh"], "scripts/check-pins.sh"),
|
|
(["make", "test-governance"], "Makefile"))
|
|
project_gates = any((formal_root / required_path).is_file() for _, required_path in gates)
|
|
for command, required_path in gates:
|
|
if project_gates and not (formal_root / required_path).is_file():
|
|
raise CompositionError(f"fixed root is missing required test gate: {required_path}")
|
|
return {"go": go_checks, "npm": npm_checks, "root": formal_root, "root_commit": root_commit,
|
|
"map_file": map_file, "rdm": rdm,
|
|
"gates": [command for command, required_path in gates if (formal_root / required_path).is_file()]}
|
|
|
|
|
|
def run_provider_tests(prepared, manifest, clean_env):
|
|
"""Execute a completed plan; this phase has no acquisition environment."""
|
|
for check in prepared["npm"][:1]:
|
|
for command in check["commands"]:
|
|
run(command, check["path"], clean_env)
|
|
manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"})
|
|
for check in prepared["go"]:
|
|
for action in ("test", "vet"):
|
|
command = ["go", action, "-mod=" + check["mode"], "./..."]
|
|
run(command, check["path"], clean_env)
|
|
identity = {key: check[key] for key in ("component", "module", "commit") if key in check}
|
|
manifest["checks"].append({**identity, "command": command, "result": "PASS"})
|
|
for check in prepared["npm"][1:]:
|
|
for command in check["commands"]:
|
|
run(command, check["path"], clean_env)
|
|
manifest["checks"].append({"component": check["component"], "command": command, "result": "PASS"})
|
|
gate_env = clean_env | {"COMPOSITION_SOURCE_MAP": str(prepared["map_file"])}
|
|
if prepared["rdm"]:
|
|
gate_env["COMPOSITION_RDM_SOURCE"] = str(prepared["rdm"])
|
|
for command in prepared["gates"]:
|
|
run(command, prepared["root"], gate_env)
|
|
manifest["checks"].append({"component": "Agent-System-Dev", "commit": prepared["root_commit"], "command": command, "result": "PASS"})
|
|
|
|
|
|
def compose(root, target, output, rdm_source=None, source_map_path=None):
|
|
tooling = tooling_identity(root)
|
|
transport_path = Path(__file__).resolve().parent / "workspace/transport.py"
|
|
spec = importlib.util.spec_from_file_location("composition_transport", transport_path)
|
|
transport = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(transport)
|
|
fetch_env = transport.environment(root)
|
|
fetch_env.update(GOWORK="off", GOFLAGS="", GOTOOLCHAIN="local")
|
|
root_commit, components, lock, input_hashes = inputs(root, fetch_env)
|
|
if root_commit != tooling["root_commit"]:
|
|
raise CompositionError("root identity changed during tooling validation")
|
|
clean_env = execution_environment(fetch_env)
|
|
manifest = {"schema": 1, "kind": "agent-system-composition/1", "target": target,
|
|
"root": {"commit": root_commit, "inputs": input_hashes}, "tooling": tooling,
|
|
"applications": {}, "go": {}, "artifacts": {}, "assets": {}}
|
|
with tempfile.TemporaryDirectory(prefix="agent-system-composition-") as directory:
|
|
temporary = Path(directory)
|
|
sources = {}
|
|
for name in (FRONTEND, BACKEND):
|
|
if target == "frontend" and name == BACKEND:
|
|
continue
|
|
source = temporary / name
|
|
manifest["applications"][name] = export_source(components[name]["repo"], lock[name]["commit"], source, temporary, fetch_env)
|
|
sources[name] = source
|
|
frontend = sources[FRONTEND]
|
|
manifest["npm"] = npm_dependencies(frontend, components, fetch_env)
|
|
products = temporary / "products"
|
|
products.mkdir()
|
|
rdm = None
|
|
admin_source = None
|
|
vendored_backend = None
|
|
test_plan = None
|
|
manifest["version"] = version(manifest, lock)
|
|
if target != "frontend":
|
|
backend = sources[BACKEND]
|
|
manifest["go"]["backend"] = go_dependencies(backend, fetch_env)
|
|
manifest["version"] = version(manifest, lock)
|
|
if rdm_source:
|
|
rdm_commit = run(["git", "rev-parse", "HEAD"], rdm_source, fetch_env, quiet=True).strip()
|
|
rdm = temporary / "Agent-System-RDM"
|
|
manifest["applications"]["Agent-System-RDM"] = export_source(str(rdm_source), rdm_commit, rdm, temporary, fetch_env)
|
|
graph = go_dependencies(rdm, fetch_env)
|
|
manifest["go"]["rdm"] = graph
|
|
backend_module = next((m["path"] for m in components[BACKEND]["modules"] if m.get("directory", ".") == "."), None)
|
|
selected = next((m for m in graph["modules"] if m["Path"] == backend_module), None)
|
|
if not selected:
|
|
raise CompositionError("RDM must consume the fixed Backend as an ordinary versioned Go module")
|
|
effective = selected.get("Replace", selected)
|
|
if effective.get("Path") != backend_module or not effective.get("Version"):
|
|
raise CompositionError("RDM Backend must resolve to the approved versioned module")
|
|
# RDM's module graph is its dependency authority. The Dev app
|
|
# source lock fixes a separate application and may differ.
|
|
revision, abbreviated = provider_revision(effective, graph["downloads"])
|
|
admin_source = temporary / "RDM-Backend"
|
|
backend_record = export_source(components[BACKEND]["repo"], revision, admin_source,
|
|
temporary, fetch_env, abbreviated=abbreviated)
|
|
graph["backend_source"] = {**backend_record, "module": backend_module, "version": effective["Version"]}
|
|
manifest["go"]["rdm_admin"] = go_dependencies(admin_source, fetch_env)
|
|
run(["go", "mod", "vendor"], rdm, fetch_env)
|
|
vendored_backend = rdm / "vendor" / backend_module
|
|
if not (vendored_backend / "web/embed.go").is_file():
|
|
raise CompositionError("RDM vendored Backend does not contain the embed package")
|
|
graph["vendor_modules_sha256"] = digest(rdm / "vendor/modules.txt")
|
|
graph["vendor_backend_source_sha256"] = tree_digest(vendored_backend)["sha256"]
|
|
if target == "test":
|
|
test_plan = prepare_provider_tests(root, root_commit, components, sources, manifest, temporary,
|
|
fetch_env, rdm, admin_source, source_map_path)
|
|
if target == "build" and run(["git", "ls-tree", root_commit, "--", "plugins"], root, fetch_env, quiet=True).strip():
|
|
archive = run(["git", "archive", "--format=tar", root_commit, "plugins"], root, fetch_env, binary=True, quiet=True)
|
|
root_assets = temporary / "root-assets"
|
|
extract_archive(archive, root_assets)
|
|
manifest["assets"]["plugins"] = tree_digest(root_assets / "plugins")
|
|
manifest["assets"]["plugins"]["root_commit"] = root_commit
|
|
|
|
# All fixed sources, package caches, vendor files and root assets are
|
|
# acquired above. Candidate commands below receive only clean_env;
|
|
# there is no acquisition environment in the execution plan.
|
|
del fetch_env
|
|
dist, manifest["frontend"] = frontend_assets(frontend, clean_env)
|
|
manifest["toolchains"] = {"go": run(["go", "version"], temporary, clean_env, quiet=True).strip(),
|
|
"node": run(["node", "--version"], temporary, clean_env, quiet=True).strip(),
|
|
"npm": run(["npm", "--version"], temporary, clean_env, quiet=True).strip()}
|
|
if target != "frontend":
|
|
inject(dist, backend / "web/dist")
|
|
manifest["go"]["backend"]["embed_sha256"] = tree_digest(backend / "web/dist")["sha256"]
|
|
if rdm:
|
|
graph = manifest["go"]["rdm"]
|
|
inject(dist, admin_source / "web/dist")
|
|
manifest["go"]["rdm_admin"]["embed_sha256"] = tree_digest(admin_source / "web/dist")["sha256"]
|
|
graph["admin_version"] = version(manifest, lock, backend_commit=graph["backend_source"]["commit"],
|
|
resolved_go={"rdm": graph["modules"], "admin": manifest["go"]["rdm_admin"]})
|
|
inject(dist, vendored_backend / "web/dist")
|
|
graph["vendor_backend_dist_sha256"] = tree_digest(vendored_backend / "web/dist")["sha256"]
|
|
if target == "test":
|
|
run_provider_tests(test_plan, manifest, clean_env)
|
|
else:
|
|
for name, package in (("agent-system", "./cmd/agent-system"), ("storage-migrate", "./cmd/storage-migrate")):
|
|
flags = "-s -w" + (" -X main.version=" + manifest["version"] if name == "agent-system" else "")
|
|
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false", "-ldflags=" + flags, "-o", str(products / name), package], backend, clean_env)
|
|
if rdm:
|
|
run(["go", "build", "-mod=vendor", "-trimpath", "-buildvcs=false", "-ldflags=-s -w", "-o", str(products / "agent-system-rdm-product"), "."], rdm, clean_env)
|
|
run(["go", "build", "-mod=readonly", "-trimpath", "-buildvcs=false",
|
|
"-ldflags=-s -w -X main.version=" + graph["admin_version"],
|
|
"-o", str(products / "agent-system-admin"), "./cmd/agent-system"], admin_source, clean_env)
|
|
for artifact in products.iterdir():
|
|
manifest["artifacts"][artifact.name] = {"sha256": digest(artifact), "size": artifact.stat().st_size}
|
|
if tooling_identity(root, clean_env) != tooling:
|
|
raise CompositionError("executing tooling changed during composition")
|
|
# Publish only after all required build/test commands succeeded.
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
for artifact in products.iterdir():
|
|
shutil.copy2(artifact, output / artifact.name)
|
|
if "plugins" in manifest["assets"]:
|
|
destination = output / "plugins"
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(root_assets / "plugins", destination)
|
|
if target == "frontend":
|
|
destination = output / "frontend-dist"
|
|
if destination.exists():
|
|
shutil.rmtree(destination)
|
|
shutil.copytree(dist, destination)
|
|
pending = output / ".resolved-manifest.json.tmp"
|
|
pending.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n")
|
|
pending.replace(output / "resolved-manifest.json")
|
|
print(f"[composition] complete: {target}; manifest={output / 'resolved-manifest.json'}", flush=True)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--target", required=True, choices=("frontend", "build", "test"))
|
|
parser.add_argument("--output", type=Path)
|
|
parser.add_argument("--rdm-source", type=Path)
|
|
parser.add_argument("--source-map", type=Path, help="Write source identity map used by test gates; paths exist only during this run")
|
|
args = parser.parse_args()
|
|
root = args.root.resolve()
|
|
try:
|
|
compose(root, args.target, (args.output or root / "bin").resolve(), args.rdm_source.resolve() if args.rdm_source else None,
|
|
args.source_map.resolve() if args.source_map else None)
|
|
except (CompositionError, OSError, ValueError, KeyError, TypeError) as error:
|
|
print(f"composition: {error}", file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|