diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 30753c30..33abf249 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -647,6 +647,24 @@ jobs: raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping") return None + def registry_package(location, entry): + resolved = entry.get("resolved", "") + if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")): + return None + # A metadata spec is name@exact-version, never a candidate URL. + source = urlsplit(resolved) + if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or + source.username or source.password or source.query or source.fragment): + raise ValueError("npm tarball source is outside the reviewed public registry") + relative = Path(location) + name = entry.get("name") or location.rsplit("node_modules/", 1)[-1] + version = entry.get("version", "") + if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or + not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or + not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)): + raise ValueError("Invalid locked registry package name or version") + return name + "@" + version + try: targets = [] selected = set() @@ -656,7 +674,8 @@ jobs: if (directory / ".npmrc").exists(): raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition") package = json.loads((directory / "package.json").read_text()) - lock = json.loads((directory / "package-lock.json").read_text()) + lock_bytes = (directory / "package-lock.json").read_bytes() + lock = json.loads(lock_bytes) if lock.get("lockfileVersion") not in {2, 3}: raise ValueError("npm acquisition requires a package lock") for group in ("dependencies", "devDependencies", "optionalDependencies"): @@ -664,7 +683,11 @@ jobs: found = git_package(value) if found: selected.add(found) - for entry in lock.get("packages", {}).values(): + registry_packages = [] + for location, entry in lock.get("packages", {}).items(): + spec = registry_package(location, entry) + if spec: + registry_packages.append((location, spec)) found = git_package(entry.get("resolved")) if found: selected.add(found) @@ -673,7 +696,7 @@ jobs: found = git_package(value) if found: selected.add(found) - targets.append(directory) + targets.append((directory, registry_packages, lock_bytes)) # Fetch only registered fixed Git package commits. npm never receives token config. for repo, sha in sorted(selected): repository = mirror(repo, sha, component_token) @@ -685,12 +708,25 @@ jobs: raise ValueError("Git package provider is missing committed package outputs") no_auth = mirror_env() empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-") - for directory in targets: + for directory, registry_packages, lock_bytes in targets: result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund", "--userconfig=/dev/null", "--globalconfig=" + empty_config.name], cwd=directory, env=no_auth, capture_output=True, text=True) if result.returncode: raise ValueError("npm dependency acquisition failed: " + result.stderr.strip()) + # npm ci caches tarballs; packed consumers also need resolver + # metadata. Only installed locked registry entries are fetched. + specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()}) + if specs: + with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root: + result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts", + "--registry=https://registry.npmjs.org", "--userconfig=/dev/null", + "--globalconfig=" + empty_config.name], cwd=metadata_root, + env=no_auth, capture_output=True, text=True) + if result.returncode: + raise ValueError("npm metadata acquisition failed: " + result.stderr.strip()) + if (directory / "package-lock.json").read_bytes() != lock_bytes: + raise ValueError("npm acquisition changed the committed package lock") except (ValueError, OSError, subprocess.CalledProcessError) as error: sys.exit("npm prefetch refused: " + str(error)) print("npm packages cached without credentials or lifecycle execution") diff --git a/components.lock.json b/components.lock.json index 1f570e45..e2d02dd5 100644 --- a/components.lock.json +++ b/components.lock.json @@ -2,7 +2,7 @@ "schema": 1, "components": { "Agent-System-Frontend": { - "commit": "6ed7b9322c5733f76b206a35cd78649ba385d83b" + "commit": "5467517e1831e7ca8b58bb5df52494a3a2c9d468" }, "Agent-System-Backend": { "commit": "fffd5790cfab317fadafb40fa55a11bab6e9a847" diff --git a/scripts/build-composition.py b/scripts/build-composition.py index 118e418d..21e06406 100644 --- a/scripts/build-composition.py +++ b/scripts/build-composition.py @@ -25,6 +25,8 @@ SHA = re.compile(r"[0-9a-f]{40}\Z") IDS = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z") BACKEND = "Agent-System-Backend" FRONTEND = "Agent-System-Frontend" +NPM_NAME = re.compile(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*\Z") +NPM_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\Z") class CompositionError(Exception): @@ -165,6 +167,11 @@ def execution_environment(env): or upper in {"SSH_AUTH_SOCK", "SSH_AGENT_PID", "NODE_OPTIONS", "BASH_ENV", "ENV", "GOFLAGS", "GIT_ASKPASS", "SSH_ASKPASS", "CONTRACTS_TARBALL", "UI_TARBALL"}): del clean[key] clean.update(GOWORK="off", GOTOOLCHAIN="local", GOFLAGS="", CGO_ENABLED="0") + # Cache location is transport-independent state, shared by acquisition and + # offline consumers. Other npm settings can carry credentials or hooks. + for key in ("npm_config_cache", "NPM_CONFIG_CACHE"): + if env.get(key): + clean[key] = env[key] return clean @@ -251,6 +258,24 @@ def go_dependencies(source, fetch_env): "downloads": [{key: value[key] for key in ("Path", "Version", "Sum", "GoModSum", "Origin") if key in value} for value in downloads]} +def npm_registry_package(location, package): + resolved = package.get("resolved", "") + if not isinstance(resolved, str): + raise CompositionError("invalid npm lock dependency") + if not resolved.startswith(("https://", "http://")): + return None + registry = urlsplit(resolved) + if (registry.scheme != "https" or registry.netloc != "registry.npmjs.org" or + registry.username or registry.password or registry.query or registry.fragment): + raise CompositionError("npm tarball source is outside the reviewed public registry") + name = package.get("name") or location.rsplit("node_modules/", 1)[-1] + version = package.get("version", "") + if (not isinstance(name, str) or not NPM_NAME.fullmatch(name) or + not isinstance(version, str) or not NPM_VERSION.fullmatch(version)): + raise CompositionError("invalid locked registry package name or version") + return name + "@" + version + + def npm_dependencies(source, components, fetch_env): path = source / "package-lock.json" lock = json.loads(path.read_text()) @@ -258,7 +283,13 @@ def npm_dependencies(source, components, fetch_env): raise CompositionError("frontend requires a complete npm lock") approved = {component["repo"] for component in components.values()} packages = {} + registry_packages = [] for name, package in lock["packages"].items(): + if name: + safe_relative(source, name) + registry_package = npm_registry_package(name, package) + if registry_package: + registry_packages.append((name, registry_package)) if package.get("link"): raise CompositionError("formal npm build forbids workspace links") for value in [package.get("resolved", ""), *package.get("dependencies", {}).values()]: @@ -273,6 +304,20 @@ def npm_dependencies(source, components, fetch_env): packages[name] = {key: package[key] for key in ("version", "resolved", "integrity") if key in package} before = digest(path) run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], source, fetch_env) + # npm ci normally stores tarballs without the packuments required when a + # disposable packed-package consumer re-resolves peers offline. Fetch the + # installed locked versions before candidate execution, from a fixed public + # registry with empty config and no acquisition credentials or candidate rc. + metadata_env = execution_environment(fetch_env) + with tempfile.TemporaryDirectory(prefix="composition-npm-metadata-") as directory: + isolated = Path(directory) + global_config = isolated / "empty-global.npmrc" + global_config.write_text("") + selected = {spec for location, spec in registry_packages if (source / location).is_dir()} + if selected: + run(["npm", "cache", "add", *sorted(selected), "--ignore-scripts", + "--registry=https://registry.npmjs.org", "--userconfig=/dev/null", + "--globalconfig=" + str(global_config)], isolated, metadata_env) if digest(path) != before: raise CompositionError("npm acquisition changed committed lock") result = {"lock_sha256": before, "packages": packages} diff --git a/tests/fixtures/component-ci/Agent-System-Frontend/ci.yml b/tests/fixtures/component-ci/Agent-System-Frontend/ci.yml index df9df1cb..428fd37a 100644 --- a/tests/fixtures/component-ci/Agent-System-Frontend/ci.yml +++ b/tests/fixtures/component-ci/Agent-System-Frontend/ci.yml @@ -408,6 +408,24 @@ jobs: raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping") return None + def registry_package(location, entry): + resolved = entry.get("resolved", "") + if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")): + return None + # A metadata spec is name@exact-version, never a candidate URL. + source = urlsplit(resolved) + if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or + source.username or source.password or source.query or source.fragment): + raise ValueError("npm tarball source is outside the reviewed public registry") + relative = Path(location) + name = entry.get("name") or location.rsplit("node_modules/", 1)[-1] + version = entry.get("version", "") + if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or + not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or + not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)): + raise ValueError("Invalid locked registry package name or version") + return name + "@" + version + try: targets = [] selected = set() @@ -417,7 +435,8 @@ jobs: if (directory / ".npmrc").exists(): raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition") package = json.loads((directory / "package.json").read_text()) - lock = json.loads((directory / "package-lock.json").read_text()) + lock_bytes = (directory / "package-lock.json").read_bytes() + lock = json.loads(lock_bytes) if lock.get("lockfileVersion") not in {2, 3}: raise ValueError("npm acquisition requires a package lock") for group in ("dependencies", "devDependencies", "optionalDependencies"): @@ -425,7 +444,11 @@ jobs: found = git_package(value) if found: selected.add(found) - for entry in lock.get("packages", {}).values(): + registry_packages = [] + for location, entry in lock.get("packages", {}).items(): + spec = registry_package(location, entry) + if spec: + registry_packages.append((location, spec)) found = git_package(entry.get("resolved")) if found: selected.add(found) @@ -434,7 +457,7 @@ jobs: found = git_package(value) if found: selected.add(found) - targets.append(directory) + targets.append((directory, registry_packages, lock_bytes)) # Fetch only registered fixed Git package commits. npm never receives token config. for repo, sha in sorted(selected): repository = mirror(repo, sha, component_token) @@ -446,12 +469,25 @@ jobs: raise ValueError("Git package provider is missing committed package outputs") no_auth = mirror_env() empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-") - for directory in targets: + for directory, registry_packages, lock_bytes in targets: result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund", "--userconfig=/dev/null", "--globalconfig=" + empty_config.name], cwd=directory, env=no_auth, capture_output=True, text=True) if result.returncode: raise ValueError("npm dependency acquisition failed: " + result.stderr.strip()) + # npm ci caches tarballs; packed consumers also need resolver + # metadata. Only installed locked registry entries are fetched. + specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()}) + if specs: + with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root: + result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts", + "--registry=https://registry.npmjs.org", "--userconfig=/dev/null", + "--globalconfig=" + empty_config.name], cwd=metadata_root, + env=no_auth, capture_output=True, text=True) + if result.returncode: + raise ValueError("npm metadata acquisition failed: " + result.stderr.strip()) + if (directory / "package-lock.json").read_bytes() != lock_bytes: + raise ValueError("npm acquisition changed the committed package lock") except (ValueError, OSError, subprocess.CalledProcessError) as error: sys.exit("npm prefetch refused: " + str(error)) print("npm packages cached without credentials or lifecycle execution") diff --git a/tests/fixtures/component-ci/Agent-System-UI/ci.yml b/tests/fixtures/component-ci/Agent-System-UI/ci.yml index 68f9f52a..970f1c44 100644 --- a/tests/fixtures/component-ci/Agent-System-UI/ci.yml +++ b/tests/fixtures/component-ci/Agent-System-UI/ci.yml @@ -406,6 +406,24 @@ jobs: raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping") return None + def registry_package(location, entry): + resolved = entry.get("resolved", "") + if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")): + return None + # A metadata spec is name@exact-version, never a candidate URL. + source = urlsplit(resolved) + if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or + source.username or source.password or source.query or source.fragment): + raise ValueError("npm tarball source is outside the reviewed public registry") + relative = Path(location) + name = entry.get("name") or location.rsplit("node_modules/", 1)[-1] + version = entry.get("version", "") + if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or + not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or + not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)): + raise ValueError("Invalid locked registry package name or version") + return name + "@" + version + try: targets = [] selected = set() @@ -415,7 +433,8 @@ jobs: if (directory / ".npmrc").exists(): raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition") package = json.loads((directory / "package.json").read_text()) - lock = json.loads((directory / "package-lock.json").read_text()) + lock_bytes = (directory / "package-lock.json").read_bytes() + lock = json.loads(lock_bytes) if lock.get("lockfileVersion") not in {2, 3}: raise ValueError("npm acquisition requires a package lock") for group in ("dependencies", "devDependencies", "optionalDependencies"): @@ -423,7 +442,11 @@ jobs: found = git_package(value) if found: selected.add(found) - for entry in lock.get("packages", {}).values(): + registry_packages = [] + for location, entry in lock.get("packages", {}).items(): + spec = registry_package(location, entry) + if spec: + registry_packages.append((location, spec)) found = git_package(entry.get("resolved")) if found: selected.add(found) @@ -432,7 +455,7 @@ jobs: found = git_package(value) if found: selected.add(found) - targets.append(directory) + targets.append((directory, registry_packages, lock_bytes)) # Fetch only registered fixed Git package commits. npm never receives token config. for repo, sha in sorted(selected): repository = mirror(repo, sha, component_token) @@ -444,12 +467,25 @@ jobs: raise ValueError("Git package provider is missing committed package outputs") no_auth = mirror_env() empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-") - for directory in targets: + for directory, registry_packages, lock_bytes in targets: result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund", "--userconfig=/dev/null", "--globalconfig=" + empty_config.name], cwd=directory, env=no_auth, capture_output=True, text=True) if result.returncode: raise ValueError("npm dependency acquisition failed: " + result.stderr.strip()) + # npm ci caches tarballs; packed consumers also need resolver + # metadata. Only installed locked registry entries are fetched. + specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()}) + if specs: + with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root: + result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts", + "--registry=https://registry.npmjs.org", "--userconfig=/dev/null", + "--globalconfig=" + empty_config.name], cwd=metadata_root, + env=no_auth, capture_output=True, text=True) + if result.returncode: + raise ValueError("npm metadata acquisition failed: " + result.stderr.strip()) + if (directory / "package-lock.json").read_bytes() != lock_bytes: + raise ValueError("npm acquisition changed the committed package lock") except (ValueError, OSError, subprocess.CalledProcessError) as error: sys.exit("npm prefetch refused: " + str(error)) print("npm packages cached without credentials or lifecycle execution") diff --git a/tests/fixtures/component-ci/provenance.json b/tests/fixtures/component-ci/provenance.json index cdc35085..c3b09fa8 100644 --- a/tests/fixtures/component-ci/provenance.json +++ b/tests/fixtures/component-ci/provenance.json @@ -53,15 +53,15 @@ }, "Agent-System-UI": { "repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-UI.git", - "source_sha": "bc0a4d33d03728e8726b5c398c4239a87def8eda", + "source_sha": "63d289e6bc15b701356cfe2d5b8740210fac0dcc", "workflow_path": ".gitea/workflows/ci.yml", - "workflow_sha256": "ef41bf7d981c37246d71fe80c09fa077341802442aad213b14c9f18a17a58694" + "workflow_sha256": "03fb580f523649db843a21ff70e3d21c5165198c1a8cf0a0bea882c3ec04f8f1" }, "Agent-System-Frontend": { "repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-Frontend.git", - "source_sha": "6ed7b9322c5733f76b206a35cd78649ba385d83b", + "source_sha": "5467517e1831e7ca8b58bb5df52494a3a2c9d468", "workflow_path": ".gitea/workflows/ci.yml", - "workflow_sha256": "51aeaa0e2ced11bb36fd6b66ac8da66a1f2efdbd6a5d6036302159646cabc3be" + "workflow_sha256": "802e5c199afd8805d87f0fd1958cd001960d8d820a4e6acb14a401007d278012" }, "Agent-System-RDM": { "repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-RDM.git", diff --git a/tests/test_composition.py b/tests/test_composition.py index c34b355b..98d4e27e 100644 --- a/tests/test_composition.py +++ b/tests/test_composition.py @@ -1,14 +1,20 @@ """Real Git/npm/Go fixtures for immutable product composition.""" +import base64 import hashlib +import importlib.util +import io import json import os from pathlib import Path import shutil import subprocess import sys +import tarfile import tempfile +import threading import unittest import zipfile +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer SCRIPT = Path(__file__).resolve().parents[1] / "scripts/build-composition.py" @@ -16,6 +22,151 @@ RDM_SCRIPTS = Path(os.environ.get("COMPOSITION_RDM_SOURCE", str(SCRIPT.parent.pa LEGACY_PRODUCT = Path(__file__).resolve().parent / "fixtures/rdm-legacy-product.sh" +class LocalNpmRegistry: + """Real npm cache/registry traffic with no public-network dependency.""" + name = "composition-cold-fixture" + + def __init__(self, base, env): + self.base, self.env = base, env + self.requests = [] + self.tarballs = {} + self.integrities = {} + for version in ("1.0.0", "1.0.1"): + payload = json.dumps({"name": self.name, "version": version, + "scripts": {"install": "node -e 'process.exit(91)'"}}).encode() + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as archive: + entry = tarfile.TarInfo("package/package.json") + entry.size = len(payload) + archive.addfile(entry, io.BytesIO(payload)) + self.tarballs[version] = stream.getvalue() + self.integrities[version] = "sha512-" + base64.b64encode(hashlib.sha512(stream.getvalue()).digest()).decode() + registry = self + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + registry.requests.append((self.path, self.headers.get("Authorization"))) + if self.path == "/" + registry.name: + versions = {v: {"name": registry.name, "version": v, "dist": { + "tarball": registry.url + "/" + registry.name + "/-/" + v + ".tgz", + "integrity": registry.integrities[v]}} for v in registry.tarballs} + body = json.dumps({"name": registry.name, "dist-tags": {"latest": "1.0.1"}, "versions": versions}).encode() + content_type = "application/json" + else: + version = self.path.rsplit("/", 1)[-1].removesuffix(".tgz") + if version not in registry.tarballs: + self.send_error(404) + return + body, content_type = registry.tarballs[version], "application/octet-stream" + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = "http://127.0.0.1:" + str(self.server.server_port) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + self.cache = base / "cold-npm-cache" + self.global_config = base / "empty-npm-globalrc" + self.global_config.write_text("") + self.home = base / "cold-home" + self.home.mkdir() + self.tools = base / "npm-tools" + self.tools.mkdir() + self.calls = base / "npm-calls.jsonl" + npm = shutil.which("npm") + wrapper = self.tools / "npm" + wrapper.write_text("#!" + sys.executable + "\n" + + "import json, os, sys\nfrom pathlib import Path\nargs = sys.argv[1:]\n" + + "with Path(" + repr(str(self.calls)) + ").open('a') as output:\n" + + " output.write(json.dumps({'args': args, 'credential_keys': [k for k in os.environ if any(w in k.upper() for w in ('TOKEN', 'SECRET', 'PASSWORD'))]}) + '\\n')\n" + + "if args[:2] == ['cache', 'add'] and args[2].startswith(" + repr(self.name + "@") + "):\n" + + " args = [a.replace('https://registry.npmjs.org', " + repr(self.url) + ") for a in args]\n" + + "if args[:1] == ['ci']: args.append('--offline')\n" + + "os.execv(" + repr(npm) + ", [" + repr(npm) + ", *args])\n") + wrapper.chmod(0o755) + self.env = env | {"HOME": str(self.home), "npm_config_cache": str(self.cache), + "PATH": str(self.tools) + os.pathsep + env["PATH"]} + # Seed only the locked tarball, exactly the state produced by npm ci. + tarball = base / "locked.tgz" + tarball.write_bytes(self.tarballs["1.0.0"]) + subprocess.run([npm, "cache", "add", str(tarball), "--ignore-scripts", "--userconfig=/dev/null", + "--globalconfig=" + str(self.global_config)], env=self.env, check=True, capture_output=True) + + def close(self): + self.server.shutdown() + self.server.server_close() + self.thread.join() + + def manifests(self, source): + source.mkdir(exist_ok=True) + manifest = {"name": "fixture", "version": "1.0.0", "dependencies": {self.name: "1.0.0"}} + (source / "package.json").write_text(json.dumps(manifest)) + lock = {"name": "fixture", "version": "1.0.0", "lockfileVersion": 3, "requires": True, + "packages": {"": manifest, "node_modules/" + self.name: { + "version": "1.0.0", "resolved": "https://registry.npmjs.org/" + self.name + "/-/1.0.0.tgz", + "integrity": self.integrities["1.0.0"]}}} + (source / "package-lock.json").write_text(json.dumps(lock)) + return (source / "package-lock.json").read_bytes() + + def consumer(self, directory, env=None): + directory.mkdir(exist_ok=True) + (directory / "package.json").write_text(json.dumps({"dependencies": {self.name: "1.0.0"}})) + return subprocess.run(["npm", "install", "--offline", "--ignore-scripts", "--no-audit", "--no-fund", + "--registry=" + self.url, "--userconfig=/dev/null", "--globalconfig=" + str(self.global_config)], + cwd=directory, env=env or self.env, capture_output=True, text=True) + + +@unittest.skipUnless(shutil.which("npm"), "npm required") +class NpmAcquisitionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="composition-cold-npm-") + self.addCleanup(self.temporary.cleanup) + self.base = Path(self.temporary.name) + self.registry = LocalNpmRegistry(self.base, os.environ.copy()) + self.addCleanup(self.registry.close) + spec = importlib.util.spec_from_file_location("cold_npm_composer", SCRIPT) + self.composer = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.composer) + + def test_locked_metadata_is_acquired_before_offline_consumer_without_credentials(self): + source = self.base / "source" + original = self.registry.manifests(source) + red = self.registry.consumer(self.base / "before") + self.assertNotEqual(red.returncode, 0) + self.assertIn("ENOTCACHED", red.stderr) + self.composer.npm_dependencies(source, {}, self.registry.env | {"GITEA_TOKEN": "synthetic-secret"}) + self.assertEqual((source / "package-lock.json").read_bytes(), original) + green = self.registry.consumer(self.base / "after", self.composer.execution_environment(self.registry.env)) + self.assertEqual(green.returncode, 0, green.stdout + green.stderr) + actual = json.loads((self.base / "after/node_modules" / self.registry.name / "package.json").read_text()) + self.assertEqual(actual["version"], "1.0.0") + calls = [json.loads(line) for line in self.registry.calls.read_text().splitlines()] + metadata = [call for call in calls if call["args"][:2] == ["cache", "add"]] + self.assertTrue(metadata, "tarball acquisition alone does not populate offline resolver metadata") + self.assertEqual(metadata[0]["args"][2], self.registry.name + "@1.0.0") + self.assertFalse(metadata[0]["credential_keys"]) + self.assertTrue(self.registry.requests) + self.assertTrue(all(auth is None for _, auth in self.registry.requests)) + + def test_unreviewed_tarball_source_is_refused_before_acquisition(self): + source = self.base / "source" + self.registry.manifests(source) + path = source / "package-lock.json" + lock = json.loads(path.read_text()) + lock["packages"]["node_modules/" + self.registry.name]["resolved"] = self.registry.url + "/untrusted.tgz" + path.write_text(json.dumps(lock)) + with self.assertRaisesRegex(self.composer.CompositionError, "reviewed public registry"): + self.composer.npm_dependencies(source, {}, self.registry.env | {"GITEA_TOKEN": "synthetic-secret"}) + self.assertFalse(self.registry.requests) + self.assertFalse(self.registry.calls.exists()) + + @unittest.skipUnless(all(shutil.which(c) for c in ("git", "go", "node", "npm")), "Git/Go/Node/npm required") class CompositionTests(unittest.TestCase): def setUp(self): @@ -118,7 +269,7 @@ args = sys.argv[1:] name = pathlib.Path(sys.argv[0]).name acquisition = (name == 'git' and args and args[0] in {'clone', 'fetch', 'archive'} or name == 'go' and args and (args[0] == 'mod' or args[0] == 'list') - or name == 'npm' and args and args[0] in {'ci', 'install'}) + or name == 'npm' and args and (args[0] in {'ci', 'install'} or args[:2] == ['cache', 'add'])) if acquisition and pathlib.Path(os.environ['COMPOSITION_EXEC_MARKER']).exists(): print('acquisition-after-execution: ' + name + ' ' + ' '.join(args[:2]), file=sys.stderr) sys.exit(86) diff --git a/tests/test_dependency_ci.py b/tests/test_dependency_ci.py index e0731a27..52b69995 100644 --- a/tests/test_dependency_ci.py +++ b/tests/test_dependency_ci.py @@ -10,6 +10,7 @@ import textwrap import unittest from tests import test_ci_credentials as fixture +from tests.test_composition import LocalNpmRegistry ROOT_TOKEN, SUB_TOKEN, SSH, WORKFLOW = fixture.ROOT_TOKEN, fixture.SUB_TOKEN, fixture.SSH, fixture.WORKFLOW @@ -242,6 +243,47 @@ class DependencyCITests(unittest.TestCase): self.assertRegex(path, r"^/weichen/Agent-System-Core\.git/") self.assertEqual(auth, "token " + SUB_TOKEN) + def test_npm_prefetch_populates_cold_offline_resolver_metadata_without_credentials(self): + workflows = {"Agent-System-Dev": WORKFLOW, **{repo: fixture.component_workflow(repo) + for repo in ("Agent-System-UI", "Agent-System-Frontend")}} + for repo, workflow in workflows.items(): + with self.subTest(repo=repo): + base = self.base / repo + base.mkdir() + registry = LocalNpmRegistry(base, self.env) + self.addCleanup(registry.close) + original = registry.manifests(self.checkout) + red = registry.consumer(base / "cold-before") + self.assertNotEqual(red.returncode, 0) + self.assertIn("ENOTCACHED", red.stderr) + result = self.run_phase("Prefetch npm packages without lifecycle scripts", workflow=workflow, extra=registry.env) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.checkout / "package-lock.json").read_bytes(), original) + green = registry.consumer(base / "cold-after") + self.assertEqual(green.returncode, 0, green.stdout + green.stderr) + installed = json.loads((base / "cold-after/node_modules" / registry.name / "package.json").read_text()) + self.assertEqual(installed["version"], "1.0.0") + calls = [json.loads(line) for line in registry.calls.read_text().splitlines()] + metadata = [call for call in calls if call["args"][:2] == ["cache", "add"]] + self.assertTrue(metadata) + self.assertEqual(metadata[0]["args"][2], registry.name + "@1.0.0") + self.assertFalse(metadata[0]["credential_keys"]) + self.assertTrue(all(auth is None for _, auth in registry.requests)) + + def test_npm_metadata_version_cannot_be_a_destination(self): + (self.checkout / "package.json").write_text(json.dumps({"name": "fixture"})) + (self.checkout / "package-lock.json").write_text(json.dumps({"lockfileVersion": 3, "packages": { + "node_modules/evil": {"version": self.url + "/untrusted.tgz", + "resolved": "https://registry.npmjs.org/evil/-/evil-1.0.0.tgz"}}})) + workflows = {"Agent-System-Dev": WORKFLOW, **{repo: fixture.component_workflow(repo) + for repo in ("Agent-System-UI", "Agent-System-Frontend")}} + for repo, workflow in workflows.items(): + with self.subTest(repo=repo): + result = self.run_phase("Prefetch npm packages without lifecycle scripts", workflow=workflow) + self.assertNotEqual(result.returncode, 0) + self.assertIn("locked registry package", result.stderr) + self.assertFalse(self.requests) + def test_npm_provider_prepare_is_rejected_without_execution(self): marker = self.base / "prepare-ran" sha = self.commit_fixture("Agent-System-Core", {