weichen/Agent-System-Dev
1
0
Fork 0
Code Issues 1 Pull Requests 5 Actions 2 Packages Projects Releases Wiki Activity

Keep offline package verification reproducible with cold caches (U5)

Browse Source 
Why:
An immutable npm lock caches tarballs but packed consumers also require resolver metadata before candidate execution.

Evidence:
SELF host replay passed 57 composition and CI credential cases; real npm 10 and 11 cold-cache consumers passed without lock changes.

Impact:
Acquire only installed exact-version npmjs metadata with clean config, preserve cache paths for offline execution, adopt the repaired Frontend source and record immutable CI fixtures.

Refs:
R6-R8, R14; dependency-workspace U4/U5; UI 63d289e and Frontend 5467517
This commit is contained in:
weichen
2026-10-14 01:44:49 +00:00
parent 222687f149
commit 01b7799663
8 changed files with 364 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/ci.yaml
+40 -4
View File
@@ -647,6 +647,24 @@ jobs:
raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping")
return None
def registry_package(location, entry):
resolved = entry.get("resolved", "")
if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")):
return None
# A metadata spec is name@exact-version, never a candidate URL.
source = urlsplit(resolved)
if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or
source.username or source.password or source.query or source.fragment):
raise ValueError("npm tarball source is outside the reviewed public registry")
relative = Path(location)
name = entry.get("name") or location.rsplit("node_modules/", 1)[-1]
version = entry.get("version", "")
if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or
not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or
not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)):
raise ValueError("Invalid locked registry package name or version")
return name + "@" + version
try:
targets = []
selected = set()
@@ -656,7 +674,8 @@ jobs:
if (directory / ".npmrc").exists():
raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition")
package = json.loads((directory / "package.json").read_text())
lock = json.loads((directory / "package-lock.json").read_text())
lock_bytes = (directory / "package-lock.json").read_bytes()
lock = json.loads(lock_bytes)
if lock.get("lockfileVersion") not in {2, 3}:
raise ValueError("npm acquisition requires a package lock")
for group in ("dependencies", "devDependencies", "optionalDependencies"):
@@ -664,7 +683,11 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
for entry in lock.get("packages", {}).values():
registry_packages = []
for location, entry in lock.get("packages", {}).items():
spec = registry_package(location, entry)
if spec:
registry_packages.append((location, spec))
found = git_package(entry.get("resolved"))
if found:
selected.add(found)
@@ -673,7 +696,7 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
targets.append(directory)
targets.append((directory, registry_packages, lock_bytes))
# Fetch only registered fixed Git package commits. npm never receives token config.
for repo, sha in sorted(selected):
repository = mirror(repo, sha, component_token)
@@ -685,12 +708,25 @@ jobs:
raise ValueError("Git package provider is missing committed package outputs")
no_auth = mirror_env()
empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-")
for directory in targets:
for directory, registry_packages, lock_bytes in targets:
result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund",
"--userconfig=/dev/null", "--globalconfig=" + empty_config.name],
cwd=directory, env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm dependency acquisition failed: " + result.stderr.strip())
# npm ci caches tarballs; packed consumers also need resolver
# metadata. Only installed locked registry entries are fetched.
specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()})
if specs:
with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root:
result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts",
"--registry=https://registry.npmjs.org", "--userconfig=/dev/null",
"--globalconfig=" + empty_config.name], cwd=metadata_root,
env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm metadata acquisition failed: " + result.stderr.strip())
if (directory / "package-lock.json").read_bytes() != lock_bytes:
raise ValueError("npm acquisition changed the committed package lock")
except (ValueError, OSError, subprocess.CalledProcessError) as error:
sys.exit("npm prefetch refused: " + str(error))
print("npm packages cached without credentials or lifecycle execution")
components.lock.json
+1 -1
View File
@@ -2,7 +2,7 @@
"schema": 1,
"components": {
"Agent-System-Frontend": {
"commit": "6ed7b9322c5733f76b206a35cd78649ba385d83b"
"commit": "5467517e1831e7ca8b58bb5df52494a3a2c9d468"
},
"Agent-System-Backend": {
"commit": "fffd5790cfab317fadafb40fa55a11bab6e9a847"
scripts/build-composition.py
+45
View File
@@ -25,6 +25,8 @@ SHA = re.compile(r"[0-9a-f]{40}\Z")
IDS = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*\Z")
BACKEND = "Agent-System-Backend"
FRONTEND = "Agent-System-Frontend"
NPM_NAME = re.compile(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*\Z")
NPM_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?\Z")
class CompositionError(Exception):
@@ -165,6 +167,11 @@ def execution_environment(env):
or upper in {"SSH_AUTH_SOCK", "SSH_AGENT_PID", "NODE_OPTIONS", "BASH_ENV", "ENV", "GOFLAGS", "GIT_ASKPASS", "SSH_ASKPASS", "CONTRACTS_TARBALL", "UI_TARBALL"}):
del clean[key]
clean.update(GOWORK="off", GOTOOLCHAIN="local", GOFLAGS="", CGO_ENABLED="0")
# Cache location is transport-independent state, shared by acquisition and
# offline consumers. Other npm settings can carry credentials or hooks.
for key in ("npm_config_cache", "NPM_CONFIG_CACHE"):
if env.get(key):
clean[key] = env[key]
return clean
@@ -251,6 +258,24 @@ def go_dependencies(source, fetch_env):
"downloads": [{key: value[key] for key in ("Path", "Version", "Sum", "GoModSum", "Origin") if key in value} for value in downloads]}
def npm_registry_package(location, package):
resolved = package.get("resolved", "")
if not isinstance(resolved, str):
raise CompositionError("invalid npm lock dependency")
if not resolved.startswith(("https://", "http://")):
return None
registry = urlsplit(resolved)
if (registry.scheme != "https" or registry.netloc != "registry.npmjs.org" or
registry.username or registry.password or registry.query or registry.fragment):
raise CompositionError("npm tarball source is outside the reviewed public registry")
name = package.get("name") or location.rsplit("node_modules/", 1)[-1]
version = package.get("version", "")
if (not isinstance(name, str) or not NPM_NAME.fullmatch(name) or
not isinstance(version, str) or not NPM_VERSION.fullmatch(version)):
raise CompositionError("invalid locked registry package name or version")
return name + "@" + version
def npm_dependencies(source, components, fetch_env):
path = source / "package-lock.json"
lock = json.loads(path.read_text())
@@ -258,7 +283,13 @@ def npm_dependencies(source, components, fetch_env):
raise CompositionError("frontend requires a complete npm lock")
approved = {component["repo"] for component in components.values()}
packages = {}
registry_packages = []
for name, package in lock["packages"].items():
if name:
safe_relative(source, name)
registry_package = npm_registry_package(name, package)
if registry_package:
registry_packages.append((name, registry_package))
if package.get("link"):
raise CompositionError("formal npm build forbids workspace links")
for value in [package.get("resolved", ""), *package.get("dependencies", {}).values()]:
@@ -273,6 +304,20 @@ def npm_dependencies(source, components, fetch_env):
packages[name] = {key: package[key] for key in ("version", "resolved", "integrity") if key in package}
before = digest(path)
run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund"], source, fetch_env)
# npm ci normally stores tarballs without the packuments required when a
# disposable packed-package consumer re-resolves peers offline. Fetch the
# installed locked versions before candidate execution, from a fixed public
# registry with empty config and no acquisition credentials or candidate rc.
metadata_env = execution_environment(fetch_env)
with tempfile.TemporaryDirectory(prefix="composition-npm-metadata-") as directory:
isolated = Path(directory)
global_config = isolated / "empty-global.npmrc"
global_config.write_text("")
selected = {spec for location, spec in registry_packages if (source / location).is_dir()}
if selected:
run(["npm", "cache", "add", *sorted(selected), "--ignore-scripts",
"--registry=https://registry.npmjs.org", "--userconfig=/dev/null",
"--globalconfig=" + str(global_config)], isolated, metadata_env)
if digest(path) != before:
raise CompositionError("npm acquisition changed committed lock")
result = {"lock_sha256": before, "packages": packages}
tests/fixtures/component-ci/Agent-System-Frontend/ci.yml
Vendored
+40 -4
View File
@@ -408,6 +408,24 @@ jobs:
raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping")
return None
def registry_package(location, entry):
resolved = entry.get("resolved", "")
if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")):
return None
# A metadata spec is name@exact-version, never a candidate URL.
source = urlsplit(resolved)
if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or
source.username or source.password or source.query or source.fragment):
raise ValueError("npm tarball source is outside the reviewed public registry")
relative = Path(location)
name = entry.get("name") or location.rsplit("node_modules/", 1)[-1]
version = entry.get("version", "")
if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or
not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or
not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)):
raise ValueError("Invalid locked registry package name or version")
return name + "@" + version
try:
targets = []
selected = set()
@@ -417,7 +435,8 @@ jobs:
if (directory / ".npmrc").exists():
raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition")
package = json.loads((directory / "package.json").read_text())
lock = json.loads((directory / "package-lock.json").read_text())
lock_bytes = (directory / "package-lock.json").read_bytes()
lock = json.loads(lock_bytes)
if lock.get("lockfileVersion") not in {2, 3}:
raise ValueError("npm acquisition requires a package lock")
for group in ("dependencies", "devDependencies", "optionalDependencies"):
@@ -425,7 +444,11 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
for entry in lock.get("packages", {}).values():
registry_packages = []
for location, entry in lock.get("packages", {}).items():
spec = registry_package(location, entry)
if spec:
registry_packages.append((location, spec))
found = git_package(entry.get("resolved"))
if found:
selected.add(found)
@@ -434,7 +457,7 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
targets.append(directory)
targets.append((directory, registry_packages, lock_bytes))
# Fetch only registered fixed Git package commits. npm never receives token config.
for repo, sha in sorted(selected):
repository = mirror(repo, sha, component_token)
@@ -446,12 +469,25 @@ jobs:
raise ValueError("Git package provider is missing committed package outputs")
no_auth = mirror_env()
empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-")
for directory in targets:
for directory, registry_packages, lock_bytes in targets:
result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund",
"--userconfig=/dev/null", "--globalconfig=" + empty_config.name],
cwd=directory, env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm dependency acquisition failed: " + result.stderr.strip())
# npm ci caches tarballs; packed consumers also need resolver
# metadata. Only installed locked registry entries are fetched.
specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()})
if specs:
with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root:
result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts",
"--registry=https://registry.npmjs.org", "--userconfig=/dev/null",
"--globalconfig=" + empty_config.name], cwd=metadata_root,
env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm metadata acquisition failed: " + result.stderr.strip())
if (directory / "package-lock.json").read_bytes() != lock_bytes:
raise ValueError("npm acquisition changed the committed package lock")
except (ValueError, OSError, subprocess.CalledProcessError) as error:
sys.exit("npm prefetch refused: " + str(error))
print("npm packages cached without credentials or lifecycle execution")
tests/fixtures/component-ci/Agent-System-UI/ci.yml
Vendored
+40 -4
View File
@@ -406,6 +406,24 @@ jobs:
raise ValueError("npm Git source is outside the reviewed fixed-SHA package mapping")
return None
def registry_package(location, entry):
resolved = entry.get("resolved", "")
if not isinstance(resolved, str) or not resolved.startswith(("https://", "http://")):
return None
# A metadata spec is name@exact-version, never a candidate URL.
source = urlsplit(resolved)
if (source.scheme != "https" or source.netloc != "registry.npmjs.org" or
source.username or source.password or source.query or source.fragment):
raise ValueError("npm tarball source is outside the reviewed public registry")
relative = Path(location)
name = entry.get("name") or location.rsplit("node_modules/", 1)[-1]
version = entry.get("version", "")
if (not location.startswith("node_modules/") or relative.is_absolute() or ".." in relative.parts or
not isinstance(name, str) or not re.fullmatch(r"(?:@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*", name) or
not isinstance(version, str) or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", version)):
raise ValueError("Invalid locked registry package name or version")
return name + "@" + version
try:
targets = []
selected = set()
@@ -415,7 +433,8 @@ jobs:
if (directory / ".npmrc").exists():
raise ValueError("Candidate .npmrc is not allowed during private dependency acquisition")
package = json.loads((directory / "package.json").read_text())
lock = json.loads((directory / "package-lock.json").read_text())
lock_bytes = (directory / "package-lock.json").read_bytes()
lock = json.loads(lock_bytes)
if lock.get("lockfileVersion") not in {2, 3}:
raise ValueError("npm acquisition requires a package lock")
for group in ("dependencies", "devDependencies", "optionalDependencies"):
@@ -423,7 +442,11 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
for entry in lock.get("packages", {}).values():
registry_packages = []
for location, entry in lock.get("packages", {}).items():
spec = registry_package(location, entry)
if spec:
registry_packages.append((location, spec))
found = git_package(entry.get("resolved"))
if found:
selected.add(found)
@@ -432,7 +455,7 @@ jobs:
found = git_package(value)
if found:
selected.add(found)
targets.append(directory)
targets.append((directory, registry_packages, lock_bytes))
# Fetch only registered fixed Git package commits. npm never receives token config.
for repo, sha in sorted(selected):
repository = mirror(repo, sha, component_token)
@@ -444,12 +467,25 @@ jobs:
raise ValueError("Git package provider is missing committed package outputs")
no_auth = mirror_env()
empty_config = tempfile.NamedTemporaryFile(prefix="ci-npm-globalrc-")
for directory in targets:
for directory, registry_packages, lock_bytes in targets:
result = subprocess.run(["npm", "ci", "--ignore-scripts", "--no-audit", "--no-fund",
"--userconfig=/dev/null", "--globalconfig=" + empty_config.name],
cwd=directory, env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm dependency acquisition failed: " + result.stderr.strip())
# npm ci caches tarballs; packed consumers also need resolver
# metadata. Only installed locked registry entries are fetched.
specs = sorted({spec for location, spec in registry_packages if (directory / location).is_dir()})
if specs:
with tempfile.TemporaryDirectory(prefix="ci-npm-metadata-") as metadata_root:
result = subprocess.run(["npm", "cache", "add", *specs, "--ignore-scripts",
"--registry=https://registry.npmjs.org", "--userconfig=/dev/null",
"--globalconfig=" + empty_config.name], cwd=metadata_root,
env=no_auth, capture_output=True, text=True)
if result.returncode:
raise ValueError("npm metadata acquisition failed: " + result.stderr.strip())
if (directory / "package-lock.json").read_bytes() != lock_bytes:
raise ValueError("npm acquisition changed the committed package lock")
except (ValueError, OSError, subprocess.CalledProcessError) as error:
sys.exit("npm prefetch refused: " + str(error))
print("npm packages cached without credentials or lifecycle execution")
tests/fixtures/component-ci/provenance.json
Vendored
+4 -4
View File
@@ -53,15 +53,15 @@
},
"Agent-System-UI": {
"repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-UI.git",
"source_sha": "bc0a4d33d03728e8726b5c398c4239a87def8eda",
"source_sha": "63d289e6bc15b701356cfe2d5b8740210fac0dcc",
"workflow_path": ".gitea/workflows/ci.yml",
"workflow_sha256": "ef41bf7d981c37246d71fe80c09fa077341802442aad213b14c9f18a17a58694"
"workflow_sha256": "03fb580f523649db843a21ff70e3d21c5165198c1a8cf0a0bea882c3ec04f8f1"
},
"Agent-System-Frontend": {
"repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-Frontend.git",
"source_sha": "6ed7b9322c5733f76b206a35cd78649ba385d83b",
"source_sha": "5467517e1831e7ca8b58bb5df52494a3a2c9d468",
"workflow_path": ".gitea/workflows/ci.yml",
"workflow_sha256": "51aeaa0e2ced11bb36fd6b66ac8da66a1f2efdbd6a5d6036302159646cabc3be"
"workflow_sha256": "802e5c199afd8805d87f0fd1958cd001960d8d820a4e6acb14a401007d278012"
},
"Agent-System-RDM": {
"repository": "ssh://git@git-ssh.u1680832.nyat.app:31411/weichen/Agent-System-RDM.git",
tests/test_composition.py
+152 -1
View File
@@ -1,14 +1,20 @@
"""Real Git/npm/Go fixtures for immutable product composition."""
import base64
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tarfile
import tempfile
import threading
import unittest
import zipfile
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
SCRIPT = Path(__file__).resolve().parents[1] / "scripts/build-composition.py"
@@ -16,6 +22,151 @@ RDM_SCRIPTS = Path(os.environ.get("COMPOSITION_RDM_SOURCE", str(SCRIPT.parent.pa
LEGACY_PRODUCT = Path(__file__).resolve().parent / "fixtures/rdm-legacy-product.sh"
class LocalNpmRegistry:
"""Real npm cache/registry traffic with no public-network dependency."""
name = "composition-cold-fixture"
def __init__(self, base, env):
self.base, self.env = base, env
self.requests = []
self.tarballs = {}
self.integrities = {}
for version in ("1.0.0", "1.0.1"):
payload = json.dumps({"name": self.name, "version": version,
"scripts": {"install": "node -e 'process.exit(91)'"}}).encode()
stream = io.BytesIO()
with tarfile.open(fileobj=stream, mode="w:gz") as archive:
entry = tarfile.TarInfo("package/package.json")
entry.size = len(payload)
archive.addfile(entry, io.BytesIO(payload))
self.tarballs[version] = stream.getvalue()
self.integrities[version] = "sha512-" + base64.b64encode(hashlib.sha512(stream.getvalue()).digest()).decode()
registry = self
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
registry.requests.append((self.path, self.headers.get("Authorization")))
if self.path == "/" + registry.name:
versions = {v: {"name": registry.name, "version": v, "dist": {
"tarball": registry.url + "/" + registry.name + "/-/" + v + ".tgz",
"integrity": registry.integrities[v]}} for v in registry.tarballs}
body = json.dumps({"name": registry.name, "dist-tags": {"latest": "1.0.1"}, "versions": versions}).encode()
content_type = "application/json"
else:
version = self.path.rsplit("/", 1)[-1].removesuffix(".tgz")
if version not in registry.tarballs:
self.send_error(404)
return
body, content_type = registry.tarballs[version], "application/octet-stream"
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *args):
pass
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
self.url = "http://127.0.0.1:" + str(self.server.server_port)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.cache = base / "cold-npm-cache"
self.global_config = base / "empty-npm-globalrc"
self.global_config.write_text("")
self.home = base / "cold-home"
self.home.mkdir()
self.tools = base / "npm-tools"
self.tools.mkdir()
self.calls = base / "npm-calls.jsonl"
npm = shutil.which("npm")
wrapper = self.tools / "npm"
wrapper.write_text("#!" + sys.executable + "\n" +
"import json, os, sys\nfrom pathlib import Path\nargs = sys.argv[1:]\n" +
"with Path(" + repr(str(self.calls)) + ").open('a') as output:\n" +
" output.write(json.dumps({'args': args, 'credential_keys': [k for k in os.environ if any(w in k.upper() for w in ('TOKEN', 'SECRET', 'PASSWORD'))]}) + '\\n')\n" +
"if args[:2] == ['cache', 'add'] and args[2].startswith(" + repr(self.name + "@") + "):\n" +
" args = [a.replace('https://registry.npmjs.org', " + repr(self.url) + ") for a in args]\n" +
"if args[:1] == ['ci']: args.append('--offline')\n" +
"os.execv(" + repr(npm) + ", [" + repr(npm) + ", *args])\n")
wrapper.chmod(0o755)
self.env = env | {"HOME": str(self.home), "npm_config_cache": str(self.cache),
"PATH": str(self.tools) + os.pathsep + env["PATH"]}
# Seed only the locked tarball, exactly the state produced by npm ci.
tarball = base / "locked.tgz"
tarball.write_bytes(self.tarballs["1.0.0"])
subprocess.run([npm, "cache", "add", str(tarball), "--ignore-scripts", "--userconfig=/dev/null",
"--globalconfig=" + str(self.global_config)], env=self.env, check=True, capture_output=True)
def close(self):
self.server.shutdown()
self.server.server_close()
self.thread.join()
def manifests(self, source):
source.mkdir(exist_ok=True)
manifest = {"name": "fixture", "version": "1.0.0", "dependencies": {self.name: "1.0.0"}}
(source / "package.json").write_text(json.dumps(manifest))
lock = {"name": "fixture", "version": "1.0.0", "lockfileVersion": 3, "requires": True,
"packages": {"": manifest, "node_modules/" + self.name: {
"version": "1.0.0", "resolved": "https://registry.npmjs.org/" + self.name + "/-/1.0.0.tgz",
"integrity": self.integrities["1.0.0"]}}}
(source / "package-lock.json").write_text(json.dumps(lock))
return (source / "package-lock.json").read_bytes()
def consumer(self, directory, env=None):
directory.mkdir(exist_ok=True)
(directory / "package.json").write_text(json.dumps({"dependencies": {self.name: "1.0.0"}}))
return subprocess.run(["npm", "install", "--offline", "--ignore-scripts", "--no-audit", "--no-fund",
"--registry=" + self.url, "--userconfig=/dev/null", "--globalconfig=" + str(self.global_config)],
cwd=directory, env=env or self.env, capture_output=True, text=True)
@unittest.skipUnless(shutil.which("npm"), "npm required")
class NpmAcquisitionTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory(prefix="composition-cold-npm-")
self.addCleanup(self.temporary.cleanup)
self.base = Path(self.temporary.name)
self.registry = LocalNpmRegistry(self.base, os.environ.copy())
self.addCleanup(self.registry.close)
spec = importlib.util.spec_from_file_location("cold_npm_composer", SCRIPT)
self.composer = importlib.util.module_from_spec(spec)
spec.loader.exec_module(self.composer)
def test_locked_metadata_is_acquired_before_offline_consumer_without_credentials(self):
source = self.base / "source"
original = self.registry.manifests(source)
red = self.registry.consumer(self.base / "before")
self.assertNotEqual(red.returncode, 0)
self.assertIn("ENOTCACHED", red.stderr)
self.composer.npm_dependencies(source, {}, self.registry.env | {"GITEA_TOKEN": "synthetic-secret"})
self.assertEqual((source / "package-lock.json").read_bytes(), original)
green = self.registry.consumer(self.base / "after", self.composer.execution_environment(self.registry.env))
self.assertEqual(green.returncode, 0, green.stdout + green.stderr)
actual = json.loads((self.base / "after/node_modules" / self.registry.name / "package.json").read_text())
self.assertEqual(actual["version"], "1.0.0")
calls = [json.loads(line) for line in self.registry.calls.read_text().splitlines()]
metadata = [call for call in calls if call["args"][:2] == ["cache", "add"]]
self.assertTrue(metadata, "tarball acquisition alone does not populate offline resolver metadata")
self.assertEqual(metadata[0]["args"][2], self.registry.name + "@1.0.0")
self.assertFalse(metadata[0]["credential_keys"])
self.assertTrue(self.registry.requests)
self.assertTrue(all(auth is None for _, auth in self.registry.requests))
def test_unreviewed_tarball_source_is_refused_before_acquisition(self):
source = self.base / "source"
self.registry.manifests(source)
path = source / "package-lock.json"
lock = json.loads(path.read_text())
lock["packages"]["node_modules/" + self.registry.name]["resolved"] = self.registry.url + "/untrusted.tgz"
path.write_text(json.dumps(lock))
with self.assertRaisesRegex(self.composer.CompositionError, "reviewed public registry"):
self.composer.npm_dependencies(source, {}, self.registry.env | {"GITEA_TOKEN": "synthetic-secret"})
self.assertFalse(self.registry.requests)
self.assertFalse(self.registry.calls.exists())
@unittest.skipUnless(all(shutil.which(c) for c in ("git", "go", "node", "npm")), "Git/Go/Node/npm required")
class CompositionTests(unittest.TestCase):
def setUp(self):
@@ -118,7 +269,7 @@ args = sys.argv[1:]
name = pathlib.Path(sys.argv[0]).name
acquisition = (name == 'git' and args and args[0] in {'clone', 'fetch', 'archive'}
or name == 'go' and args and (args[0] == 'mod' or args[0] == 'list')
or name == 'npm' and args and args[0] in {'ci', 'install'})
or name == 'npm' and args and (args[0] in {'ci', 'install'} or args[:2] == ['cache', 'add']))
if acquisition and pathlib.Path(os.environ['COMPOSITION_EXEC_MARKER']).exists():
print('acquisition-after-execution: ' + name + ' ' + ' '.join(args[:2]), file=sys.stderr)
sys.exit(86)
tests/test_dependency_ci.py
+42
View File
@@ -10,6 +10,7 @@ import textwrap
import unittest
from tests import test_ci_credentials as fixture
from tests.test_composition import LocalNpmRegistry
ROOT_TOKEN, SUB_TOKEN, SSH, WORKFLOW = fixture.ROOT_TOKEN, fixture.SUB_TOKEN, fixture.SSH, fixture.WORKFLOW
@@ -242,6 +243,47 @@ class DependencyCITests(unittest.TestCase):
self.assertRegex(path, r"^/weichen/Agent-System-Core\.git/")
self.assertEqual(auth, "token " + SUB_TOKEN)
def test_npm_prefetch_populates_cold_offline_resolver_metadata_without_credentials(self):
workflows = {"Agent-System-Dev": WORKFLOW, **{repo: fixture.component_workflow(repo)
for repo in ("Agent-System-UI", "Agent-System-Frontend")}}
for repo, workflow in workflows.items():
with self.subTest(repo=repo):
base = self.base / repo
base.mkdir()
registry = LocalNpmRegistry(base, self.env)
self.addCleanup(registry.close)
original = registry.manifests(self.checkout)
red = registry.consumer(base / "cold-before")
self.assertNotEqual(red.returncode, 0)
self.assertIn("ENOTCACHED", red.stderr)
result = self.run_phase("Prefetch npm packages without lifecycle scripts", workflow=workflow, extra=registry.env)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual((self.checkout / "package-lock.json").read_bytes(), original)
green = registry.consumer(base / "cold-after")
self.assertEqual(green.returncode, 0, green.stdout + green.stderr)
installed = json.loads((base / "cold-after/node_modules" / registry.name / "package.json").read_text())
self.assertEqual(installed["version"], "1.0.0")
calls = [json.loads(line) for line in registry.calls.read_text().splitlines()]
metadata = [call for call in calls if call["args"][:2] == ["cache", "add"]]
self.assertTrue(metadata)
self.assertEqual(metadata[0]["args"][2], registry.name + "@1.0.0")
self.assertFalse(metadata[0]["credential_keys"])
self.assertTrue(all(auth is None for _, auth in registry.requests))
def test_npm_metadata_version_cannot_be_a_destination(self):
(self.checkout / "package.json").write_text(json.dumps({"name": "fixture"}))
(self.checkout / "package-lock.json").write_text(json.dumps({"lockfileVersion": 3, "packages": {
"node_modules/evil": {"version": self.url + "/untrusted.tgz",
"resolved": "https://registry.npmjs.org/evil/-/evil-1.0.0.tgz"}}}))
workflows = {"Agent-System-Dev": WORKFLOW, **{repo: fixture.component_workflow(repo)
for repo in ("Agent-System-UI", "Agent-System-Frontend")}}
for repo, workflow in workflows.items():
with self.subTest(repo=repo):
result = self.run_phase("Prefetch npm packages without lifecycle scripts", workflow=workflow)
self.assertNotEqual(result.returncode, 0)
self.assertIn("locked registry package", result.stderr)
self.assertFalse(self.requests)
def test_npm_provider_prepare_is_rejected_without_execution(self):
marker = self.base / "prepare-ran"
sha = self.commit_fixture("Agent-System-Core", {