feat(infra): Wave 2C same-image digest pin for test/main promote

Build/apply candidate images once, require digests on promote, and keep
pinned env-service from rebuilding divergent images.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
weichen
2026-09-16 14:44:26 +00:00
co-authored by Cursor
parent 7c808bdcfc
commit da15eb91b6
18 changed files with 546 additions and 27 deletions
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env bash
# Apply candidate image digests from a promotion receipt to test|main.
# Retags local images and writes configs/.env.<env>.images for pinned compose.
#
# Usage:
# ./scripts/candidate-images-apply.sh <test|main> [--commit SHA]
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${ROOT}"
ENV_NAME="${1:-}"
shift || true
COMMIT=""
while [[ $# -gt 0 ]]; do
case "$1" in
--commit) COMMIT="${2:-}"; shift 2 ;;
-h|--help)
sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//'
exit 0
;;
*) echo "unknown arg: $1" >&2; exit 1 ;;
esac
done
case "${ENV_NAME}" in
test|main) ;;
*) echo "usage: $0 <test|main> [--commit SHA]" >&2; exit 1 ;;
esac
if [[ -z "${COMMIT}" ]]; then
if [[ "${ENV_NAME}" == "test" ]]; then
COMMIT="$(git rev-parse dev)"
else
COMMIT="$(git rev-parse test)"
fi
fi
COMMIT="$(git rev-parse --verify "${COMMIT}^{commit}" 2>/dev/null || echo "${COMMIT}")"
COMMIT="$(echo "${COMMIT}" | tr 'A-F' 'a-f')"
RECEIPT_DIR="${ROOT}/.doc/test-reports/assets/promotion-receipts"
RECEIPT=""
if [[ -f "${RECEIPT_DIR}/${COMMIT}.json" ]]; then
RECEIPT="${RECEIPT_DIR}/${COMMIT}.json"
else
shopt -s nullglob
for f in "${RECEIPT_DIR}"/*.json; do
base="$(basename "${f}" .json)"
[[ "${base}" == *.waiver ]] && continue
[[ "${base}" == *.images ]] && continue
if [[ "${COMMIT}" == "${base}"* || "${base}" == "${COMMIT}"* ]]; then
RECEIPT="${f}"
break
fi
done
fi
if [[ -z "${RECEIPT}" || ! -f "${RECEIPT}" ]]; then
echo "error: no promotion receipt for ${COMMIT}" >&2
exit 1
fi
export RECEIPT ENV_NAME ROOT
python3 <<'PY'
import json, os, subprocess, sys
receipt_path = os.environ["RECEIPT"]
env_name = os.environ["ENV_NAME"]
root = os.environ["ROOT"]
with open(receipt_path, encoding="utf-8") as f:
receipt = json.load(f)
verdict = receipt.get("verdict")
if verdict != "pass":
# allow apply when digests present on a pass-only path; still warn
print(f"warning: receipt verdict={verdict!r} (expected pass)", file=sys.stderr)
digests = receipt.get("image_digests") or {}
app_d = digests.get("app") or ""
events_d = digests.get("events") or ""
pi_d = digests.get("pi-agent") or ""
missing = [k for k, v in (("app", app_d), ("events", events_d), ("pi-agent", pi_d)) if not v]
if missing:
print(f"error: receipt missing image_digests: {', '.join(missing)}", file=sys.stderr)
sys.exit(1)
def resolve_local_id(ref: str) -> str:
# ref may be sha256:ID, name@sha256:..., or full image id
candidates = [ref]
if ref.startswith("sha256:"):
candidates.append(ref)
elif "@sha256:" in ref:
candidates.append(ref.split("@", 1)[1])
for c in candidates:
try:
out = subprocess.check_output(
["docker", "image", "inspect", "--format", "{{.Id}}", c],
stderr=subprocess.DEVNULL,
text=True,
).strip()
if out:
return out
except subprocess.CalledProcessError:
continue
print(f"error: image not found locally for digest/ref: {ref}", file=sys.stderr)
print("hint: run scripts/candidate-images-build.sh for this commit first", file=sys.stderr)
sys.exit(1)
app_id = resolve_local_id(app_d)
events_id = resolve_local_id(events_d)
pi_id = resolve_local_id(pi_d)
tags = {
"app": f"shuimu-agent-app:{env_name}",
"events": f"shuimu-agent-events:{env_name}",
"pi-agent": f"shuimu-pi-agent:{env_name}",
}
for img_id, tag in ((app_id, tags["app"]), (events_id, tags["events"]), (pi_id, tags["pi-agent"])):
subprocess.check_call(["docker", "tag", img_id, tag])
got = subprocess.check_output(
["docker", "image", "inspect", "--format", "{{.Id}}", tag], text=True
).strip()
if got != img_id:
print(f"error: retag mismatch for {tag}: want {img_id} got {got}", file=sys.stderr)
sys.exit(1)
print(f"tagged {tag} -> {got}")
# Pin compose to retagged tags (same IDs as receipt).
env_path = os.path.join(root, f"configs/.env.{env_name}.images")
lines = [
f"# generated by candidate-images-apply.sh from {os.path.basename(receipt_path)}",
f"APP_IMAGE={tags['app']}",
f"EVENTS_IMAGE={tags['events']}",
f"PI_AGENT_IMAGE={tags['pi-agent']}",
f"PINNED_IMAGE_MODE=1",
f"PINNED_COMMIT={receipt.get('commit') or ''}",
f"PINNED_APP_DIGEST={app_d}",
f"PINNED_EVENTS_DIGEST={events_d}",
f"PINNED_PI_AGENT_DIGEST={pi_d}",
"",
]
with open(env_path, "w", encoding="utf-8") as f:
f.write("\n".join(lines))
print(f"wrote {env_path}")
PY
+110
View File
@@ -0,0 +1,110 @@
#!/usr/bin/env bash
# Build candidate app (+events) and pi-agent images once for a commit.
# Writes .doc/test-reports/assets/promotion-receipts/<sha>.images.json
#
# Usage:
# ./scripts/candidate-images-build.sh [--commit SHA] [--force]
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "${ROOT}"
COMMIT=""
FORCE=0
while [[ $# -gt 0 ]]; do
case "$1" in
--commit) COMMIT="${2:-}"; shift 2 ;;
--force) FORCE=1; shift ;;
-h|--help)
sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//'
exit 0
;;
*) echo "unknown arg: $1" >&2; exit 1 ;;
esac
done
if [[ -z "${COMMIT}" ]]; then
COMMIT="$(git rev-parse HEAD)"
fi
COMMIT="$(git rev-parse --verify "${COMMIT}^{commit}")"
SHORT="${COMMIT:0:12}"
RECEIPT_DIR="${ROOT}/.doc/test-reports/assets/promotion-receipts"
OUT="${RECEIPT_DIR}/${COMMIT}.images.json"
mkdir -p "${RECEIPT_DIR}"
if [[ -f "${OUT}" && "${FORCE}" -ne 1 ]]; then
echo "images.json already exists (immutable): ${OUT}" >&2
echo "hint: use --force to rebuild local fixtures" >&2
cat "${OUT}"
exit 0
fi
APP_TAG="shuimu-agent-app:candidate-${SHORT}"
EVENTS_TAG="shuimu-agent-events:candidate-${SHORT}"
PI_TAG="shuimu-pi-agent:candidate-${SHORT}"
echo "==> building app/events image ${APP_TAG}..."
docker build -f deploy/app/Dockerfile -t "${APP_TAG}" .
# events binary lives in the same image
docker tag "${APP_TAG}" "${EVENTS_TAG}"
echo "==> building pi-agent image ${PI_TAG}..."
docker build -f deploy/pi-agent/Dockerfile -t "${PI_TAG}" .
image_id() {
docker inspect --format '{{.Id}}' "$1"
}
repo_digest_or_id() {
local name="$1"
local repo
repo="$(docker inspect --format '{{index .RepoDigests 0}}' "${name}" 2>/dev/null || true)"
if [[ -n "${repo}" && "${repo}" != "<no value>" ]]; then
printf '%s' "${repo}"
else
image_id "${name}"
fi
}
APP_ID="$(image_id "${APP_TAG}")"
EVENTS_ID="$(image_id "${EVENTS_TAG}")"
PI_ID="$(image_id "${PI_TAG}")"
APP_DIGEST="$(repo_digest_or_id "${APP_TAG}")"
EVENTS_DIGEST="$(repo_digest_or_id "${EVENTS_TAG}")"
PI_DIGEST="$(repo_digest_or_id "${PI_TAG}")"
RECORDED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
export COMMIT SHORT RECORDED_AT APP_TAG EVENTS_TAG PI_TAG
export APP_ID EVENTS_ID PI_ID APP_DIGEST EVENTS_DIGEST PI_DIGEST OUT
python3 -c '
import json, os
payload = {
"commit": os.environ["COMMIT"],
"recorded_at": os.environ["RECORDED_AT"],
"tags": {
"app": os.environ["APP_TAG"],
"events": os.environ["EVENTS_TAG"],
"pi-agent": os.environ["PI_TAG"],
},
"image_ids": {
"app": os.environ["APP_ID"],
"events": os.environ["EVENTS_ID"],
"pi-agent": os.environ["PI_ID"],
},
"image_digests": {
"app": os.environ["APP_DIGEST"],
"events": os.environ["EVENTS_DIGEST"],
"pi-agent": os.environ["PI_DIGEST"],
},
}
with open(os.environ["OUT"], "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
f.write("\n")
print(os.environ["OUT"])
'
echo "wrote candidate images manifest: ${OUT}"
echo "app=${APP_DIGEST}"
echo "events=${EVENTS_DIGEST}"
echo "pi-agent=${PI_DIGEST}"
+27 -4
View File
@@ -100,12 +100,26 @@ run_compose() {
if [[ -f "${env_file}" ]]; then
env_args+=(--env-file "${env_file}")
fi
# Wave 2C: optional pinned image digests from candidate-images-apply.sh
local images_file="configs/.env.${target}.images"
local pinned=0
if [[ -f "${images_file}" ]]; then
env_args+=(--env-file "${images_file}")
if grep -q '^PINNED_IMAGE_MODE=1' "${images_file}" 2>/dev/null; then
pinned=1
fi
fi
case "${act}" in
start)
ensure_shared_network
echo "==> [${target}] 启动服务 (${compose_file})..."
docker compose -f "${compose_file}" "${env_args[@]}" up -d
if [[ "${pinned}" -eq 1 && ( "${target}" == "test" || "${target}" == "main" ) ]]; then
echo "==> [${target}] pinned image mode: up -d --no-build"
docker compose -f "${compose_file}" "${env_args[@]}" up -d --no-build
else
docker compose -f "${compose_file}" "${env_args[@]}" up -d
fi
echo "==> [${target}] 服务已启动"
;;
stop)
@@ -119,13 +133,22 @@ run_compose() {
;;
rebuild)
ensure_shared_network
echo "==> [${target}] 重建镜像并重建容器..."
docker compose -f "${compose_file}" "${env_args[@]}" up -d --build
echo "==> [${target}] 镜像与容器已更新"
if [[ "${pinned}" -eq 1 && ( "${target}" == "test" || "${target}" == "main" ) ]]; then
echo "==> [${target}] pinned image mode: recreate from pinned digests (refusing docker build)"
docker compose -f "${compose_file}" "${env_args[@]}" up -d --no-build --force-recreate
echo "==> [${target}] 已按 pin 重建容器(未重新 build 镜像)"
else
echo "==> [${target}] 重建镜像并重建容器..."
docker compose -f "${compose_file}" "${env_args[@]}" up -d --build
echo "==> [${target}] 镜像与容器已更新"
fi
;;
status|ps)
echo "------------------------------------------------------------"
echo "环境: [${target}] (编排: ${compose_file})"
if [[ "${pinned}" -eq 1 ]]; then
echo "image pin: ${images_file}"
fi
echo "------------------------------------------------------------"
docker compose -f "${compose_file}" "${env_args[@]}" ps
echo ""
+10 -4
View File
@@ -100,16 +100,19 @@ case "${COMMAND}" in
exit 1
fi
# Wave 2B / PU5B: require PASS receipt (or timed waiver) for local `dev` HEAD.
# Wave 2C: require image_digests for same-image promote.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEV_SHA="$(git rev-parse dev)"
echo "==> promotion-receipt gate for dev@${DEV_SHA:0:12}..."
"${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${DEV_SHA}"
"${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${DEV_SHA}" --require-digests
echo "==> 切换至 test 分支并合并本地 dev 当前稳定节点..."
git checkout test
# 本地晋升默认不 rebase origin(分叉时会把数百提交重放并易冲突)。需要同步远端时请显式 pull --ff-only/merge。
git merge --no-ff dev -m "release-candidate: 汇集 dev 功能至 test 分支,准备长期实测 [Tag: ${TAG}]"
git tag -a "${TAG}" -m "实测候选版本: ${TAG}"
echo "已成功合并至 test 并打上标签: ${TAG}!请启动 test 环境进行运维与功能测试。"
echo "==> applying candidate image digests to test..."
"${SCRIPT_DIR}/candidate-images-apply.sh" test --commit "${DEV_SHA}"
echo "已成功合并至 test 并打上标签: ${TAG}!请用 env-service.sh test start/rebuild(pinned 不重建镜像)。"
;;
release-to-main)
@@ -119,16 +122,19 @@ case "${COMMAND}" in
exit 1
fi
# Wave 2B / PU5B: require PASS receipt (or timed waiver) for local `test` HEAD.
# Wave 2C: require image_digests for same-image promote.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TEST_SHA="$(git rev-parse test)"
echo "==> promotion-receipt gate for test@${TEST_SHA:0:12}..."
"${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${TEST_SHA}"
"${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${TEST_SHA}" --require-digests
echo "==> 切换至 main 分支并合并 test 验证节点..."
git checkout main
git pull --rebase origin main 2>/dev/null || true
git merge --no-ff test -m "release: 经长期测试与运维验证通过,正式发布 ${TAG}"
git tag -a "${TAG}" -m "正式发布版本: ${TAG}"
echo "已成功合并至 main 并打上正式发布标签: ${TAG}!"
echo "==> applying candidate image digests to main..."
"${SCRIPT_DIR}/candidate-images-apply.sh" main --commit "${TEST_SHA}"
echo "已成功合并至 main 并打上正式发布标签: ${TAG}!请用 env-service.sh main start/rebuild(pinned 不重建镜像)。"
;;
*)
+19 -9
View File
@@ -3,19 +3,21 @@
# or a non-expired audited waiver binding a successful receipt.
#
# Usage:
# ./scripts/promotion-receipt-check.sh [--commit SHA]
# ./scripts/promotion-receipt-check.sh [--commit SHA] [--require-digests]
#
# Exit 0: eligible to promote
# Exit 1: blocked (missing / product_fail / infra_fail / stale waiver / mismatch)
# Exit 1: blocked (missing / product_fail / infra_fail / stale waiver / mismatch / missing digests)
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RECEIPT_DIR="${ROOT}/.doc/test-reports/assets/promotion-receipts"
COMMIT=""
REQUIRE_DIGESTS=0
while [[ $# -gt 0 ]]; do
case "$1" in
--commit) COMMIT="${2:-}"; shift 2 ;;
--require-digests) REQUIRE_DIGESTS=1; shift ;;
-h|--help)
sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'
exit 0
@@ -48,6 +50,7 @@ find_receipt() {
local base
base="$(basename "${candidate}" .json)"
[[ "${base}" == *.waiver ]] && continue
[[ "${base}" == *.images ]] && continue
if [[ "${sha}" == "${base}"* || "${base}" == "${sha}"* ]]; then
echo "${candidate}"
return 0
@@ -88,7 +91,7 @@ else
WAIVER_PATH=""
fi
export RECEIPT_PATH WAIVER_PATH COMMIT RECEIPT_DIR ROOT
export RECEIPT_PATH WAIVER_PATH COMMIT RECEIPT_DIR ROOT REQUIRE_DIGESTS
python3 <<'PY'
import json, os, sys
@@ -99,6 +102,7 @@ receipt_path = os.environ.get("RECEIPT_PATH") or ""
waiver_path = os.environ.get("WAIVER_PATH") or ""
receipt_dir = os.environ["RECEIPT_DIR"]
root = os.environ.get("ROOT") or "."
require_digests = os.environ.get("REQUIRE_DIGESTS") == "1"
def load(path):
with open(path, encoding="utf-8") as f:
@@ -112,6 +116,14 @@ def ok(msg):
print(f"promotion-receipt-check: OK — {msg}")
sys.exit(0)
def require_image_digests(r, label):
if not require_digests:
return
digests = r.get("image_digests") or {}
missing = [k for k in ("app", "events", "pi-agent") if not digests.get(k)]
if missing:
fail(f"{label} missing image_digests for same-image promote: {', '.join(missing)}")
def resolve_bound_receipt(binds):
if not binds:
return None
@@ -123,10 +135,10 @@ def resolve_bound_receipt(binds):
candidates.append(os.path.join(receipt_dir, binds))
candidates.append(os.path.join(receipt_dir, f"{binds}.json"))
for c in candidates:
if os.path.isfile(c):
if os.path.isfile(c) and not c.endswith(".images.json") and not c.endswith(".waiver.json"):
return c
for name in os.listdir(receipt_dir):
if name.endswith(".waiver.json") or not name.endswith(".json"):
if name.endswith(".waiver.json") or name.endswith(".images.json") or not name.endswith(".json"):
continue
stem = name[:-5]
if binds.startswith(stem) or stem.startswith(binds):
@@ -142,18 +154,16 @@ if receipt_path:
if r_commit and not (commit.startswith(r_commit) or r_commit.startswith(commit)):
fail(f"receipt commit mismatch: receipt={r_commit} target={commit}")
if verdict == "pass":
require_image_digests(r, "PASS receipt")
ok(f"PASS receipt for {commit[:12]} ({receipt_path})")
elif verdict == "product_fail":
fail(f"product_fail receipt rejects promotion ({receipt_path}); failure_class={r.get('failure_class')}")
elif verdict == "infra_fail":
# infra_fail freezes next promote unless a valid timed waiver is present
if not waiver_path:
fail(f"infra_fail receipt freezes promotion ({receipt_path}); provide timed waiver or re-run nightly")
# fall through to waiver validation
else:
fail(f"unknown verdict {verdict!r} in {receipt_path}")
# No PASS receipt — require a valid waiver
if not waiver_path:
fail(f"missing promotion receipt for {commit[:12]} under {receipt_dir}")
@@ -166,7 +176,6 @@ expires = w.get("expires_at")
if not expires:
fail(f"waiver missing expires_at ({waiver_path})")
try:
# support Z and offset
exp = datetime.fromisoformat(expires.replace("Z", "+00:00"))
except Exception as e:
fail(f"waiver expires_at unparseable: {expires} ({e})")
@@ -187,6 +196,7 @@ if not bound_path:
br = load(bound_path)
if br.get("verdict") != "pass":
fail(f"waiver bound receipt is not PASS: {bound_path}")
require_image_digests(br, "waiver-bound PASS receipt")
ok(
f"audited waiver for {commit[:12]} by {owner} until {expires}; "
+30 -3
View File
@@ -90,9 +90,36 @@ digest_for() {
docker inspect --format '{{.Id}}' "${name}" 2>/dev/null || true
}
APP_DIGEST="$(digest_for shuimu-agent-app-dev "${RECEIPT_IMAGE_APP:-}")"
EVENTS_DIGEST="$(digest_for shuimu-agent-events-dev "${RECEIPT_IMAGE_EVENTS:-}")"
PI_DIGEST="$(digest_for shuimu-pi-agent-dev "${RECEIPT_IMAGE_PI_AGENT:-}")"
# Prefer candidate-images-build manifest when present.
IMAGES_JSON=""
if [[ -f "${RECEIPT_DIR}/${COMMIT}.images.json" ]]; then
IMAGES_JSON="${RECEIPT_DIR}/${COMMIT}.images.json"
else
# Resolve abbreviated commit to images.json via prefix.
shopt -s nullglob
for f in "${RECEIPT_DIR}"/*.images.json; do
base="$(basename "${f}" .images.json)"
if [[ "${COMMIT}" == "${base}"* || "${base}" == "${COMMIT}"* ]]; then
IMAGES_JSON="${f}"
break
fi
done
fi
if [[ -n "${IMAGES_JSON}" ]]; then
APP_DIGEST="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("image_digests",{}).get("app") or "")' "${IMAGES_JSON}")"
EVENTS_DIGEST="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("image_digests",{}).get("events") or "")' "${IMAGES_JSON}")"
PI_DIGEST="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("image_digests",{}).get("pi-agent") or "")' "${IMAGES_JSON}")"
else
APP_DIGEST="$(digest_for shuimu-agent-app-dev "${RECEIPT_IMAGE_APP:-}")"
EVENTS_DIGEST="$(digest_for shuimu-agent-events-dev "${RECEIPT_IMAGE_EVENTS:-}")"
PI_DIGEST="$(digest_for shuimu-pi-agent-dev "${RECEIPT_IMAGE_PI_AGENT:-}")"
# Also try candidate tags for this commit.
SHORT12="${COMMIT:0:12}"
[[ -z "${APP_DIGEST}" ]] && APP_DIGEST="$(digest_for "shuimu-agent-app:candidate-${SHORT12}" "")"
[[ -z "${EVENTS_DIGEST}" ]] && EVENTS_DIGEST="$(digest_for "shuimu-agent-events:candidate-${SHORT12}" "")"
[[ -z "${PI_DIGEST}" ]] && PI_DIGEST="$(digest_for "shuimu-pi-agent:candidate-${SHORT12}" "")"
fi
mkdir -p "${RECEIPT_DIR}"
OUT="${RECEIPT_DIR}/${COMMIT}.json"
+16
View File
@@ -71,6 +71,21 @@ classify_test_log() {
echo "==> nightly real-PI suite commit=${COMMIT:0:12} runner=${RUNNER_URL}"
# Wave 2C: ensure candidate image digests exist for PASS receipts / promote gate.
ensure_candidate_images() {
local images_json="${ROOT}/.doc/test-reports/assets/promotion-receipts/${COMMIT}.images.json"
if [[ -f "${images_json}" ]]; then
echo "==> reusing candidate images manifest ${images_json}"
return 0
fi
if [[ "${NIGHTLY_SKIP_IMAGE_BUILD:-0}" == "1" ]]; then
echo "warning: NIGHTLY_SKIP_IMAGE_BUILD=1 and no images.json; PASS may lack digests" >&2
return 0
fi
echo "==> building candidate images for digest pin..."
"${ROOT}/scripts/candidate-images-build.sh" --commit "${COMMIT}"
}
if [[ "${DRY_INFRA_FAIL}" -eq 1 ]]; then
echo "forced infra failure (--dry-infra-fail)" | tee "${EVIDENCE_DIR}/infra-forced.txt"
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-forced.txt"
@@ -123,6 +138,7 @@ TEST_RC=${PIPESTATUS[0]}
set -e
if [[ "${TEST_RC}" -eq 0 ]]; then
ensure_candidate_images
write_receipt pass none "${LOG_FILE}" "${EVIDENCE_DIR}/runner-health.json"
echo "nightly real-PI PASS; receipt written for ${COMMIT:0:12}"
exit 0