From fda159c995bb713cf93dbe6096131147b350ad98 Mon Sep 17 00:00:00 2001 From: weichen Date: Wed, 16 Sep 2026 14:02:24 +0000 Subject: [PATCH] feat(ci): nightly real-PI workflow and promote/release receipt gates Add schedule/dispatch nightly entry that classifies infra vs product failures, and require a valid receipt (or waiver) before promote/release. Co-authored-by: Cursor --- .github/workflows/nightly-real-pi.yml | 76 ++++++++++++++ scripts/git-workflow-helper.sh | 10 ++ scripts/promote-to-test.sh | 6 ++ scripts/release-to-main.sh | 6 ++ scripts/run-nightly-real-pi.sh | 140 ++++++++++++++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 .github/workflows/nightly-real-pi.yml create mode 100755 scripts/promote-to-test.sh create mode 100755 scripts/release-to-main.sh create mode 100755 scripts/run-nightly-real-pi.sh diff --git a/.github/workflows/nightly-real-pi.yml b/.github/workflows/nightly-real-pi.yml new file mode 100644 index 00000000..17246915 --- /dev/null +++ b/.github/workflows/nightly-real-pi.yml @@ -0,0 +1,76 @@ +# Nightly real PI regression (Wave 2B / PU5B). +# PR CI (ci.yml) intentionally does NOT run real PI. +name: Nightly Real PI + +on: + schedule: + # Daily 16:00 UTC ≈ 00:00 CST + - cron: "0 16 * * *" + workflow_dispatch: + +concurrency: + group: nightly-real-pi + cancel-in-progress: true + +jobs: + nightly-real-pi: + runs-on: ubuntu-latest + timeout-minutes: 120 + permissions: + contents: read + env: + # Optional overrides via repo vars; script defaults apply when empty. + PI_AGENT_RUNNER_URL: ${{ vars.PI_AGENT_RUNNER_URL }} + PI_AGENT_MODEL: ${{ vars.PI_AGENT_MODEL }} + PI_AGENT_PROTOCOL: typed-result-v1 + # Prefer dedicated secret; empty → infra_fail via run-nightly-real-pi.sh (never PASS). + RUNNER_AUTH_TOKEN: ${{ secrets.RUNNER_AUTH_TOKEN }} + PI_AGENT_RUNNER_AUTH_TOKEN: ${{ secrets.RUNNER_AUTH_TOKEN }} + TEST_DATABASE_URL: ${{ secrets.NIGHTLY_TEST_DATABASE_URL }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: "1.27.1" + cache-dependency-path: go.sum + + - name: Run nightly real-PI suite (classified receipt) + id: nightly + run: | + chmod +x scripts/run-nightly-real-pi.sh scripts/promotion-receipt-write.sh scripts/promotion-receipt-check.sh + set +e + ./scripts/run-nightly-real-pi.sh + rc=$? + set -e + echo "rc=${rc}" >> "$GITHUB_OUTPUT" + # Always upload receipts/evidence; never treat missing secret as PASS. + exit "${rc}" + + - name: Upload promotion receipt artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: promotion-receipt-${{ github.sha }} + path: | + .doc/test-reports/assets/promotion-receipts/${{ github.sha }}.json + .doc/test-reports/assets/promotion-receipts/evidence-* + if-no-files-found: warn + retention-days: 14 + + - name: Summarize receipt verdict + if: always() + run: | + f=".doc/test-reports/assets/promotion-receipts/${GITHUB_SHA}.json" + if [[ -f "$f" ]]; then + echo "Receipt:" + cat "$f" + verdict="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["verdict"])' "$f")" + echo "verdict=${verdict}" + if [[ "${verdict}" != "pass" ]]; then + echo "::error::nightly real-PI verdict=${verdict} (infrastructure failures must not be reported as PASS)" + fi + else + echo "::error::no promotion receipt written" + exit 1 + fi diff --git a/scripts/git-workflow-helper.sh b/scripts/git-workflow-helper.sh index 30606b07..b6e78ef1 100755 --- a/scripts/git-workflow-helper.sh +++ b/scripts/git-workflow-helper.sh @@ -99,6 +99,11 @@ case "${COMMAND}" in echo "错误: 请提供测试候选 Tag 名称，例如: $0 promote-to-test rc-v1.0.0-beta.1" exit 1 fi + # Wave 2B / PU5B: require PASS receipt (or timed waiver) for local `dev` HEAD. + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + DEV_SHA="$(git rev-parse dev)" + echo "==> promotion-receipt gate for dev@${DEV_SHA:0:12}..." + "${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${DEV_SHA}" echo "==> 切换至 test 分支并合并本地 dev 当前稳定节点..." git checkout test # 本地晋升默认不 rebase origin（分叉时会把数百提交重放并易冲突）。需要同步远端时请显式 pull --ff-only/merge。 @@ -113,6 +118,11 @@ case "${COMMAND}" in echo "错误: 请提供正式发布 Tag 名称，例如: $0 release-to-main v1.0.0" exit 1 fi + # Wave 2B / PU5B: require PASS receipt (or timed waiver) for local `test` HEAD. + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + TEST_SHA="$(git rev-parse test)" + echo "==> promotion-receipt gate for test@${TEST_SHA:0:12}..." + "${SCRIPT_DIR}/promotion-receipt-check.sh" --commit "${TEST_SHA}" echo "==> 切换至 main 分支并合并 test 验证节点..." git checkout main git pull --rebase origin main 2>/dev/null || true diff --git a/scripts/promote-to-test.sh b/scripts/promote-to-test.sh new file mode 100755 index 00000000..f15e5c9b --- /dev/null +++ b/scripts/promote-to-test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thin wrapper: promote current local `dev` into `test` with a candidate tag. +# Enforces promotion-receipt gate via git-workflow-helper. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec "${ROOT}/scripts/git-workflow-helper.sh" promote-to-test "$@" diff --git a/scripts/release-to-main.sh b/scripts/release-to-main.sh new file mode 100755 index 00000000..53b91dcd --- /dev/null +++ b/scripts/release-to-main.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thin wrapper: release verified `test` into `main` with a release tag. +# Enforces promotion-receipt gate via git-workflow-helper. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec "${ROOT}/scripts/git-workflow-helper.sh" release-to-main "$@" diff --git a/scripts/run-nightly-real-pi.sh b/scripts/run-nightly-real-pi.sh new file mode 100755 index 00000000..2da4353f --- /dev/null +++ b/scripts/run-nightly-real-pi.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Local / Actions entry for Wave 2B nightly real-PI suite. +# Classifies infrastructure vs product failures and writes a promotion receipt. +# +# Usage: +# ./scripts/run-nightly-real-pi.sh [--dry-infra-fail] +# +# Environment: +# PI_AGENT_RUNNER_URL default http://127.0.0.1:18865 +# RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN +# TEST_DATABASE_URL required for product run (isolated PG) +# PI_AGENT_MODEL default MiniMax-M2.7-highspeed +# NIGHTLY_SUITE_TIMEOUT go test timeout (default 90m) +# NIGHTLY_EVIDENCE_DIR override evidence dir +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT}" + +DRY_INFRA_FAIL=0 +if [[ "${1:-}" == "--dry-infra-fail" ]]; then + DRY_INFRA_FAIL=1 +fi + +RUNNER_URL="${PI_AGENT_RUNNER_URL:-http://127.0.0.1:18865}" +TOKEN="${PI_AGENT_RUNNER_AUTH_TOKEN:-${RUNNER_AUTH_TOKEN:-}}" +MODEL_ID="${PI_AGENT_MODEL:-MiniMax-M2.7-highspeed}" +PROTOCOL="${PI_AGENT_PROTOCOL:-typed-result-v1}" +TIMEOUT="${NIGHTLY_SUITE_TIMEOUT:-90m}" +COMMIT="$(git rev-parse HEAD)" +SUITE="nightly-real-pi-u8-fullnone" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +EVIDENCE_DIR="${NIGHTLY_EVIDENCE_DIR:-${ROOT}/.doc/test-reports/assets/promotion-receipts/evidence-${COMMIT:0:12}-${STAMP}}" +mkdir -p "${EVIDENCE_DIR}" + +write_receipt() { + local verdict="$1" + local failure_class="$2" + shift 2 + local args=( + --verdict "${verdict}" + --commit "${COMMIT}" + --suite "${SUITE}" + --failure-class "${failure_class}" + --runner-url "${RUNNER_URL}" + --model-id "${MODEL_ID}" + --protocol "${PROTOCOL}" + --force + ) + local ref + for ref in "$@"; do + args+=(--evidence "${ref}") + done + PI_AGENT_RUNNER_URL="${RUNNER_URL}" PI_AGENT_MODEL="${MODEL_ID}" PI_AGENT_PROTOCOL="${PROTOCOL}" \ + "${ROOT}/scripts/promotion-receipt-write.sh" "${args[@]}" +} + +classify_test_log() { + local log_file="$1" + # Heuristics: auth/network/timeout/unavailable → infra; assertion/FAIL → product + if grep -Eiq 'context deadline exceeded|connection refused|401 Unauthorized|Unauthorized|runner not available|no such host|i/o timeout|TLS handshake|dial tcp' "${log_file}"; then + echo infra + return + fi + if grep -Eiq '^--- FAIL:|FAIL\t|assertion|Expected|Fatalf|require\.' "${log_file}"; then + echo product + return + fi + echo product +} + +echo "==> nightly real-PI suite commit=${COMMIT:0:12} runner=${RUNNER_URL}" + +if [[ "${DRY_INFRA_FAIL}" -eq 1 ]]; then + echo "forced infra failure (--dry-infra-fail)" | tee "${EVIDENCE_DIR}/infra-forced.txt" + write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-forced.txt" + echo "wrote infra_fail receipt (dry)" + exit 1 +fi + +if [[ -z "${TOKEN}" ]]; then + echo "missing RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN" | tee "${EVIDENCE_DIR}/infra-missing-secret.txt" + write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-secret.txt" + echo "infrastructure failure: missing runner secret" >&2 + exit 1 +fi + +HEALTH_CODE=0 +HEALTH_BODY="" +if ! HEALTH_BODY="$(curl -fsS --max-time 5 "${RUNNER_URL}/health" 2>"${EVIDENCE_DIR}/health.err")"; then + HEALTH_CODE=$? + { + echo "runner health check failed code=${HEALTH_CODE} url=${RUNNER_URL}/health" + cat "${EVIDENCE_DIR}/health.err" 2>/dev/null || true + } | tee "${EVIDENCE_DIR}/infra-runner-down.txt" + write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-runner-down.txt" + echo "infrastructure failure: runner unreachable" >&2 + exit 1 +fi +printf '%s\n' "${HEALTH_BODY}" > "${EVIDENCE_DIR}/runner-health.json" + +if [[ -z "${TEST_DATABASE_URL:-}" ]]; then + echo "TEST_DATABASE_URL not set" | tee "${EVIDENCE_DIR}/infra-missing-db.txt" + write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-db.txt" + echo "infrastructure failure: missing TEST_DATABASE_URL" >&2 + exit 1 +fi + +export PI_AGENT_RUNNER_URL="${RUNNER_URL}" +export PI_AGENT_RUNNER_AUTH_TOKEN="${TOKEN}" +export RUNNER_AUTH_TOKEN="${TOKEN}" +export PI_AGENT_MODEL="${MODEL_ID}" +export RUN_PI_E2E=1 + +# Clear env pollution that flakes short/config tests elsewhere. +unset AGENT_TIMEOUT KB_DATABASE_URL KB_EMBEDDING_DIM 2>/dev/null || true + +RUN_FILTER='TestMultiIntentU8_RealPI_ConfirmGenerate_PG|TestWorkflowGovernance_PiAgent_PG_FullNoneMixedChain' +LOG_FILE="${EVIDENCE_DIR}/go-test.txt" +set +e +go test -count=1 -timeout "${TIMEOUT}" -v ./internal/server/ -run "${RUN_FILTER}" 2>&1 | tee "${LOG_FILE}" +TEST_RC=${PIPESTATUS[0]} +set -e + +if [[ "${TEST_RC}" -eq 0 ]]; then + write_receipt pass none "${LOG_FILE}" "${EVIDENCE_DIR}/runner-health.json" + echo "nightly real-PI PASS; receipt written for ${COMMIT:0:12}" + exit 0 +fi + +CLASS="$(classify_test_log "${LOG_FILE}")" +if [[ "${CLASS}" == "infra" ]]; then + write_receipt infra_fail infra "${LOG_FILE}" + echo "infrastructure failure during product suite" >&2 + exit 1 +fi + +write_receipt product_fail product "${LOG_FILE}" +echo "product failure; promotion must reject this commit" >&2 +exit 1