weichen
fed87ad14b
Consolidate approved policies and add local lease, document CAS, commit, candidate and environment gates. Preserve conservative recovery and cleanup limits.
219 lines
8.8 KiB
Bash
Executable File
219 lines
8.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Local / Actions entry for Wave 2B nightly real-PI suite.
|
|
# Classifies infrastructure vs product failures and writes a promotion receipt.
|
|
#
|
|
# Usage:
|
|
# ./scripts/run-nightly-real-pi.sh [--dry-infra-fail]
|
|
#
|
|
# Environment:
|
|
# PI_AGENT_RUNNER_URL default http://127.0.0.1:18865
|
|
# RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN
|
|
# TEST_DATABASE_URL required for product run (isolated PG)
|
|
# PI_AGENT_MODEL default MiniMax-M2.7-highspeed
|
|
# NIGHTLY_SUITE_TIMEOUT go test timeout (default 90m)
|
|
# NIGHTLY_EVIDENCE_DIR override evidence dir
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "${ROOT}"
|
|
|
|
DRY_INFRA_FAIL=0
|
|
if [[ "${1:-}" == "--dry-infra-fail" ]]; then
|
|
DRY_INFRA_FAIL=1
|
|
fi
|
|
|
|
RUNNER_URL="${PI_AGENT_RUNNER_URL:-http://127.0.0.1:18865}"
|
|
TOKEN="${PI_AGENT_RUNNER_AUTH_TOKEN:-${RUNNER_AUTH_TOKEN:-}}"
|
|
MODEL_ID="${PI_AGENT_MODEL:-MiniMax-M2.7-highspeed}"
|
|
PROTOCOL="${PI_AGENT_PROTOCOL:-typed-result-v1}"
|
|
TIMEOUT="${NIGHTLY_SUITE_TIMEOUT:-90m}"
|
|
COMMIT="$(git rev-parse HEAD)"
|
|
SUITE="nightly-real-pi-u8-fullnone"
|
|
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
EVIDENCE_DIR="${NIGHTLY_EVIDENCE_DIR:-${ROOT}/.doc/test-reports/assets/promotion-receipts/evidence-${COMMIT:0:12}-${STAMP}}"
|
|
mkdir -p "${EVIDENCE_DIR}"
|
|
|
|
write_receipt() {
|
|
local verdict="$1"
|
|
local failure_class="$2"
|
|
shift 2
|
|
local args=(
|
|
--verdict "${verdict}"
|
|
--commit "${COMMIT}"
|
|
--suite "${SUITE}"
|
|
--failure-class "${failure_class}"
|
|
--runner-url "${RUNNER_URL}"
|
|
--model-id "${MODEL_ID}"
|
|
--protocol "${PROTOCOL}"
|
|
--force
|
|
)
|
|
local ref
|
|
for ref in "$@"; do
|
|
args+=(--evidence "${ref}")
|
|
done
|
|
PI_AGENT_RUNNER_URL="${RUNNER_URL}" PI_AGENT_MODEL="${MODEL_ID}" PI_AGENT_PROTOCOL="${PROTOCOL}" \
|
|
"${ROOT}/scripts/promotion-receipt-write.sh" "${args[@]}"
|
|
}
|
|
|
|
classify_test_log() {
|
|
local log_file="$1"
|
|
# Heuristics: auth/network/timeout/unavailable → infra; assertion/FAIL → product
|
|
if grep -Eiq 'context deadline exceeded|connection refused|401 Unauthorized|Unauthorized|runner not available|no such host|i/o timeout|TLS handshake|dial tcp|read-only file system|mkdir .* read-only' "${log_file}"; then
|
|
echo infra
|
|
return
|
|
fi
|
|
if grep -Eiq '^--- FAIL:|FAIL\t|assertion|Expected|Fatalf|require\.' "${log_file}"; then
|
|
echo product
|
|
return
|
|
fi
|
|
echo product
|
|
}
|
|
|
|
echo "==> nightly real-PI suite commit=${COMMIT:0:12} runner=${RUNNER_URL}"
|
|
|
|
# Wave 2C: ensure candidate image digests exist for PASS receipts / promote gate.
|
|
ensure_candidate_images() {
|
|
local images_json="${ROOT}/.doc/test-reports/assets/promotion-receipts/${COMMIT}.images.json"
|
|
if [[ -f "${images_json}" ]]; then
|
|
echo "==> reusing candidate images manifest ${images_json}"
|
|
return 0
|
|
fi
|
|
if [[ "${NIGHTLY_SKIP_IMAGE_BUILD:-0}" == "1" ]]; then
|
|
echo "warning: NIGHTLY_SKIP_IMAGE_BUILD=1 and no images.json; PASS may lack digests" >&2
|
|
return 0
|
|
fi
|
|
echo "==> building candidate images for digest pin..."
|
|
"${ROOT}/scripts/candidate-images-build.sh" --commit "${COMMIT}"
|
|
}
|
|
|
|
if [[ "${DRY_INFRA_FAIL}" -eq 1 ]]; then
|
|
echo "forced infra failure (--dry-infra-fail)" | tee "${EVIDENCE_DIR}/infra-forced.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-forced.txt"
|
|
echo "wrote infra_fail receipt (dry)"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${TOKEN}" ]]; then
|
|
echo "missing RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN" | tee "${EVIDENCE_DIR}/infra-missing-secret.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-secret.txt"
|
|
echo "infrastructure failure: missing runner secret" >&2
|
|
exit 1
|
|
fi
|
|
|
|
HEALTH_CODE=0
|
|
HEALTH_BODY=""
|
|
if ! HEALTH_BODY="$(curl -fsS --max-time 5 "${RUNNER_URL}/health" 2>"${EVIDENCE_DIR}/health.err")"; then
|
|
HEALTH_CODE=$?
|
|
{
|
|
echo "runner health check failed code=${HEALTH_CODE} url=${RUNNER_URL}/health"
|
|
cat "${EVIDENCE_DIR}/health.err" 2>/dev/null || true
|
|
} | tee "${EVIDENCE_DIR}/infra-runner-down.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-runner-down.txt"
|
|
echo "infrastructure failure: runner unreachable" >&2
|
|
exit 1
|
|
fi
|
|
printf '%s\n' "${HEALTH_BODY}" > "${EVIDENCE_DIR}/runner-health.json"
|
|
|
|
if [[ -z "${TEST_DATABASE_URL:-}" ]]; then
|
|
echo "TEST_DATABASE_URL not set" | tee "${EVIDENCE_DIR}/infra-missing-db.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-db.txt"
|
|
echo "infrastructure failure: missing TEST_DATABASE_URL" >&2
|
|
exit 1
|
|
fi
|
|
|
|
export PI_AGENT_RUNNER_URL="${RUNNER_URL}"
|
|
export PI_AGENT_RUNNER_AUTH_TOKEN="${TOKEN}"
|
|
export RUNNER_AUTH_TOKEN="${TOKEN}"
|
|
export PI_AGENT_MODEL="${MODEL_ID}"
|
|
export RUN_PI_E2E=1
|
|
export KB_REQUIRE_ONLINE="${KB_REQUIRE_ONLINE:-1}"
|
|
|
|
# Shared KB preflight (host :5432). Fail closed unless KB_ALLOW_OFFLINE=1.
|
|
preflight_shared_kb() {
|
|
if [[ "${KB_ALLOW_OFFLINE:-0}" == "1" ]]; then
|
|
echo "NOTE: KB offline degradation test (KB_ALLOW_OFFLINE=1)" | tee "${EVIDENCE_DIR}/kb-allow-offline.txt"
|
|
return 0
|
|
fi
|
|
local kb_host="${KB_DB_HOST:-127.0.0.1}"
|
|
local kb_port="${KB_DB_PORT:-5432}"
|
|
local kb_user="${KB_DB_USER:-regkb}"
|
|
local kb_name="${KB_DB_NAME:-regulations}"
|
|
local kb_pass="${KB_DB_PASSWORD:-}"
|
|
if [[ -z "${kb_pass}" && -f "${ROOT}/.env" ]]; then
|
|
kb_pass="$(grep -E '^KB_DB_PASSWORD=' "${ROOT}/.env" | head -1 | cut -d= -f2- | tr -d '"' | tr -d "'")"
|
|
fi
|
|
if [[ -z "${kb_pass}" ]]; then
|
|
echo "KB_DB_PASSWORD missing (root .env or env); cannot preflight shared KB" | tee "${EVIDENCE_DIR}/infra-kb-missing-password.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-kb-missing-password.txt"
|
|
echo "infrastructure failure: shared KB password missing" >&2
|
|
exit 1
|
|
fi
|
|
if ! command -v psql >/dev/null 2>&1; then
|
|
echo "psql not found; skip SQL preflight, rely on go test KB_REQUIRE_ONLINE" | tee "${EVIDENCE_DIR}/kb-preflight-skipped-no-psql.txt"
|
|
return 0
|
|
fi
|
|
if ! PGPASSWORD="${kb_pass}" psql -h "${kb_host}" -p "${kb_port}" -U "${kb_user}" -d "${kb_name}" -c 'SELECT 1' >/dev/null 2>"${EVIDENCE_DIR}/kb-preflight.err"; then
|
|
{
|
|
echo "shared KB preflight failed host=${kb_host} port=${kb_port} db=${kb_name} user=${kb_user}"
|
|
cat "${EVIDENCE_DIR}/kb-preflight.err" 2>/dev/null || true
|
|
} | tee "${EVIDENCE_DIR}/infra-kb-down.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-kb-down.txt"
|
|
echo "infrastructure failure: shared KB unreachable" >&2
|
|
exit 1
|
|
fi
|
|
echo "shared KB preflight OK ${kb_host}:${kb_port}/${kb_name}" | tee "${EVIDENCE_DIR}/kb-preflight-ok.txt"
|
|
}
|
|
preflight_shared_kb
|
|
|
|
# Clear env pollution that flakes short/config tests elsewhere.
|
|
unset AGENT_TIMEOUT KB_DATABASE_URL KB_EMBEDDING_DIM 2>/dev/null || true
|
|
|
|
RUN_FILTER='TestMultiIntentU8_RealPI_ConfirmGenerate_PG|TestWorkflowGovernance_PiAgent_PG_FullNoneMixedChain'
|
|
LOG_FILE="${EVIDENCE_DIR}/go-test.txt"
|
|
set +e
|
|
go test -count=1 -timeout "${TIMEOUT}" -v ./internal/server/ -run "${RUN_FILTER}" 2>&1 | tee "${LOG_FILE}"
|
|
TEST_RC=${PIPESTATUS[0]}
|
|
RESULT_LOG="${LOG_FILE}"
|
|
# One retry for known flakes: model quality_failed, or PI hang / client deadline.
|
|
if [[ "${TEST_RC}" -ne 0 ]] && grep -Eq 'quality_failed want completed|status=quality_failed|context deadline exceeded|Client\.Timeout exceeded|MISSING_TARGET|missing target deliverable' "${LOG_FILE}"; then
|
|
echo "==> retry once after quality_failed/deadline/MISSING_TARGET flake..." | tee -a "${LOG_FILE}"
|
|
RETRY_LOG="${EVIDENCE_DIR}/go-test-retry.txt"
|
|
go test -count=1 -timeout "${TIMEOUT}" -v ./internal/server/ -run "${RUN_FILTER}" 2>&1 | tee "${RETRY_LOG}"
|
|
TEST_RC=${PIPESTATUS[0]}
|
|
RESULT_LOG="${RETRY_LOG}"
|
|
cat "${RETRY_LOG}" >> "${LOG_FILE}"
|
|
fi
|
|
set -e
|
|
|
|
if [[ "${TEST_RC}" -eq 0 ]]; then
|
|
# Exit zero alone also covers no tests and partial skips; require both named tests.
|
|
if ! python3 "${ROOT}/scripts/verify_go_test_evidence.py" --log "${RESULT_LOG}" \
|
|
--expected TestMultiIntentU8_RealPI_ConfirmGenerate_PG \
|
|
--expected TestWorkflowGovernance_PiAgent_PG_FullNoneMixedChain \
|
|
> "${EVIDENCE_DIR}/expected-tests.json"; then
|
|
write_receipt infra_fail infra "${LOG_FILE}" "${EVIDENCE_DIR}/expected-tests.json"
|
|
echo "infrastructure failure: expected real-PI tests did not all execute and pass" >&2
|
|
exit 1
|
|
fi
|
|
if ! ensure_candidate_images; then
|
|
echo "candidate image build failed" | tee "${EVIDENCE_DIR}/infra-image-build.txt"
|
|
write_receipt infra_fail infra "${LOG_FILE}" "${EVIDENCE_DIR}/infra-image-build.txt"
|
|
echo "infrastructure failure: candidate image build" >&2
|
|
exit 1
|
|
fi
|
|
write_receipt pass none "${LOG_FILE}" "${EVIDENCE_DIR}/runner-health.json"
|
|
echo "nightly real-PI PASS; receipt written for ${COMMIT:0:12}"
|
|
exit 0
|
|
fi
|
|
|
|
CLASS="$(classify_test_log "${LOG_FILE}")"
|
|
if [[ "${CLASS}" == "infra" ]]; then
|
|
write_receipt infra_fail infra "${LOG_FILE}"
|
|
echo "infrastructure failure during product suite" >&2
|
|
exit 1
|
|
fi
|
|
|
|
write_receipt product_fail product "${LOG_FILE}"
|
|
echo "product failure; promotion must reject this commit" >&2
|
|
exit 1
|