weichen
3a23fb4533
Relative DATA_DIR from .env resolves under go test package cwd (internal/server/data/dev) and diverges from the pi-agent bind mount, leaving agents without template.md/blackboard.json.
244 lines
10 KiB
Bash
Executable File
244 lines
10 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Local / Actions entry for Wave 2B nightly real-PI suite.
|
|
# Classifies infrastructure vs product failures and writes a promotion receipt.
|
|
#
|
|
# Note: product jev-service integration defaults to JEV_DECIDE_MODE=off (compose),
|
|
# so U8 / workflow nightly filters below are unchanged by the jevdecide shadow work.
|
|
#
|
|
# Usage:
|
|
# ./scripts/run-nightly-real-pi.sh [--dry-infra-fail]
|
|
#
|
|
# Environment:
|
|
# PI_AGENT_RUNNER_URL default http://127.0.0.1:18865
|
|
# RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN
|
|
# TEST_DATABASE_URL required for product run (isolated PG)
|
|
# PI_AGENT_MODEL default MiniMax-M2.7-highspeed
|
|
# NIGHTLY_SUITE_TIMEOUT go test timeout (default 90m)
|
|
# NIGHTLY_EVIDENCE_DIR override evidence dir
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "${ROOT}"
|
|
|
|
DRY_INFRA_FAIL=0
|
|
if [[ "${1:-}" == "--dry-infra-fail" ]]; then
|
|
DRY_INFRA_FAIL=1
|
|
fi
|
|
|
|
RUNNER_URL="${PI_AGENT_RUNNER_URL:-http://127.0.0.1:18865}"
|
|
TOKEN="${PI_AGENT_RUNNER_AUTH_TOKEN:-${RUNNER_AUTH_TOKEN:-}}"
|
|
MODEL_ID="${PI_AGENT_MODEL:-MiniMax-M2.7-highspeed}"
|
|
PROTOCOL="${PI_AGENT_PROTOCOL:-typed-result-v1}"
|
|
TIMEOUT="${NIGHTLY_SUITE_TIMEOUT:-180m}"
|
|
COMMIT="$(git rev-parse HEAD)"
|
|
SUITE="nightly-real-pi-u8-four-workflows"
|
|
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
EVIDENCE_DIR="${NIGHTLY_EVIDENCE_DIR:-${ROOT}/.doc/test-reports/assets/promotion-receipts/evidence-${COMMIT:0:12}-${STAMP}}"
|
|
mkdir -p "${EVIDENCE_DIR}"
|
|
|
|
write_receipt() {
|
|
local verdict="$1"
|
|
local failure_class="$2"
|
|
shift 2
|
|
local args=(
|
|
--verdict "${verdict}"
|
|
--commit "${COMMIT}"
|
|
--suite "${SUITE}"
|
|
--failure-class "${failure_class}"
|
|
--runner-url "${RUNNER_URL}"
|
|
--model-id "${MODEL_ID}"
|
|
--protocol "${PROTOCOL}"
|
|
--force
|
|
)
|
|
local ref
|
|
for ref in "$@"; do
|
|
args+=(--evidence "${ref}")
|
|
done
|
|
PI_AGENT_RUNNER_URL="${RUNNER_URL}" PI_AGENT_MODEL="${MODEL_ID}" PI_AGENT_PROTOCOL="${PROTOCOL}" \
|
|
"${ROOT}/scripts/promotion-receipt-write.sh" "${args[@]}"
|
|
}
|
|
|
|
classify_test_log() {
|
|
local log_file="$1"
|
|
# Heuristics: auth/network/timeout/unavailable → infra; assertion/FAIL → product
|
|
if grep -Eiq 'context deadline exceeded|connection refused|401 Unauthorized|Unauthorized|runner not available|no such host|i/o timeout|TLS handshake|dial tcp|read-only file system|mkdir .* read-only' "${log_file}"; then
|
|
echo infra
|
|
return
|
|
fi
|
|
if grep -Eiq '^--- FAIL:|FAIL\t|assertion|Expected|Fatalf|require\.' "${log_file}"; then
|
|
echo product
|
|
return
|
|
fi
|
|
echo product
|
|
}
|
|
|
|
echo "==> nightly real-PI suite commit=${COMMIT:0:12} runner=${RUNNER_URL}"
|
|
|
|
# Wave 2C: ensure candidate image digests exist for PASS receipts / promote gate.
|
|
ensure_candidate_images() {
|
|
local images_json="${ROOT}/.doc/test-reports/assets/promotion-receipts/${COMMIT}.images.json"
|
|
if [[ -f "${images_json}" ]]; then
|
|
echo "==> reusing candidate images manifest ${images_json}"
|
|
return 0
|
|
fi
|
|
if [[ "${NIGHTLY_SKIP_IMAGE_BUILD:-0}" == "1" ]]; then
|
|
echo "warning: NIGHTLY_SKIP_IMAGE_BUILD=1 and no images.json; PASS may lack digests" >&2
|
|
return 0
|
|
fi
|
|
echo "==> building candidate images for digest pin..."
|
|
"${ROOT}/scripts/candidate-images-build.sh" --commit "${COMMIT}"
|
|
}
|
|
|
|
if [[ "${DRY_INFRA_FAIL}" -eq 1 ]]; then
|
|
echo "forced infra failure (--dry-infra-fail)" | tee "${EVIDENCE_DIR}/infra-forced.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-forced.txt"
|
|
echo "wrote infra_fail receipt (dry)"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${TOKEN}" ]]; then
|
|
echo "missing RUNNER_AUTH_TOKEN / PI_AGENT_RUNNER_AUTH_TOKEN" | tee "${EVIDENCE_DIR}/infra-missing-secret.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-secret.txt"
|
|
echo "infrastructure failure: missing runner secret" >&2
|
|
exit 1
|
|
fi
|
|
|
|
HEALTH_CODE=0
|
|
HEALTH_BODY=""
|
|
if ! HEALTH_BODY="$(curl -fsS --max-time 5 "${RUNNER_URL}/health" 2>"${EVIDENCE_DIR}/health.err")"; then
|
|
HEALTH_CODE=$?
|
|
{
|
|
echo "runner health check failed code=${HEALTH_CODE} url=${RUNNER_URL}/health"
|
|
cat "${EVIDENCE_DIR}/health.err" 2>/dev/null || true
|
|
} | tee "${EVIDENCE_DIR}/infra-runner-down.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-runner-down.txt"
|
|
echo "infrastructure failure: runner unreachable" >&2
|
|
exit 1
|
|
fi
|
|
printf '%s\n' "${HEALTH_BODY}" > "${EVIDENCE_DIR}/runner-health.json"
|
|
|
|
if [[ -z "${TEST_DATABASE_URL:-}" ]]; then
|
|
echo "TEST_DATABASE_URL not set" | tee "${EVIDENCE_DIR}/infra-missing-db.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-missing-db.txt"
|
|
echo "infrastructure failure: missing TEST_DATABASE_URL" >&2
|
|
exit 1
|
|
fi
|
|
|
|
export PI_AGENT_RUNNER_URL="${RUNNER_URL}"
|
|
export PI_AGENT_RUNNER_AUTH_TOKEN="${TOKEN}"
|
|
export RUNNER_AUTH_TOKEN="${TOKEN}"
|
|
export PI_AGENT_MODEL="${MODEL_ID}"
|
|
export RUN_PI_E2E=1
|
|
export KB_REQUIRE_ONLINE="${KB_REQUIRE_ONLINE:-1}"
|
|
# Host Go tests must write workspaces into the same tree the runner mounts
|
|
# (docker-compose.dev.yml: ./data/dev/workspaces → /app/data/workspaces).
|
|
#
|
|
# CRITICAL: `go test ./internal/server/...` runs with cwd=internal/server.
|
|
# A relative DATA_DIR (e.g. ./data/dev from configs/.env.dev) resolves to
|
|
# internal/server/data/dev and diverges from the pi-agent bind mount, so the
|
|
# agent sees an empty workspace (missing template.md / blackboard.json) while
|
|
# PrepareWorkspace wrote a complete tree under the package-local path.
|
|
if [[ -n "${DATA_DIR:-}" && "${DATA_DIR}" != /* ]]; then
|
|
echo "warning: ignoring relative DATA_DIR=${DATA_DIR}; forcing ${ROOT}/data/dev" >&2
|
|
DATA_DIR="${ROOT}/data/dev"
|
|
fi
|
|
export DATA_DIR="${DATA_DIR:-${ROOT}/data/dev}"
|
|
# Absolutize even if caller exported a relative path via other means.
|
|
DATA_DIR="$(cd "${DATA_DIR}" 2>/dev/null && pwd -P || echo "${ROOT}/data/dev")"
|
|
export DATA_DIR
|
|
echo "==> DATA_DIR=${DATA_DIR}"
|
|
|
|
# Shared KB preflight (host :5432). Fail closed unless KB_ALLOW_OFFLINE=1.
|
|
preflight_shared_kb() {
|
|
if [[ "${KB_ALLOW_OFFLINE:-0}" == "1" ]]; then
|
|
echo "NOTE: KB offline degradation test (KB_ALLOW_OFFLINE=1)" | tee "${EVIDENCE_DIR}/kb-allow-offline.txt"
|
|
return 0
|
|
fi
|
|
local kb_host="${KB_DB_HOST:-127.0.0.1}"
|
|
local kb_port="${KB_DB_PORT:-5432}"
|
|
local kb_user="${KB_DB_USER:-regkb}"
|
|
local kb_name="${KB_DB_NAME:-regulations}"
|
|
local kb_pass="${KB_DB_PASSWORD:-}"
|
|
if [[ -z "${kb_pass}" && -f "${ROOT}/.env" ]]; then
|
|
kb_pass="$(grep -E '^KB_DB_PASSWORD=' "${ROOT}/.env" | head -1 | cut -d= -f2- | tr -d '"' | tr -d "'")"
|
|
fi
|
|
if [[ -z "${kb_pass}" ]]; then
|
|
echo "KB_DB_PASSWORD missing (root .env or env); cannot preflight shared KB" | tee "${EVIDENCE_DIR}/infra-kb-missing-password.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-kb-missing-password.txt"
|
|
echo "infrastructure failure: shared KB password missing" >&2
|
|
exit 1
|
|
fi
|
|
if ! command -v psql >/dev/null 2>&1; then
|
|
echo "psql not found; skip SQL preflight, rely on go test KB_REQUIRE_ONLINE" | tee "${EVIDENCE_DIR}/kb-preflight-skipped-no-psql.txt"
|
|
return 0
|
|
fi
|
|
if ! PGPASSWORD="${kb_pass}" psql -h "${kb_host}" -p "${kb_port}" -U "${kb_user}" -d "${kb_name}" -c 'SELECT 1' >/dev/null 2>"${EVIDENCE_DIR}/kb-preflight.err"; then
|
|
{
|
|
echo "shared KB preflight failed host=${kb_host} port=${kb_port} db=${kb_name} user=${kb_user}"
|
|
cat "${EVIDENCE_DIR}/kb-preflight.err" 2>/dev/null || true
|
|
} | tee "${EVIDENCE_DIR}/infra-kb-down.txt"
|
|
write_receipt infra_fail infra "${EVIDENCE_DIR}/infra-kb-down.txt"
|
|
echo "infrastructure failure: shared KB unreachable" >&2
|
|
exit 1
|
|
fi
|
|
echo "shared KB preflight OK ${kb_host}:${kb_port}/${kb_name}" | tee "${EVIDENCE_DIR}/kb-preflight-ok.txt"
|
|
}
|
|
preflight_shared_kb
|
|
|
|
# Clear env pollution that flakes short/config tests elsewhere.
|
|
unset AGENT_TIMEOUT KB_DATABASE_URL KB_EMBEDDING_DIM 2>/dev/null || true
|
|
|
|
# Four workflow governance chains (main / product_manager / test_manager / FullNone)
|
|
# plus U8 confirm-generate. Rdev P3 remains the Python harness (not yet a Go PG top-level).
|
|
RUN_FILTER='TestMultiIntentU8_RealPI_ConfirmGenerate_PG|TestWorkflowGovernance_PiAgent_PG_(FullChain|ProductManager_FullChain|TestManager_FullChain|FullNoneMixedChain)$'
|
|
LOG_FILE="${EVIDENCE_DIR}/go-test.txt"
|
|
set +e
|
|
go test -count=1 -timeout "${TIMEOUT}" -v ./internal/server/ -run "${RUN_FILTER}" 2>&1 | tee "${LOG_FILE}"
|
|
TEST_RC=${PIPESTATUS[0]}
|
|
RESULT_LOG="${LOG_FILE}"
|
|
# One retry for known flakes: model quality_failed, or PI hang / client deadline.
|
|
if [[ "${TEST_RC}" -ne 0 ]] && grep -Eq 'quality_failed want completed|status=quality_failed|context deadline exceeded|Client\.Timeout exceeded|MISSING_TARGET|missing target deliverable' "${LOG_FILE}"; then
|
|
echo "==> retry once after quality_failed/deadline/MISSING_TARGET flake..." | tee -a "${LOG_FILE}"
|
|
RETRY_LOG="${EVIDENCE_DIR}/go-test-retry.txt"
|
|
go test -count=1 -timeout "${TIMEOUT}" -v ./internal/server/ -run "${RUN_FILTER}" 2>&1 | tee "${RETRY_LOG}"
|
|
TEST_RC=${PIPESTATUS[0]}
|
|
RESULT_LOG="${RETRY_LOG}"
|
|
cat "${RETRY_LOG}" >> "${LOG_FILE}"
|
|
fi
|
|
set -e
|
|
|
|
if [[ "${TEST_RC}" -eq 0 ]]; then
|
|
# Exit zero alone also covers no tests and partial skips; require all named top-level tests.
|
|
if ! python3 "${ROOT}/scripts/verify_go_test_evidence.py" --log "${RESULT_LOG}" \
|
|
--expected TestMultiIntentU8_RealPI_ConfirmGenerate_PG \
|
|
--expected TestWorkflowGovernance_PiAgent_PG_FullChain \
|
|
--expected TestWorkflowGovernance_PiAgent_PG_ProductManager_FullChain \
|
|
--expected TestWorkflowGovernance_PiAgent_PG_TestManager_FullChain \
|
|
--expected TestWorkflowGovernance_PiAgent_PG_FullNoneMixedChain \
|
|
> "${EVIDENCE_DIR}/expected-tests.json"; then
|
|
write_receipt infra_fail infra "${LOG_FILE}" "${EVIDENCE_DIR}/expected-tests.json"
|
|
echo "infrastructure failure: expected real-PI tests did not all execute and pass" >&2
|
|
exit 1
|
|
fi
|
|
if ! ensure_candidate_images; then
|
|
echo "candidate image build failed" | tee "${EVIDENCE_DIR}/infra-image-build.txt"
|
|
write_receipt infra_fail infra "${LOG_FILE}" "${EVIDENCE_DIR}/infra-image-build.txt"
|
|
echo "infrastructure failure: candidate image build" >&2
|
|
exit 1
|
|
fi
|
|
write_receipt pass none "${LOG_FILE}" "${EVIDENCE_DIR}/runner-health.json"
|
|
echo "nightly real-PI PASS; receipt written for ${COMMIT:0:12}"
|
|
exit 0
|
|
fi
|
|
|
|
CLASS="$(classify_test_log "${LOG_FILE}")"
|
|
if [[ "${CLASS}" == "infra" ]]; then
|
|
write_receipt infra_fail infra "${LOG_FILE}"
|
|
echo "infrastructure failure during product suite" >&2
|
|
exit 1
|
|
fi
|
|
|
|
write_receipt product_fail product "${LOG_FILE}"
|
|
echo "product failure; promotion must reject this commit" >&2
|
|
exit 1
|