#!/bin/bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" . "$SCRIPT_DIR/verify_common.sh" . "$SCRIPT_DIR/secret_gate_lib.sh" DB_URL="${DATABASE_URL:-host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable}" SERVER_BIN="/tmp/llm_phase6_server" SERVER_LOG="/tmp/llm_phase6_server.log" SERVER_PORT="${PHASE6_PORT:-}" SERVER_PID="" API_AUTH_TOKEN="${API_AUTH_TOKEN:-phase6-local-token}" ROOT_CAUSE_CLASS="none" ROOT_CAUSE_SOURCE="none" ROOT_CAUSE_SUMMARY="none" RELEASE_SEMANTICS_CLASS="release-ready" RELEASE_SEMANTICS_GATE="phase6" RELEASE_SEMANTICS_POLICY="release-allowed" BLOCKER_SWITCH_CLASS="none" BLOCKER_SWITCH_OLD="none" BLOCKER_SWITCH_NEW="none" cleanup() { if [ -n "${SERVER_PID:-}" ] && kill -0 "$SERVER_PID" >/dev/null 2>&1; then kill "$SERVER_PID" >/dev/null 2>&1 || true wait "$SERVER_PID" >/dev/null 2>&1 || true fi rm -f "$SERVER_BIN" } trap cleanup EXIT port_in_use() { local port="$1" (echo >"/dev/tcp/127.0.0.1/$port") >/dev/null 2>&1 } reserve_server_port() { if [ -n "${SERVER_PORT:-}" ]; then return 0 fi for candidate in $(seq 18080 18120); do if ! port_in_use "$candidate"; then SERVER_PORT="$candidate" return 0 fi done return 1 } start_server() { DATABASE_URL="$DB_URL" PORT="$SERVER_PORT" API_AUTH_TOKEN="$API_AUTH_TOKEN" "$SERVER_BIN" >"$SERVER_LOG" 2>&1 & SERVER_PID=$! for _ in $(seq 1 20); do if ! kill -0 "$SERVER_PID" >/dev/null 2>&1; then return 1 fi if curl -fsS "http://127.0.0.1:${SERVER_PORT}/health" >/tmp/llm_phase6_health.out 2>/tmp/llm_phase6_health.err && grep -q '"status":"ok"' /tmp/llm_phase6_health.out; then return 0 fi sleep 0.5 done return 1 } last_nonempty_line() { awk 'NF { line=$0 } END { print line }' } last_meaningful_failure_line() { awk 'NF && $0 !~ /^exit status [0-9]+$/ { line=$0 } END { print line }' } extract_window_metric() { local name="$1" local payload="$2" printf '%s\n' "$payload" | awk -v key="$name" ' $0 ~ key"=" { for (i = 1; i <= NF; i++) { split($i, parts, "=") if (parts[1] == key) { print parts[2] exit } } } ' } classify_window_failure() { local payload="$1" local precondition_missing external_provider_failure collector_runtime_failure unknown_failure precondition_missing="$(extract_window_metric precondition_missing "$payload")" external_provider_failure="$(extract_window_metric external_provider_failure "$payload")" collector_runtime_failure="$(extract_window_metric collector_runtime_failure "$payload")" unknown_failure="$(extract_window_metric unknown_failure "$payload")" precondition_missing="${precondition_missing:-0}" external_provider_failure="${external_provider_failure:-0}" collector_runtime_failure="${collector_runtime_failure:-0}" unknown_failure="${unknown_failure:-0}" if [ "$precondition_missing" -gt 0 ] && [ "$external_provider_failure" -eq 0 ] && [ "$collector_runtime_failure" -eq 0 ] && [ "$unknown_failure" -eq 0 ]; then echo "precondition_missing_only" elif [ "$external_provider_failure" -gt 0 ] && [ "$precondition_missing" -eq 0 ] && [ "$collector_runtime_failure" -eq 0 ] && [ "$unknown_failure" -eq 0 ]; then echo "external_provider_failure_only" else echo "mixed" fi } set_root_cause_once() { local class="$1" local source="$2" local summary="$3" if [ "$ROOT_CAUSE_CLASS" != "none" ]; then return fi ROOT_CAUSE_CLASS="$class" ROOT_CAUSE_SOURCE="$source" ROOT_CAUSE_SUMMARY="$summary" } set_release_semantics() { local class="$1" local gate="$2" local policy="$3" RELEASE_SEMANTICS_CLASS="$class" RELEASE_SEMANTICS_GATE="$gate" RELEASE_SEMANTICS_POLICY="$policy" } set_blocker_switch_once() { local class="$1" local old="$2" local new="$3" if [ "$BLOCKER_SWITCH_CLASS" != "none" ]; then return fi BLOCKER_SWITCH_CLASS="$class" BLOCKER_SWITCH_OLD="$old" BLOCKER_SWITCH_NEW="$new" } classify_live_run_failure() { local live_tail="$1" local normalized normalized="$(printf '%s' "$live_tail" | tr '[:upper:]' '[:lower:]')" case "$normalized" in *"api key"*|*"database_url"*|*"must provide"*|*"未设置"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*) printf '%s\n' "precondition_missing" ;; *"signature_guard"*|*"unexpected status 403"*|*"unexpected status 502"*|*"unexpected status 503"*|*"unexpected status 504"*|*"no pricing cards found"*|*"no model rows parsed"*|*"no model overview cards parsed"*|*"context deadline exceeded"*|*"client.timeout"*|*"i/o timeout"*|*"tls handshake timeout"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"no such host"*) printf '%s\n' "external_provider_failure" ;; *) printf '%s\n' "primary_pipeline_failure" ;; esac } classify_live_run_provider() { local live_tail="$1" local normalized normalized="$(printf '%s' "$live_tail" | tr '[:upper:]' '[:lower:]')" case "$normalized" in *"import_vertex_pricing"*) printf '%s\n' 'vertex_pricing' ;; *"import_cloudflare_pricing"*|*"cloudflare_pricing"*) printf '%s\n' 'cloudflare_pricing' ;; *"import_perplexity_pricing"*|*"perplexity_pricing"*) printf '%s\n' 'perplexity_pricing' ;; *"import_xfyun_pricing"*|*"xfyun_pricing"*) printf '%s\n' 'xfyun_pricing' ;; *) printf '%s\n' 'unknown_external_provider' ;; esac } run_live_pipeline_gate() { local live_output live_rc live_tail set +e live_output="$(bash scripts/run_real_pipeline.sh 2>&1)" live_rc=$? set -e printf '%s\n' "$live_output" >/tmp/llm_phase6_live_pipeline.out live_tail="$(printf '%s\n' "$live_output" | last_meaningful_failure_line)" if [ "$live_rc" -eq 0 ]; then pass "live_run_result=PASS 主链路真实采集并输出今日日报" else live_failure_class="$(classify_live_run_failure "${live_tail:-}")" case "$live_failure_class" in precondition_missing) set_root_cause_once "precondition_missing" "live_run" "主链路因前置条件缺失未执行" fail "live_run_result=FAIL 主链路因前置条件缺失未执行 (${live_tail:-see /tmp/llm_phase6_live_pipeline.out})" ;; external_provider_failure) live_provider="$(classify_live_run_provider "${live_tail:-}")" set_root_cause_once "external_provider_failure" "live_run:${live_provider}" "外部文档站/价格页异常阻断主链路" fail "live_run_result=FAIL 外部文档站/价格页异常阻断主链路 (${live_tail:-see /tmp/llm_phase6_live_pipeline.out})" ;; *) set_root_cause_once "primary_pipeline_failure" "live_run" "主链路真实采集失败" fail "live_run_result=FAIL 主链路真实采集失败 (${live_tail:-see /tmp/llm_phase6_live_pipeline.out})" ;; esac fi } run_importer_smoke_gate() { local smoke_output smoke_rc smoke_tail set +e smoke_output="$(bash scripts/verify_importer_smoke.sh 2>&1)" smoke_rc=$? set -e printf '%s\n' "$smoke_output" printf '%s\n' "$smoke_output" >/tmp/llm_phase6_importer_smoke.out if [ "$smoke_rc" -eq 0 ]; then pass "importer_smoke_gate_result=PASS 新增导入器 smoke gate 通过" return 0 fi smoke_tail="$(printf '%s\n' "$smoke_output" | last_meaningful_failure_line)" set_root_cause_once "importer_smoke_gate_failure" "importer_smoke_gate" "新增导入器 smoke gate 未通过" fail "importer_smoke_gate_result=FAIL 新增导入器 smoke gate 未通过 (${smoke_tail:-see /tmp/llm_phase6_importer_smoke.out})" return 1 } run_window_gate() { local collector_window_output collector_window_rc window_failure_class set +e collector_window_output="$(bash scripts/collector_stats_window_audit.sh --db "$DB_URL" --limit 7 --assert-success-rate 95 2>&1)" collector_window_rc=$? set -e echo "$collector_window_output" if [ "$collector_window_rc" -eq 0 ]; then set_release_semantics "release-ready" "window_gate" "release-allowed" pass "window_gate_result=PASS 最近 7 次采集成功率达到 95%(已输出分类摘要;stability_label=stable-window)" return fi window_failure_class="$(classify_window_failure "$collector_window_output")" if [ "$window_failure_class" = "precondition_missing_only" ]; then set_release_semantics "precondition-degraded" "window_gate" "release-allowed-with-warning" pass "window_gate_result=PASS 最近 7 次采集成功率达到 95%(环境纪律问题:precondition_missing_only,调度环境缺 OPENROUTER_API_KEY,非系统缺陷;stability_label=precondition-only-window)" elif [ "$window_failure_class" = "external_provider_failure_only" ]; then set_release_semantics "degraded-external-provider" "window_gate" "release-allowed-with-warning" set_root_cause_once "external_provider_failure_only" "window_gate" "最近 7 次采集窗口仅被外部依赖失败拖低" warn "window_gate_result=WARN 最近 7 次采集成功率未达 95%(仅外部文档站失败:external_provider_failure_only,需要 release 语义降级而非误判为 collector bug;stability_label=recovered-external-incident)" else set_release_semantics "release-blocked" "window_gate" "release-blocked" set_root_cause_once "mixed_window_failure" "window_gate" "最近 7 次采集窗口存在混合失败" fail "window_gate_result=FAIL 最近 7 次采集成功率达到 95%(window_failure_class=${window_failure_class};stability_label=unstable-window)" fi } echo "=== Phase 6 综合验收检查 ===" check_shell "Phase 1~5 总门禁通过" "bash scripts/verify_pre_phase6.sh" check_shell "全仓 Go 测试通过" "go test ./..." check_shell "脚本级采集器单测通过" "bash scripts/test.sh" if run_importer_smoke_gate; then run_live_pipeline_gate else set_blocker_switch_once "global-blocker-shift" "importer_smoke_gate" "live_run" warn "live_run_result=SKIPPED 因 importer_smoke_gate_result=FAIL" fi if [ "$BLOCKER_SWITCH_CLASS" = "none" ] && [ "$ROOT_CAUSE_CLASS" != "none" ] && grep -q 'importer_smoke_gate_result=PASS' /tmp/llm_phase6_importer_smoke.out 2>/dev/null; then set_blocker_switch_once "global-blocker-shift" "importer_smoke_gate" "$ROOT_CAUSE_SOURCE" fi check_shell "API Server 可构建" "go build -o /dev/null ./cmd/server" check_shell "健康检查脚本通过" "DATABASE_URL='$DB_URL' bash healthcheck.sh" check_shell "源码与环境文件未包含明显硬编码密钥" "source scripts/secret_gate_lib.sh && secret_scan_paths . cmd internal frontend/src scripts .github/workflows && secret_env_files .dockerignore" run_window_gate if go build -o "$SERVER_BIN" ./cmd/server >/tmp/llm_phase6_server_build.out 2>/tmp/llm_phase6_server_build.err; then if reserve_server_port && start_server; then pass "API /health 可用" set +e api_metrics="$(curl -sS -H "Authorization: Bearer ${API_AUTH_TOKEN}" -o /tmp/llm_phase6_models.json -w '%{http_code} %{time_total}' "http://127.0.0.1:${SERVER_PORT}/api/v1/models")" api_rc=$? set -e if [ "$api_rc" -eq 0 ]; then api_code="$(printf '%s' "$api_metrics" | awk '{print $1}')" api_time="$(printf '%s' "$api_metrics" | awk '{print $2}')" if [ "$api_code" = "200" ]; then pass "API /api/v1/models 返回 200" else fail "API /api/v1/models 返回异常状态 (HTTP ${api_code:-unknown})" fi if awk "BEGIN { exit !($api_time < 0.5) }"; then pass "API 响应 < 500ms (当前: ${api_time}s)" else fail "API 响应 >= 500ms (当前: ${api_time}s)" fi if grep -q '"data"' /tmp/llm_phase6_models.json; then pass "API 返回模型数据载荷" else fail "API 返回体缺少 data 字段" fi else fail "API /api/v1/models 请求失败" fi set +e plan_metrics="$(curl -sS -H "Authorization: Bearer ${API_AUTH_TOKEN}" -o /tmp/llm_phase6_subscription_plans.json -w '%{http_code} %{time_total}' "http://127.0.0.1:${SERVER_PORT}/api/v1/subscription-plans")" plan_rc=$? set -e if [ "$plan_rc" -eq 0 ]; then plan_code="$(printf '%s' "$plan_metrics" | awk '{print $1}')" if [ "$plan_code" = "200" ]; then pass "API /api/v1/subscription-plans 返回 200" else fail "API /api/v1/subscription-plans 返回异常状态 (HTTP ${plan_code:-unknown})" fi if grep -q '"data"' /tmp/llm_phase6_subscription_plans.json; then pass "API 返回套餐数据载荷" else fail "套餐 API 返回体缺少 data 字段" fi else fail "API /api/v1/subscription-plans 请求失败" fi printf 'RELEASE_SEMANTICS class=%s gate=%s policy=%s\n' "$RELEASE_SEMANTICS_CLASS" "$RELEASE_SEMANTICS_GATE" "$RELEASE_SEMANTICS_POLICY" printf 'BLOCKER_SWITCH class=%s old=%s new=%s\n' "$BLOCKER_SWITCH_CLASS" "$BLOCKER_SWITCH_OLD" "$BLOCKER_SWITCH_NEW" else details="$(tr '\n' ' ' <"$SERVER_LOG" | sed 's/[[:space:]]\+/ /g' | sed 's/ $//')" set_root_cause_once "api_server_start_failure" "api_server" "API Server 启动失败" fail "API Server 启动失败 (${details:-no server log})" fi else details="$(tr '\n' ' ' /tmp/llm_phase6_frontend_test.out 2>/tmp/llm_phase6_frontend_test.err" check_shell "secret gate 独立测试通过" "bash scripts/secret_gate_test.sh" printf 'ROOT_CAUSE class=%s source=%s summary=%s\n' "$ROOT_CAUSE_CLASS" "$ROOT_CAUSE_SOURCE" "$ROOT_CAUSE_SUMMARY" finish_phase