niuniu/llm-intelligence
1
0
Fork 1
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(pipeline): wire collectors into real pipeline gates

Browse Source 
Wire the new subscription and official pricing collectors into the daily, real, and intel pipeline entrypoints.

This commit also upgrades Phase 6 verification with recent-window collector classification so gate failures distinguish preconditions from true runtime or provider issues.
This commit is contained in:
phamnazage-jpg
2026-05-15 22:37:06 +08:00
parent 256975e10c
commit d5d18e987e
6 changed files with 739 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
scripts/collector_stats_window_audit.sh Normal file
View File

@@ -0,0 +1,164 @@
#!/usr/bin/env bash
set -euo pipefail
LIMIT=7
DB_URL="${DATABASE_URL:-}"
INPUT_PATH=""
THRESHOLD=""
FIELD_SEP=$'\x1f'
usage() {
cat <<'EOF'
用法:
bash scripts/collector_stats_window_audit.sh --db <DATABASE_URL> [--limit N] [--assert-success-rate PCT]
bash scripts/collector_stats_window_audit.sh --input <tsv-file> [--limit N] [--assert-success-rate PCT]
输入 TSV 列顺序:
source<TAB>success<TAB>error_message<TAB>created_at
EOF
}
classify_failure() {
local message normalized
message="${1:-}"
normalized="$(printf '%s' "$message" | tr '[:upper:]' '[:lower:]')"
if [[ -z "${normalized// }" ]]; then
printf '%s\n' "collector_runtime_failure"
return
fi
case "$normalized" in
*"api key"*|*"strict real mode"*|*"database_url"*|*"password authentication failed"*|*"permission denied"*|*"role does not exist"*|*"relation does not exist"*|*"must provide"*)
printf '%s\n' "precondition_missing"
;;
*"429"*|*"rate limit"*|*"too many requests"*|*"timeout"*|*"temporarily unavailable"*|*"transport closed"*|*"connection reset"*|*"connection refused"*|*"eof"*|*"tls handshake timeout"*|*"no such host"*|*"i/o timeout"*)
printf '%s\n' "external_provider_failure"
;;
*)
printf '%s\n' "collector_runtime_failure"
;;
esac
}
fetch_rows_from_db() {
if [[ -z "${DB_URL:-}" ]]; then
echo "missing --db / DATABASE_URL" >&2
return 1
fi
psql "$DB_URL" -F "$FIELD_SEP" -Atqc "
SELECT
COALESCE(source, ''),
CASE WHEN success THEN 't' ELSE 'f' END,
COALESCE(error_message, ''),
TO_CHAR(created_at, 'YYYY-MM-DD HH24:MI:SS')
FROM collector_stats
ORDER BY created_at DESC
LIMIT ${LIMIT};
"
}
fetch_rows_from_file() {
if [[ -z "${INPUT_PATH:-}" ]]; then
echo "missing --input" >&2
return 1
fi
head -n "$LIMIT" "$INPUT_PATH"
}
while [[ $# -gt 0 ]]; do
case "$1" in
--db)
DB_URL="$2"
shift 2
;;
--input)
INPUT_PATH="$2"
shift 2
;;
--limit)
LIMIT="$2"
shift 2
;;
--assert-success-rate)
THRESHOLD="$2"
shift 2
;;
--help|-h)
usage
exit 0
;;
*)
echo "unknown arg: $1" >&2
usage >&2
exit 1
;;
esac
done
if [[ -n "$INPUT_PATH" ]]; then
ROWS="$(fetch_rows_from_file)"
else
ROWS="$(fetch_rows_from_db)"
fi
SUCCESS_COUNT=0
FAILURE_COUNT=0
PRECONDITION_COUNT=0
EXTERNAL_COUNT=0
RUNTIME_COUNT=0
UNKNOWN_COUNT=0
ROW_COUNT=0
DETAIL_LINES=""
while IFS= read -r raw_line; do
[[ -z "${raw_line}" ]] && continue
normalized_line="${raw_line//$'\t'/$FIELD_SEP}"
IFS="$FIELD_SEP" read -r source success error_message created_at <<< "$normalized_line"
ROW_COUNT=$((ROW_COUNT + 1))
if [[ "$success" == "t" || "$success" == "true" ]]; then
SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
category="success"
rendered_error="-"
else
FAILURE_COUNT=$((FAILURE_COUNT + 1))
category="$(classify_failure "$error_message")"
rendered_error="${error_message:-unknown}"
case "$category" in
precondition_missing)
PRECONDITION_COUNT=$((PRECONDITION_COUNT + 1))
;;
external_provider_failure)
EXTERNAL_COUNT=$((EXTERNAL_COUNT + 1))
;;
collector_runtime_failure)
RUNTIME_COUNT=$((RUNTIME_COUNT + 1))
;;
*)
UNKNOWN_COUNT=$((UNKNOWN_COUNT + 1))
;;
esac
fi
DETAIL_LINES+=$'sample_'"${ROW_COUNT}"$' created_at='"${created_at:-unknown}"$' source='"${source:-unknown}"$' outcome='"$([[ "$category" == "success" ]] && printf '%s' "success" || printf '%s' "failure")"$' category='"${category}"$' error='"${rendered_error}"$'\n'
done <<< "$ROWS"
if [[ "$ROW_COUNT" -eq 0 ]]; then
echo "window_size=0 success_count=0 failure_count=0 success_rate=0.00 threshold=${THRESHOLD:-n/a} precondition_missing=0 external_provider_failure=0 collector_runtime_failure=0 unknown_failure=0"
echo "sample_window=empty"
if [[ -n "$THRESHOLD" ]]; then
exit 1
fi
exit 0
fi
SUCCESS_RATE="$(awk -v success="$SUCCESS_COUNT" -v total="$ROW_COUNT" 'BEGIN { printf "%.2f", (success * 100) / total }')"
echo "window_size=${ROW_COUNT} success_count=${SUCCESS_COUNT} failure_count=${FAILURE_COUNT} success_rate=${SUCCESS_RATE} threshold=${THRESHOLD:-n/a} precondition_missing=${PRECONDITION_COUNT} external_provider_failure=${EXTERNAL_COUNT} collector_runtime_failure=${RUNTIME_COUNT} unknown_failure=${UNKNOWN_COUNT}"
printf '%s' "$DETAIL_LINES"
if [[ -n "$THRESHOLD" ]]; then
if awk -v actual="$SUCCESS_RATE" -v threshold="$THRESHOLD" 'BEGIN { exit !(actual >= threshold) }'; then
exit 0
fi
exit 1
fi

52
scripts/collector_stats_window_audit_test.sh Normal file
View File

@@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
TMP_DIR="$(mktemp -d)"
trap 'rm -rf "$TMP_DIR"' EXIT
FIXTURE_FAIL="$TMP_DIR/collector_stats_fail.tsv"
cat > "$FIXTURE_FAIL" <<'EOF'
openrouter f 严格真实模式下必须提供 API Key 2026-05-15 20:00:00
openrouter f 429 Too Many Requests 2026-05-15 19:59:00
openrouter t 2026-05-15 19:58:00
openrouter t 2026-05-15 19:57:00
openrouter t 2026-05-15 19:56:00
openrouter t 2026-05-15 19:55:00
openrouter f insert models failed 2026-05-15 19:54:00
EOF
set +e
FAIL_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_FAIL" --limit 7 --assert-success-rate 95 2>&1)"
FAIL_RC=$?
set -e
if [[ "$FAIL_RC" -eq 0 ]]; then
echo "expected failing fixture to exit non-zero"
exit 1
fi
printf '%s' "$FAIL_OUTPUT" | grep -q 'success_rate=57.14'
printf '%s' "$FAIL_OUTPUT" | grep -q 'precondition_missing=1'
printf '%s' "$FAIL_OUTPUT" | grep -q 'external_provider_failure=1'
printf '%s' "$FAIL_OUTPUT" | grep -q 'collector_runtime_failure=1'
printf '%s' "$FAIL_OUTPUT" | grep -q 'sample_1 created_at=2026-05-15 20:00:00'
FIXTURE_PASS="$TMP_DIR/collector_stats_pass.tsv"
cat > "$FIXTURE_PASS" <<'EOF'
openrouter t 2026-05-15 20:00:00
openrouter t 2026-05-15 19:59:00
openrouter t 2026-05-15 19:58:00
openrouter t 2026-05-15 19:57:00
openrouter t 2026-05-15 19:56:00
openrouter t 2026-05-15 19:55:00
openrouter t 2026-05-15 19:54:00
EOF
PASS_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_PASS" --limit 7 --assert-success-rate 95 2>&1)"
printf '%s' "$PASS_OUTPUT" | grep -q 'success_rate=100.00'
printf '%s' "$PASS_OUTPUT" | grep -q 'failure_count=0'
printf '%s' "$PASS_OUTPUT" | grep -q 'sample_7 created_at=2026-05-15 19:54:00'

211
scripts/run_daily.sh
View File

@@ -14,14 +14,15 @@ if [[ -f "$PROJECT_DIR/.env" ]]; then
source "$PROJECT_DIR/.env"
fi
DB_URL="${DATABASE_URL:-host=/var/run/postgresql dbname=llm_intelligence user=long sslmode=disable}"
export DATABASE_URL="$DB_URL"
REPORT_DATE="$(report_date_value)"
LOG_FILE="/tmp/llm_hub_daily_${REPORT_DATE}.log"
FEISHU_WEBHOOK="${FEISHU_WEBHOOK:-}"
MODEL_COUNT=""
FETCH_OUT="${PROJECT_DIR}/models.json"
FETCH_TOTAL="0"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
PIPELINE_FAILED_SOURCE_SET="none"
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
PIPELINE_AUDIT_SUMMARY=""
@@ -181,6 +182,210 @@ if ! go run -tags llm_script scripts/import_bytedance_data.go >> "$LOG_FILE" 2>&
merge_failed_source_keys "bytedance"
error_exit "字节官方导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/aliyun_subscription_lib.go \
scripts/import_aliyun_subscription.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "aliyun_subscription"
error_exit "阿里云套餐导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/baidu_subscription_lib.go \
scripts/import_baidu_subscription.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "baidu_subscription"
error_exit "百度套餐导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/ctyun_subscription_lib.go \
scripts/import_ctyun_subscription.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "ctyun_subscription"
error_exit "天翼云套餐导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/bytedance_subscription_lib.go \
scripts/import_bytedance_subscription.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "bytedance_subscription"
error_exit "火山方舟套餐导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/huawei_package_lib.go \
scripts/import_huawei_package.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "huawei_package"
error_exit "华为云套餐包导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/zhipu_coding_plan_lib.go \
scripts/import_zhipu_coding_plan.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "zhipu_coding_plan"
error_exit "智谱 Coding Plan 导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/minimax_subscription_lib.go \
scripts/import_minimax_subscription.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "minimax_subscription"
error_exit "MiniMax Token Plan 导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/catalog_verification_common.go \
scripts/import_cucloud_catalog.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "cucloud_catalog"
error_exit "联通云目录校验失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/catalog_verification_common.go \
scripts/import_mobile_cloud_catalog.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "mobile_cloud_catalog"
error_exit "移动云目录校验失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/youdao_pricing_lib.go \
scripts/import_youdao_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "youdao_pricing"
error_exit "网易有道价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/platform360_pricing_lib.go \
scripts/import_360_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "platform360_pricing"
error_exit "360 智脑价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/siliconflow_pricing_lib.go \
scripts/import_siliconflow_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "siliconflow_pricing"
error_exit "硅基流动价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/ppio_pricing_lib.go \
scripts/import_ppio_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "ppio_pricing"
error_exit "PPIO 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/ucloud_pricing_lib.go \
scripts/import_ucloud_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "ucloud_pricing"
error_exit "UCloud 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/pricing_markdown_snapshot_lib.go \
scripts/cloudflare_pricing_snapshot_lib.go \
scripts/signature_guard_common.go \
scripts/official_import_signature_audit_lib.go \
scripts/cloudflare_pricing_signature_guard_lib.go \
scripts/cloudflare_pricing_import_runner.go \
scripts/cloudflare_pricing_lib.go \
scripts/cloudflare_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "cloudflare_pricing_signature"
error_exit "Cloudflare Workers AI 价格页结构签名漂移"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/pricing_markdown_snapshot_lib.go \
scripts/cloudflare_pricing_snapshot_lib.go \
scripts/cloudflare_pricing_import_runner.go \
scripts/cloudflare_pricing_lib.go \
scripts/import_cloudflare_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "cloudflare_pricing"
error_exit "Cloudflare Workers AI 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/pricing_markdown_snapshot_lib.go \
scripts/perplexity_pricing_snapshot_lib.go \
scripts/signature_guard_common.go \
scripts/official_import_signature_audit_lib.go \
scripts/perplexity_pricing_signature_guard_lib.go \
scripts/perplexity_pricing_import_runner.go \
scripts/perplexity_pricing_lib.go \
scripts/perplexity_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "perplexity_pricing_signature"
error_exit "Perplexity API 价格页结构签名漂移"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/pricing_markdown_snapshot_lib.go \
scripts/perplexity_pricing_snapshot_lib.go \
scripts/perplexity_pricing_import_runner.go \
scripts/perplexity_pricing_lib.go \
scripts/import_perplexity_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "perplexity_pricing"
error_exit "Perplexity API 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/pricing_markdown_snapshot_lib.go \
scripts/signature_guard_common.go \
scripts/official_import_signature_audit_lib.go \
scripts/vertex_pricing_snapshot_lib.go \
scripts/vertex_pricing_signature_guard_lib.go \
scripts/vertex_pricing_import_runner.go \
scripts/vertex_pricing_lib.go \
scripts/vertex_pricing_signature_guard.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "vertex_pricing_signature"
error_exit "Vertex AI 价格页结构签名漂移"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/vertex_pricing_snapshot_lib.go \
scripts/vertex_pricing_import_runner.go \
scripts/vertex_pricing_lib.go \
scripts/import_vertex_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "vertex_pricing"
error_exit "Vertex AI 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/bedrock_pricing_lib.go \
scripts/import_bedrock_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "bedrock_pricing"
error_exit "Amazon Bedrock 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/official_pricing_import_common.go \
scripts/azure_openai_pricing_lib.go \
scripts/import_azure_openai_pricing.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "azure_openai_pricing"
error_exit "Azure OpenAI 价格导入失败"
fi
if ! go run -tags llm_script \
scripts/subscription_import_common.go \
scripts/import_catalog_seed_verification.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "catalog_seed_verification"
error_exit "目录级官方入口核验失败"
fi
if ! SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script \
scripts/materialize_daily_signals.go >> "$LOG_FILE" 2>&1; then
merge_failed_source_keys "daily_signal_snapshot"
error_exit "每日关键信号物化失败"
fi
refresh_pipeline_audit
log "✅ 多源补充同步完成"
@@ -195,7 +400,7 @@ log "✅ 数据质量检查通过 (模型数: ${MODEL_COUNT})"
# 3. 生成日报
log "3️⃣ 生成日报..."
export DATABASE_URL="$DB_URL"
if ! REPORT_RUN_KIND="scheduled" REPORT_TRIGGER_SOURCE="cron" REPORT_IS_OFFICIAL_DAILY="true" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run scripts/generate_daily_report.go >> "$LOG_FILE" 2>&1; then
if ! REPORT_RUN_KIND="scheduled" REPORT_TRIGGER_SOURCE="cron" REPORT_IS_OFFICIAL_DAILY="true" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run scripts/generate_daily_report.go scripts/official_import_signature_audit_query_lib.go >> "$LOG_FILE" 2>&1; then
error_exit "日报生成失败"
fi
log "✅ 日报生成完成"

181
scripts/run_intel_pipeline.sh Executable file
View File

@@ -0,0 +1,181 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
if [[ -f ".env.local" ]]; then
# shellcheck disable=SC1091
source ".env.local"
fi
if [[ -f ".env" ]]; then
# shellcheck disable=SC1091
source ".env"
fi
if [[ -z "${DATABASE_URL:-}" ]]; then
echo "DATABASE_URL 未设置" >&2
exit 1
fi
if [[ -z "${OPENROUTER_API_KEY:-}" ]]; then
echo "OPENROUTER_API_KEY 未设置，无法执行真实采集" >&2
exit 1
fi
REPORT_DATE="${REPORT_DATE:-$(date +%F)}"
FETCH_OUT="$ROOT_DIR/models.json"
FETCH_TOTAL="0"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
PIPELINE_FAILED_SOURCE_SET="none"
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
PIPELINE_AUDIT_SUMMARY=""
normalize_summary_file() {
local path="$1"
if [[ ! -f "$path" ]]; then
return
fi
tr '\n' ' ' < "$path" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//'
}
extract_failed_source_keys() {
local summary="$1"
printf '%s\n' "$summary" | sed -n 's/.*failed_source_keys=\([^ ]*\).*/\1/p'
}
merge_failed_source_keys() {
local keys="$1"
if [[ -z "$keys" || "$keys" == "none" ]]; then
return
fi
if [[ "$PIPELINE_FAILED_SOURCE_SET" == "none" ]]; then
PIPELINE_FAILED_SOURCE_SET="$keys"
return
fi
PIPELINE_FAILED_SOURCE_SET="${PIPELINE_FAILED_SOURCE_SET},${keys}"
}
refresh_pipeline_audit() {
PIPELINE_AUDIT_SUMMARY="runtime_audit stage_set=${PIPELINE_STAGE_SET} selected_source_keys=${PIPELINE_SOURCE_SET} failed_source_keys=${PIPELINE_FAILED_SOURCE_SET} openrouter_total=${FETCH_TOTAL:-0} ${MULTI_SOURCE_AUDIT}"
}
run_or_fail() {
local source_key="$1"
local error_message="$2"
shift 2
if ! "$@"; then
merge_failed_source_keys "$source_key"
refresh_pipeline_audit
echo "$error_message" >&2
exit 1
fi
}
refresh_pipeline_audit
bash "$ROOT_DIR/scripts/apply_migration.sh"
run_or_fail "openrouter" "OpenRouter 真实采集失败" \
go run "./scripts/fetch_openrouter.go" -api-key "$OPENROUTER_API_KEY" -db "$DATABASE_URL" -out "$FETCH_OUT" -strict-real
FETCH_TOTAL=$(python3 - <<'PY' "$FETCH_OUT"
import json, sys
path = sys.argv[1]
with open(path, 'r', encoding='utf-8') as f:
data = json.load(f)
print(int(data.get("total", 0)))
PY
)
if [[ "${FETCH_TOTAL:-0}" -lt 10 ]]; then
merge_failed_source_keys "openrouter"
refresh_pipeline_audit
echo "本次采集结果异常: total=${FETCH_TOTAL:-0} < 10" >&2
exit 1
fi
refresh_pipeline_audit
MULTI_SOURCE_OUTPUT="$(mktemp)"
if ! go run "./scripts/fetch_multi_source.go" --sources moonshot,deepseek,openai > "$MULTI_SOURCE_OUTPUT"; then
MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
if [[ -n "$MULTI_SOURCE_SUMMARY" ]]; then
MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY}"
merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
else
MULTI_SOURCE_AUDIT="multi_source_audit=stage_failed"
merge_failed_source_keys "moonshot,deepseek,openai"
fi
cat "$MULTI_SOURCE_OUTPUT"
rm -f "$MULTI_SOURCE_OUTPUT"
refresh_pipeline_audit
echo "多源补充同步失败" >&2
exit 1
fi
MULTI_SOURCE_SUMMARY="$(normalize_summary_file "$MULTI_SOURCE_OUTPUT")"
MULTI_SOURCE_AUDIT="multi_source_audit=${MULTI_SOURCE_SUMMARY:-none}"
merge_failed_source_keys "$(extract_failed_source_keys "$MULTI_SOURCE_SUMMARY")"
refresh_pipeline_audit
cat "$MULTI_SOURCE_OUTPUT"
rm -f "$MULTI_SOURCE_OUTPUT"
run_or_fail "zhipu" "智谱官方导入失败" go run -tags llm_script "./scripts/import_zhipu_data.go"
run_or_fail "official_seed_export" "官方种子导出失败" go run -tags llm_script "./scripts/export_official_seed_json.go"
run_or_fail "baidu" "百度官方导入失败" go run -tags llm_script "./scripts/import_phase2_data.go"
run_or_fail "bytedance" "字节官方导入失败" go run -tags llm_script "./scripts/import_bytedance_data.go"
run_or_fail "aliyun_subscription" "阿里云套餐导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/aliyun_subscription_lib.go ./scripts/import_aliyun_subscription.go
run_or_fail "baidu_subscription" "百度套餐导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/baidu_subscription_lib.go ./scripts/import_baidu_subscription.go
run_or_fail "ctyun_subscription" "天翼云套餐导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/ctyun_subscription_lib.go ./scripts/import_ctyun_subscription.go
run_or_fail "bytedance_subscription" "火山方舟套餐导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/bytedance_subscription_lib.go ./scripts/import_bytedance_subscription.go
run_or_fail "huawei_package" "华为云套餐包导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/huawei_package_lib.go ./scripts/import_huawei_package.go
run_or_fail "zhipu_coding_plan" "智谱 Coding Plan 导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/zhipu_coding_plan_lib.go ./scripts/import_zhipu_coding_plan.go
run_or_fail "minimax_subscription" "MiniMax Token Plan 导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/minimax_subscription_lib.go ./scripts/import_minimax_subscription.go
run_or_fail "cucloud_catalog" "联通云目录校验失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_cucloud_catalog.go
run_or_fail "mobile_cloud_catalog" "移动云目录校验失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/catalog_verification_common.go ./scripts/import_mobile_cloud_catalog.go
run_or_fail "youdao_pricing" "网易有道价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/youdao_pricing_lib.go ./scripts/import_youdao_pricing.go
run_or_fail "platform360_pricing" "360 智脑价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/platform360_pricing_lib.go ./scripts/import_360_pricing.go
run_or_fail "siliconflow_pricing" "硅基流动价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/siliconflow_pricing_lib.go ./scripts/import_siliconflow_pricing.go
run_or_fail "ppio_pricing" "PPIO 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/ppio_pricing_lib.go ./scripts/import_ppio_pricing.go
run_or_fail "ucloud_pricing" "UCloud 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/ucloud_pricing_lib.go ./scripts/import_ucloud_pricing.go
run_or_fail "cloudflare_pricing_signature" "Cloudflare Workers AI 价格页结构签名漂移" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/cloudflare_pricing_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/cloudflare_pricing_signature_guard_lib.go ./scripts/cloudflare_pricing_import_runner.go ./scripts/cloudflare_pricing_lib.go ./scripts/cloudflare_pricing_signature_guard.go
run_or_fail "cloudflare_pricing" "Cloudflare Workers AI 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/cloudflare_pricing_snapshot_lib.go ./scripts/cloudflare_pricing_import_runner.go ./scripts/cloudflare_pricing_lib.go ./scripts/import_cloudflare_pricing.go
run_or_fail "perplexity_pricing_signature" "Perplexity API 价格页结构签名漂移" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/perplexity_pricing_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/perplexity_pricing_signature_guard_lib.go ./scripts/perplexity_pricing_import_runner.go ./scripts/perplexity_pricing_lib.go ./scripts/perplexity_pricing_signature_guard.go
run_or_fail "perplexity_pricing" "Perplexity API 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/perplexity_pricing_snapshot_lib.go ./scripts/perplexity_pricing_import_runner.go ./scripts/perplexity_pricing_lib.go ./scripts/import_perplexity_pricing.go
run_or_fail "vertex_pricing_signature" "Vertex AI 价格页结构签名漂移" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/pricing_markdown_snapshot_lib.go ./scripts/signature_guard_common.go ./scripts/official_import_signature_audit_lib.go ./scripts/vertex_pricing_snapshot_lib.go ./scripts/vertex_pricing_signature_guard_lib.go ./scripts/vertex_pricing_import_runner.go ./scripts/vertex_pricing_lib.go ./scripts/vertex_pricing_signature_guard.go
run_or_fail "vertex_pricing" "Vertex AI 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/vertex_pricing_snapshot_lib.go ./scripts/vertex_pricing_import_runner.go ./scripts/vertex_pricing_lib.go ./scripts/import_vertex_pricing.go
run_or_fail "bedrock_pricing" "Amazon Bedrock 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/bedrock_pricing_lib.go ./scripts/import_bedrock_pricing.go
run_or_fail "azure_openai_pricing" "Azure OpenAI 价格导入失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/official_pricing_import_common.go ./scripts/azure_openai_pricing_lib.go ./scripts/import_azure_openai_pricing.go
refresh_pipeline_audit
run_or_fail "catalog_seed_verification" "目录级官方入口核验失败" \
go run -tags llm_script ./scripts/subscription_import_common.go ./scripts/import_catalog_seed_verification.go
refresh_pipeline_audit
run_or_fail "daily_signal_snapshot" "每日关键信号物化失败" \
env SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script "./scripts/materialize_daily_signals.go"
echo "$PIPELINE_AUDIT_SUMMARY"

126
scripts/run_real_pipeline.sh
View File

@@ -27,8 +27,8 @@ fi
REPORT_DATE="$(report_date_value)"
FETCH_OUT="$ROOT_DIR/models.json"
FETCH_TOTAL="0"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance"
PIPELINE_STAGE_SET="openrouter,multi_source,official_imports,daily_signal_snapshot,daily_report"
PIPELINE_SOURCE_SET="openrouter,moonshot,deepseek,openai,zhipu,baidu,bytedance,aliyun_subscription,baidu_subscription,ctyun_subscription,bytedance_subscription,huawei_package,zhipu_coding_plan,minimax_subscription,cucloud_catalog,mobile_cloud_catalog,youdao_pricing,platform360_pricing,siliconflow_pricing,ppio_pricing,ucloud_pricing,cloudflare_pricing,perplexity_pricing,vertex_pricing,bedrock_pricing,azure_openai_pricing,catalog_seed_verification"
PIPELINE_FAILED_SOURCE_SET="none"
MULTI_SOURCE_AUDIT="multi_source_audit=unavailable"
PIPELINE_AUDIT_SUMMARY=""
@@ -149,9 +149,129 @@ if ! go run -tags llm_script "./scripts/import_bytedance_data.go"; then
record_failure "字节官方导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/aliyun_subscription_lib.go" "./scripts/import_aliyun_subscription.go"; then
merge_failed_source_keys "aliyun_subscription"
record_failure "阿里云套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/baidu_subscription_lib.go" "./scripts/import_baidu_subscription.go"; then
merge_failed_source_keys "baidu_subscription"
record_failure "百度套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/ctyun_subscription_lib.go" "./scripts/import_ctyun_subscription.go"; then
merge_failed_source_keys "ctyun_subscription"
record_failure "天翼云套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/bytedance_subscription_lib.go" "./scripts/import_bytedance_subscription.go"; then
merge_failed_source_keys "bytedance_subscription"
record_failure "火山方舟套餐导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/huawei_package_lib.go" "./scripts/import_huawei_package.go"; then
merge_failed_source_keys "huawei_package"
record_failure "华为云套餐包导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/zhipu_coding_plan_lib.go" "./scripts/import_zhipu_coding_plan.go"; then
merge_failed_source_keys "zhipu_coding_plan"
record_failure "智谱 Coding Plan 导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/minimax_subscription_lib.go" "./scripts/import_minimax_subscription.go"; then
merge_failed_source_keys "minimax_subscription"
record_failure "MiniMax Token Plan 导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_cucloud_catalog.go"; then
merge_failed_source_keys "cucloud_catalog"
record_failure "联通云目录校验失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/catalog_verification_common.go" "./scripts/import_mobile_cloud_catalog.go"; then
merge_failed_source_keys "mobile_cloud_catalog"
record_failure "移动云目录校验失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/youdao_pricing_lib.go" "./scripts/import_youdao_pricing.go"; then
merge_failed_source_keys "youdao_pricing"
record_failure "网易有道价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/platform360_pricing_lib.go" "./scripts/import_360_pricing.go"; then
merge_failed_source_keys "platform360_pricing"
record_failure "360 智脑价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/siliconflow_pricing_lib.go" "./scripts/import_siliconflow_pricing.go"; then
merge_failed_source_keys "siliconflow_pricing"
record_failure "硅基流动价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ppio_pricing_lib.go" "./scripts/import_ppio_pricing.go"; then
merge_failed_source_keys "ppio_pricing"
record_failure "PPIO 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/ucloud_pricing_lib.go" "./scripts/import_ucloud_pricing.go"; then
merge_failed_source_keys "ucloud_pricing"
record_failure "UCloud 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/cloudflare_pricing_signature_guard_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/cloudflare_pricing_signature_guard.go"; then
merge_failed_source_keys "cloudflare_pricing_signature"
record_failure "Cloudflare Workers AI 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/cloudflare_pricing_snapshot_lib.go" "./scripts/cloudflare_pricing_import_runner.go" "./scripts/cloudflare_pricing_lib.go" "./scripts/import_cloudflare_pricing.go"; then
merge_failed_source_keys "cloudflare_pricing"
record_failure "Cloudflare Workers AI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/perplexity_pricing_signature_guard_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/perplexity_pricing_signature_guard.go"; then
merge_failed_source_keys "perplexity_pricing_signature"
record_failure "Perplexity API 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/perplexity_pricing_snapshot_lib.go" "./scripts/perplexity_pricing_import_runner.go" "./scripts/perplexity_pricing_lib.go" "./scripts/import_perplexity_pricing.go"; then
merge_failed_source_keys "perplexity_pricing"
record_failure "Perplexity API 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/pricing_markdown_snapshot_lib.go" "./scripts/signature_guard_common.go" "./scripts/official_import_signature_audit_lib.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_signature_guard_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/vertex_pricing_signature_guard.go"; then
merge_failed_source_keys "vertex_pricing_signature"
record_failure "Vertex AI 价格页结构签名漂移"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/vertex_pricing_snapshot_lib.go" "./scripts/vertex_pricing_import_runner.go" "./scripts/vertex_pricing_lib.go" "./scripts/import_vertex_pricing.go"; then
merge_failed_source_keys "vertex_pricing"
record_failure "Vertex AI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/bedrock_pricing_lib.go" "./scripts/import_bedrock_pricing.go"; then
merge_failed_source_keys "bedrock_pricing"
record_failure "Amazon Bedrock 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/official_pricing_import_common.go" "./scripts/azure_openai_pricing_lib.go" "./scripts/import_azure_openai_pricing.go"; then
merge_failed_source_keys "azure_openai_pricing"
record_failure "Azure OpenAI 价格导入失败"
exit 1
fi
if ! go run -tags llm_script "./scripts/subscription_import_common.go" "./scripts/import_catalog_seed_verification.go"; then
merge_failed_source_keys "catalog_seed_verification"
record_failure "目录级官方入口核验失败"
exit 1
fi
if ! SIGNAL_SOURCE_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run -tags llm_script "./scripts/materialize_daily_signals.go"; then
merge_failed_source_keys "daily_signal_snapshot"
record_failure "每日关键信号物化失败"
exit 1
fi
refresh_pipeline_audit
if ! REPORT_RUN_KIND="manual" REPORT_TRIGGER_SOURCE="pipeline" REPORT_IS_OFFICIAL_DAILY="false" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run "./scripts/generate_daily_report.go"; then
if ! REPORT_RUN_KIND="manual" REPORT_TRIGGER_SOURCE="pipeline" REPORT_IS_OFFICIAL_DAILY="false" REPORT_RUNTIME_AUDIT="$PIPELINE_AUDIT_SUMMARY" go run "./scripts/generate_daily_report.go" "./scripts/official_import_signature_audit_query_lib.go"; then
record_failure "日报生成失败"
exit 1
fi

12
scripts/verify_phase6.sh
View File

@@ -64,7 +64,17 @@ check_shell "真实采集并输出今日日报" "bash scripts/run_real_pipeline.
check_shell "API Server 可构建" "go build -o /dev/null ./cmd/server"
check_shell "健康检查脚本通过" "DATABASE_URL='$DB_URL' bash healthcheck.sh"
check_shell "密钥未硬编码进源码" "grep -R -n 'sk-' cmd internal frontend/src scripts .github/workflows --include='*.go' --include='*.ts' --include='*.tsx' --include='*.sh' --include='*.yml' --include='*.yaml' --exclude='verify_phase6.sh' >/tmp/llm_phase6_secret_scan.out 2>/dev/null; test ! -s /tmp/llm_phase6_secret_scan.out"
check_shell "最近 7 次采集成功率达到 95%" "psql \"$DB_URL\" -Atqc \"select coalesce(round(avg(case when success then 100 else 0 end),2),0) from (select success from collector_stats order by created_at desc limit 7) t;\" | awk '{ exit !(\$1 >= 95) }'"
set +e
collector_window_output="$(bash scripts/collector_stats_window_audit.sh --db "$DB_URL" --limit 7 --assert-success-rate 95 2>&1)"
collector_window_rc=$?
set -e
echo "$collector_window_output"
if [ "$collector_window_rc" -eq 0 ]; then
pass "最近 7 次采集成功率达到 95%（已输出分类摘要）"
else
fail "最近 7 次采集成功率达到 95%（见上方分类摘要）"
fi
if go build -o "$SERVER_BIN" ./cmd/server >/tmp/llm_phase6_server_build.out 2>/tmp/llm_phase6_server_build.err; then
if reserve_server_port && start_server; then