89 lines
3.2 KiB
Bash
89 lines
3.2 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT_DIR"
|
|
|
|
TMP_DIR="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP_DIR"' EXIT
|
|
|
|
FIXTURE_FAIL="$TMP_DIR/collector_stats_fail.tsv"
|
|
cat > "$FIXTURE_FAIL" <<'EOF'
|
|
openrouter f 严格真实模式下必须提供 API Key 2026-05-15 20:00:00
|
|
openrouter f 429 Too Many Requests 2026-05-15 19:59:00
|
|
openrouter t 2026-05-15 19:58:00
|
|
openrouter t 2026-05-15 19:57:00
|
|
openrouter t 2026-05-15 19:56:00
|
|
openrouter t 2026-05-15 19:55:00
|
|
openrouter f insert models failed 2026-05-15 19:54:00
|
|
EOF
|
|
|
|
set +e
|
|
FAIL_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_FAIL" --limit 7 --assert-success-rate 95 2>&1)"
|
|
FAIL_RC=$?
|
|
set -e
|
|
|
|
if [[ "$FAIL_RC" -eq 0 ]]; then
|
|
echo "expected failing fixture to exit non-zero"
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'success_rate=66.67'
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'precondition_missing=0'
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'aged_precondition_missing=1'
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'external_provider_failure=1'
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'collector_runtime_failure=1'
|
|
printf '%s' "$FAIL_OUTPUT" | grep -q 'sample_1 created_at=2026-05-15 20:00:00'
|
|
|
|
FIXTURE_PASS="$TMP_DIR/collector_stats_pass.tsv"
|
|
cat > "$FIXTURE_PASS" <<'EOF'
|
|
openrouter t 2026-05-15 20:00:00
|
|
openrouter t 2026-05-15 19:59:00
|
|
openrouter t 2026-05-15 19:58:00
|
|
openrouter t 2026-05-15 19:57:00
|
|
openrouter t 2026-05-15 19:56:00
|
|
openrouter t 2026-05-15 19:55:00
|
|
openrouter t 2026-05-15 19:54:00
|
|
EOF
|
|
|
|
PASS_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_PASS" --limit 7 --assert-success-rate 95 2>&1)"
|
|
printf '%s' "$PASS_OUTPUT" | grep -q 'success_rate=100.00'
|
|
printf '%s' "$PASS_OUTPUT" | grep -q 'failure_count=0'
|
|
printf '%s' "$PASS_OUTPUT" | grep -q 'sample_7 created_at=2026-05-15 19:54:00'
|
|
|
|
|
|
FIXTURE_AGED_PRECONDITION="$TMP_DIR/collector_stats_aged_precondition.tsv"
|
|
cat > "$FIXTURE_AGED_PRECONDITION" <<'EOF'
|
|
openrouter f OPENROUTER_API_KEY 未设置 2026-05-10 08:00:00
|
|
openrouter t 2026-05-15 20:00:00
|
|
openrouter t 2026-05-15 19:59:00
|
|
openrouter t 2026-05-15 19:58:00
|
|
openrouter t 2026-05-15 19:57:00
|
|
openrouter t 2026-05-15 19:56:00
|
|
openrouter t 2026-05-15 19:55:00
|
|
EOF
|
|
|
|
AGED_OUTPUT="$(LLM_NOW='2026-05-15 20:00' bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_AGED_PRECONDITION" --limit 7 --assert-success-rate 95 2>&1)"
|
|
printf '%s' "$AGED_OUTPUT" | grep -q 'aged_precondition_missing=1'
|
|
printf '%s' "$AGED_OUTPUT" | grep -q 'precondition_missing=0'
|
|
FIXTURE_EXTERNAL_ONLY="$TMP_DIR/collector_stats_external_only.tsv"
|
|
cat > "$FIXTURE_EXTERNAL_ONLY" <<'EOF'
|
|
perplexity f unexpected perplexity pricing content: no model rows parsed 2026-05-15 20:00:00
|
|
vertex f fetch https://example.com: unexpected status 403 2026-05-15 19:59:00
|
|
cloudflare t 2026-05-15 19:58:00
|
|
EOF
|
|
|
|
set +e
|
|
EXTERNAL_OUTPUT="$(bash scripts/collector_stats_window_audit.sh --input "$FIXTURE_EXTERNAL_ONLY" --limit 3 --assert-success-rate 95 2>&1)"
|
|
EXTERNAL_RC=$?
|
|
set -e
|
|
|
|
if [[ "$EXTERNAL_RC" -eq 0 ]]; then
|
|
echo "expected external-only fixture to exit non-zero"
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s' "$EXTERNAL_OUTPUT" | grep -q 'external_provider_failure=2'
|
|
printf '%s' "$EXTERNAL_OUTPUT" | grep -q 'collector_runtime_failure=0'
|