138 lines
4.5 KiB
Bash
Executable File
138 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
require_var() {
|
|
local name="$1"
|
|
if [[ -z "${!name:-}" ]]; then
|
|
echo "missing required env: $name" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
json_has_model() {
|
|
local file="$1"
|
|
local model="$2"
|
|
python3 - "$file" "$model" <<'PY'
|
|
import json, pathlib, sys
|
|
path = pathlib.Path(sys.argv[1])
|
|
model = sys.argv[2].strip()
|
|
obj = json.loads(path.read_text(encoding='utf-8'))
|
|
for item in obj.get('data', []):
|
|
if str(item.get('id', '')).strip() == model:
|
|
print('true')
|
|
raise SystemExit(0)
|
|
print('false')
|
|
PY
|
|
}
|
|
|
|
status_from_headers() {
|
|
local file="$1"
|
|
python3 - "$file" <<'PY'
|
|
import pathlib, re, sys
|
|
text = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')
|
|
for line in text.splitlines():
|
|
m = re.match(r'^HTTP/\S+\s+(\d{3})\b', line.strip())
|
|
if m:
|
|
print(m.group(1))
|
|
raise SystemExit(0)
|
|
print('0')
|
|
PY
|
|
}
|
|
|
|
content_type_from_headers() {
|
|
local file="$1"
|
|
python3 - "$file" <<'PY'
|
|
import pathlib, sys
|
|
text = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')
|
|
for line in text.splitlines():
|
|
if ':' not in line:
|
|
continue
|
|
k, v = line.split(':', 1)
|
|
if k.strip().lower() == 'content-type':
|
|
print(v.strip())
|
|
raise SystemExit(0)
|
|
print('')
|
|
PY
|
|
}
|
|
|
|
require_var ARTIFACT_DIR
|
|
require_var HOST_BASE
|
|
require_var HOST_MANAGED_KEY
|
|
require_var UPSTREAM_BASE
|
|
require_var UPSTREAM_API_KEY
|
|
|
|
MODEL="${MODEL:-deepseek-v4-flash}"
|
|
PROMPT="${PROMPT:-ping}"
|
|
ARTIFACT_DIR="${ARTIFACT_DIR%/}"
|
|
mkdir -p "$ARTIFACT_DIR"
|
|
|
|
host_models_headers="$ARTIFACT_DIR/01-host-models.headers.txt"
|
|
host_models_body="$ARTIFACT_DIR/02-host-models.body.json"
|
|
host_chat_headers="$ARTIFACT_DIR/03-host-chat.headers.txt"
|
|
host_chat_body="$ARTIFACT_DIR/04-host-chat.body.json"
|
|
upstream_chat_headers="$ARTIFACT_DIR/05-upstream-chat.headers.txt"
|
|
upstream_chat_body="$ARTIFACT_DIR/06-upstream-chat.body.txt"
|
|
summary_file="$ARTIFACT_DIR/summary.json"
|
|
|
|
chat_payload="$(python3 - "$MODEL" "$PROMPT" <<'PY'
|
|
import json, sys
|
|
print(json.dumps({
|
|
'model': sys.argv[1],
|
|
'messages': [{'role': 'user', 'content': sys.argv[2]}],
|
|
'max_tokens': 8,
|
|
'temperature': 0,
|
|
}, ensure_ascii=False))
|
|
PY
|
|
)"
|
|
|
|
curl -sS -D "$host_models_headers" -o "$host_models_body" \
|
|
-H "Authorization: Bearer $HOST_MANAGED_KEY" \
|
|
"${HOST_BASE%/}/v1/models"
|
|
|
|
curl -sS -D "$host_chat_headers" -o "$host_chat_body" \
|
|
-H "Authorization: Bearer $HOST_MANAGED_KEY" \
|
|
-H 'Content-Type: application/json' \
|
|
"${HOST_BASE%/}/v1/chat/completions" \
|
|
-d "$chat_payload"
|
|
|
|
curl -sS -D "$upstream_chat_headers" -o "$upstream_chat_body" \
|
|
-H "Authorization: Bearer $UPSTREAM_API_KEY" \
|
|
-H 'Content-Type: application/json' \
|
|
"${UPSTREAM_BASE%/}/chat/completions" \
|
|
-d "$chat_payload"
|
|
|
|
host_models_status="$(status_from_headers "$host_models_headers")"
|
|
host_chat_status="$(status_from_headers "$host_chat_headers")"
|
|
upstream_chat_status="$(status_from_headers "$upstream_chat_headers")"
|
|
host_has_expected_model="$(json_has_model "$host_models_body" "$MODEL")"
|
|
upstream_content_type="$(content_type_from_headers "$upstream_chat_headers")"
|
|
|
|
python3 - "$summary_file" "$host_models_status" "$host_has_expected_model" "$host_chat_status" "$upstream_chat_status" "$upstream_content_type" "$host_chat_body" "$upstream_chat_body" <<'PY'
|
|
import json, pathlib, sys
|
|
summary_path = pathlib.Path(sys.argv[1])
|
|
host_models_status = int(sys.argv[2])
|
|
host_has_expected_model = sys.argv[3].strip().lower() == 'true'
|
|
host_chat_status = int(sys.argv[4])
|
|
upstream_chat_status = int(sys.argv[5])
|
|
upstream_content_type = sys.argv[6].strip()
|
|
host_chat_body = pathlib.Path(sys.argv[7]).read_text(encoding='utf-8').strip()
|
|
upstream_chat_body = pathlib.Path(sys.argv[8]).read_text(encoding='utf-8').strip()
|
|
classification = 'unknown'
|
|
if host_models_status == 200 and host_has_expected_model and host_chat_status == 502 and upstream_chat_status == 200:
|
|
classification = 'host_compatibility_gap'
|
|
elif host_models_status == 200 and host_has_expected_model and upstream_chat_status == 403 and 'insufficient_user_quota' in upstream_chat_body:
|
|
classification = 'upstream_key_quota_issue'
|
|
summary = {
|
|
'host_models_status': host_models_status,
|
|
'host_has_expected_model': host_has_expected_model,
|
|
'host_chat_status': host_chat_status,
|
|
'upstream_chat_status': upstream_chat_status,
|
|
'upstream_chat_content_type': upstream_content_type,
|
|
'classification': classification,
|
|
'host_chat_body': host_chat_body,
|
|
'upstream_chat_body_preview': upstream_chat_body[:400],
|
|
}
|
|
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
PY
|