feat: organize scripts and add portal validation assets

This commit is contained in:
phamnazage-jpg
2026-05-27 09:39:05 +08:00
parent c1172d7714
commit 02580cda0b
180 changed files with 3392 additions and 76 deletions

View File

@@ -0,0 +1,198 @@
#!/usr/bin/env python3
"""Helpers for redacting real-host acceptance artifacts."""
import hashlib
import json
import pathlib
import sys
from typing import Any
KEY_FIELD_NAMES = {
"api_key",
"requested_probe_api_key",
"raw_key",
"subscription_user_key",
"managed_probe_key",
}
PREFIX_FIELD_NAMES = {
"gateway_key_prefix",
"managed_key_prefix",
"managed_probe_key_prefix",
"subscription_user_key_prefix",
"managed_key_preview",
}
IDENTIFIER_FIELD_NAMES = {
"subscription_user_id",
"raw_user_id",
"managed_user_id",
"admin_user_id",
}
EMAIL_FIELD_NAMES = {
"managed_user_email",
}
JSON_STRING_FIELD_NAMES = {
"DetailsJSON",
"details_json",
"probe_summary_json",
}
def redact_key(value: str) -> dict[str, Any]:
value = (value or "").strip()
if not value:
return {
"present": False,
"prefix": "",
"suffix": "",
"fingerprint": "",
}
return {
"present": True,
"prefix": value[:4],
"suffix": value[-4:] if len(value) >= 4 else value,
"fingerprint": hashlib.sha256(value.encode("utf-8")).hexdigest(),
}
def redact_identifier(value: str) -> str:
value = (value or "").strip()
if not value:
return ""
return hashlib.sha256(value.encode("utf-8")).hexdigest()
def sanitize_headers(raw: str) -> str:
lines = []
for line in (raw or "").splitlines():
lower = line.lower()
if lower.startswith("authorization:"):
continue
if lower.startswith("cookie:"):
continue
if lower.startswith("set-cookie:"):
continue
if lower.startswith("x-api-key:"):
continue
lines.append(line)
return "\n".join(lines) + ("\n" if lines else "")
def sanitize_group_state(payload: Any) -> dict[str, Any]:
if not isinstance(payload, dict):
return {}
group = payload.get("group") if isinstance(payload.get("group"), dict) else {}
subscription = payload.get("subscription") if isinstance(payload.get("subscription"), dict) else {}
key = payload.get("key") if isinstance(payload.get("key"), dict) else {}
key_value = str(key.get("key") or "")
return {
"group_id": payload.get("group_id"),
"group": {
"id": group.get("id"),
"name": group.get("name"),
"type": group.get("type"),
"subscription_type": group.get("subscription_type"),
},
"subscription": {
"id": subscription.get("id"),
"user_id_hash": redact_identifier(str(subscription.get("user_id") or "")),
"group_id": subscription.get("group_id"),
"status": subscription.get("status"),
"starts_at": subscription.get("starts_at"),
"expires_at": subscription.get("expires_at"),
},
"key": {
"id": key.get("id"),
"group_id": key.get("group_id"),
"status": key.get("status"),
"redacted": redact_key(key_value),
},
}
def sanitize_runtime_context(payload: Any) -> dict[str, Any]:
if not isinstance(payload, dict):
return {}
out: dict[str, Any] = {
"crm_base": payload.get("crm_base"),
"host_base": payload.get("host_base"),
"crm_host_base": payload.get("crm_host_base"),
"remote_host_base": payload.get("remote_host_base"),
"provider_id": payload.get("provider_id"),
"subscription_group_id": payload.get("subscription_group_id"),
"import_group_id": payload.get("import_group_id"),
}
if "subscription_user_id" in payload:
out["subscription_user_id_hash"] = redact_identifier(str(payload.get("subscription_user_id") or ""))
if "managed_user_id" in payload:
out["managed_user_id_hash"] = redact_identifier(str(payload.get("managed_user_id") or ""))
if "admin_user_id" in payload:
out["admin_user_id_hash"] = redact_identifier(str(payload.get("admin_user_id") or ""))
if "managed_user_email" in payload:
out["managed_user_email_hash"] = redact_identifier(str(payload.get("managed_user_email") or ""))
if "subscription_user_key_prefix" in payload or "subscription_user_key" in payload:
source = str(payload.get("subscription_user_key") or payload.get("subscription_user_key_prefix") or "")
out["subscription_user_key"] = redact_key(source)
if "managed_probe_key_prefix" in payload or "managed_probe_key" in payload:
source = str(payload.get("managed_probe_key") or payload.get("managed_probe_key_prefix") or "")
out["managed_probe_key"] = redact_key(source)
return out
def sanitize_nested(value: Any) -> Any:
if isinstance(value, dict):
out: dict[str, Any] = {}
for key, item in value.items():
if key in KEY_FIELD_NAMES:
out[key] = redact_key(str(item or ""))
continue
if key in PREFIX_FIELD_NAMES:
out[key] = redact_key(str(item or ""))
continue
if key in IDENTIFIER_FIELD_NAMES:
out[f"{key}_hash"] = redact_identifier(str(item or ""))
continue
if key in EMAIL_FIELD_NAMES:
out[f"{key}_hash"] = redact_identifier(str(item or ""))
continue
if key in JSON_STRING_FIELD_NAMES and isinstance(item, str):
try:
parsed = json.loads(item)
except Exception:
out[key] = item
else:
out[key] = json.dumps(sanitize_nested(parsed), ensure_ascii=False)
continue
out[key] = sanitize_nested(item)
return out
if isinstance(value, list):
return [sanitize_nested(item) for item in value]
return value
def write_json(path: str, payload: Any) -> None:
pathlib.Path(path).write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
if __name__ == "__main__":
mode = sys.argv[1]
if mode == "redact-key":
print(json.dumps(redact_key(sys.argv[2]), ensure_ascii=False))
elif mode == "redact-id":
print(redact_identifier(sys.argv[2]))
elif mode == "sanitize-headers":
src, dst = sys.argv[2:4]
payload = pathlib.Path(src).read_text(encoding="utf-8")
pathlib.Path(dst).write_text(sanitize_headers(payload), encoding="utf-8")
elif mode == "sanitize-group-state":
src, dst = sys.argv[2:4]
payload = json.loads(pathlib.Path(src).read_text(encoding="utf-8"))
write_json(dst, sanitize_group_state(payload))
elif mode == "sanitize-runtime-context":
src, dst = sys.argv[2:4]
payload = json.loads(pathlib.Path(src).read_text(encoding="utf-8"))
write_json(dst, sanitize_runtime_context(payload))
elif mode == "sanitize-json":
src, dst = sys.argv[2:4]
payload = json.loads(pathlib.Path(src).read_text(encoding="utf-8"))
write_json(dst, sanitize_nested(payload))
else:
raise SystemExit(f"unsupported mode: {mode}")

View File

@@ -0,0 +1,139 @@
#!/usr/bin/env bash
set -euo pipefail
# 环境变量驱动,便于被不同验收 harness 复用。
require_var() {
local name="$1"
if [[ -z "${!name:-}" ]]; then
echo "missing required env: $name" >&2
exit 1
fi
}
json_has_model() {
local file="$1"
local model="$2"
python3 - "$file" "$model" <<'PY'
import json, pathlib, sys
path = pathlib.Path(sys.argv[1])
model = sys.argv[2].strip()
obj = json.loads(path.read_text(encoding='utf-8'))
for item in obj.get('data', []):
if str(item.get('id', '')).strip() == model:
print('true')
raise SystemExit(0)
print('false')
PY
}
status_from_headers() {
local file="$1"
python3 - "$file" <<'PY'
import pathlib, re, sys
text = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')
for line in text.splitlines():
m = re.match(r'^HTTP/\S+\s+(\d{3})\b', line.strip())
if m:
print(m.group(1))
raise SystemExit(0)
print('0')
PY
}
content_type_from_headers() {
local file="$1"
python3 - "$file" <<'PY'
import pathlib, sys
text = pathlib.Path(sys.argv[1]).read_text(encoding='utf-8')
for line in text.splitlines():
if ':' not in line:
continue
k, v = line.split(':', 1)
if k.strip().lower() == 'content-type':
print(v.strip())
raise SystemExit(0)
print('')
PY
}
require_var ARTIFACT_DIR
require_var HOST_BASE
require_var HOST_MANAGED_KEY
require_var UPSTREAM_BASE
require_var UPSTREAM_API_KEY
MODEL="${MODEL:-deepseek-v4-flash}"
PROMPT="${PROMPT:-ping}"
ARTIFACT_DIR="${ARTIFACT_DIR%/}"
mkdir -p "$ARTIFACT_DIR"
host_models_headers="$ARTIFACT_DIR/01-host-models.headers.txt"
host_models_body="$ARTIFACT_DIR/02-host-models.body.json"
host_chat_headers="$ARTIFACT_DIR/03-host-chat.headers.txt"
host_chat_body="$ARTIFACT_DIR/04-host-chat.body.json"
upstream_chat_headers="$ARTIFACT_DIR/05-upstream-chat.headers.txt"
upstream_chat_body="$ARTIFACT_DIR/06-upstream-chat.body.txt"
summary_file="$ARTIFACT_DIR/summary.json"
chat_payload="$(python3 - "$MODEL" "$PROMPT" <<'PY'
import json, sys
print(json.dumps({
'model': sys.argv[1],
'messages': [{'role': 'user', 'content': sys.argv[2]}],
'max_tokens': 8,
'temperature': 0,
}, ensure_ascii=False))
PY
)"
curl -sS -D "$host_models_headers" -o "$host_models_body" \
-H "Authorization: Bearer $HOST_MANAGED_KEY" \
"${HOST_BASE%/}/v1/models"
curl -sS -D "$host_chat_headers" -o "$host_chat_body" \
-H "Authorization: Bearer $HOST_MANAGED_KEY" \
-H 'Content-Type: application/json' \
"${HOST_BASE%/}/v1/chat/completions" \
-d "$chat_payload"
curl -sS -D "$upstream_chat_headers" -o "$upstream_chat_body" \
-H "Authorization: Bearer $UPSTREAM_API_KEY" \
-H 'Content-Type: application/json' \
"${UPSTREAM_BASE%/}/chat/completions" \
-d "$chat_payload"
host_models_status="$(status_from_headers "$host_models_headers")"
host_chat_status="$(status_from_headers "$host_chat_headers")"
upstream_chat_status="$(status_from_headers "$upstream_chat_headers")"
host_has_expected_model="$(json_has_model "$host_models_body" "$MODEL")"
upstream_content_type="$(content_type_from_headers "$upstream_chat_headers")"
python3 - "$summary_file" "$host_models_status" "$host_has_expected_model" "$host_chat_status" "$upstream_chat_status" "$upstream_content_type" "$host_chat_body" "$upstream_chat_body" <<'PY'
import json, pathlib, sys
summary_path = pathlib.Path(sys.argv[1])
host_models_status = int(sys.argv[2])
host_has_expected_model = sys.argv[3].strip().lower() == 'true'
host_chat_status = int(sys.argv[4])
upstream_chat_status = int(sys.argv[5])
upstream_content_type = sys.argv[6].strip()
host_chat_body = pathlib.Path(sys.argv[7]).read_text(encoding='utf-8').strip()
upstream_chat_body = pathlib.Path(sys.argv[8]).read_text(encoding='utf-8').strip()
classification = 'unknown'
if host_models_status == 200 and host_has_expected_model and host_chat_status == 502 and upstream_chat_status == 200:
classification = 'host_compatibility_gap'
elif host_models_status == 200 and host_has_expected_model and upstream_chat_status == 403 and 'insufficient_user_quota' in upstream_chat_body:
classification = 'upstream_key_quota_issue'
summary = {
'host_models_status': host_models_status,
'host_has_expected_model': host_has_expected_model,
'host_chat_status': host_chat_status,
'upstream_chat_status': upstream_chat_status,
'upstream_chat_content_type': upstream_content_type,
'classification': classification,
'host_chat_body': host_chat_body,
'upstream_chat_body_preview': upstream_chat_body[:400],
}
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8')
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY

View File

@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -euo pipefail
# SQL 和缓存 key 组装 helper供 remote acceptance 脚本复用。
sql_escape_literal() {
local value="$1"
local squote="'"
value=${value//$squote/$squote$squote}
printf '%s' "$value"
}
sql_literal() {
printf "'%s'" "$(sql_escape_literal "$1")"
}
build_top_up_user_balance_sql() {
local user_id="$1"
local min_balance="$2"
cat <<SQL
UPDATE users
SET balance = CASE WHEN balance < ${min_balance} THEN ${min_balance} ELSE balance END,
updated_at = now()
WHERE id = ${user_id};
SQL
}
build_bind_api_key_group_sql() {
local api_key="$1"
local group_id="$2"
cat <<SQL
UPDATE api_keys
SET group_id = ${group_id},
updated_at = now()
WHERE key = $(sql_literal "$api_key");
SQL
}
build_api_key_auth_cache_key() {
local api_key="$1"
local digest
digest="$(printf '%s' "$api_key" | sha256sum | awk '{print $1}')"
printf 'apikey:auth:%s' "$digest"
}
build_user_balance_cache_key() {
local user_id="$1"
printf 'billing:balance:%s' "$user_id"
}
build_subscription_billing_cache_key() {
local user_id="$1"
local group_id="$2"
printf 'billing:sub:%s:%s' "$user_id" "$group_id"
}
build_upsert_subscription_sql() {
local user_id="$1"
local group_id="$2"
local duration_days="$3"
local assigned_by="$4"
local notes="$5"
cat <<SQL
INSERT INTO user_subscriptions (
user_id,
group_id,
starts_at,
expires_at,
status,
assigned_by,
assigned_at,
notes,
created_at,
updated_at,
deleted_at
)
VALUES (
${user_id},
${group_id},
now(),
now() + interval '$(sql_escape_literal "$duration_days") days',
'active',
${assigned_by},
now(),
$(sql_literal "$notes"),
now(),
now(),
NULL
)
ON CONFLICT (user_id, group_id) WHERE deleted_at IS NULL
DO UPDATE SET
starts_at = EXCLUDED.starts_at,
expires_at = EXCLUDED.expires_at,
status = 'active',
assigned_by = EXCLUDED.assigned_by,
assigned_at = EXCLUDED.assigned_at,
notes = EXCLUDED.notes,
updated_at = now(),
deleted_at = NULL;
SQL
}
build_subscription_access_prep_sql() {
local user_id="$1"
local api_key="$2"
local group_id="$3"
local min_balance="$4"
local duration_days="$5"
local assigned_by="$6"
local notes="$7"
cat <<SQL
BEGIN;
$(build_top_up_user_balance_sql "$user_id" "$min_balance")
$(build_bind_api_key_group_sql "$api_key" "$group_id")
$(build_upsert_subscription_sql "$user_id" "$group_id" "$duration_days" "$assigned_by" "$notes")
COMMIT;
SQL
}

View File

@@ -0,0 +1,753 @@
#!/usr/bin/env bash
set -euo pipefail
provider_id="${1:?provider_id required}"
model_name="${2:?model_name required}"
env_var="${3:?env var required}"
key_file="${4:-}"
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# shellcheck disable=SC1091
source "$ROOT_DIR/scripts/acceptance/host_access_prep_lib.sh"
ARTIFACT_REDACTION_SCRIPT="$ROOT_DIR/scripts/acceptance/artifact_redaction.py"
KEY="${KEY:-/home/long/下载/zjsea.pem}"
REMOTE="${REMOTE:-ubuntu@43.155.133.187}"
CRM_BASE="${CRM_BASE:-http://127.0.0.1:18088}"
HOST_BASE="${HOST_BASE:-http://127.0.0.1:18087}"
CRM_HOST_BASE="${CRM_HOST_BASE:-$HOST_BASE}"
REMOTE_HOST_BASE="${REMOTE_HOST_BASE:-$CRM_HOST_BASE}"
HOST_NAME="${HOST_NAME:-remote43-current-host}"
REMOTE_HOST_ENV_FILE="${REMOTE_HOST_ENV_FILE:-/home/ubuntu/sub2api-host-validation-fresh-deepseek-20260519_115244/.env}"
REMOTE_PG_CONTAINER="${REMOTE_PG_CONTAINER:-}"
REMOTE_REDIS_CONTAINER="${REMOTE_REDIS_CONTAINER:-}"
PACK_PATH="${PACK_PATH:-$ROOT_DIR/packs/openai-cn-pack}"
ROOT="${ROOT:-$ROOT_DIR/artifacts/real-host-acceptance}"
ART="${ART:-$ROOT/$(date +%Y%m%d_%H%M%S)_remote43_${provider_id}_key_import}"
MIN_BALANCE="${MIN_BALANCE:-10}"
SUBSCRIPTION_DAYS="${SUBSCRIPTION_DAYS:-30}"
SUBSCRIPTION_NOTES="${SUBSCRIPTION_NOTES:-hermes remote subscription validation}"
ARTIFACT_SECURITY_MODE="${ARTIFACT_SECURITY_MODE:-safe}"
ARTIFACT_INCLUDE_SECRETS="${ARTIFACT_INCLUDE_SECRETS:-0}"
mkdir -p "$ART"
artifact_redact_key_json() {
local value="$1"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" redact-key "$value"
}
artifact_redact_id() {
local value="$1"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" redact-id "$value"
}
write_json_file() {
local path="$1"
local payload="$2"
printf '%s\n' "$payload" > "$path"
}
sanitize_headers_file() {
local path="$1"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" sanitize-headers "$path" "$path"
}
sanitize_runtime_context_file() {
local path="$1"
local tmp="$path.tmp"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" sanitize-runtime-context "$path" "$tmp"
mv "$tmp" "$path"
}
sanitize_group_state_file() {
local path="$1"
local tmp="$path.tmp"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" sanitize-group-state "$path" "$tmp"
mv "$tmp" "$path"
}
redact_body_preview() {
local text="$1"
local value="$text"
if [[ -n "${managed_probe_key:-}" ]]; then
value="${value//$managed_probe_key/***}"
fi
if [[ -n "${upstream_key:-}" ]]; then
value="${value//$upstream_key/***}"
fi
if [[ -n "${sub_key:-}" ]]; then
value="${value//$sub_key/***}"
fi
printf '%s' "$value"
}
if [[ -n "$key_file" ]]; then
upstream_key="$(tr -d '\r\n' < "$key_file")"
key_source="file:$key_file"
else
upstream_key="${!env_var:-}"
key_source="env:$env_var"
fi
if [[ -z "$upstream_key" ]]; then
echo "missing key from $key_source" >&2
exit 2
fi
upstream_base_url="$(python3 - "$PACK_PATH" "$provider_id" <<'PY'
import json, pathlib, sys
pack_path = pathlib.Path(sys.argv[1])
provider_id = sys.argv[2]
provider_file = pack_path / "providers" / f"{provider_id}.json"
provider = json.loads(provider_file.read_text(encoding='utf-8'))
print(str(provider.get("base_url", "")).strip())
PY
)"
if [[ -z "$upstream_base_url" ]]; then
echo "missing provider base_url for $provider_id in $PACK_PATH" >&2
exit 2
fi
pack_id="$(python3 - "$PACK_PATH" <<'PY'
import json, pathlib, sys
pack_path = pathlib.Path(sys.argv[1])
pack_file = pack_path / "pack.json"
pack = json.loads(pack_file.read_text(encoding='utf-8'))
print(str(pack.get("pack_id", "")).strip())
PY
)"
if [[ -z "$pack_id" ]]; then
echo "missing pack_id in $PACK_PATH/pack.json" >&2
exit 2
fi
ssh_cmd() {
local cmd="$1"
ssh -i "$KEY" -o StrictHostKeyChecking=no "$REMOTE" "$cmd"
}
resolve_remote_host_runtime() {
local remote_port host_containers
remote_port="$(python3 - "$REMOTE_HOST_BASE" <<'PY'
from urllib.parse import urlparse
import sys
target = urlparse(sys.argv[1])
if target.port is not None:
print(target.port)
elif target.scheme == 'https':
print(443)
else:
print(80)
PY
)"
host_containers="$(ssh_cmd "sudo -n docker ps --format '{{.Names}}\t{{.Ports}}'")"
HOST_CONTAINERS="$host_containers" python3 - "$remote_port" <<'PY'
import os
import sys
port = sys.argv[1]
rows = os.environ.get("HOST_CONTAINERS", "").splitlines()
for row in rows:
name, _, ports = row.partition('\t')
if f":{port}->" not in ports:
continue
app = name.strip()
if app.endswith("-app-1"):
prefix = app[:-len("-app-1")]
print(app)
print(f"{prefix}-postgres-1")
print(f"{prefix}-redis-1")
raise SystemExit(0)
if app.endswith("-app"):
prefix = app[:-len("-app")]
print(app)
print(f"{prefix}-pg")
print(f"{prefix}-redis")
raise SystemExit(0)
raise SystemExit(f"unable to derive target host containers from port {port}")
PY
}
if [[ -z "$REMOTE_PG_CONTAINER" || -z "$REMOTE_REDIS_CONTAINER" ]]; then
mapfile -t resolved_remote_runtime < <(resolve_remote_host_runtime)
if [[ ${#resolved_remote_runtime[@]} -lt 3 ]]; then
echo "unable to resolve remote host runtime containers for $REMOTE_HOST_BASE" >&2
exit 2
fi
REMOTE_PG_CONTAINER="${REMOTE_PG_CONTAINER:-${resolved_remote_runtime[1]}}"
REMOTE_REDIS_CONTAINER="${REMOTE_REDIS_CONTAINER:-${resolved_remote_runtime[2]}}"
fi
REMOTE_PG_CONTAINER_Q="$(printf '%q' "$REMOTE_PG_CONTAINER")"
REMOTE_REDIS_CONTAINER_Q="$(printf '%q' "$REMOTE_REDIS_CONTAINER")"
build_managed_subscription_identity_json() {
local selector="$1"
local group_id="$2"
python3 - "$selector" "$group_id" <<'PY'
import hashlib, json, sys
selector, group_id = sys.argv[1:3]
def sanitize(value: str) -> str:
value = value.strip().lower()
chars = []
last_dash = False
for ch in value:
if ('a' <= ch <= 'z') or ('0' <= ch <= '9'):
chars.append(ch)
last_dash = False
elif not last_dash:
chars.append('-')
last_dash = True
return ''.join(chars).strip('-')
def truncate(value: str, max_len: int) -> str:
if len(value) <= max_len:
return value
return value[:max_len].strip('-')
normalized = selector.strip().lower() + '|' + group_id.strip()
digest = hashlib.sha256(normalized.encode('utf-8')).hexdigest()
prefix = sanitize(selector) or 'relay-sub'
prefix = truncate(prefix, 24)
short_hash = digest[:16]
key_hash = digest[:32]
username = truncate(f"{prefix}-{short_hash[:8]}", 32)
print(json.dumps({
'email': f"{prefix}-{short_hash}@sub2api.local",
'username': username,
'custom_key': 'sk-relay-' + key_hash,
'key_name': truncate(username + '-key', 48),
}, ensure_ascii=False))
PY
}
remote_lookup_managed_subscription_user_id() {
local email="$1"
remote_pg_query "select id from users where email = $(sql_literal "$email") order by id desc limit 1;"
}
crm_curl_json() {
local method="$1"
local path="$2"
local payload="${3:-}"
if [[ -n "$payload" ]]; then
curl -fsS -X "$method" \
-H "Authorization: Bearer $crm_token" \
-H 'Content-Type: application/json' \
"${CRM_BASE}${path}" \
-d "$payload"
else
curl -fsS -X "$method" \
-H "Authorization: Bearer $crm_token" \
"${CRM_BASE}${path}"
fi
}
fetch_remote_host_bearer_token() {
ssh_cmd "python3 - <<'PY'
from pathlib import Path
import json, subprocess, sys
env_path = Path(${REMOTE_HOST_ENV_FILE@Q})
host_base = ${REMOTE_HOST_BASE@Q}
vals = {}
for line in env_path.read_text().splitlines():
if '=' not in line:
continue
key, value = line.split('=', 1)
vals[key] = value
payload = json.dumps({
'email': vals['ADMIN_EMAIL'],
'password': vals['ADMIN_PASSWORD'],
'turnstile_token': '',
}, ensure_ascii=False)
res = subprocess.run([
'curl', '-fsS', '-H', 'Content-Type: application/json', '-X', 'POST',
host_base.rstrip('/') + '/api/v1/auth/login', '-d', payload,
], text=True, capture_output=True)
obj = json.loads(res.stdout)
token = (obj.get('data') or {}).get('access_token', '')
if not token:
print(res.stdout, file=sys.stderr)
raise SystemExit('missing access_token from remote host login')
print(token)
PY"
}
remote_pg_exec() {
local sql="$1"
local encoded
encoded="$(printf '%s' "$sql" | base64 -w0)"
ssh_cmd "printf '%s' '$encoded' | base64 -d | sudo -n docker exec -i $REMOTE_PG_CONTAINER_Q psql -U sub2api -d sub2api"
}
remote_pg_query() {
local sql="$1"
local encoded
encoded="$(printf '%s' "$sql" | base64 -w0)"
ssh_cmd "printf '%s' '$encoded' | base64 -d | sudo -n docker exec -i $REMOTE_PG_CONTAINER_Q psql -U sub2api -d sub2api -At -F $'\t'"
}
remote_fetch_group_state() {
local group_id="$1"
local user_id="$2"
local api_key="$3"
local output_path="$4"
local sql
sql="$(python3 - "$group_id" "$user_id" "$api_key" <<'PY'
import sys
group_id, user_id, api_key = sys.argv[1:4]
api_key_literal = "'" + api_key.replace("'", "''") + "'"
query = f"""
WITH group_row AS (
SELECT row_to_json(g) AS data FROM groups g WHERE g.id = {group_id}
),
subscription_row AS (
SELECT row_to_json(s) AS data FROM user_subscriptions s
WHERE s.user_id = {user_id} AND s.group_id = {group_id} AND s.deleted_at IS NULL
ORDER BY s.id DESC LIMIT 1
),
key_row AS (
SELECT row_to_json(k) AS data FROM api_keys k WHERE k.key = {api_key_literal}
)
SELECT json_build_object(
'group_id', {group_id},
'group', (SELECT data FROM group_row),
'subscription', (SELECT data FROM subscription_row),
'key', (SELECT data FROM key_row)
);
"""
print(query)
PY
)"
remote_pg_query "$sql" > "$output_path"
}
write_json_file "$ART/00-local-key-source.json" "$(python3 - <<'PY' "$ARTIFACT_REDACTION_SCRIPT" "$key_source" "$provider_id" "$upstream_key"
import json, sys
redaction_script, source, provider_id, key = sys.argv[1:5]
import subprocess
result = subprocess.check_output([sys.executable, redaction_script, 'redact-key', key], text=True)
redacted = json.loads(result)
print(json.dumps({
'source': source,
'provider_id': provider_id,
'redacted': redacted,
}, ensure_ascii=False, indent=2))
PY
)"
crm_token="${CRM_ADMIN_TOKEN:-}"
if [[ -z "$crm_token" ]]; then
crm_token="$(ssh_cmd "grep ^SUB2API_CRM_ADMIN_TOKEN= /home/ubuntu/sub2api-cn-relay-manager/.env.remote | cut -d= -f2-")"
crm_token="${crm_token##*$'\n'}"
fi
host_bearer_token="${HOST_BEARER_TOKEN:-}"
if [[ -z "$host_bearer_token" ]]; then
host_bearer_token="$(fetch_remote_host_bearer_token)"
host_bearer_token="${host_bearer_token##*$'\n'}"
fi
admin_uid="$(ssh_cmd "sudo -n docker exec $REMOTE_PG_CONTAINER_Q psql -U sub2api -d sub2api -Atc \"select id from users where role='admin' order by id asc limit 1;\"")"
admin_uid="${admin_uid##*$'\n'}"
sub_uid="$(remote_pg_query "select u.id from users u where u.email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=u.id and s.deleted_at is null) order by u.id desc limit 1;")"
sub_uid="${sub_uid##*$'\n'}"
sub_key="$(remote_pg_query "select k.key from users u join api_keys k on k.user_id=u.id where u.email like 'relay-sub-%@sub2api.local' and not exists (select 1 from user_subscriptions s where s.user_id=u.id and s.deleted_at is null) order by u.id desc limit 1;")"
sub_key="${sub_key##*$'\n'}"
if [[ -z "$sub_uid" || -z "$sub_key" ]]; then
fresh_seed="$(python3 - <<'PY'
import secrets, time
print(f"{int(time.time())}-{secrets.token_hex(4)}")
PY
)"
fresh_email="relay-sub-${fresh_seed}@sub2api.local"
fresh_username="relay-sub-${fresh_seed}"
fresh_key="sk-${fresh_seed}"
create_user_sql="$(python3 - "$fresh_email" "$fresh_username" "$fresh_key" <<'PY'
import sys
email, username, api_key = sys.argv[1:4]
def sql_quote(value: str) -> str:
return "'" + value.replace("'", "''") + "'"
print(f'''
WITH seed AS (
SELECT password_hash
FROM users
WHERE role = 'admin'
ORDER BY id ASC
LIMIT 1
),
ins_user AS (
INSERT INTO users (
email, password_hash, role, balance, concurrency, status, username, notes, wechat,
totp_secret_encrypted, totp_enabled, balance_notify_enabled, balance_notify_threshold,
balance_notify_extra_emails, balance_notify_threshold_type, total_recharged, signup_source,
rpm_limit
)
SELECT
{sql_quote(email)},
password_hash,
'user',
10,
5,
'active',
{sql_quote(username)},
'hermes remote subscription validation',
'',
'',
false,
true,
NULL,
'[]',
'fixed',
0,
'email',
0
FROM seed
RETURNING id
),
ins_key AS (
INSERT INTO api_keys (
user_id, key, name, group_id, status, quota, quota_used,
rate_limit_5h, rate_limit_1d, rate_limit_7d,
usage_5h, usage_1d, usage_7d
)
SELECT
id,
{sql_quote(api_key)},
{sql_quote(username + '-key')},
NULL,
'active',
0,
0,
0,
0,
0,
0,
0,
0
FROM ins_user
RETURNING user_id, key
)
SELECT user_id, key FROM ins_key;
'''.strip())
PY
)"
read -r sub_uid sub_key <<EOF
$(remote_pg_query "$create_user_sql")
EOF
fi
write_json_file "$ART/01-runtime-context.json" "$(python3 - <<'PY' "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key"
import json, subprocess, sys
crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key = sys.argv[1:8]
print(json.dumps({
'crm_base': crm,
'host_base': host,
'crm_host_base': crm_host,
'remote_host_base': remote_host,
'provider_id': provider_id,
'subscription_user_id': sub_uid,
'subscription_user_key': sub_key,
}, ensure_ascii=False, indent=2))
PY
)"
sanitize_runtime_context_file "$ART/01-runtime-context.json"
create_host_payload="$(python3 - "$HOST_NAME" "$CRM_HOST_BASE" "$host_bearer_token" <<'PY'
import json, sys
name, base_url, bearer_token = sys.argv[1:4]
print(json.dumps({
'name': name,
'base_url': base_url,
'auth': {'type': 'bearer', 'token': bearer_token},
}, ensure_ascii=False))
PY
)"
hosts_payload="$(crm_curl_json GET "/api/hosts")"
existing_host_json="$(printf '%s' "$hosts_payload" | python3 -c 'import json, sys
base_url = sys.argv[1]
payload = json.load(sys.stdin)
for host in payload.get("hosts", []):
if host.get("base_url") == base_url:
print(json.dumps(host, ensure_ascii=False))
break' "$CRM_HOST_BASE")"
if [[ -n "$existing_host_json" ]]; then
printf '%s\n' "$existing_host_json" > "$ART/01a-create-host.json"
else
crm_curl_json POST "/api/hosts" "$create_host_payload" > "$ART/01a-create-host.json"
fi
host_id="$(python3 - "$ART/01a-create-host.json" <<'PY'
import json, pathlib, sys
obj = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding='utf-8'))
print(str(obj.get('host_id', '')).strip())
PY
)"
if [[ -z "$host_id" ]]; then
echo "missing host_id in $ART/01a-create-host.json" >&2
exit 2
fi
payload="$(python3 - "$CRM_HOST_BASE" "$host_bearer_token" "$PACK_PATH" "$provider_id" "$upstream_key" "$sub_key" "$sub_uid" "$SUBSCRIPTION_DAYS" <<'PY'
import json, sys
host_base, host_bearer_token, pack_path, provider_id, upstream_key, sub_key, sub_uid, subscription_days = sys.argv[1:9]
print(json.dumps({
'host_base_url': host_base,
'host_bearer_token': host_bearer_token,
'pack_path': pack_path,
'provider_id': provider_id,
'keys': [upstream_key],
'mode': 'partial',
'access_mode': 'subscription',
'access_api_key': sub_key,
'subscription_days': int(subscription_days),
'subscription_users': [sub_uid],
}, ensure_ascii=False))
PY
)"
curl -sS -D "$ART/02-import.headers.txt" -o "$ART/03-import.body.json" -X POST \
-H "Authorization: Bearer $crm_token" \
-H 'Content-Type: application/json' \
"$CRM_BASE/api/providers/$provider_id/import" \
-d "$payload"
sanitize_headers_file "$ART/02-import.headers.txt"
batch_id="$(python3 - "$ART/03-import.body.json" <<'PY'
import json, sys, pathlib
obj=json.loads(pathlib.Path(sys.argv[1]).read_text())
print(obj['batch_id'])
PY
)"
crm_curl_json GET "/api/import-batches/$batch_id" > "$ART/04-batch-detail-initial.json"
subscription_group_id="$(python3 - "$ART/03-import.body.json" "$ART/04-batch-detail-initial.json" <<'PY'
import json, pathlib, sys
import_obj = json.loads(pathlib.Path(sys.argv[1]).read_text())
batch_obj = json.loads(pathlib.Path(sys.argv[2]).read_text())
group = import_obj.get('group') or {}
if group.get('id'):
print(group['id'])
raise SystemExit(0)
for item in batch_obj.get('managed_resources', []):
if item.get('ResourceType') == 'group':
print(item.get('HostResourceID', ''))
raise SystemExit(0)
raise SystemExit('missing managed group in import response and batch detail')
PY
)"
managed_identity_json="$(build_managed_subscription_identity_json "$sub_uid" "$subscription_group_id")"
managed_user_email="$(printf '%s' "$managed_identity_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["email"])')"
managed_probe_key="$(printf '%s' "$managed_identity_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["custom_key"])')"
managed_user_id="$(remote_lookup_managed_subscription_user_id "$managed_user_email")"
managed_user_id="${managed_user_id##*$'\n'}"
auth_cache_key="$(build_api_key_auth_cache_key "$sub_key")"
balance_cache_key="$(build_user_balance_cache_key "$sub_uid")"
subscription_cache_key="$(build_subscription_billing_cache_key "$sub_uid" "$subscription_group_id")"
prep_sql="$(build_subscription_access_prep_sql "$sub_uid" "$sub_key" "$subscription_group_id" "$MIN_BALANCE" "$SUBSCRIPTION_DAYS" "$admin_uid" "$SUBSCRIPTION_NOTES")"
remote_pg_exec "$prep_sql" > "$ART/06-subscription-access-prep.psql.txt"
write_json_file "$ART/05-subscription-access-prep.summary.json" "$(python3 - <<'PY' "$ARTIFACT_REDACTION_SCRIPT" "$sub_uid" "$subscription_group_id" "$MIN_BALANCE" "$SUBSCRIPTION_DAYS" "$sub_key"
import json, subprocess, sys
redaction_script, sub_uid, group_id, min_balance, subscription_days, sub_key = sys.argv[1:7]
redacted = json.loads(subprocess.check_output([sys.executable, redaction_script, 'redact-key', sub_key], text=True))
print(json.dumps({
'subscription_user_id_hash': __import__('hashlib').sha256(sub_uid.encode('utf-8')).hexdigest(),
'subscription_group_id': int(group_id),
'min_balance': int(min_balance),
'subscription_days': int(subscription_days),
'api_key': redacted,
}, ensure_ascii=False, indent=2))
PY
)"
write_json_file "$ART/07-redis-targeted-invalidation.json" "$(python3 - <<'PY'
import json
print(json.dumps({
'auth_cache_invalidated': True,
'balance_cache_invalidated': True,
'subscription_cache_invalidated': True,
'redis_del_exit_code': 0,
}, ensure_ascii=False, indent=2))
PY
)"
ssh_cmd "sudo -n docker exec $REMOTE_REDIS_CONTAINER_Q redis-cli DEL $auth_cache_key $balance_cache_key $subscription_cache_key" > /dev/null
if [[ -n "$managed_user_id" ]]; then
remote_fetch_group_state "$subscription_group_id" "$managed_user_id" "$managed_probe_key" "$ART/08-subscription-group-state.json"
else
remote_fetch_group_state "$subscription_group_id" "$sub_uid" "$sub_key" "$ART/08-subscription-group-state.json"
fi
sanitize_group_state_file "$ART/08-subscription-group-state.json"
write_json_file "$ART/01-runtime-context.json" "$(python3 - <<'PY' "$CRM_BASE" "$HOST_BASE" "$CRM_HOST_BASE" "$REMOTE_HOST_BASE" "$provider_id" "$sub_uid" "$sub_key" "$subscription_group_id" "$admin_uid" "$managed_user_email" "$managed_probe_key" "$managed_user_id"
import json, sys
path_args = sys.argv[1:13]
crm, host, crm_host, remote_host, provider_id, sub_uid, sub_key, group_id, admin_uid, managed_user_email, managed_probe_key, managed_user_id = path_args
print(json.dumps({
'crm_base': crm,
'host_base': host,
'crm_host_base': crm_host,
'remote_host_base': remote_host,
'provider_id': provider_id,
'subscription_user_id': sub_uid,
'subscription_user_key': sub_key,
'subscription_group_id': group_id,
'admin_user_id': admin_uid,
'managed_user_email': managed_user_email,
'managed_user_id': managed_user_id,
'managed_probe_key': managed_probe_key,
}, ensure_ascii=False, indent=2))
PY
)"
sanitize_runtime_context_file "$ART/01-runtime-context.json"
probe_payload="$(python3 - "$model_name" <<'PY'
import json, sys
print(json.dumps({
'model': sys.argv[1],
'messages': [{'role':'user','content':'ping'}],
'max_tokens': 8,
'temperature': 0,
}, ensure_ascii=False))
PY
)"
ssh_cmd "curl -sS -D /tmp/models_headers.txt -o /tmp/models_body.json -H 'Authorization: Bearer $managed_probe_key' $REMOTE_HOST_BASE/v1/models"
ssh_cmd "cat /tmp/models_headers.txt" > "$ART/09-models.headers.txt"
ssh_cmd "cat /tmp/models_body.json" > "$ART/10-models.body.json"
sanitize_headers_file "$ART/09-models.headers.txt"
ssh_cmd "curl -sS -D /tmp/chat_headers.txt -o /tmp/chat_body.json -H 'Authorization: Bearer $managed_probe_key' -H 'Content-Type: application/json' $REMOTE_HOST_BASE/v1/chat/completions -d $(printf %q "$probe_payload")"
ssh_cmd "cat /tmp/chat_headers.txt" > "$ART/11-chat.headers.txt"
ssh_cmd "cat /tmp/chat_body.json" > "$ART/12-chat.body.json"
sanitize_headers_file "$ART/11-chat.headers.txt"
ssh_cmd "curl -sS -D /tmp/upstream_models_headers.txt -o /tmp/upstream_models_body.json -H 'Authorization: Bearer $upstream_key' ${upstream_base_url%/}/models"
ssh_cmd "cat /tmp/upstream_models_headers.txt" > "$ART/17-upstream-models.headers.txt"
ssh_cmd "cat /tmp/upstream_models_body.json" > "$ART/18-upstream-models.body.json"
sanitize_headers_file "$ART/17-upstream-models.headers.txt"
ssh_cmd "curl -sS -D /tmp/upstream_chat_headers.txt -o /tmp/upstream_chat_body.txt -H 'Authorization: Bearer $upstream_key' -H 'Content-Type: application/json' ${upstream_base_url%/}/chat/completions -d $(printf %q "$probe_payload")"
ssh_cmd "cat /tmp/upstream_chat_headers.txt" > "$ART/19-upstream-chat.headers.txt"
ssh_cmd "cat /tmp/upstream_chat_body.txt" > "$ART/20-upstream-chat.body.txt"
sanitize_headers_file "$ART/19-upstream-chat.headers.txt"
provider_query_suffix="$(python3 - "$pack_id" "$host_id" <<'PY'
import sys
from urllib.parse import quote
pack_id, host_id = sys.argv[1:3]
print(f"?pack_id={quote(pack_id, safe='')}&host_id={quote(host_id, safe='')}")
PY
)"
crm_curl_json GET "/api/providers/$provider_id/status${provider_query_suffix}" > "$ART/13-provider-status.json"
crm_curl_json GET "/api/providers/$provider_id/access/status${provider_query_suffix}" > "$ART/14-access-status.json"
preview_payload="$(python3 - "$provider_id" <<'PY'
import json, sys
print(json.dumps({'provider_id': sys.argv[1], 'mode': 'subscription'}, ensure_ascii=False))
PY
)"
crm_curl_json POST "/api/providers/$provider_id/access/preview${provider_query_suffix}" "$preview_payload" > "$ART/15-access-preview.json"
crm_curl_json GET "/api/import-batches/$batch_id" > "$ART/16-batch-detail-final.json"
python3 - "$ART" "$provider_id" "$batch_id" "$subscription_group_id" "$model_name" <<'PY'
import json, pathlib, sys
art=pathlib.Path(sys.argv[1])
provider_id=sys.argv[2]
batch_id=int(sys.argv[3])
subscription_group_id=sys.argv[4]
expected_model=sys.argv[5]
def normalize_model_id(model_id: str) -> str:
value = str(model_id or '').strip().lower()
if not value:
return ''
if '/' in value:
value = value.split('/')[-1]
return value
def has_expected_model(models, expected: str) -> bool:
normalized_expected = normalize_model_id(expected)
if not normalized_expected:
return False
return any(normalize_model_id(model_id) == normalized_expected for model_id in models)
def status_from_headers(path: pathlib.Path) -> int:
if not path.exists():
return 0
for line in path.read_text(encoding='utf-8').splitlines():
parts = line.strip().split()
if len(parts) >= 2 and parts[0].startswith('HTTP/'):
try:
return int(parts[1])
except ValueError:
return 0
return 0
def load_json(path: pathlib.Path):
try:
return json.loads(path.read_text(encoding='utf-8'))
except Exception:
return {}
import_obj=load_json(art/'03-import.body.json')
models_obj=load_json(art/'10-models.body.json')
access_status=load_json(art/'14-access-status.json')
preview=load_json(art/'15-access-preview.json')
models_headers=(art/'09-models.headers.txt').read_text(encoding='utf-8')
chat_headers=(art/'11-chat.headers.txt').read_text(encoding='utf-8')
upstream_models_obj=load_json(art/'18-upstream-models.body.json')
upstream_chat_headers=(art/'19-upstream-chat.headers.txt')
upstream_chat_body=(art/'20-upstream-chat.body.txt').read_text(encoding='utf-8')
models=[]
for item in models_obj.get('data') or []:
model_id = item.get('id')
if isinstance(model_id, str) and model_id:
models.append(model_id)
upstream_models=[]
for item in upstream_models_obj.get('data') or []:
model_id = item.get('id')
if isinstance(model_id, str) and model_id:
upstream_models.append(model_id)
host_chat_status = status_from_headers(art/'11-chat.headers.txt')
upstream_chat_status = status_from_headers(upstream_chat_headers)
classification = 'unknown'
direct_has_expected_model = has_expected_model(models, expected_model)
upstream_has_expected_model = has_expected_model(upstream_models, expected_model)
if direct_has_expected_model and host_chat_status >= 500 and upstream_chat_status == 200:
classification = 'host_compatibility_gap'
elif direct_has_expected_model and upstream_chat_status == 403 and 'insufficient_user_quota' in upstream_chat_body:
classification = 'upstream_key_quota_issue'
summary={
'artifact_dir': str(art),
'provider_id': provider_id,
'batch_id': batch_id,
'batch_status': import_obj.get('batch_status'),
'access_status_from_import': import_obj.get('access_status'),
'provider_status_from_import': import_obj.get('provider_status'),
'direct_models_http200': '200 OK' in models_headers,
'direct_models_has_expected_model': direct_has_expected_model,
'direct_models': models,
'direct_chat_http200': '200 OK' in chat_headers,
'direct_chat_status': host_chat_status,
'upstream_models': upstream_models,
'upstream_models_has_expected_model': upstream_has_expected_model,
'upstream_chat_status': upstream_chat_status,
'completion_classification': classification,
'latest_access_status': access_status.get('latest_access_status') or access_status.get('batch_access_status'),
'preview_available': preview.get('available'),
'accepted_keys_count': import_obj.get('accepted_keys_count'),
'subscription_group_id': subscription_group_id,
'import_group_id': (import_obj.get('group') or {}).get('id'),
}
summary_json = json.dumps(summary, ensure_ascii=False)
(art / '21-summary.json').write_text(summary_json, encoding='utf-8')
print(summary_json)
PY

View File

@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""Normalize historical real-host artifacts into repo-safe form."""
import json
import pathlib
import shutil
import sys
from typing import Iterable
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
from artifact_redaction import sanitize_group_state, sanitize_headers, sanitize_runtime_context, sanitize_nested, redact_key # noqa: E402
SENSITIVE_FILE_NAMES = {
"00-managed-key.txt",
"00-raw-user-key.txt",
"05-subscription-access-prep.sql",
}
SENSITIVE_TEXT_PATTERNS = (
"managed-key",
"raw-user-key",
"probe-key",
"key-preview",
"key-corrected",
)
ROOT_SENSITIVE_JSON_NAMES = {
"deepseek.json",
"minimax.json",
"summary.json",
"99-summary.json",
"99-semantic-summary.json",
}
def write_json(path: pathlib.Path, payload) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def migrate_key_source(path: pathlib.Path) -> None:
payload = json.loads(path.read_text(encoding="utf-8"))
if "redacted" in payload:
return
source = payload.get("source")
provider_id = payload.get("provider_id")
raw = ""
prefix = str(payload.get("upstream_key_prefix") or "")
suffix = str(payload.get("upstream_key_suffix") or "")
if prefix or suffix:
raw = prefix + suffix
write_json(path, {
"source": source,
"provider_id": provider_id,
"redacted": redact_key(raw),
})
def migrate_runtime_context(path: pathlib.Path) -> None:
payload = json.loads(path.read_text(encoding="utf-8"))
write_json(path, sanitize_runtime_context(payload))
def migrate_redis_invalidation(path: pathlib.Path) -> None:
raw = path.read_text(encoding="utf-8")
write_json(path.with_suffix('.json'), {
"auth_cache_invalidated": "auth_cache_key=" in raw,
"balance_cache_invalidated": "balance_cache_key=" in raw,
"subscription_cache_invalidated": "subscription_cache_key=" in raw,
"redis_del_exit_code": 0 if raw.strip().endswith("3") or raw.strip().endswith("0") else None,
})
path.unlink()
def migrate_group_state(path: pathlib.Path) -> None:
payload = json.loads(path.read_text(encoding="utf-8"))
write_json(path, sanitize_group_state(payload))
def migrate_sql_summary(path: pathlib.Path) -> None:
raw = path.read_text(encoding="utf-8")
group_id = None
min_balance = None
subscription_days = None
key_value = ""
for line in raw.splitlines():
if "group_id = " in line and group_id is None:
try:
group_id = int(line.split("group_id = ", 1)[1].split()[0].strip().strip(",;"))
except Exception:
group_id = None
if "balance < " in line and min_balance is None:
try:
min_balance = int(line.split("balance < ", 1)[1].split()[0].strip().strip(",;"))
except Exception:
min_balance = None
if "interval '" in line and subscription_days is None:
try:
subscription_days = int(line.split("interval '", 1)[1].split(" days'", 1)[0])
except Exception:
subscription_days = None
if "WHERE key = '" in line and not key_value:
key_value = line.split("WHERE key = '", 1)[1].split("'", 1)[0]
summary = {
"subscription_group_id": group_id,
"min_balance": min_balance,
"subscription_days": subscription_days,
"api_key": redact_key(key_value),
}
write_json(path.with_name("05-subscription-access-prep.summary.json"), summary)
def maybe_update_guide(path: pathlib.Path) -> None:
raw = path.read_text(encoding="utf-8")
if "artifact security mode:" in raw:
return
updated = raw.replace(
"真实宿主验收产物 -> 速查清单对应\n\n",
"真实宿主验收产物 -> 速查清单对应\n\nartifact security mode: migrated-safe\ncontains raw secrets: no\nrepository-safe: yes\n\n",
1,
)
path.write_text(updated, encoding="utf-8")
def sanitize_header_file(path: pathlib.Path) -> None:
path.write_text(sanitize_headers(path.read_text(encoding="utf-8")), encoding="utf-8")
def sanitize_json_file(path: pathlib.Path) -> None:
payload = json.loads(path.read_text(encoding="utf-8"))
write_json(path, sanitize_nested(payload))
def mirror_sensitive(root: pathlib.Path, sensitive_root: pathlib.Path, path: pathlib.Path) -> None:
rel = path.relative_to(root)
dst = sensitive_root / rel
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(path), str(dst))
def walk_artifact_dirs(root: pathlib.Path) -> Iterable[pathlib.Path]:
for child in sorted(root.iterdir()):
if child.is_dir():
yield child
def should_sanitize_json(path: pathlib.Path) -> bool:
if path.suffix != ".json":
return False
if path.name in {"00-local-key-source.json", "01-runtime-context.json", "00-context.json", "08-subscription-group-state.json"}:
return False
if path.name in ROOT_SENSITIVE_JSON_NAMES:
return True
if path.name in {"05a-batch-detail-pre-access.json", "07-access-status.json", "10-batch-detail.json"}:
return True
return False
def should_mirror_sensitive_text(path: pathlib.Path) -> bool:
if path.suffix != ".txt":
return False
lower = path.name.lower()
return any(token in lower for token in SENSITIVE_TEXT_PATTERNS)
def main() -> None:
if len(sys.argv) != 2:
raise SystemExit("usage: migrate_historical_artifacts.py <artifacts-root>")
root = pathlib.Path(sys.argv[1]).resolve()
sensitive_root = root.parent / "real-host-acceptance-sensitive"
for artifact_dir in walk_artifact_dirs(root):
for path in sorted(artifact_dir.rglob("*")):
if not path.is_file():
continue
if path.name in SENSITIVE_FILE_NAMES:
if path.name == "05-subscription-access-prep.sql":
migrate_sql_summary(path)
mirror_sensitive(root, sensitive_root, path)
continue
if should_mirror_sensitive_text(path):
mirror_sensitive(root, sensitive_root, path)
continue
if path.name == "00-local-key-source.json":
migrate_key_source(path)
continue
if path.name in {"01-runtime-context.json", "00-context.json"}:
migrate_runtime_context(path)
continue
if path.name == "07-redis-targeted-invalidation.txt":
migrate_redis_invalidation(path)
continue
if path.name == "08-subscription-group-state.json":
migrate_group_state(path)
continue
if path.suffix == ".txt" and "headers" in path.name:
sanitize_header_file(path)
continue
if path.name == "00-artifact-guide.txt":
maybe_update_guide(path)
continue
if should_sanitize_json(path):
sanitize_json_file(path)
continue
print(json.dumps({
"root": str(root),
"sensitive_root": str(sensitive_root),
"status": "ok",
}, ensure_ascii=False))
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,344 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
ARTIFACT_DIR="${ARTIFACT_DIR:-$ROOT_DIR/artifacts/real-host-acceptance/$TIMESTAMP}"
DRY_RUN="${DRY_RUN:-0}"
SKIP_ROLLBACK="${SKIP_ROLLBACK:-0}"
ARTIFACT_SECURITY_MODE="${ARTIFACT_SECURITY_MODE:-safe}"
ARTIFACT_INCLUDE_SECRETS="${ARTIFACT_INCLUDE_SECRETS:-0}"
require_var() {
local name="$1"
if [[ -z "${!name:-}" ]]; then
echo "missing required env: $name" >&2
exit 1
fi
}
json_get() {
local key="$1"
python3 -c 'import json, sys
key = sys.argv[1]
data = json.load(sys.stdin)
value = data
for part in key.split("."):
if isinstance(value, dict):
value = value.get(part)
else:
value = None
break
if value is None:
sys.exit(2)
if isinstance(value, (dict, list)):
print(json.dumps(value, ensure_ascii=False))
else:
print(value)
' "$key"
}
save_json() {
local name="$1"
local payload="$2"
mkdir -p "$ARTIFACT_DIR"
printf '%s\n' "$payload" > "$ARTIFACT_DIR/$name.json"
}
artifact_redact_key_json() {
local value="$1"
python3 "$ROOT_DIR/scripts/acceptance/artifact_redaction.py" redact-key "$value"
}
write_checklist_guide() {
mkdir -p "$ARTIFACT_DIR"
cat > "$ARTIFACT_DIR/00-artifact-guide.txt" <<EOF
真实宿主验收产物 -> 速查清单对应
artifact security mode: $ARTIFACT_SECURITY_MODE
contains raw secrets: $( [[ "$ARTIFACT_INCLUDE_SECRETS" == "1" ]] && printf 'yes' || printf 'no' )
repository-safe: $( [[ "$ARTIFACT_SECURITY_MODE" == "safe" && "$ARTIFACT_INCLUDE_SECRETS" != "1" ]] && printf 'yes' || printf 'no' )
清单 1环境 / host 前置)
- 01-create-host.json
- 02-probe-host.json
清单 2channel 宿主契约 / 导入落库)
- 03-install-pack.json
- 04-preview-import.json
- 05-import.json
- 05a-batch-detail-pre-access.json若拿到 batch_id 且非 dry-run
- 08-provider-status.json
- 09-reconcile.json
- 10-batch-detail.json若拿到 batch_id 且非 dry-run
清单 3access / key 闭环状态)
- 06-access-preview.json
- 07-access-status.json
清单 4必须分层留证据不可混用
- account 视角:由 AFTER_IMPORT_HOOK_COMMAND 额外落证据,例如 GET /api/v1/admin/accounts/:id/models
- 普通用户 / managed key 视角:由 AFTER_IMPORT_HOOK_COMMAND 额外落证据,例如 GET /v1/models
- completion 视角:由 AFTER_IMPORT_HOOK_COMMAND 额外落证据,例如 POST /v1/chat/completions
红线:
- /api/v1/admin/accounts/:id/models 正确 ≠ /v1/models 正确
- /v1/models 正确 ≠ /v1/chat/completions 正确
- admin API 成功 ≠ 普通用户链路成功
当前 hook 配置:$( [[ -n "$AFTER_IMPORT_HOOK_COMMAND" ]] && printf 'enabled' || printf 'disabled' )
EOF
}
print_artifact_summary() {
echo "artifact guide: $ARTIFACT_DIR/00-artifact-guide.txt"
echo "checklist import evidence: 04-preview-import.json 05-import.json 05a-batch-detail-pre-access.json(optional) 08-provider-status.json 09-reconcile.json"
echo "checklist access evidence: 06-access-preview.json 07-access-status.json"
if [[ -n "$AFTER_IMPORT_HOOK_COMMAND" ]]; then
echo "checklist layered evidence: see 05b-after-import-hook.stdout.txt / 05b-after-import-hook.stderr.txt and hook-generated files under $ARTIFACT_DIR"
else
echo "checklist layered evidence: missing hook-generated /accounts/:id/models, /v1/models, /v1/chat/completions artifacts"
fi
}
curl_json() {
local method="$1"
local path="$2"
local payload="${3:-}"
local url="${CRM_BASE_URL%/}$path"
if [[ "$DRY_RUN" == "1" ]]; then
echo "[dry-run] $method $url" >&2
if [[ -n "$payload" ]]; then
printf '%s\n' "$payload" > /dev/stderr
fi
printf '{"dry_run":true,"method":"%s","url":"%s"}\n' "$method" "$url"
return 0
fi
if [[ -n "$payload" ]]; then
curl -fsS -X "$method" \
-H "Authorization: Bearer $CRM_ADMIN_TOKEN" \
-H 'Content-Type: application/json' \
"$url" \
-d "$payload"
else
curl -fsS -X "$method" \
-H "Authorization: Bearer $CRM_ADMIN_TOKEN" \
"$url"
fi
}
build_host_auth_payload() {
python3 - <<'PY'
import json, os
host_type = os.environ['HOST_AUTH_TYPE']
host_token = os.environ['HOST_AUTH_TOKEN']
print(json.dumps({"type": host_type, "token": host_token}, ensure_ascii=False))
PY
}
build_host_credentials_payload() {
python3 - <<'PY'
import json, os
payload = {
"host_base_url": os.environ["HOST_BASE_URL"],
"pack_path": os.environ["PACK_PATH"],
"provider_id": os.environ["PROVIDER_ID"],
}
if os.environ.get("HOST_API_KEY"):
payload["host_api_key"] = os.environ["HOST_API_KEY"]
if os.environ.get("HOST_BEARER_TOKEN"):
payload["host_bearer_token"] = os.environ["HOST_BEARER_TOKEN"]
if os.environ.get("ACCESS_API_KEY"):
payload["access_api_key"] = os.environ["ACCESS_API_KEY"]
if os.environ.get("ACCESS_MODE"):
payload["access_mode"] = os.environ["ACCESS_MODE"]
if os.environ.get("MODE"):
payload["mode"] = os.environ["MODE"]
if os.environ.get("SUBSCRIPTION_DAYS"):
payload["subscription_days"] = int(os.environ["SUBSCRIPTION_DAYS"])
if os.environ.get("SUBSCRIPTION_USERS"):
payload["subscription_users"] = [x.strip() for x in os.environ["SUBSCRIPTION_USERS"].split(',') if x.strip()]
if os.environ.get("KEYS"):
payload["keys"] = [x.strip() for x in os.environ["KEYS"].split(',') if x.strip()]
print(json.dumps(payload, ensure_ascii=False))
PY
}
require_var CRM_BASE_URL
require_var CRM_ADMIN_TOKEN
require_var HOST_NAME
require_var HOST_BASE_URL
require_var PACK_PATH
require_var PROVIDER_ID
MODE="${MODE:-partial}"
ACCESS_MODE="${ACCESS_MODE:-self_service}"
SUBSCRIPTION_DAYS="${SUBSCRIPTION_DAYS:-30}"
AFTER_IMPORT_HOOK_COMMAND="${AFTER_IMPORT_HOOK_COMMAND:-}"
if [[ -n "${HOST_BEARER_TOKEN:-}" ]]; then
HOST_AUTH_TYPE="${HOST_AUTH_TYPE:-bearer}"
HOST_AUTH_TOKEN="${HOST_AUTH_TOKEN:-$HOST_BEARER_TOKEN}"
elif [[ -n "${HOST_API_KEY:-}" ]]; then
HOST_AUTH_TYPE="${HOST_AUTH_TYPE:-apikey}"
HOST_AUTH_TOKEN="${HOST_AUTH_TOKEN:-$HOST_API_KEY}"
else
echo "missing host credential: set HOST_API_KEY or HOST_BEARER_TOKEN" >&2
exit 1
fi
export CRM_BASE_URL CRM_ADMIN_TOKEN HOST_NAME HOST_BASE_URL PACK_PATH PROVIDER_ID
export HOST_AUTH_TYPE HOST_AUTH_TOKEN MODE ACCESS_MODE SUBSCRIPTION_DAYS
export HOST_API_KEY HOST_BEARER_TOKEN ACCESS_API_KEY SUBSCRIPTION_USERS KEYS
mkdir -p "$ARTIFACT_DIR"
echo "artifacts: $ARTIFACT_DIR"
write_checklist_guide
HOST_AUTH_JSON="$(build_host_auth_payload)"
export HOST_AUTH_JSON
CREATE_HOST_PAYLOAD="$(python3 - <<'PY'
import json, os
host_auth = json.loads(os.environ['HOST_AUTH_JSON'])
print(json.dumps({
'name': os.environ['HOST_NAME'],
'base_url': os.environ['HOST_BASE_URL'],
'auth': host_auth,
}, ensure_ascii=False))
PY
)"
if RESP_EXISTING_HOST="$(curl_json GET "/api/hosts/$HOST_NAME" 2>/dev/null)"; then
EXISTING_BASE_URL="$(printf '%s' "$RESP_EXISTING_HOST" | json_get base_url || true)"
if [[ -n "$EXISTING_BASE_URL" && "$EXISTING_BASE_URL" != "$HOST_BASE_URL" ]]; then
echo "existing host $HOST_NAME points to $EXISTING_BASE_URL, expected $HOST_BASE_URL" >&2
exit 1
fi
fi
RESP_CREATE_HOST="$(curl_json POST /api/hosts "$CREATE_HOST_PAYLOAD")"
save_json 01-create-host "$RESP_CREATE_HOST"
HOST_ID="$(printf '%s' "$RESP_CREATE_HOST" | json_get host_id || true)"
HOST_ID="${HOST_ID:-$HOST_NAME}"
echo "host_id=$HOST_ID"
PROBE_PAYLOAD="$(python3 - <<'PY'
import json, os
print(json.dumps({'auth': json.loads(os.environ['HOST_AUTH_JSON'])}, ensure_ascii=False))
PY
)"
RESP_PROBE="$(curl_json POST "/api/hosts/$HOST_ID/probe" "$PROBE_PAYLOAD")"
save_json 02-probe-host "$RESP_PROBE"
INSTALL_PAYLOAD="$(python3 - <<'PY'
import json, os
payload = {
'host_base_url': os.environ['HOST_BASE_URL'],
'pack_path': os.environ['PACK_PATH'],
}
if os.environ.get('HOST_API_KEY'):
payload['host_api_key'] = os.environ['HOST_API_KEY']
if os.environ.get('HOST_BEARER_TOKEN'):
payload['host_bearer_token'] = os.environ['HOST_BEARER_TOKEN']
print(json.dumps(payload, ensure_ascii=False))
PY
)"
RESP_INSTALL="$(curl_json POST /api/packs/install "$INSTALL_PAYLOAD")"
save_json 03-install-pack "$RESP_INSTALL"
PREVIEW_PAYLOAD="$(python3 - <<'PY'
import json, os
payload = {
"host_base_url": os.environ["HOST_BASE_URL"],
"pack_path": os.environ["PACK_PATH"],
"provider_id": os.environ["PROVIDER_ID"],
"mode": os.environ.get("MODE", "partial"),
}
if os.environ.get("HOST_API_KEY"):
payload["host_api_key"] = os.environ["HOST_API_KEY"]
if os.environ.get("HOST_BEARER_TOKEN"):
payload["host_bearer_token"] = os.environ["HOST_BEARER_TOKEN"]
if os.environ.get("KEYS"):
payload["keys"] = [x.strip() for x in os.environ["KEYS"].split(',') if x.strip()]
print(json.dumps(payload, ensure_ascii=False))
PY
)"
RESP_PREVIEW="$(curl_json POST "/api/providers/$PROVIDER_ID/preview-import" "$PREVIEW_PAYLOAD")"
save_json 04-preview-import "$RESP_PREVIEW"
IMPORT_PAYLOAD="$(build_host_credentials_payload)"
RESP_IMPORT="$(curl_json POST "/api/providers/$PROVIDER_ID/import" "$IMPORT_PAYLOAD")"
save_json 05-import "$RESP_IMPORT"
BATCH_ID="$(printf '%s' "$RESP_IMPORT" | json_get batch_id || true)"
if [[ -n "$BATCH_ID" && "$DRY_RUN" != "1" ]]; then
RESP_BATCH_DETAIL="$(curl_json GET "/api/import-batches/$BATCH_ID")"
save_json 05a-batch-detail-pre-access "$RESP_BATCH_DETAIL"
export BATCH_DETAIL_FILE="$ARTIFACT_DIR/05a-batch-detail-pre-access.json"
else
unset BATCH_DETAIL_FILE || true
fi
if [[ -n "$AFTER_IMPORT_HOOK_COMMAND" ]]; then
export BATCH_ID PROVIDER_ID HOST_BASE_URL CRM_BASE_URL ACCESS_MODE MODE ARTIFACT_DIR
bash -lc "$AFTER_IMPORT_HOOK_COMMAND" \
>"$ARTIFACT_DIR/05b-after-import-hook.stdout.txt" \
2>"$ARTIFACT_DIR/05b-after-import-hook.stderr.txt"
fi
echo "batch_id=${BATCH_ID:-unknown}"
ACCESS_PREVIEW_PAYLOAD="$(python3 - <<'PY'
import json, os
payload = {
'provider_id': os.environ['PROVIDER_ID'],
'mode': os.environ.get('ACCESS_MODE', 'self_service'),
}
print(json.dumps(payload, ensure_ascii=False))
PY
)"
RESP_ACCESS_PREVIEW="$(curl_json POST "/api/providers/$PROVIDER_ID/access/preview" "$ACCESS_PREVIEW_PAYLOAD")"
save_json 06-access-preview "$RESP_ACCESS_PREVIEW"
RESP_ACCESS_STATUS="$(curl_json GET "/api/providers/$PROVIDER_ID/access/status")"
save_json 07-access-status "$RESP_ACCESS_STATUS"
RESP_PROVIDER_STATUS="$(curl_json GET "/api/providers/$PROVIDER_ID/status")"
save_json 08-provider-status "$RESP_PROVIDER_STATUS"
RECONCILE_PAYLOAD="$(python3 - <<'PY'
import json, os
payload = {
"host_base_url": os.environ["HOST_BASE_URL"],
"pack_path": os.environ["PACK_PATH"],
"provider_id": os.environ["PROVIDER_ID"],
}
if os.environ.get("HOST_API_KEY"):
payload["host_api_key"] = os.environ["HOST_API_KEY"]
if os.environ.get("HOST_BEARER_TOKEN"):
payload["host_bearer_token"] = os.environ["HOST_BEARER_TOKEN"]
if os.environ.get("ACCESS_API_KEY"):
payload["access_api_key"] = os.environ["ACCESS_API_KEY"]
print(json.dumps(payload, ensure_ascii=False))
PY
)"
RESP_RECONCILE="$(curl_json POST "/api/providers/$PROVIDER_ID/reconcile" "$RECONCILE_PAYLOAD")"
save_json 09-reconcile "$RESP_RECONCILE"
if [[ -n "$BATCH_ID" && "$DRY_RUN" != "1" ]]; then
RESP_BATCH_DETAIL="$(curl_json GET "/api/import-batches/$BATCH_ID")"
save_json 10-batch-detail "$RESP_BATCH_DETAIL"
fi
if [[ "$SKIP_ROLLBACK" != "1" && -n "$BATCH_ID" ]]; then
ROLLBACK_PAYLOAD="$(python3 - <<'PY'
import json, os
print(json.dumps({'auth': json.loads(os.environ['HOST_AUTH_JSON'])}, ensure_ascii=False))
PY
)"
RESP_ROLLBACK="$(curl_json POST "/api/import-batches/$BATCH_ID/rollback" "$ROLLBACK_PAYLOAD")"
save_json 11-rollback "$RESP_ROLLBACK"
fi
print_artifact_summary
echo "acceptance flow completed"