#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" # shellcheck disable=SC1091 source "$ROOT_DIR/scripts/acceptance/route_acceptance_lib.sh" CRM_BASE="${CRM_BASE:-https://sub.tksea.top/portal-admin-api}" ROUTE_HEALTH_PAGE_URL="${ROUTE_HEALTH_PAGE_URL:-https://sub.tksea.top/portal/admin/route-health.html}" TS="${TS:-$(timestamp_token)}" ARTIFACT_DIR="${ARTIFACT_DIR:-$ROUTE_MATRIX_ROOT/${TS}_route_health_ui}" GROUP_ID="${GROUP_ID:-p2t4-health-${TS}}" PRIMARY_ROUTE="${PRIMARY_ROUTE:-primary-${TS}}" FALLBACK_ROUTE="${FALLBACK_ROUTE:-fallback-${TS}}" FAILING_ROUTE="${FAILING_ROUTE:-failing-${TS}}" PUBLIC_MODEL="${PUBLIC_MODEL:-gpt-5.4}" REQUEST_ID="${REQUEST_ID:-req-p2t4-health-${TS}}" SUBJECT_ID="${SUBJECT_ID:-conv-p2t4-health-${TS}}" crm_auth_init ensure_artifact_dir curl_status_to_file "$ROUTE_HEALTH_PAGE_URL" "$ARTIFACT_DIR/00-route-health.html" create_group_payload="$(python3 - "$GROUP_ID" <<'PY' import json, sys group_id = sys.argv[1] print(json.dumps({ "logical_group_id": group_id, "display_name": f"P2T4 Health {group_id}", "status": "active", "description": "P2-T4 health verification group", "route_policy": "priority", "sticky_mode": "conversation_preferred", "conversation_ttl_seconds": 1200, "user_model_ttl_seconds": 600, "failover_threshold": 2, "cooldown_seconds": 300, }, ensure_ascii=False)) PY )" save_json 01-create-group "$(crm_curl_json POST "/api/logical-groups" "$create_group_payload")" save_json 02-add-group-model "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")" create_route() { local route_id="$1" local route_name="$2" local priority="$3" local shadow_group_id="$4" local shadow_host_id="$5" crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes" \ "{\"route_id\":\"$route_id\",\"name\":\"$route_name\",\"status\":\"active\",\"priority\":$priority,\"weight\":100,\"shadow_group_id\":\"$shadow_group_id\",\"shadow_host_id\":\"$shadow_host_id\",\"upstream_base_url_hint\":\"https://$route_id.example/v1\"}" } save_json 03-create-primary-route "$(create_route "$PRIMARY_ROUTE" "Primary Route" 10 "shadow-primary-$TS" "shadow-host-primary-$TS")" save_json 04-create-fallback-route "$(create_route "$FALLBACK_ROUTE" "Fallback Route" 20 "shadow-fallback-$TS" "shadow-host-fallback-$TS")" save_json 05-create-failing-route "$(create_route "$FAILING_ROUTE" "Failing Route" 30 "shadow-failing-$TS" "shadow-host-failing-$TS")" for route_id in "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE"; do save_json "route-model-${route_id}" "$(crm_curl_json POST "/api/logical-groups/$GROUP_ID/routes/$route_id/models" "{\"public_model\":\"$PUBLIC_MODEL\",\"shadow_model\":\"$PUBLIC_MODEL\",\"status\":\"active\"}")" done save_json 06-set-cooldown "$(crm_curl_json POST "/api/routing/sticky/cooldowns" "{\"route_id\":\"$PRIMARY_ROUTE\",\"reason\":\"degraded\",\"ttl_seconds\":600}")" save_json 07-set-failure "$(crm_curl_json POST "/api/routing/sticky/route-failures" "{\"route_id\":\"$FAILING_ROUTE\",\"failure_count\":2,\"last_error_class\":\"timeout\",\"ttl_seconds\":600}")" save_json 08-health-before "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")" save_json 09-resolve "$(crm_curl_json POST "/api/routing/resolve" "{\"request_id\":\"$REQUEST_ID\",\"logical_group_id\":\"$GROUP_ID\",\"public_model\":\"$PUBLIC_MODEL\",\"scope\":\"conversation\",\"subject_id\":\"$SUBJECT_ID\",\"sync\":true}")" save_json 10-health-after "$(crm_curl_json GET "/api/routing/routes/health?logical_group_id=$GROUP_ID")" save_json 11-failovers "$(crm_curl_json GET "/api/routing/logs/failovers?request_id=$REQUEST_ID&limit=5")" python3 - "$ARTIFACT_DIR" "$GROUP_ID" "$PRIMARY_ROUTE" "$FALLBACK_ROUTE" "$FAILING_ROUTE" "$REQUEST_ID" >"$ARTIFACT_DIR/12-summary.json" <<'PY' import json import sys from pathlib import Path art_dir, group_id, primary_route, fallback_route, failing_route, request_id = sys.argv[1:7] art = Path(art_dir) page = (art / "00-route-health.html").read_text() before = json.loads((art / "08-health-before.json").read_text())["route_health"] resolve = json.loads((art / "09-resolve.json").read_text())["resolve"] after = json.loads((art / "10-health-after.json").read_text())["route_health"] failovers = json.loads((art / "11-failovers.json").read_text())["failover_events"] def by_id(items): return {item["route_id"]: item for item in items} before_map = by_id(before) after_map = by_id(after) assert "Route Health Admin" in page assert before_map[primary_route]["runtime_status"] == "cooldown" assert before_map[failing_route]["runtime_status"] == "failing" assert resolve["route_id"] == fallback_route assert resolve["fallback_used"] is True assert after_map[primary_route]["runtime_status"] == "cooldown" assert after_map[fallback_route]["runtime_status"] == "healthy" assert after_map[fallback_route]["recent_failover_count"] >= 1 assert failovers and failovers[0]["from_route_id"] == primary_route assert failovers[0]["to_route_id"] == fallback_route assert failovers[0]["reason"] == "active_cooldown:degraded" summary = { "group_id": group_id, "request_id": request_id, "resolve_route_id": resolve["route_id"], "resolve_fallback_used": resolve["fallback_used"], "before_statuses": {k: v["runtime_status"] for k, v in before_map.items()}, "after_statuses": {k: v["runtime_status"] for k, v in after_map.items()}, "fallback_recent_failover_count": after_map[fallback_route]["recent_failover_count"], } print(json.dumps(summary, ensure_ascii=False, indent=2)) PY cat "$ARTIFACT_DIR/12-summary.json"