chore: initial snapshot for gitea/github upload
This commit is contained in:
Your Name
@@ -0,0 +1,6 @@
|
||||
# RunwayML integration for LiteLLM
|
||||
|
||||
from .cost_calculator import cost_calculator
|
||||
from .videos.transformation import RunwayMLVideoConfig
|
||||
|
||||
__all__ = ["RunwayMLVideoConfig", "cost_calculator"]
|
||||
@@ -0,0 +1,30 @@
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
|
||||
def cost_calculator(
|
||||
model: str,
|
||||
image_response: Any,
|
||||
) -> float:
|
||||
"""
|
||||
RunwayML image generation cost calculator.
|
||||
|
||||
RunwayML charges per image generated, not per pixel.
|
||||
Pricing is stored in model_prices_and_context_window.json with output_cost_per_image.
|
||||
"""
|
||||
_model_info = litellm.get_model_info(
|
||||
model=model,
|
||||
custom_llm_provider=litellm.LlmProviders.RUNWAYML.value,
|
||||
)
|
||||
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
else:
|
||||
raise ValueError(
|
||||
f"image_response must be of type ImageResponse, got type={type(image_response)}"
|
||||
)
|
||||
@@ -0,0 +1,13 @@
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .transformation import RunwayMLImageGenerationConfig
|
||||
|
||||
__all__ = [
|
||||
"RunwayMLImageGenerationConfig",
|
||||
]
|
||||
|
||||
|
||||
def get_runwayml_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
||||
return RunwayMLImageGenerationConfig()
|
||||
@@ -0,0 +1,515 @@
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
RUNWAYML_DEFAULT_API_VERSION,
|
||||
RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
OpenAIImageGenerationOptionalParams,
|
||||
)
|
||||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class RunwayMLImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
Configuration for RunwayML image generation models.
|
||||
"""
|
||||
|
||||
DEFAULT_BASE_URL: str = "https://api.dev.runwayml.com"
|
||||
IMAGE_GENERATION_ENDPOINT: str = "v1/text_to_image"
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
model: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
stream: Optional[bool] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Get the complete url for the request
|
||||
|
||||
Some providers need `model` in `api_base`
|
||||
"""
|
||||
complete_url: str = (
|
||||
api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL
|
||||
)
|
||||
|
||||
complete_url = complete_url.rstrip("/")
|
||||
if self.IMAGE_GENERATION_ENDPOINT:
|
||||
complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}"
|
||||
return complete_url
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
final_api_key: Optional[str] = (
|
||||
api_key
|
||||
or get_secret_str("RUNWAYML_API_SECRET")
|
||||
or get_secret_str("RUNWAYML_API_KEY")
|
||||
)
|
||||
if not final_api_key:
|
||||
raise ValueError("RUNWAYML_API_SECRET or RUNWAYML_API_KEY is not set")
|
||||
|
||||
headers["Authorization"] = f"Bearer {final_api_key}"
|
||||
headers["X-Runway-Version"] = RUNWAYML_DEFAULT_API_VERSION
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def _transform_runwayml_response_to_openai(
|
||||
response_data: Dict[str, Any],
|
||||
model_response: ImageResponse,
|
||||
) -> ImageResponse:
|
||||
"""
|
||||
Transform RunwayML response format to OpenAI ImageResponse format.
|
||||
|
||||
RunwayML response format (after polling):
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "SUCCEEDED",
|
||||
"output": ["https://cloudfront.net/.../image.png"],
|
||||
"completedAt": "2025-11-13T..."
|
||||
}
|
||||
|
||||
OpenAI ImageResponse format:
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"url": "https://cloudfront.net/.../image.png",
|
||||
"b64_json": null
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Args:
|
||||
response_data: JSON response from RunwayML (after polling completes)
|
||||
model_response: ImageResponse object to populate
|
||||
|
||||
Returns:
|
||||
Populated ImageResponse in OpenAI format
|
||||
"""
|
||||
if not model_response.data:
|
||||
model_response.data = []
|
||||
|
||||
# Handle RunwayML response format
|
||||
# Response contains task.output with image URL(s)
|
||||
output = response_data.get("output", [])
|
||||
|
||||
if isinstance(output, list):
|
||||
for image_item in output:
|
||||
if isinstance(image_item, str):
|
||||
# If output is a list of URL strings
|
||||
model_response.data.append(
|
||||
ImageObject(
|
||||
url=image_item,
|
||||
b64_json=None,
|
||||
)
|
||||
)
|
||||
elif isinstance(image_item, dict):
|
||||
# If output contains dict with url/b64_json
|
||||
model_response.data.append(
|
||||
ImageObject(
|
||||
url=image_item.get("url", None),
|
||||
b64_json=image_item.get("b64_json", None),
|
||||
)
|
||||
)
|
||||
|
||||
return model_response
|
||||
|
||||
@staticmethod
|
||||
def _check_timeout(start_time: float, timeout_secs: float) -> None:
|
||||
"""
|
||||
Check if operation has timed out.
|
||||
|
||||
Args:
|
||||
start_time: Start time of the operation
|
||||
timeout_secs: Timeout duration in seconds
|
||||
|
||||
Raises:
|
||||
TimeoutError: If operation has exceeded timeout
|
||||
"""
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise TimeoutError(
|
||||
f"RunwayML task polling timed out after {timeout_secs} seconds"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _check_task_status(response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Check RunwayML task status from response.
|
||||
|
||||
RunwayML statuses: PENDING, RUNNING, SUCCEEDED, FAILED, CANCELLED, THROTTLED
|
||||
|
||||
Args:
|
||||
response_data: JSON response from RunwayML task endpoint
|
||||
|
||||
Returns:
|
||||
Normalized status string: "running", "succeeded", or raises on failure
|
||||
|
||||
Raises:
|
||||
ValueError: If task failed or status is unknown
|
||||
"""
|
||||
status = response_data.get("status", "").upper()
|
||||
|
||||
verbose_logger.debug(f"RunwayML task status: {status}")
|
||||
|
||||
if status == "SUCCEEDED":
|
||||
return "succeeded"
|
||||
elif status == "FAILED":
|
||||
failure_reason = response_data.get("failure", "Unknown error")
|
||||
failure_code = response_data.get("failureCode", "unknown")
|
||||
raise ValueError(
|
||||
f"RunwayML image generation failed: {failure_reason} (code: {failure_code})"
|
||||
)
|
||||
elif status == "CANCELLED":
|
||||
raise ValueError("RunwayML image generation was cancelled")
|
||||
elif status in ["PENDING", "RUNNING", "THROTTLED"]:
|
||||
return "running"
|
||||
else:
|
||||
raise ValueError(f"Unknown RunwayML task status: {status}")
|
||||
|
||||
def _poll_task_sync(
|
||||
self,
|
||||
task_id: str,
|
||||
api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout_secs: float = 600,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Poll RunwayML task until completion (sync).
|
||||
|
||||
RunwayML POST returns immediately with a task that has status PENDING/RUNNING.
|
||||
We need to poll GET /v1/tasks/{task_id} until status is SUCCEEDED or FAILED.
|
||||
|
||||
Args:
|
||||
task_id: The task ID to poll
|
||||
api_base: Base URL for RunwayML API
|
||||
headers: Request headers (including auth)
|
||||
timeout_secs: Total timeout in seconds (default: 600s = 10 minutes)
|
||||
|
||||
Returns:
|
||||
Final response with completed task
|
||||
"""
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
|
||||
client = _get_httpx_client()
|
||||
start_time = time.time()
|
||||
|
||||
# Build task status URL
|
||||
api_base = api_base.rstrip("/")
|
||||
task_url = f"{api_base}/v1/tasks/{task_id}"
|
||||
|
||||
verbose_logger.debug(f"Polling RunwayML task: {task_url}")
|
||||
|
||||
while True:
|
||||
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
|
||||
|
||||
# Poll the task status
|
||||
response = client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
||||
if status == "succeeded":
|
||||
return response
|
||||
elif status == "running":
|
||||
# Wait before polling again (RunwayML recommends 1-2 second intervals)
|
||||
time.sleep(2)
|
||||
|
||||
async def _poll_task_async(
|
||||
self,
|
||||
task_id: str,
|
||||
api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout_secs: float = 600,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Poll RunwayML task until completion (async).
|
||||
|
||||
Args:
|
||||
task_id: The task ID to poll
|
||||
api_base: Base URL for RunwayML API
|
||||
headers: Request headers (including auth)
|
||||
timeout_secs: Total timeout in seconds (default: 600s = 10 minutes)
|
||||
|
||||
Returns:
|
||||
Final response with completed task
|
||||
"""
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
||||
client = get_async_httpx_client(llm_provider=litellm.LlmProviders.RUNWAYML)
|
||||
start_time = time.time()
|
||||
|
||||
# Build task status URL
|
||||
api_base = api_base.rstrip("/")
|
||||
task_url = f"{api_base}/v1/tasks/{task_id}"
|
||||
|
||||
verbose_logger.debug(f"Polling RunwayML task (async): {task_url}")
|
||||
|
||||
while True:
|
||||
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
|
||||
|
||||
# Poll the task status
|
||||
response = await client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
||||
if status == "succeeded":
|
||||
return response
|
||||
elif status == "running":
|
||||
# Wait before polling again (RunwayML recommends 1-2 second intervals)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ImageResponse:
|
||||
"""
|
||||
Transform the image generation response to the litellm image response.
|
||||
|
||||
RunwayML returns a task immediately with status PENDING/RUNNING.
|
||||
We need to poll the task until it completes (status SUCCEEDED).
|
||||
|
||||
Initial response:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "PENDING" | "RUNNING",
|
||||
"createdAt": "2025-11-13T..."
|
||||
}
|
||||
|
||||
After polling:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "SUCCEEDED",
|
||||
"output": ["https://cloudfront.net/.../image.png"],
|
||||
"completedAt": "2025-11-13T..."
|
||||
}
|
||||
"""
|
||||
try:
|
||||
response_data = raw_response.json()
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming image generation response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
verbose_logger.debug("RunwayML starting polling...")
|
||||
|
||||
# Get task ID
|
||||
task_id = response_data.get("id")
|
||||
if not task_id:
|
||||
raise ValueError("RunwayML response missing task ID")
|
||||
|
||||
# Get headers for polling (need auth)
|
||||
poll_headers = {
|
||||
"Authorization": raw_response.request.headers.get("Authorization", ""),
|
||||
"X-Runway-Version": raw_response.request.headers.get(
|
||||
"X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION
|
||||
),
|
||||
}
|
||||
|
||||
# Poll until task completes
|
||||
raw_response = self._poll_task_sync(
|
||||
task_id=task_id,
|
||||
api_base=self.DEFAULT_BASE_URL,
|
||||
headers=poll_headers,
|
||||
timeout_secs=RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
|
||||
# Update response_data with polled result
|
||||
response_data = raw_response.json()
|
||||
|
||||
verbose_logger.debug("RunwayML polling complete, transforming to OpenAI format")
|
||||
|
||||
# Transform RunwayML response to OpenAI format
|
||||
return self._transform_runwayml_response_to_openai(
|
||||
response_data=response_data,
|
||||
model_response=model_response,
|
||||
)
|
||||
|
||||
async def async_transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: Any,
|
||||
api_key: Optional[str] = None,
|
||||
json_mode: Optional[bool] = None,
|
||||
) -> ImageResponse:
|
||||
"""
|
||||
Async transform the image generation response to the litellm image response.
|
||||
|
||||
RunwayML returns a task immediately with status PENDING/RUNNING.
|
||||
We need to poll the task until it completes (status SUCCEEDED) using async polling.
|
||||
"""
|
||||
try:
|
||||
response_data = raw_response.json()
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error transforming image generation response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
verbose_logger.debug("RunwayML starting polling (async)...")
|
||||
|
||||
# Get task ID
|
||||
task_id = response_data.get("id")
|
||||
if not task_id:
|
||||
raise ValueError("RunwayML response missing task ID")
|
||||
|
||||
# Get headers for polling (need auth)
|
||||
poll_headers = {
|
||||
"Authorization": raw_response.request.headers.get("Authorization", ""),
|
||||
"X-Runway-Version": raw_response.request.headers.get(
|
||||
"X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION
|
||||
),
|
||||
}
|
||||
|
||||
# Poll until task completes (async)
|
||||
raw_response = await self._poll_task_async(
|
||||
task_id=task_id,
|
||||
api_base=self.DEFAULT_BASE_URL,
|
||||
headers=poll_headers,
|
||||
timeout_secs=RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
|
||||
# Update response_data with polled result
|
||||
response_data = raw_response.json()
|
||||
|
||||
verbose_logger.debug(
|
||||
"RunwayML polling complete (async), transforming to OpenAI format"
|
||||
)
|
||||
|
||||
# Transform RunwayML response to OpenAI format
|
||||
return self._transform_runwayml_response_to_openai(
|
||||
response_data=response_data,
|
||||
model_response=model_response,
|
||||
)
|
||||
|
||||
def get_supported_openai_params(
|
||||
self, model: str
|
||||
) -> List[OpenAIImageGenerationOptionalParams]:
|
||||
"""
|
||||
Get supported OpenAI parameters for RunwayML image generation
|
||||
"""
|
||||
return [
|
||||
"size",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params = self.get_supported_openai_params(model)
|
||||
|
||||
# Map OpenAI 'size' parameter to RunwayML 'ratio' parameter
|
||||
if "size" in non_default_params:
|
||||
size = non_default_params["size"]
|
||||
# Map common OpenAI sizes to RunwayML ratios
|
||||
size_to_ratio_map = {
|
||||
"1024x1024": "1024:1024",
|
||||
"1792x1024": "1792:1024",
|
||||
"1024x1792": "1024:1792",
|
||||
"1920x1080": "1920:1080",
|
||||
"1080x1920": "1080:1920",
|
||||
}
|
||||
optional_params["ratio"] = size_to_ratio_map.get(size, "1920:1080")
|
||||
|
||||
for k in non_default_params.keys():
|
||||
if k not in optional_params.keys():
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Transform the image generation request to the RunwayML image generation request body
|
||||
|
||||
RunwayML expects:
|
||||
- model: The model to use (e.g., 'gen4_image')
|
||||
- promptText: The text prompt
|
||||
- ratio: The aspect ratio (e.g., '1920:1080', '1080:1920', '1024:1024')
|
||||
"""
|
||||
runwayml_request_body = {
|
||||
"model": model or "gen4_image",
|
||||
"promptText": prompt,
|
||||
}
|
||||
|
||||
# Add any RunwayML-specific parameters
|
||||
if "ratio" in optional_params:
|
||||
runwayml_request_body["ratio"] = optional_params["ratio"]
|
||||
else:
|
||||
# Set default ratio if not provided
|
||||
runwayml_request_body["ratio"] = "1920:1080"
|
||||
|
||||
# Add any other optional parameters
|
||||
for k, v in optional_params.items():
|
||||
if k not in runwayml_request_body and k not in ["size"]:
|
||||
runwayml_request_body[k] = v
|
||||
|
||||
return runwayml_request_body
|
||||
@@ -0,0 +1,4 @@
|
||||
"""RunwayML Text-to-Speech implementation."""
|
||||
from .transformation import RunwayMLTextToSpeechConfig
|
||||
|
||||
__all__ = ["RunwayMLTextToSpeechConfig"]
|
||||
@@ -0,0 +1,590 @@
|
||||
"""
|
||||
RunwayML Text-to-Speech transformation
|
||||
|
||||
Maps OpenAI TTS spec to RunwayML Text-to-Speech API
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
RUNWAYML_DEFAULT_API_VERSION,
|
||||
RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
HttpxBinaryResponseContent = Any
|
||||
|
||||
|
||||
class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
"""
|
||||
Configuration for RunwayML Text-to-Speech
|
||||
|
||||
Reference: https://api.dev.runwayml.com/v1/text_to_speech
|
||||
"""
|
||||
|
||||
DEFAULT_BASE_URL: str = "https://api.dev.runwayml.com"
|
||||
TTS_ENDPOINT_PATH: str = "v1/text_to_speech"
|
||||
DEFAULT_MODEL: str = "eleven_multilingual_v2"
|
||||
DEFAULT_VOICE_TYPE: str = "runway-preset"
|
||||
DEFAULT_VOICE_PRESET_ID: str = "Bernard"
|
||||
|
||||
# Voice mappings from OpenAI voices to RunwayML preset IDs
|
||||
# OpenAI voices mapped to similar-sounding RunwayML voices
|
||||
VOICE_MAPPINGS = {
|
||||
"alloy": "Maya", # Neutral, balanced female voice
|
||||
"echo": "James", # Male voice
|
||||
"fable": "Bernard", # Warm, storytelling voice
|
||||
"onyx": "Vincent", # Deep male voice
|
||||
"nova": "Serene", # Warm, expressive female voice
|
||||
"shimmer": "Ella", # Clear, friendly female voice
|
||||
}
|
||||
|
||||
def dispatch_text_to_speech(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: Optional[Union[str, Dict]],
|
||||
optional_params: Dict,
|
||||
litellm_params_dict: Dict,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
extra_headers: Optional[Dict[str, Any]],
|
||||
base_llm_http_handler: Any,
|
||||
aspeech: bool,
|
||||
api_base: Optional[str],
|
||||
api_key: Optional[str],
|
||||
**kwargs: Any,
|
||||
) -> Union[
|
||||
"HttpxBinaryResponseContent",
|
||||
Coroutine[Any, Any, "HttpxBinaryResponseContent"],
|
||||
]:
|
||||
"""
|
||||
Dispatch method to handle RunwayML TTS requests
|
||||
|
||||
This method encapsulates RunwayML-specific credential resolution and parameter handling
|
||||
|
||||
Args:
|
||||
base_llm_http_handler: The BaseLLMHTTPHandler instance from main.py
|
||||
"""
|
||||
# Resolve api_base from multiple sources
|
||||
api_base = (
|
||||
api_base
|
||||
or litellm_params_dict.get("api_base")
|
||||
or litellm.api_base
|
||||
or get_secret_str("RUNWAYML_API_BASE")
|
||||
or self.DEFAULT_BASE_URL
|
||||
)
|
||||
|
||||
# Resolve api_key from multiple sources
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm_params_dict.get("api_key")
|
||||
or litellm.api_key
|
||||
or get_secret_str("RUNWAYML_API_SECRET")
|
||||
or get_secret_str("RUNWAYML_API_KEY")
|
||||
)
|
||||
|
||||
# Convert voice to appropriate format
|
||||
voice_param: Optional[Union[str, Dict]] = voice
|
||||
if isinstance(voice, str):
|
||||
# Keep as string, will be processed in map_openai_params
|
||||
voice_param = voice
|
||||
elif isinstance(voice, dict):
|
||||
# Already in dict format, pass through
|
||||
voice_param = voice
|
||||
|
||||
litellm_params_dict.update(
|
||||
{
|
||||
"api_key": api_key,
|
||||
"api_base": api_base,
|
||||
}
|
||||
)
|
||||
|
||||
# Call the text_to_speech_handler
|
||||
response = base_llm_http_handler.text_to_speech_handler(
|
||||
model=model,
|
||||
input=input,
|
||||
voice=voice_param,
|
||||
text_to_speech_provider_config=self,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
custom_llm_provider="runwayml",
|
||||
litellm_params=litellm_params_dict,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
extra_headers=extra_headers,
|
||||
client=None,
|
||||
_is_async=aspeech,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
RunwayML TTS supports these OpenAI parameters
|
||||
"""
|
||||
return ["voice"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: Dict,
|
||||
voice: Optional[Union[str, Dict]] = None,
|
||||
drop_params: bool = False,
|
||||
kwargs: Dict = {},
|
||||
) -> Tuple[Optional[str], Dict]:
|
||||
"""
|
||||
Map OpenAI parameters to RunwayML TTS parameters
|
||||
|
||||
Returns:
|
||||
Tuple of (mapped_voice_string, mapped_params)
|
||||
|
||||
Note: Since RunwayML requires voice as a dict, we store it in
|
||||
mapped_params["runwayml_voice"] and return None for the voice string.
|
||||
"""
|
||||
mapped_params = {}
|
||||
|
||||
# Map voice parameter to RunwayML format dict
|
||||
voice_dict: Optional[Dict] = None
|
||||
if isinstance(voice, str):
|
||||
# Check if it's an OpenAI voice name that needs mapping
|
||||
if voice in self.VOICE_MAPPINGS:
|
||||
preset_id = self.VOICE_MAPPINGS[voice]
|
||||
voice_dict = {
|
||||
"type": self.DEFAULT_VOICE_TYPE,
|
||||
"presetId": preset_id,
|
||||
}
|
||||
else:
|
||||
# Assume it's a RunwayML preset ID
|
||||
voice_dict = {
|
||||
"type": self.DEFAULT_VOICE_TYPE,
|
||||
"presetId": voice,
|
||||
}
|
||||
elif isinstance(voice, dict):
|
||||
# Already in RunwayML format, use as-is
|
||||
voice_dict = voice
|
||||
|
||||
# Store the voice dict in optional_params for later use
|
||||
if voice_dict is not None:
|
||||
mapped_params["runwayml_voice"] = voice_dict
|
||||
|
||||
# No other OpenAI params are currently supported by RunwayML TTS
|
||||
# (response_format, speed, etc. are not supported)
|
||||
|
||||
# Return None for voice string since RunwayML uses dict format
|
||||
return None, mapped_params
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Validate RunwayML environment and set up authentication headers
|
||||
"""
|
||||
validated_headers = headers.copy()
|
||||
|
||||
final_api_key = (
|
||||
api_key
|
||||
or get_secret_str("RUNWAYML_API_SECRET")
|
||||
or get_secret_str("RUNWAYML_API_KEY")
|
||||
)
|
||||
|
||||
if not final_api_key:
|
||||
raise ValueError("RUNWAYML_API_SECRET or RUNWAYML_API_KEY is not set")
|
||||
|
||||
validated_headers["Authorization"] = f"Bearer {final_api_key}"
|
||||
validated_headers["X-Runway-Version"] = RUNWAYML_DEFAULT_API_VERSION
|
||||
validated_headers["Content-Type"] = "application/json"
|
||||
|
||||
return validated_headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
"""
|
||||
Get the complete URL for RunwayML TTS request
|
||||
"""
|
||||
complete_url = (
|
||||
api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL
|
||||
)
|
||||
|
||||
complete_url = complete_url.rstrip("/")
|
||||
return f"{complete_url}/{self.TTS_ENDPOINT_PATH}"
|
||||
|
||||
@staticmethod
|
||||
def _check_timeout(start_time: float, timeout_secs: float) -> None:
|
||||
"""
|
||||
Check if operation has timed out.
|
||||
|
||||
Args:
|
||||
start_time: Start time of the operation
|
||||
timeout_secs: Timeout duration in seconds
|
||||
|
||||
Raises:
|
||||
TimeoutError: If operation has exceeded timeout
|
||||
"""
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise TimeoutError(
|
||||
f"RunwayML TTS task polling timed out after {timeout_secs} seconds"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _check_task_status(response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Check RunwayML task status from response.
|
||||
|
||||
RunwayML statuses: PENDING, RUNNING, SUCCEEDED, FAILED, CANCELLED, THROTTLED
|
||||
|
||||
Args:
|
||||
response_data: JSON response from RunwayML task endpoint
|
||||
|
||||
Returns:
|
||||
Normalized status string: "running", "succeeded", or raises on failure
|
||||
|
||||
Raises:
|
||||
ValueError: If task failed or status is unknown
|
||||
"""
|
||||
status = response_data.get("status", "").upper()
|
||||
|
||||
verbose_logger.debug(f"RunwayML TTS task status: {status}")
|
||||
|
||||
if status == "SUCCEEDED":
|
||||
return "succeeded"
|
||||
elif status == "FAILED":
|
||||
failure_reason = response_data.get("failure", "Unknown error")
|
||||
failure_code = response_data.get("failureCode", "unknown")
|
||||
raise ValueError(
|
||||
f"RunwayML TTS failed: {failure_reason} (code: {failure_code})"
|
||||
)
|
||||
elif status == "CANCELLED":
|
||||
raise ValueError("RunwayML TTS was cancelled")
|
||||
elif status in ["PENDING", "RUNNING", "THROTTLED"]:
|
||||
return "running"
|
||||
else:
|
||||
raise ValueError(f"Unknown RunwayML task status: {status}")
|
||||
|
||||
def _poll_task_sync(
|
||||
self,
|
||||
task_id: str,
|
||||
api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout_secs: float = 600,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Poll RunwayML task until completion (sync).
|
||||
|
||||
RunwayML POST returns immediately with a task that has status PENDING/RUNNING.
|
||||
We need to poll GET /v1/tasks/{task_id} until status is SUCCEEDED or FAILED.
|
||||
|
||||
Args:
|
||||
task_id: The task ID to poll
|
||||
api_base: Base URL for RunwayML API
|
||||
headers: Request headers (including auth)
|
||||
timeout_secs: Total timeout in seconds (default: 600s = 10 minutes)
|
||||
|
||||
Returns:
|
||||
Final response with completed task
|
||||
"""
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
|
||||
client = _get_httpx_client()
|
||||
start_time = time.time()
|
||||
|
||||
# Build task status URL
|
||||
api_base = api_base.rstrip("/")
|
||||
task_url = f"{api_base}/v1/tasks/{task_id}"
|
||||
|
||||
verbose_logger.debug(f"Polling RunwayML TTS task: {task_url}")
|
||||
|
||||
while True:
|
||||
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
|
||||
|
||||
# Poll the task status
|
||||
response = client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
||||
if status == "succeeded":
|
||||
return response
|
||||
elif status == "running":
|
||||
# Wait before polling again (RunwayML recommends 1-2 second intervals)
|
||||
time.sleep(2)
|
||||
|
||||
async def _poll_task_async(
|
||||
self,
|
||||
task_id: str,
|
||||
api_base: str,
|
||||
headers: Dict[str, str],
|
||||
timeout_secs: float = 600,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Poll RunwayML task until completion (async).
|
||||
|
||||
Args:
|
||||
task_id: The task ID to poll
|
||||
api_base: Base URL for RunwayML API
|
||||
headers: Request headers (including auth)
|
||||
timeout_secs: Total timeout in seconds (default: 600s = 10 minutes)
|
||||
|
||||
Returns:
|
||||
Final response with completed task
|
||||
"""
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
||||
client = get_async_httpx_client(llm_provider=litellm.LlmProviders.RUNWAYML)
|
||||
start_time = time.time()
|
||||
|
||||
# Build task status URL
|
||||
api_base = api_base.rstrip("/")
|
||||
task_url = f"{api_base}/v1/tasks/{task_id}"
|
||||
|
||||
verbose_logger.debug(f"Polling RunwayML TTS task (async): {task_url}")
|
||||
|
||||
while True:
|
||||
self._check_timeout(start_time=start_time, timeout_secs=timeout_secs)
|
||||
|
||||
# Poll the task status
|
||||
response = await client.get(url=task_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
# Check task status
|
||||
status = self._check_task_status(response_data=response_data)
|
||||
|
||||
if status == "succeeded":
|
||||
return response
|
||||
elif status == "running":
|
||||
# Wait before polling again (RunwayML recommends 1-2 second intervals)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
def transform_text_to_speech_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: Optional[Union[str, Dict]],
|
||||
optional_params: Dict,
|
||||
litellm_params: Dict,
|
||||
headers: dict,
|
||||
) -> TextToSpeechRequestData:
|
||||
"""
|
||||
Transform OpenAI TTS request to RunwayML TTS format
|
||||
|
||||
RunwayML expects:
|
||||
- model: The model to use (e.g., 'eleven_multilingual_v2')
|
||||
- promptText: The text to convert to speech
|
||||
- voice: Voice configuration object
|
||||
{
|
||||
"type": "runway-preset",
|
||||
"presetId": "Bernard"
|
||||
}
|
||||
|
||||
Returns:
|
||||
TextToSpeechRequestData: Contains JSON body and headers
|
||||
"""
|
||||
# Get voice from optional_params (mapped in map_openai_params)
|
||||
runwayml_voice = optional_params.get("runwayml_voice")
|
||||
if runwayml_voice is None:
|
||||
# Use default voice if not provided
|
||||
runwayml_voice = {
|
||||
"type": self.DEFAULT_VOICE_TYPE,
|
||||
"presetId": self.DEFAULT_VOICE_PRESET_ID,
|
||||
}
|
||||
|
||||
# Build request body
|
||||
request_body = {
|
||||
"model": model or self.DEFAULT_MODEL,
|
||||
"promptText": input,
|
||||
"voice": runwayml_voice,
|
||||
}
|
||||
|
||||
# Add any other optional parameters (except runwayml_voice which we already used)
|
||||
for k, v in optional_params.items():
|
||||
if k not in request_body and k != "runwayml_voice":
|
||||
request_body[k] = v
|
||||
|
||||
return {
|
||||
"dict_body": request_body,
|
||||
"headers": headers,
|
||||
}
|
||||
|
||||
def transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
"""
|
||||
Transform RunwayML TTS response to standard format
|
||||
|
||||
RunwayML returns a task immediately with status PENDING/RUNNING.
|
||||
We need to poll the task until it completes, then download the audio.
|
||||
|
||||
Initial response:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "PENDING" | "RUNNING",
|
||||
"createdAt": "2025-11-13T..."
|
||||
}
|
||||
|
||||
After polling:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "SUCCEEDED",
|
||||
"output": ["https://storage.googleapis.com/.../audio.mp3"],
|
||||
"completedAt": "2025-11-13T..."
|
||||
}
|
||||
"""
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data = raw_response.json()
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=dict(raw_response.headers),
|
||||
)
|
||||
|
||||
verbose_logger.debug("RunwayML TTS starting polling...")
|
||||
|
||||
# Get task ID
|
||||
task_id = response_data.get("id")
|
||||
if not task_id:
|
||||
raise ValueError("RunwayML TTS response missing task ID")
|
||||
|
||||
# Get headers for polling (need auth)
|
||||
poll_headers = {
|
||||
"Authorization": raw_response.request.headers.get("Authorization", ""),
|
||||
"X-Runway-Version": raw_response.request.headers.get(
|
||||
"X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION
|
||||
),
|
||||
}
|
||||
|
||||
# Poll until task completes
|
||||
polled_response = self._poll_task_sync(
|
||||
task_id=task_id,
|
||||
api_base=self.DEFAULT_BASE_URL,
|
||||
headers=poll_headers,
|
||||
timeout_secs=RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data = polled_response.json()
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete, downloading audio")
|
||||
|
||||
# Get audio URL from output
|
||||
output = task_data.get("output", [])
|
||||
if not output or not isinstance(output, list) or len(output) == 0:
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url = output[0]
|
||||
if not isinstance(audio_url, str):
|
||||
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")
|
||||
|
||||
# Download the audio file
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
|
||||
client = _get_httpx_client()
|
||||
audio_response = client.get(url=audio_url)
|
||||
audio_response.raise_for_status()
|
||||
|
||||
verbose_logger.debug("RunwayML TTS audio downloaded successfully")
|
||||
|
||||
# Return the audio data wrapped in HttpxBinaryResponseContent
|
||||
return HttpxBinaryResponseContent(audio_response)
|
||||
|
||||
async def async_transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
"""
|
||||
Async transform RunwayML TTS response to standard format
|
||||
|
||||
Same as sync version but uses async polling and download
|
||||
"""
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_data = raw_response.json()
|
||||
except Exception as e:
|
||||
raise self.get_error_class(
|
||||
error_message=f"Error parsing RunwayML TTS response: {e}",
|
||||
status_code=raw_response.status_code,
|
||||
headers=dict(raw_response.headers),
|
||||
)
|
||||
|
||||
verbose_logger.debug("RunwayML TTS starting polling (async)...")
|
||||
|
||||
# Get task ID
|
||||
task_id = response_data.get("id")
|
||||
if not task_id:
|
||||
raise ValueError("RunwayML TTS response missing task ID")
|
||||
|
||||
# Get headers for polling (need auth)
|
||||
poll_headers = {
|
||||
"Authorization": raw_response.request.headers.get("Authorization", ""),
|
||||
"X-Runway-Version": raw_response.request.headers.get(
|
||||
"X-Runway-Version", RUNWAYML_DEFAULT_API_VERSION
|
||||
),
|
||||
}
|
||||
|
||||
# Poll until task completes (async)
|
||||
polled_response = await self._poll_task_async(
|
||||
task_id=task_id,
|
||||
api_base=self.DEFAULT_BASE_URL,
|
||||
headers=poll_headers,
|
||||
timeout_secs=RUNWAYML_POLLING_TIMEOUT,
|
||||
)
|
||||
|
||||
# Get the completed task data
|
||||
task_data = polled_response.json()
|
||||
|
||||
verbose_logger.debug("RunwayML TTS polling complete (async), downloading audio")
|
||||
|
||||
# Get audio URL from output
|
||||
output = task_data.get("output", [])
|
||||
if not output or not isinstance(output, list) or len(output) == 0:
|
||||
raise ValueError("RunwayML TTS response missing audio URL in output")
|
||||
|
||||
audio_url = output[0]
|
||||
if not isinstance(audio_url, str):
|
||||
raise ValueError(f"RunwayML TTS audio URL is not a string: {audio_url}")
|
||||
|
||||
# Download the audio file (async)
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
|
||||
client = get_async_httpx_client(llm_provider=litellm.LlmProviders.RUNWAYML)
|
||||
audio_response = await client.get(url=audio_url)
|
||||
audio_response.raise_for_status()
|
||||
|
||||
verbose_logger.debug("RunwayML TTS audio downloaded successfully (async)")
|
||||
|
||||
# Return the audio data wrapped in HttpxBinaryResponseContent
|
||||
return HttpxBinaryResponseContent(audio_response)
|
||||
@@ -0,0 +1 @@
|
||||
# RunwayML video generation
|
||||
@@ -0,0 +1,604 @@
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
import litellm
|
||||
from litellm.constants import RUNWAYML_DEFAULT_API_VERSION
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
|
||||
from litellm.types.videos.utils import (
|
||||
encode_video_id_with_provider,
|
||||
extract_original_video_id,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class RunwayMLVideoConfig(BaseVideoConfig):
|
||||
"""
|
||||
Configuration class for RunwayML video generation.
|
||||
|
||||
RunwayML uses a task-based API where:
|
||||
1. POST /v1/image_to_video creates a task
|
||||
2. The task returns immediately with a task ID
|
||||
3. Client must poll or wait for task completion
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Get the list of supported OpenAI parameters for video generation.
|
||||
Maps OpenAI params to RunwayML equivalents:
|
||||
- prompt -> promptText
|
||||
- input_reference -> promptImage
|
||||
- size -> ratio (e.g., "1280x720" -> "1280:720")
|
||||
- seconds -> duration
|
||||
"""
|
||||
return [
|
||||
"model",
|
||||
"prompt",
|
||||
"input_reference",
|
||||
"seconds",
|
||||
"size",
|
||||
"user",
|
||||
"extra_headers",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
video_create_optional_params: VideoCreateOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> Dict:
|
||||
"""
|
||||
Map OpenAI parameters to RunwayML format.
|
||||
|
||||
Mappings:
|
||||
- prompt -> promptText
|
||||
- input_reference -> promptImage
|
||||
- size -> ratio (convert "WIDTHxHEIGHT" to "WIDTH:HEIGHT")
|
||||
- seconds -> duration (convert to integer)
|
||||
"""
|
||||
mapped_params: Dict[str, Any] = {}
|
||||
|
||||
# Handle input_reference parameter - map to promptImage
|
||||
if "input_reference" in video_create_optional_params:
|
||||
input_reference = video_create_optional_params["input_reference"]
|
||||
# RunwayML supports URLs and data URIs directly
|
||||
mapped_params["promptImage"] = input_reference
|
||||
|
||||
# Handle size parameter - convert "1280x720" to "1280:720"
|
||||
if "size" in video_create_optional_params:
|
||||
size = video_create_optional_params["size"]
|
||||
if isinstance(size, str) and "x" in size:
|
||||
mapped_params["ratio"] = size.replace("x", ":")
|
||||
|
||||
# Handle seconds parameter - convert to integer
|
||||
if "seconds" in video_create_optional_params:
|
||||
seconds = video_create_optional_params["seconds"]
|
||||
if seconds is not None:
|
||||
try:
|
||||
mapped_params["duration"] = (
|
||||
int(float(seconds))
|
||||
if isinstance(seconds, str)
|
||||
else int(seconds)
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
# If conversion fails, use default duration
|
||||
pass
|
||||
|
||||
# Pass through other parameters that aren't OpenAI-specific
|
||||
supported_openai_params = self.get_supported_openai_params(model)
|
||||
for key, value in video_create_optional_params.items():
|
||||
if key not in supported_openai_params:
|
||||
mapped_params[key] = value
|
||||
|
||||
return mapped_params
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
api_key: Optional[str] = None,
|
||||
litellm_params: Optional[GenericLiteLLMParams] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Validate environment and set up authentication headers.
|
||||
RunwayML uses Bearer token authentication via RUNWAYML_API_SECRET.
|
||||
"""
|
||||
# Use api_key from litellm_params if available, otherwise fall back to other sources
|
||||
if litellm_params and litellm_params.api_key:
|
||||
api_key = api_key or litellm_params.api_key
|
||||
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or get_secret_str("RUNWAYML_API_SECRET")
|
||||
or get_secret_str("RUNWAYML_API_KEY")
|
||||
)
|
||||
|
||||
if api_key is None:
|
||||
raise ValueError(
|
||||
"RunwayML API key is required. Set RUNWAYML_API_SECRET environment variable "
|
||||
"or pass api_key parameter."
|
||||
)
|
||||
|
||||
headers.update(
|
||||
{
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"X-Runway-Version": RUNWAYML_DEFAULT_API_VERSION,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
)
|
||||
return headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: Optional[str],
|
||||
litellm_params: dict,
|
||||
) -> str:
|
||||
"""
|
||||
Get the base URL for RunwayML API.
|
||||
The specific endpoint path will be added in the transform methods.
|
||||
"""
|
||||
if api_base is None:
|
||||
api_base = "https://api.dev.runwayml.com/v1"
|
||||
|
||||
return api_base.rstrip("/")
|
||||
|
||||
def transform_video_create_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
video_create_optional_request_params: Dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[Dict, RequestFiles, str]:
|
||||
"""
|
||||
Transform the video creation request for RunwayML API.
|
||||
|
||||
RunwayML expects:
|
||||
{
|
||||
"model": "gen4_turbo",
|
||||
"promptImage": "https://... or data:image/...",
|
||||
"promptText": "description",
|
||||
"ratio": "1280:720",
|
||||
"duration": 5
|
||||
}
|
||||
"""
|
||||
# Build the request data
|
||||
request_data: Dict[str, Any] = {
|
||||
"model": model,
|
||||
"promptText": prompt,
|
||||
}
|
||||
|
||||
# Add mapped parameters
|
||||
request_data.update(video_create_optional_request_params)
|
||||
|
||||
# RunwayML uses JSON body, no files multipart
|
||||
files_list: List[Tuple[str, Any]] = []
|
||||
|
||||
# Append the specific endpoint for video generation
|
||||
full_api_base = f"{api_base}/image_to_video"
|
||||
|
||||
return request_data, files_list, full_api_base
|
||||
|
||||
def transform_video_create_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict] = None,
|
||||
) -> VideoObject:
|
||||
"""
|
||||
Transform the RunwayML video creation response.
|
||||
|
||||
RunwayML returns a task object that looks like:
|
||||
{
|
||||
"id": "task_123...",
|
||||
"status": "PENDING" | "RUNNING" | "SUCCEEDED" | "FAILED",
|
||||
"output": ["https://...video.mp4"] (when succeeded)
|
||||
}
|
||||
|
||||
We map this to OpenAI VideoObject format.
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
|
||||
# Map RunwayML task response to VideoObject format
|
||||
video_data: Dict[str, Any] = {
|
||||
"id": response_data.get("id", ""),
|
||||
"object": "video",
|
||||
"status": self._map_runway_status(response_data.get("status", "pending")),
|
||||
"created_at": self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
}
|
||||
|
||||
# Add optional fields if present
|
||||
if "output" in response_data and response_data["output"]:
|
||||
# RunwayML returns output as array of URLs when task succeeds
|
||||
video_data["output_url"] = (
|
||||
response_data["output"][0]
|
||||
if isinstance(response_data["output"], list)
|
||||
else response_data["output"]
|
||||
)
|
||||
|
||||
if "completedAt" in response_data:
|
||||
video_data["completed_at"] = self._parse_runway_timestamp(
|
||||
response_data.get("completedAt")
|
||||
)
|
||||
|
||||
if "failureCode" in response_data or "failure" in response_data:
|
||||
video_data["error"] = {
|
||||
"code": response_data.get("failureCode", "unknown"),
|
||||
"message": response_data.get("failure", "Video generation failed"),
|
||||
}
|
||||
|
||||
# Add model and size info if available from request
|
||||
if request_data:
|
||||
if "model" in request_data:
|
||||
video_data["model"] = request_data["model"]
|
||||
if "ratio" in request_data:
|
||||
# Convert ratio back to size format
|
||||
ratio = request_data["ratio"]
|
||||
if isinstance(ratio, str) and ":" in ratio:
|
||||
video_data["size"] = ratio.replace(":", "x")
|
||||
if "duration" in request_data:
|
||||
video_data["seconds"] = str(request_data["duration"])
|
||||
|
||||
video_obj = VideoObject(**video_data) # type: ignore[arg-type]
|
||||
|
||||
if custom_llm_provider and video_obj.id:
|
||||
video_obj.id = encode_video_id_with_provider(
|
||||
video_obj.id, custom_llm_provider, model
|
||||
)
|
||||
|
||||
# Add usage data for cost tracking
|
||||
usage_data = {}
|
||||
if video_obj and hasattr(video_obj, "seconds") and video_obj.seconds:
|
||||
try:
|
||||
usage_data["duration_seconds"] = float(video_obj.seconds)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
video_obj.usage = usage_data
|
||||
|
||||
return video_obj
|
||||
|
||||
def _map_runway_status(self, runway_status: str) -> str:
|
||||
"""
|
||||
Map RunwayML status to OpenAI status format.
|
||||
|
||||
RunwayML statuses: PENDING, RUNNING, SUCCEEDED, FAILED, CANCELLED
|
||||
OpenAI statuses: queued, in_progress, completed, failed
|
||||
"""
|
||||
status_map = {
|
||||
"PENDING": "queued",
|
||||
"RUNNING": "in_progress",
|
||||
"SUCCEEDED": "completed",
|
||||
"FAILED": "failed",
|
||||
"CANCELLED": "failed",
|
||||
"THROTTLED": "queued",
|
||||
}
|
||||
return status_map.get(runway_status.upper(), "queued")
|
||||
|
||||
def _parse_runway_timestamp(self, timestamp_str: Optional[str]) -> int:
|
||||
"""
|
||||
Convert RunwayML ISO 8601 timestamp to Unix timestamp.
|
||||
|
||||
RunwayML returns timestamps like: "2025-11-11T21:48:50.448Z"
|
||||
We need to convert to Unix timestamp (seconds since epoch).
|
||||
"""
|
||||
if not timestamp_str:
|
||||
return 0
|
||||
|
||||
try:
|
||||
# Parse ISO 8601 timestamp
|
||||
dt = datetime.fromisoformat(timestamp_str.replace("Z", "+00:00"))
|
||||
# Convert to Unix timestamp
|
||||
return int(dt.timestamp())
|
||||
except (ValueError, AttributeError):
|
||||
return 0
|
||||
|
||||
def transform_video_content_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
variant: Optional[str] = None,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video content request for RunwayML API.
|
||||
|
||||
RunwayML doesn't have a separate content download endpoint.
|
||||
The video URL is returned in the task output field.
|
||||
We'll retrieve the task and extract the video URL.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Get task status to retrieve video URL
|
||||
url = f"{api_base}/tasks/{original_video_id}"
|
||||
|
||||
params: Dict[str, Any] = {}
|
||||
|
||||
return url, params
|
||||
|
||||
def _extract_video_url_from_response(self, response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
Helper method to extract video URL from RunwayML response.
|
||||
Shared between sync and async transforms.
|
||||
"""
|
||||
# Extract video URL from the output field
|
||||
video_url = None
|
||||
if "output" in response_data and response_data["output"]:
|
||||
output = response_data["output"]
|
||||
video_url = output[0] if isinstance(output, list) else output
|
||||
|
||||
if not video_url:
|
||||
# Check if the video generation failed or is still processing
|
||||
status = response_data.get("status", "UNKNOWN")
|
||||
if status in ["PENDING", "RUNNING", "THROTTLED"]:
|
||||
raise ValueError(
|
||||
f"Video is still processing (status: {status}). Please wait and try again."
|
||||
)
|
||||
elif status == "FAILED":
|
||||
failure_reason = response_data.get("failure", "Unknown error")
|
||||
raise ValueError(f"Video generation failed: {failure_reason}")
|
||||
else:
|
||||
raise ValueError(
|
||||
"Video URL not found in response. Video may not be ready yet."
|
||||
)
|
||||
|
||||
return video_url
|
||||
|
||||
def transform_video_content_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> bytes:
|
||||
"""
|
||||
Transform the RunwayML video content download response (synchronous).
|
||||
|
||||
RunwayML's task endpoint returns JSON with a video URL in the output field.
|
||||
We need to extract the URL and download the video.
|
||||
|
||||
Example response:
|
||||
{
|
||||
"id":"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"createdAt":"2025-11-11T21:48:50.448Z",
|
||||
"status":"SUCCEEDED",
|
||||
"output":["https://dnznrvs05pmza.cloudfront.net/.../video.mp4?_jwt=..."]
|
||||
}
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
video_url = self._extract_video_url_from_response(response_data)
|
||||
|
||||
# Download the video from the CloudFront URL synchronously
|
||||
httpx_client: HTTPHandler = _get_httpx_client()
|
||||
video_response = httpx_client.get(video_url)
|
||||
video_response.raise_for_status()
|
||||
|
||||
return video_response.content
|
||||
|
||||
async def async_transform_video_content_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> bytes:
|
||||
"""
|
||||
Transform the RunwayML video content download response (asynchronous).
|
||||
|
||||
RunwayML's task endpoint returns JSON with a video URL in the output field.
|
||||
We need to extract the URL and download the video asynchronously.
|
||||
|
||||
Example response:
|
||||
{
|
||||
"id":"63fd0f13-f29d-4e58-99d3-1cb9efa14a5b",
|
||||
"createdAt":"2025-11-11T21:48:50.448Z",
|
||||
"status":"SUCCEEDED",
|
||||
"output":["https://dnznrvs05pmza.cloudfront.net/.../video.mp4?_jwt=..."]
|
||||
}
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
video_url = self._extract_video_url_from_response(response_data)
|
||||
|
||||
# Download the video from the CloudFront URL asynchronously
|
||||
async_httpx_client: AsyncHTTPHandler = get_async_httpx_client(
|
||||
llm_provider=litellm.LlmProviders.RUNWAYML,
|
||||
)
|
||||
video_response = await async_httpx_client.get(video_url)
|
||||
video_response.raise_for_status()
|
||||
|
||||
return video_response.content
|
||||
|
||||
def transform_video_remix_request(
|
||||
self,
|
||||
video_id: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
extra_body: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video remix request for RunwayML API.
|
||||
|
||||
RunwayML doesn't have a direct remix endpoint in their current API.
|
||||
This would need to be implemented when/if they add this feature.
|
||||
"""
|
||||
raise NotImplementedError("Video remix is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_remix_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> VideoObject:
|
||||
"""Transform the RunwayML video remix response."""
|
||||
raise NotImplementedError("Video remix is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_list_request(
|
||||
self,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
after: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
order: Optional[str] = None,
|
||||
extra_query: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video list request for RunwayML API.
|
||||
|
||||
RunwayML doesn't expose a list endpoint in their public API yet.
|
||||
"""
|
||||
raise NotImplementedError("Video listing is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_list_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Transform the RunwayML video list response."""
|
||||
raise NotImplementedError("Video listing is not yet supported by RunwayML API")
|
||||
|
||||
def transform_video_delete_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the video delete request for RunwayML API.
|
||||
|
||||
RunwayML uses task cancellation.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Construct the URL for task cancellation
|
||||
url = f"{api_base}/tasks/{original_video_id}/cancel"
|
||||
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
return url, data
|
||||
|
||||
def transform_video_delete_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> VideoObject:
|
||||
"""Transform the RunwayML video delete/cancel response."""
|
||||
response_data = raw_response.json()
|
||||
|
||||
video_obj = VideoObject(
|
||||
id=response_data.get("id", ""),
|
||||
object="video",
|
||||
status="cancelled",
|
||||
created_at=self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
) # type: ignore[arg-type]
|
||||
|
||||
return video_obj
|
||||
|
||||
def transform_video_status_retrieve_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Transform the RunwayML video status retrieve request.
|
||||
|
||||
RunwayML uses GET /v1/tasks/{task_id} to retrieve task status.
|
||||
"""
|
||||
original_video_id = extract_original_video_id(video_id)
|
||||
|
||||
# Construct the full URL for task status retrieval
|
||||
url = f"{api_base}/tasks/{original_video_id}"
|
||||
|
||||
# Empty dict for GET request (no body)
|
||||
data: Dict[str, Any] = {}
|
||||
|
||||
return url, data
|
||||
|
||||
def transform_video_status_retrieve_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
) -> VideoObject:
|
||||
"""
|
||||
Transform the RunwayML video status retrieve response.
|
||||
"""
|
||||
response_data = raw_response.json()
|
||||
|
||||
# Map RunwayML task response to VideoObject format
|
||||
video_data: Dict[str, Any] = {
|
||||
"id": response_data.get("id", ""),
|
||||
"object": "video",
|
||||
"status": self._map_runway_status(response_data.get("status", "pending")),
|
||||
"created_at": self._parse_runway_timestamp(response_data.get("createdAt")),
|
||||
}
|
||||
|
||||
# Add optional fields if present
|
||||
if "output" in response_data and response_data["output"]:
|
||||
video_data["output_url"] = (
|
||||
response_data["output"][0]
|
||||
if isinstance(response_data["output"], list)
|
||||
else response_data["output"]
|
||||
)
|
||||
|
||||
if "completedAt" in response_data:
|
||||
video_data["completed_at"] = self._parse_runway_timestamp(
|
||||
response_data.get("completedAt")
|
||||
)
|
||||
|
||||
if "progress" in response_data:
|
||||
video_data["progress"] = response_data["progress"]
|
||||
|
||||
if "failureCode" in response_data or "failure" in response_data:
|
||||
video_data["error"] = {
|
||||
"code": response_data.get("failureCode", "unknown"),
|
||||
"message": response_data.get("failure", "Video generation failed"),
|
||||
}
|
||||
|
||||
video_obj = VideoObject(**video_data) # type: ignore[arg-type]
|
||||
|
||||
if custom_llm_provider and video_obj.id:
|
||||
video_obj.id = encode_video_id_with_provider(
|
||||
video_obj.id, custom_llm_provider, None
|
||||
)
|
||||
|
||||
return video_obj
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
from ...base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
raise BaseLLMException(
|
||||
status_code=status_code,
|
||||
message=error_message,
|
||||
headers=headers,
|
||||
)
|
||||
Reference in New Issue
Block a user