niuniu/lijiaoqiao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initial public snapshot for github upload

Browse Source
This commit is contained in:
Your Name
2026-03-26 20:06:14 +08:00
commit 0e5ecd930e
3497 changed files with 1586236 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

578
llm-gateway-competitors/litellm-wheel-src/litellm/llms/ollama/chat/transformation.py Normal file
View File

@@ -0,0 +1,578 @@
import json
import time
from litellm._uuid import uuid
from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Iterator,
List,
Optional,
Union,
cast,
)
from httpx._models import Headers, Response
from pydantic import BaseModel
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
convert_content_list_to_str,
extract_images_from_message,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.ollama import (
OllamaChatCompletionMessage,
OllamaToolCall,
OllamaToolCallFunction,
)
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantToolCall,
ChatCompletionUsageBlock,
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from ..common_utils import OllamaError
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OllamaChatConfig(BaseConfig):
"""
Reference: https://github.com/ollama/ollama/blob/main/docs/api.md#parameters
The class `OllamaConfig` provides the configuration for the Ollama's API interface. Below are the parameters:
- `mirostat` (int): Enable Mirostat sampling for controlling perplexity. Default is 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0. Example usage: mirostat 0
- `mirostat_eta` (float): Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. Default: 0.1. Example usage: mirostat_eta 0.1
- `mirostat_tau` (float): Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. Default: 5.0. Example usage: mirostat_tau 5.0
- `num_ctx` (int): Sets the size of the context window used to generate the next token. Default: 2048. Example usage: num_ctx 4096
- `num_gqa` (int): The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b. Example usage: num_gqa 1
- `num_gpu` (int): The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. Example usage: num_gpu 0
- `num_thread` (int): Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Example usage: num_thread 8
- `repeat_last_n` (int): Sets how far back for the model to look back to prevent repetition. Default: 64, 0 = disabled, -1 = num_ctx. Example usage: repeat_last_n 64
- `repeat_penalty` (float): Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. Default: 1.1. Example usage: repeat_penalty 1.1
- `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7
- `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42
- `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:"
- `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1
- `num_predict` (int): Maximum number of tokens to predict when generating text. Default: 128, -1 = infinite generation, -2 = fill context. Example usage: num_predict 42
- `top_k` (int): Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. Default: 40. Example usage: top_k 40
- `top_p` (float): Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. Default: 0.9. Example usage: top_p 0.9
- `system` (string): system prompt for model (overrides what is defined in the Modelfile)
- `template` (string): the full prompt or prompt template (overrides what is defined in the Modelfile)
"""
mirostat: Optional[int] = None
mirostat_eta: Optional[float] = None
mirostat_tau: Optional[float] = None
num_ctx: Optional[int] = None
num_gqa: Optional[int] = None
num_thread: Optional[int] = None
repeat_last_n: Optional[int] = None
repeat_penalty: Optional[float] = None
seed: Optional[int] = None
tfs_z: Optional[float] = None
num_predict: Optional[int] = None
top_k: Optional[int] = None
system: Optional[str] = None
template: Optional[str] = None
def __init__(
self,
mirostat: Optional[int] = None,
mirostat_eta: Optional[float] = None,
mirostat_tau: Optional[float] = None,
num_ctx: Optional[int] = None,
num_gqa: Optional[int] = None,
num_thread: Optional[int] = None,
repeat_last_n: Optional[int] = None,
repeat_penalty: Optional[float] = None,
temperature: Optional[float] = None,
seed: Optional[int] = None,
stop: Optional[list] = None,
tfs_z: Optional[float] = None,
num_predict: Optional[int] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
system: Optional[str] = None,
template: Optional[str] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@classmethod
def get_config(cls):
return super().get_config()
def get_supported_openai_params(self, model: str):
return [
"max_tokens",
"max_completion_tokens",
"stream",
"top_p",
"temperature",
"seed",
"frequency_penalty",
"stop",
"tools",
"tool_choice",
"functions",
"response_format",
"reasoning_effort",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
for param, value in non_default_params.items():
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["num_predict"] = value
if param == "stream":
optional_params["stream"] = value
if param == "temperature":
optional_params["temperature"] = value
if param == "seed":
optional_params["seed"] = value
if param == "top_p":
optional_params["top_p"] = value
if param == "frequency_penalty":
optional_params["repeat_penalty"] = value
if param == "stop":
optional_params["stop"] = value
if (
param == "response_format"
and isinstance(value, dict)
and value.get("type") == "json_object"
):
optional_params["format"] = "json"
if (
param == "response_format"
and isinstance(value, dict)
and value.get("type") == "json_schema"
):
if value.get("json_schema") and value["json_schema"].get("schema"):
optional_params["format"] = value["json_schema"]["schema"]
if param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
### FUNCTION CALLING LOGIC ###
# Ollama 0.4+ supports native tool calling - pass tools directly
# and let Ollama handle model capability detection
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if param == "tools":
optional_params["tools"] = value
if param == "functions":
optional_params["tools"] = value
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
non_default_params.pop("functions", None) # causes ollama requests to hang
return optional_params
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
if api_key is not None and "Authorization" not in headers:
headers["Authorization"] = f"Bearer {api_key}"
return headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
OPTIONAL
Get the complete url for the request
Some providers need `model` in `api_base`
"""
if api_base is None:
api_base = "http://localhost:11434"
if api_base.endswith("/api/chat"):
url = api_base
else:
url = f"{api_base}/api/chat"
return url
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
stream = optional_params.pop("stream", False)
format = optional_params.pop("format", None)
keep_alive = optional_params.pop("keep_alive", None)
think = optional_params.pop("think", None)
function_name = optional_params.pop("function_name", None)
litellm_params["function_name"] = function_name
tools = optional_params.pop("tools", None)
new_messages = []
for m in messages:
if isinstance(
m, BaseModel
): # avoid message serialization issues - https://github.com/BerriAI/litellm/issues/5319
m = m.model_dump(exclude_none=True)
tool_calls = m.get("tool_calls")
if tool_calls is not None and isinstance(tool_calls, list):
new_tools: List[OllamaToolCall] = []
for tool in tool_calls:
typed_tool = ChatCompletionAssistantToolCall(**tool) # type: ignore
if typed_tool["type"] == "function":
arguments = {}
if "arguments" in typed_tool["function"]:
arguments = json.loads(typed_tool["function"]["arguments"])
ollama_tool_call = OllamaToolCall(
function=OllamaToolCallFunction(
name=typed_tool["function"].get("name") or "",
arguments=arguments,
)
)
new_tools.append(ollama_tool_call)
cast(dict, m)["tool_calls"] = new_tools
reasoning_content, parsed_content = _extract_reasoning_content(
cast(dict, m)
)
content_str = convert_content_list_to_str(cast(AllMessageValues, m))
images = extract_images_from_message(cast(AllMessageValues, m))
ollama_message = OllamaChatCompletionMessage(
role=cast(str, m.get("role")),
)
if reasoning_content is not None:
ollama_message["thinking"] = reasoning_content
if content_str is not None:
ollama_message["content"] = content_str
if images is not None:
ollama_message["images"] = images
new_messages.append(ollama_message)
# Load Config
config = self.get_config()
for k, v in config.items():
if k not in optional_params:
optional_params[k] = v
data = {
"model": model,
"messages": new_messages,
"options": optional_params,
"stream": stream,
}
if format is not None:
data["format"] = format
if tools is not None:
data["tools"] = tools
if keep_alive is not None:
data["keep_alive"] = keep_alive
if think is not None:
data["think"] = think
return data
def transform_response(
self,
model: str,
raw_response: Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: str,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
## LOGGING
logging_obj.post_call(
input=messages,
api_key="",
original_response=raw_response.text,
additional_args={
"headers": None,
"api_base": litellm_params.get("api_base"),
},
)
response_json = raw_response.json()
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
response_json_message = response_json.get("message")
if response_json_message is not None:
if "thinking" in response_json_message:
# remap 'thinking' to 'reasoning_content'
response_json_message["reasoning_content"] = response_json_message[
"thinking"
]
del response_json_message["thinking"]
elif response_json_message.get("content") is not None:
# parse reasoning content from content
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
reasoning_content, content = _parse_content_for_reasoning(
response_json_message["content"]
)
response_json_message["reasoning_content"] = reasoning_content
response_json_message["content"] = content
if (
request_data.get("format", "") == "json"
and litellm_params.get("function_name") is not None
):
function_call = json.loads(response_json_message["content"])
message = litellm.Message(
content=None,
tool_calls=[
{
"id": f"call_{str(uuid.uuid4())}",
"function": {
"name": function_call.get(
"name", litellm_params.get("function_name")
),
"arguments": json.dumps(
function_call.get("arguments", function_call)
),
},
"type": "function",
}
],
reasoning_content=response_json_message.get("reasoning_content"),
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "tool_calls"
else:
_message = litellm.Message(**response_json_message)
model_response.choices[0].message = _message # type: ignore
# Set finish_reason to "tool_calls" when tool_calls are present
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if _message.tool_calls:
model_response.choices[0].finish_reason = "tool_calls"
model_response.created = int(time.time())
model_response.model = "ollama_chat/" + model
prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore
completion_tokens = response_json.get(
"eval_count",
litellm.token_counter(text=response_json["message"]["content"]),
)
setattr(
model_response,
"usage",
litellm.Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
),
)
return model_response
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return OllamaError(
status_code=status_code, message=error_message, headers=headers
)
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
):
return OllamaChatCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
started_reasoning_content: bool = False
finished_reasoning_content: bool = False
def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
if isinstance(function_args, dict):
return True
try:
json.loads(function_args)
return True
except Exception:
return False
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
try:
"""
Expected chunk format:
{
"model": "llama3.1",
"created_at": "2025-05-24T02:12:05.859654Z",
"message": {
"role": "assistant",
"content": "",
"tool_calls": [{
"function": {
"name": "get_latest_album_ratings",
"arguments": {
"artist_name": "Taylor Swift"
}
}
}]
},
"done_reason": "stop",
"done": true,
...
}
Need to:
- convert 'message' to 'delta'
- return finish_reason when done is true
- return usage when done is true
"""
from litellm.types.utils import Delta, StreamingChoices
# process tool calls - if complete function arg - add id to tool call
tool_calls = chunk["message"].get("tool_calls")
if tool_calls is not None:
for tool_call in tool_calls:
function_args = tool_call.get("function").get("arguments")
if function_args is not None and len(function_args) > 0:
is_function_call_complete = self._is_function_call_complete(
function_args
)
if is_function_call_complete:
tool_call["id"] = str(uuid.uuid4())
# PROCESS REASONING CONTENT
reasoning_content: Optional[str] = None
content: Optional[str] = None
if chunk["message"].get("thinking") is not None:
reasoning_content = chunk["message"].get("thinking")
self.started_reasoning_content = True
elif chunk["message"].get("content") is not None:
if (
self.started_reasoning_content
and not self.finished_reasoning_content
):
self.finished_reasoning_content = True
message_content = chunk["message"].get("content")
if "<think>" in message_content:
message_content = message_content.replace("<think>", "")
self.started_reasoning_content = True
if "</think>" in message_content and self.started_reasoning_content:
message_content = message_content.replace("</think>", "")
self.finished_reasoning_content = True
if (
self.started_reasoning_content
and not self.finished_reasoning_content
):
reasoning_content = message_content
else:
content = message_content
delta = Delta(
content=content,
reasoning_content=reasoning_content,
tool_calls=tool_calls,
)
if chunk["done"] is True:
finish_reason = chunk.get("done_reason", "stop")
# Override finish_reason when tool_calls are present
# Fixes: https://github.com/BerriAI/litellm/issues/18922
if tool_calls is not None:
finish_reason = "tool_calls"
choices = [
StreamingChoices(
delta=delta,
finish_reason=finish_reason,
)
]
else:
choices = [
StreamingChoices(
delta=delta,
)
]
usage = ChatCompletionUsageBlock(
prompt_tokens=chunk.get("prompt_eval_count", 0),
completion_tokens=chunk.get("eval_count", 0),
total_tokens=chunk.get("prompt_eval_count", 0)
+ chunk.get("eval_count", 0),
)
return ModelResponseStream(
id=str(uuid.uuid4()),
object="chat.completion.chunk",
created=int(time.time()), # ollama created_at is in UTC
usage=usage,
model=chunk["model"],
choices=choices,
)
except KeyError as e:
raise OllamaError(
message=f"KeyError: {e}, Got unexpected response from Ollama: {chunk}",
status_code=400,
headers={"Content-Type": "application/json"},
)
except Exception as e:
raise e

149
llm-gateway-competitors/litellm-wheel-src/litellm/llms/ollama/common_utils.py Normal file
View File

@@ -0,0 +1,149 @@
from typing import List, Optional, Union
import httpx
from litellm import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
class OllamaError(BaseLLMException):
def __init__(
self, status_code: int, message: str, headers: Union[dict, httpx.Headers]
):
super().__init__(status_code=status_code, message=message, headers=headers)
def _convert_image(image):
"""
Convert image to base64 encoded image if not already in base64 format
If image is already in base64 format AND is a jpeg/png, return it
If image is not JPEG/PNG, convert it to JPEG base64 format
"""
import base64
import io
try:
from PIL import Image
except Exception:
raise Exception(
"ollama image conversion failed please run `pip install Pillow`"
)
orig = image
if image.startswith("data:"):
image = image.split(",")[-1]
try:
image_data = Image.open(io.BytesIO(base64.b64decode(image)))
if image_data.format in ["JPEG", "PNG"]:
return image
except Exception:
return orig
jpeg_image = io.BytesIO()
image_data.convert("RGB").save(jpeg_image, "JPEG")
jpeg_image.seek(0)
return base64.b64encode(jpeg_image.getvalue()).decode("utf-8")
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
class OllamaModelInfo(BaseLLMModelInfo):
"""
Dynamic model listing for Ollama server.
Fetches /api/models and /api/tags, then for each tag also /api/models?tag=...
Returns the union of all model names.
"""
@staticmethod
def get_api_key(api_key=None) -> Optional[str]:
"""Get API key from environment variables or litellm configuration"""
import os
import litellm
from litellm.secret_managers.main import get_secret_str
return (
os.environ.get("OLLAMA_API_KEY")
or litellm.api_key
or litellm.openai_key
or get_secret_str("OLLAMA_API_KEY")
)
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> str:
from litellm.secret_managers.main import get_secret_str
# env var OLLAMA_API_BASE or default
return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]:
"""
List all models available on the Ollama server via /api/tags endpoint.
"""
base = self.get_api_base(api_base)
api_key = self.get_api_key()
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
names: set[str] = set()
try:
resp = httpx.get(f"{base}/api/tags", headers=headers)
resp.raise_for_status()
data = resp.json()
# Expecting a dict with a 'models' list
models_list = []
if (
isinstance(data, dict)
and "models" in data
and isinstance(data["models"], list)
):
models_list = data["models"]
elif isinstance(data, list):
models_list = data
# Extract model names
for entry in models_list:
if not isinstance(entry, dict):
continue
nm = entry.get("name") or entry.get("model")
if isinstance(nm, str):
names.add(nm)
except Exception as e:
verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}")
# If tags endpoint fails, fall back to static list
try:
from litellm import models_by_provider
static = models_by_provider.get("ollama", []) or []
return [f"ollama/{m}" for m in static]
except Exception as e1:
verbose_logger.warning(
f"Error retrieving static ollama models as fallback: {e1}"
)
return []
# assemble full model names
result = sorted(names)
return result
def validate_environment(
self,
headers: dict,
model: str,
messages: list,
optional_params: dict,
litellm_params: dict,
api_key=None,
api_base=None,
) -> dict:
"""
No-op environment validation for Ollama.
"""
return {}
@staticmethod
def get_base_model(model: str) -> str:
"""
Return the base model name for Ollama (no-op).
"""
return model

123
llm-gateway-competitors/litellm-wheel-src/litellm/llms/ollama/completion/handler.py Normal file
View File

@@ -0,0 +1,123 @@
"""
Ollama /chat/completion calls handled in llm_http_handler.py
[TODO]: migrate embeddings to a base handler as well.
"""
from typing import Any, Dict, List
import litellm
from litellm.types.utils import EmbeddingResponse
def _prepare_ollama_embedding_payload(
model: str, prompts: List[str], optional_params: Dict[str, Any]
) -> Dict[str, Any]:
data: Dict[str, Any] = {"model": model, "input": prompts}
special_optional_params = ["truncate", "options", "keep_alive", "dimensions"]
for k, v in optional_params.items():
if k in special_optional_params:
data[k] = v
else:
data.setdefault("options", {})
if isinstance(data["options"], dict):
data["options"].update({k: v})
return data
def _process_ollama_embedding_response(
response_json: dict,
prompts: List[str],
model: str,
model_response: EmbeddingResponse,
logging_obj: Any,
encoding: Any,
) -> EmbeddingResponse:
output_data = []
embeddings: List[List[float]] = response_json["embeddings"]
for idx, emb in enumerate(embeddings):
output_data.append({"object": "embedding", "index": idx, "embedding": emb})
input_tokens = response_json.get("prompt_eval_count", None)
if input_tokens is None:
if encoding is not None:
input_tokens = len(encoding.encode("".join(prompts)))
if logging_obj:
logging_obj.debug(
"Ollama response missing prompt_eval_count; estimated with encoding."
)
else:
input_tokens = 0
if logging_obj:
logging_obj.warning(
"Missing prompt_eval_count and no encoding provided; defaulted to 0."
)
model_response.object = "list"
model_response.data = output_data
model_response.model = "ollama/" + model
model_response.usage = litellm.Usage(
prompt_tokens=input_tokens,
completion_tokens=0,
total_tokens=input_tokens,
prompt_tokens_details=None,
completion_tokens_details=None,
)
return model_response
async def ollama_aembeddings(
api_base: str,
model: str,
prompts: List[str],
model_response: EmbeddingResponse,
optional_params: dict,
logging_obj: Any,
encoding: Any,
):
if not api_base.endswith("/api/embed"):
api_base += "/api/embed"
data = _prepare_ollama_embedding_payload(model, prompts, optional_params)
response = await litellm.module_level_aclient.post(url=api_base, json=data)
response_json = response.json()
return _process_ollama_embedding_response(
response_json=response_json,
prompts=prompts,
model=model,
model_response=model_response,
logging_obj=logging_obj,
encoding=encoding,
)
def ollama_embeddings(
api_base: str,
model: str,
prompts: List[str],
optional_params: dict,
model_response: EmbeddingResponse,
logging_obj: Any,
encoding: Any = None,
):
if not api_base.endswith("/api/embed"):
api_base += "/api/embed"
data = _prepare_ollama_embedding_payload(model, prompts, optional_params)
response = litellm.module_level_client.post(url=api_base, json=data)
response_json = response.json()
return _process_ollama_embedding_response(
response_json=response_json,
prompts=prompts,
model=model,
model_response=model_response,
logging_obj=logging_obj,
encoding=encoding,
)

612
llm-gateway-competitors/litellm-wheel-src/litellm/llms/ollama/completion/transformation.py Normal file
View File

@@ -0,0 +1,612 @@
import json
import time
from litellm._uuid import uuid
from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union
from httpx._models import Headers, Response
import litellm
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_ollama_image,
custom_prompt,
ollama_pt,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
from litellm.types.utils import (
Delta,
GenericStreamingChunk,
ModelInfoBase,
ModelResponse,
ModelResponseStream,
ProviderField,
StreamingChoices,
)
from ..common_utils import OllamaError, _convert_image
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class OllamaConfig(BaseConfig):
"""
Reference: https://github.com/ollama/ollama/blob/main/docs/api.md#parameters
The class `OllamaConfig` provides the configuration for the Ollama's API interface. Below are the parameters:
- `mirostat` (int): Enable Mirostat sampling for controlling perplexity. Default is 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0. Example usage: mirostat 0
- `mirostat_eta` (float): Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. Default: 0.1. Example usage: mirostat_eta 0.1
- `mirostat_tau` (float): Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. Default: 5.0. Example usage: mirostat_tau 5.0
- `num_ctx` (int): Sets the size of the context window used to generate the next token. Default: 2048. Example usage: num_ctx 4096
- `num_gqa` (int): The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b. Example usage: num_gqa 1
- `num_gpu` (int): The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. Example usage: num_gpu 0
- `num_thread` (int): Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Example usage: num_thread 8
- `repeat_last_n` (int): Sets how far back for the model to look back to prevent repetition. Default: 64, 0 = disabled, -1 = num_ctx. Example usage: repeat_last_n 64
- `repeat_penalty` (float): Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. Default: 1.1. Example usage: repeat_penalty 1.1
- `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7
- `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42
- `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:"
- `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1
- `num_predict` (int): Maximum number of tokens to predict when generating text. Default: 128, -1 = infinite generation, -2 = fill context. Example usage: num_predict 42
- `top_k` (int): Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. Default: 40. Example usage: top_k 40
- `top_p` (float): Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. Default: 0.9. Example usage: top_p 0.9
- `system` (string): system prompt for model (overrides what is defined in the Modelfile)
- `template` (string): the full prompt or prompt template (overrides what is defined in the Modelfile)
"""
mirostat: Optional[int] = None
mirostat_eta: Optional[float] = None
mirostat_tau: Optional[float] = None
num_ctx: Optional[int] = None
num_gqa: Optional[int] = None
num_gpu: Optional[int] = None
num_thread: Optional[int] = None
repeat_last_n: Optional[int] = None
repeat_penalty: Optional[float] = None
temperature: Optional[float] = None
seed: Optional[int] = None
stop: Optional[
list
] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
tfs_z: Optional[float] = None
num_predict: Optional[int] = None
top_k: Optional[int] = None
top_p: Optional[float] = None
system: Optional[str] = None
template: Optional[str] = None
def __init__(
self,
mirostat: Optional[int] = None,
mirostat_eta: Optional[float] = None,
mirostat_tau: Optional[float] = None,
num_ctx: Optional[int] = None,
num_gqa: Optional[int] = None,
num_gpu: Optional[int] = None,
num_thread: Optional[int] = None,
repeat_last_n: Optional[int] = None,
repeat_penalty: Optional[float] = None,
temperature: Optional[float] = None,
seed: Optional[int] = None,
stop: Optional[list] = None,
tfs_z: Optional[float] = None,
num_predict: Optional[int] = None,
top_k: Optional[int] = None,
top_p: Optional[float] = None,
system: Optional[str] = None,
template: Optional[str] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
setattr(self.__class__, key, value)
@classmethod
def get_config(cls):
return super().get_config()
def get_required_params(self) -> List[ProviderField]:
"""For a given provider, return it's required fields with a description"""
return [
ProviderField(
field_name="base_url",
field_type="string",
field_description="Your Ollama API Base",
field_value="http://10.10.11.249:11434",
)
]
def get_supported_openai_params(self, model: str):
return [
"max_tokens",
"stream",
"top_p",
"temperature",
"seed",
"frequency_penalty",
"stop",
"response_format",
"max_completion_tokens",
"reasoning_effort",
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
for param, value in non_default_params.items():
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["num_predict"] = value
elif param == "stream":
optional_params["stream"] = value
elif param == "temperature":
optional_params["temperature"] = value
elif param == "seed":
optional_params["seed"] = value
elif param == "top_p":
optional_params["top_p"] = value
elif param == "frequency_penalty":
optional_params["frequency_penalty"] = value
elif param == "stop":
optional_params["stop"] = value
elif param == "reasoning_effort" and value is not None:
if model.startswith("gpt-oss"):
optional_params["think"] = value
else:
optional_params["think"] = value in {"low", "medium", "high"}
elif param == "response_format" and isinstance(value, dict):
if value["type"] == "json_object":
optional_params["format"] = "json"
elif value["type"] == "json_schema":
optional_params["format"] = value["json_schema"]["schema"]
return optional_params
def _supports_function_calling(self, ollama_model_info: dict) -> bool:
"""
Check if the 'template' field in the ollama_model_info contains a 'tools' or 'function' key.
"""
_template: str = str(ollama_model_info.get("template", "") or "")
return "tools" in _template.lower()
def _get_max_tokens(self, ollama_model_info: dict) -> Optional[int]:
_model_info: dict = ollama_model_info.get("model_info", {})
for k, v in _model_info.items():
if "context_length" in k:
return v
return None
@staticmethod
def get_api_key() -> Optional[str]:
"""Get API key from environment variables or litellm configuration"""
import os
import litellm
from litellm.secret_managers.main import get_secret_str
return (
os.environ.get("OLLAMA_API_KEY")
or litellm.api_key
or litellm.openai_key
or get_secret_str("OLLAMA_API_KEY")
)
def get_model_info(
self, model: str, api_base: Optional[str] = None
) -> ModelInfoBase:
"""
curl http://localhost:11434/api/show -d '{
"name": "mistral"
}'
"""
if model.startswith("ollama/") or model.startswith("ollama_chat/"):
model = model.split("/", 1)[1]
api_base = (
api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
)
api_key = self.get_api_key()
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
try:
response = litellm.module_level_client.post(
url=f"{api_base}/api/show",
json={"name": model},
headers=headers,
)
except Exception as e:
verbose_logger.debug(
"OllamaError: Could not get model info for %s from %s. Error: %s",
model,
api_base,
e,
)
return ModelInfoBase(
key=model,
litellm_provider="ollama",
mode="chat",
input_cost_per_token=0.0,
output_cost_per_token=0.0,
max_tokens=None,
max_input_tokens=None,
max_output_tokens=None,
)
model_info = response.json()
_max_tokens: Optional[int] = self._get_max_tokens(model_info)
return ModelInfoBase(
key=model,
litellm_provider="ollama",
mode="chat",
supports_function_calling=self._supports_function_calling(model_info),
input_cost_per_token=0.0,
output_cost_per_token=0.0,
max_tokens=_max_tokens,
max_input_tokens=_max_tokens,
max_output_tokens=_max_tokens,
)
def get_error_class(
self, error_message: str, status_code: int, headers: Union[dict, Headers]
) -> BaseLLMException:
return OllamaError(
status_code=status_code, message=error_message, headers=headers
)
def transform_response(
self,
model: str,
raw_response: Response,
model_response: ModelResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: str,
api_key: Optional[str] = None,
json_mode: Optional[bool] = None,
) -> ModelResponse:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_parse_content_for_reasoning,
)
response_json = raw_response.json()
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
if request_data.get("format", "") == "json":
# Check if response field exists and is not empty before parsing JSON
response_text = response_json.get("response", "")
if not response_text or not response_text.strip():
# Handle empty response gracefully - set empty content
message = litellm.Message(content="")
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
else:
try:
response_content = json.loads(response_text)
# Check if this is a function call format with name/arguments structure
if (
isinstance(response_content, dict)
and "name" in response_content
and "arguments" in response_content
):
# Handle as function call (original behavior)
function_call = response_content
message = litellm.Message(
content=None,
tool_calls=[
{
"id": f"call_{str(uuid.uuid4())}",
"function": {
"name": function_call["name"],
"arguments": json.dumps(
function_call["arguments"]
),
},
"type": "function",
}
],
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "tool_calls"
else:
# Handle as regular JSON (new behavior)
message = litellm.Message(
content=json.dumps(response_content),
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
except json.JSONDecodeError:
# If JSON parsing fails, treat as regular text response
## output parse reasoning content from response_text
reasoning_content: Optional[str] = None
content: Optional[str] = None
if response_text is not None:
reasoning_content, content = _parse_content_for_reasoning(
response_text
)
message = litellm.Message(
content=content, reasoning_content=reasoning_content
)
model_response.choices[0].message = message # type: ignore
model_response.choices[0].finish_reason = "stop"
else:
response_text = response_json.get("response", "")
content = None
reasoning_content = None
if response_text is not None and isinstance(response_text, str):
reasoning_content, content = _parse_content_for_reasoning(response_text)
else:
content = response_text # type: ignore
model_response.choices[0].message.content = content # type: ignore
model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore
model_response.created = int(time.time())
model_response.model = "ollama/" + model
_prompt = request_data.get("prompt", "")
prompt_tokens = response_json.get(
"prompt_eval_count", len(encoding.encode(_prompt, disallowed_special=())) # type: ignore
)
completion_tokens = response_json.get(
"eval_count", len(response_json.get("message", dict()).get("content", ""))
)
setattr(
model_response,
"usage",
litellm.Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
),
)
return model_response
def transform_request(
self,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> dict:
custom_prompt_dict = (
litellm_params.get("custom_prompt_dict") or litellm.custom_prompt_dict
)
text_completion_request = litellm_params.get("text_completion")
if model in custom_prompt_dict:
# check if the model has a registered custom prompt
model_prompt_details = custom_prompt_dict[model]
ollama_prompt = custom_prompt(
role_dict=model_prompt_details["roles"],
initial_prompt_value=model_prompt_details["initial_prompt_value"],
final_prompt_value=model_prompt_details["final_prompt_value"],
messages=messages,
)
elif text_completion_request: # handle `/completions` requests
ollama_prompt = get_str_from_messages(messages=messages)
else: # handle `/chat/completions` requests
modified_prompt = ollama_pt(model=model, messages=messages)
if isinstance(modified_prompt, dict):
ollama_prompt, images = (
modified_prompt["prompt"],
modified_prompt["images"],
)
optional_params["images"] = images
else:
ollama_prompt = modified_prompt
stream = optional_params.pop("stream", False)
format = optional_params.pop("format", None)
images = optional_params.pop("images", None)
think = optional_params.pop("think", None)
data = {
"model": model,
"prompt": ollama_prompt,
"options": optional_params,
"stream": stream,
}
if format is not None:
data["format"] = format
if images is not None:
data["images"] = [
_convert_image(convert_to_ollama_image(image)) for image in images
]
if think is not None:
data["think"] = think
return data
def validate_environment(
self,
headers: dict,
model: str,
messages: List[AllMessageValues],
optional_params: dict,
litellm_params: dict,
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
return headers
def get_complete_url(
self,
api_base: Optional[str],
api_key: Optional[str],
model: str,
optional_params: dict,
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
"""
OPTIONAL
Get the complete url for the request
Some providers need `model` in `api_base`
"""
if api_base is None:
api_base = "http://localhost:11434"
if api_base.endswith("/api/generate"):
url = api_base
else:
url = f"{api_base}/api/generate"
return url
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
):
return OllamaTextCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
def __init__(
self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
):
super().__init__(streaming_response, sync_stream, json_mode)
self.started_reasoning_content: bool = False
self.finished_reasoning_content: bool = False
def _handle_string_chunk(
self, str_line: str
) -> Union[GenericStreamingChunk, ModelResponseStream]:
return self.chunk_parser(json.loads(str_line))
def chunk_parser(
self, chunk: dict
) -> Union[GenericStreamingChunk, ModelResponseStream]:
try:
if "error" in chunk:
raise Exception(f"Ollama Error - {chunk}")
text = ""
is_finished = False
finish_reason = None
if chunk["done"] is True:
text = ""
is_finished = True
finish_reason = "stop"
prompt_eval_count: Optional[int] = chunk.get("prompt_eval_count", None)
eval_count: Optional[int] = chunk.get("eval_count", None)
usage: Optional[ChatCompletionUsageBlock] = None
if prompt_eval_count is not None and eval_count is not None:
usage = ChatCompletionUsageBlock(
prompt_tokens=prompt_eval_count,
completion_tokens=eval_count,
total_tokens=prompt_eval_count + eval_count,
)
return GenericStreamingChunk(
text=text,
is_finished=is_finished,
finish_reason=finish_reason,
usage=usage,
)
elif chunk["response"]:
text = chunk["response"]
reasoning_content: Optional[str] = None
content: Optional[str] = None
if text is not None:
if "<think>" in text:
text = text.replace("<think>", "")
self.started_reasoning_content = True
elif "</think>" in text:
text = text.replace("</think>", "")
self.finished_reasoning_content = True
if (
self.started_reasoning_content
and not self.finished_reasoning_content
):
reasoning_content = text
else:
content = text
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(
reasoning_content=reasoning_content, content=content
),
)
],
finish_reason=finish_reason,
usage=None,
)
# return GenericStreamingChunk(
# text=text,
# is_finished=is_finished,
# finish_reason="stop",
# usage=None,
# )
elif "thinking" in chunk and not chunk["response"]:
# Return reasoning content as ModelResponseStream so UIs can render it
thinking_content = chunk.get("thinking") or ""
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(reasoning_content=thinking_content),
)
]
)
else:
# In this case, 'thinking' is not present in the chunk, chunk["done"] is false,
# and chunk["response"] is falsy (None or empty string),
# but Ollama is just starting to stream, so it should be processed as a normal dict
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(reasoning_content=""),
)
]
)
# raise Exception(f"Unable to parse ollama chunk - {chunk}")
except Exception as e:
verbose_proxy_logger.error(f"Unable to parse ollama chunk - {chunk}")
raise e