uPjrVUdZddlmZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZddlmZmZddl m!Z!ddl m"Z"ddl m#Z#dd l$m%Z%m&Z&m'Z'dd l(m)Z)ddl*Z*ddl+Z+ddl,Z,ddl-Z-dd l+m.Z.dd l/m0Z0dd l1m2Z2m3Z3ddl4m5Z5ddl6m7Z7ddl-m8Z8ddl9m:Z:ddl;Z;ddlm?Z?ddl@mAZAmBZBmCZCddlDmEZEddlFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWddlXmYZYdaZda[da\da]dZ^edd{dZ_d|dZ`dZadZbdZcdaddZeddlfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZndd lompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZeqDcic]}|j$|c}Zd!ed"< e!j*d#j-d$j/d%d&'5Zej2eZdddej>eZddlZ dd(lmZmZmZmZdd)lmZmZmZmZmZmZmZmZdd*l;mZdd+lmZmZerdd,lmZmZdd-lmZdd.lmZmZmZdd/lmZmZmZdd0lmZdd1lmZmZdd2lmZdd3lmZmZdd4lmZmZdd5lmZdd6lmZdd7lmZmZmZmZmZdd8lmZdd9lmZdd:lmZdd;lmZmZddlmZdd?lmZdd@lmZddAlmZddBlmZddClmZddDlmZddElmZddFlmZddGlmZddHlmZddIlmZddJlmZddKlmZddLlmZddMlmZddNlmZddOl m Z ddPl m Z ddQl mZddRlmZddSlmZddTlmZddUlmZddVlmZddWlmZddXlmZddYlmZddZlm Z dd[l!m"Z"dd\l#m$Z$dd]l%m&Z&dd^l'm(Z(m)Z)dd_l*m+Z+dd`l,m-Z-ddal.m/Z/m0Z0ddbl1m2Z2m3Z3ddclfm4Z4m5Z5m6Z6dddl7m8Z8ddel9m:Z:ddfl;m<Z<ddgl=m>Z>ddhl?m@Z@ddilAmBZBddjlCmDZDddklEmFZFdldmlGmHZHmIZIdldnlJmKZKmLZLmMZMmNZNmOZOmPZPdldolQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`erddpl;maZadZbdqedr<dZcdqeds<daddZedqedt<dZfdqedu<dZgdqedv<dZhdqedw<dZidqedx<dZjdqedy<dZkdqedz<dZldqed{<dZmdqed|<dZndqed}<dZodqed~<dZpdqed<dZqdqed<dZrdqed<dZsdqed<dZtdqed<dZudqed<dZvdqed<dZwdqed<dZxdqed<dZydqed<gazded<da{iZ|ded<iZ}ded<dZ~dqed<dZdqed< d} d~dZdZddZ ddZ ddZddZddZddZ ddZddZddZddZddZddZddZddZdd ddZ ddZddZddZddZddZ ddZ ddZ ddZdZ d ddZeeqj8eqj:eqj8j$eqj:j$hZdqed< ddZdddZeeIddZddZddZdddZ d ddZddZdddZddZ d ddZddZddZdddZdddZddZdddZddd„ZdddÄZdddĄZddńZddƄZddDŽZddȄZdddɄZdddʄZddd˄Zddd̄Zddd̈́Zddd΄ZdddτZdddЄZdddфZddd҄ZdddӄZddԄZdՄZd֐Zdqed<dؐZdqed<ddڄZƐddۄZiZded<Gdބd߫ZɐddZʐddZddd ddZ̐ddZ͐ddZ d ddZϐddZ d ddZ d ddZdZdZԐddZՐddZ֐ddZאddZGddZ d ddZڐddZېddZ d ddZ d ddZސddZehdZdqed<ddZddZ ddZddZdddZ d ddZdʐdZdːdZd̐dZd̐dZd͐dZdΐdZdϐdZdАdZdѐdZdҐd Zd ZdӐd ZdԐd ZdԐd ZdՐdZd֐dZdaded<daded<dאdZddZdؐdZ dِdZ dِdZdڐdZdېdZdܐdZddlmZmZGddeZ dݐdZdސdZdߐd ZeeI d dd!Zdd"Z dd#Z ejd$Z dqed%< d dd&Z d dd'ZeeI d dd(Z d dd)Zej"e_ej$e_dd*Zdd+Zd,Z d dd-Zdd.Zd/Z d dd0Zd1Zd2Zdd3Zdd4Z d dd5Z d dd6Ziddif dd7Z Gd8d9Z!ddd:Z" d dd;Z#dd<Z$d=Z%d>Z&d?Z'd@Z(dAZ)dddBZ*deLddf ddCZ+ddDl,m-Z-GdEdFe-Z.e.Z/dqedG< d ddHZ0 d ddIZ1 d ddJZ2dKZ3dLZ4ddMZ5GdNdOZ6GdPdQZ7GdRdSeZ8ddTZ9ddUZ:ddVZ;ddWZ< ddXZ= ddYZ>ddZZ?dd[lfm@Z@mAZAmBZBmCZCmDZDmEZEdd\ZFdd]ZGdd^ZHdd_ZI dd`ZJddaZKddbZL d dcZMdd ddZNedld deZO d dfZPGdgdhZQ d ddiZRddjZSddkZT d ddlZUddmZVddnZWddoZXddpZYddqZZ ddrZ[ddsZ\ ddtZ]dduZ^ddvZ_ddwZ`ddxZa ddyZbddzZcycc}w#1swY xYw#eeef$rAe!j<d#d$5Zej2eZdddn #1swYnxYwY wxYw(zGUtility helpers for LiteLLM core request handling and provider support.) annotationsN) dataclassfield) lru_cachewraps) resources) iscoroutine)StringIO)abspathdirnamejoin)MappingProxyType)Proxy)get_environment_proxies)_parsing _pydantic)ResponseFormat) BaseModel)Encoding) Tokenizer)is_internal_call)_get_default_encoding_get_modified_max_tokens_get_token_counter_new)uuid)$DEFAULT_CHAT_COMPLETION_PARAM_VALUESDEFAULT_EMBEDDING_PARAM_VALUESDEFAULT_MAX_LRU_CACHE_SIZE(DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT/DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGETDEFAULT_TRIM_RATIOFUNCTION_DEFINITION_TOKEN_COUNTHF_CONFIG_FETCH_TIMEOUT_SECONDSINITIAL_RETRY_DELAYJITTERMAX_RETRY_DELAYMAX_TOKEN_TRIMMING_ATTEMPTS)MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDENON_INFERENCE_CALL_TYPESOPENAI_EMBEDDING_PARAMS,PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFOTOOL_CHOICE_OBJECT_TOKEN_COUNT) match_capability_generalizationsc*tddlm}|atS)z Get cached CustomLogger class. Lazy imports on first call to avoid loading custom_logger at import time. Subsequent calls use cached class for better performance. r CustomLogger) _CustomLogger"litellm.integrations.custom_loggerr0r/s R/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/utils.py_get_cached_custom_loggerr4]sC$ maxsizectj|jj}d|vxs t d|j DS)aP Whether cls's async_post_call_failure_deployment_hook override accepts a fallback_depth keyword, cached per class so a signature the base class added after a subscriber's override was written (e.g. the PR's own earlier 3-arg proof-of-fix example) doesn't raise TypeError - swallowed at debug level - on every call. fallback_depthc3hK|]*}|jtjju,ywN)kindinspect Parameter VAR_KEYWORD).0ps r3 z:_accepts_fallback_depth_kwarg_for_class..ts&,nYZQVVw7H7H7T7T-T,ns02)r= signature'async_post_call_failure_deployment_hook parametersanyvalues)clsparamss r3'_accepts_fallback_depth_kwarg_for_classrJksE%%c&Q&QR]]F v % n,n^d^k^k^m,n)nnr5cV t|}|j|}|jj|j|j|_|j |_|j |_|j|_|j|_|S#t$r|cYSwxYw)a A same-class copy of exception that skips __init__ (many litellm exceptions require constructor args beyond what .args carries, so copy.copy's pickle-based reconstruction fails on them). Handed to failure-hook callbacks instead of the live object so a callback setting e.g. exception.status_code cannot change the status code the real caller actually receives. Falls back to the live object if snapshotting fails for a type this doesn't anticipate, since the real exception must still reach the callback. ) type__new____dict__updateargs __traceback__ __cause__ __context____suppress_context__ Exception) exceptionrHsnapshots r3_snapshot_exception_for_hookrXws )_++c*  !3!34! !*!8!8&00(44)2(F(F% sBB B('B(c*tddlm}|atS)z Get cached CustomGuardrail class. Lazy imports on first call to avoid loading custom_guardrail at import time. Subsequent calls use cached class for better performance. rCustomGuardrail)_CustomGuardrail%litellm.integrations.custom_guardrailr[rZs r3_get_cached_custom_guardrailr^sI* r5c*tddlm}|atS)z Get cached CachingHandlerResponse class. Lazy imports on first call to avoid loading caching_handler at import time. Subsequent calls use cached class for better performance. rCachingHandlerResponse)_CachingHandlerResponselitellm.caching.caching_handlerrar`s r3$_get_cached_caching_handler_responserds&J"8 ""r5c*tddlm}|atS)z Get cached LLMCachingHandler class. Lazy imports on first call to avoid loading caching_handler at import time. Subsequent calls use cached class for better performance. rLLMCachingHandler)_LLMCachingHandlerrcrgrfs r3_get_cached_llm_caching_handlerris!E. r5cbt$ddl}|jjjatS)z Get cached audio_utils.utils module. Lazy imports on first call to avoid loading audio_utils.utils at import time. Subsequent calls use cached module for better performance. Nr)_audio_utils_module,litellm.litellm_core_utils.audio_utils.utilslitellm_core_utils audio_utilsutils)litellms r3_get_cached_audio_utilsrqs+";%88DDJJ r5)AllMessageValuesAllPromptValuesChatCompletionAssistantToolCall"ChatCompletionNamedToolChoiceParamChatCompletionToolParam$ChatCompletionToolParamFunctionChunkOpenAITextCompletionUserMessageOpenAIWebSearchOptions)"OPENAI_RESPONSE_HEADERS CallTypesChatCompletionDeltaToolCallChatCompletionMessageToolCallChoices CostPerTokenCredentialItemCustomHuggingfaceTokenizerDelta EmbeddingEmbeddingResponse FileTypesFunction ImageResponse LlmProvidersLlmProvidersSetLLMResponseTypesMessage ModelInfo ModelInfoBase ModelResponseModelResponseStream ProviderFieldProviderSpecificModelInfoRawRequestTypedDictSandboxProvidersSearchProvidersSelectTokenizerResponseStreamingChoices TextChoicesTextCompletionResponseTranscriptionResponseUsageall_litellm_paramsz Final[dict]_CALL_TYPE_ENUM_MAPz%litellm.litellm_core_utils.tokenizerszanthropic_tokenizer.jsonrutf-8)encoding)CallableIterableMappingSequence) TYPE_CHECKINGAnyFinalLiteralOptionalUnioncastget_args)ro)BaseLLMModelInfotype_to_response_format_param)rargr/)get_coroutine_checkerget_litellm_logging_classget_set_callbacks) get_litellm_metadata_from_kwargsmap_finish_reasonprocess_response_headers)CredentialAccessor)delete_nested_valueis_nested_path)exception_type)*_get_base_model_from_litellm_call_metadataget_litellm_params)_is_non_openai_azure_modelget_llm_provider)get_supported_openai_params)_ensure_extra_body_is_safe)LiteLLMResponseObjectHandler#_handle_invalid_parallel_tool_calls convert_to_model_response_objectconvert_to_streaming_response#convert_to_streaming_response_async) get_api_base)get_formatted_prompt)get_response_headers)ResponseMetadataupdate_response_metadata)_parse_content_for_reasoning)LiteLLMLoggingObject(redact_message_input_output_from_logging)RulesCustomStreamWrapper)executor)BaseAnthropicMessagesConfig)BaseAudioTranscriptionConfig)BaseBatchesConfig)BaseContainerConfig)BaseEmbeddingConfig)BaseFilesConfig)$BaseGoogleGenAIGenerateContentConfig)BaseImageEditConfig)BaseImageGenerationConfig)BaseImageVariationConfig) BaseOCRConfig)BasePassthroughConfig)BaseRealtimeHTTPConfig)BaseRealtimeConfig)BaseRerankConfig)BaseSandboxConfig)BaseSearchConfigBaseTextToSpeechConfig)BaseVectorStoreConfig)BaseVectorStoreFilesConfig)BaseVideoConfigBedrockModelInfo)AmazonNovaEmbeddingConfig)AmazonTitanG1Config)&AmazonTitanMultimodalEmbeddingG1Config)AmazonTitanV2Config)BedrockCohereEmbeddingConfig) TwelveLabsMarengoEmbeddingConfig)CohereModelInfo)AsyncHTTPHandler HTTPHandler)MistralOCRConfig)AllowedModelRegion)!get_num_retries_from_retry_policyreset_retry_policy)ANTHROPIC_API_ONLY_HEADERSAnthropicThinkingParam) ChatCompletionDeltaToolCallChunkChatCompletionToolCallChunk#ChatCompletionToolCallFunctionChunk)RerankResponseLiteLLM_Params) BaseConfig)BaseTextCompletionConfig)BaseEvalsAPIConfig)BaseResponsesAPIConfig)BaseSkillsAPIConfig) get_secret)_is_debugging_onverbose_logger)AzureBlobCacheCacheQdrantSemanticCache RedisCacheRedisSemanticCacheS3Cache)APIConnectionErrorAPIErrorAuthenticationErrorBadRequestErrorBudgetExceededErrorContentPolicyViolationErrorContextWindowExceededError NotFoundError OpenAIErrorPermissionDeniedErrorRateLimitErrorServiceUnavailableErrorTimeoutUnprocessableEntityErrorUnsupportedParamsError) MockExceptionrsentry_sdk_instancecapture_exceptionposthog slack_appalerts_channelheliconeLogger athinaLoggerpromptLayerLoggerlangsmithLogger logfireLoggerweightsBiasesLogger customLoggerlangFuseLoggeropenMeterLogger lagoLogger dataDogLoggerprometheusLogger dynamoLoggers3LoggergreenscaleLogger lunaryLogger aispendLoggersupabaseClientlist[str] | None callback_listzFinal[dict[str, str] | None]additional_details local_cachelast_fetched_atlast_fetched_at_keysFc  |dk(rtj|n5|dk(rtj|n|dk(rtj|tj dur|dur t |yyy#t$rYywxYw)NDEBUGINFOERRORTF)rdebuginfoerrorrp set_verboseprintrU)print_statement logger_only log_levels r3 print_verboserDs      1 &    0 ' !   1   $ &;%+? / ",@ &    sA1A66 BBc tjD]q}|dtjvr"tjj|d|dtjvsPtjj|dsy)z2 Add custom_llm provider to provider list providerN)rpcustom_provider_map provider_listappend_custom_providers) custom_llms r3custom_llm_setuprL$ss11E j !)>)> >  ! ! ( (J)? @ j !)B)B B  % % , ,Z -C D Er5cddlm}ddlm}|tjvr&t j d|tjy|t||dd}|rj|dk(rt|durtjj|tjj||tjvrtjj||tjvr tjj|yy|d k(rt|durtjj!|tjj#||tj$vrtj$j||tj&vr tj&j|yyyyy) z< Add a custom logger callback to the specific event r)+_custom_logger_compatible_callbacks_literal)$_init_custom_logger_compatible_classzMCallback %s is not a valid custom logger compatible callback. Known list - %sNinternal_usage_cache llm_routersuccessFfailure)rprN*litellm.litellm_core_utils.litellm_loggingrO)_known_custom_logger_compatible_callbacksrr<r0_custom_logger_class_exists_in_success_callbackslogging_callback_manageradd_litellm_success_callback"add_litellm_async_success_callbacksuccess_callbackremove_async_success_callback0_custom_logger_class_exists_in_failure_callbacksadd_litellm_failure_callback"add_litellm_async_failure_callbackfailure_callback_async_failure_callback)callback logging_eventrNrOcallback_classs r3-_add_custom_logger_callback_to_specific_eventrf0sDwHHH [   = = @ 8(C!N  I %*Z[i*jns*s  , , I I. Y  , , O OP^ _7333((//97::://66x@; i ',\]k,lpu,u  , , I I. Y  , , O OP^ _7333((//97::://66x@; -v 'r5cjtfdtjtjzDS)a Returns True if an instance of the custom logger exists in litellm.success_callback or litellm._async_success_callback e.g if `LangfusePromptManagement` is passed in, it will return True if an instance of `LangfusePromptManagement` exists in litellm.success_callback or litellm._async_success_callback Prevents double adding a custom logger callback to the litellm callbacks Matches on the exact class; an instance of a subclass does not count as registered c3JK|]}t|tuywr;rLr@cbres r3rBzC_custom_logger_class_exists_in_success_callbacks..duBtBx4//u #)rFrpr[r]res`r3rWrWX( u':R:RU\UtUt:tu uur5cjtfdtjtjzDS)a Returns True if an instance of the custom logger exists in litellm.failure_callback or litellm._async_failure_callback e.g if `LangfusePromptManagement` is passed in, it will return True if an instance of `LangfusePromptManagement` exists in litellm.failure_callback or litellm._async_failure_callback Prevents double adding a custom logger callback to the litellm callbacks Matches on the exact class; an instance of a subclass does not count as registered c3JK|]}t|tuywr;rirjs r3rBzC_custom_logger_class_exists_in_failure_callbacks..srlrm)rFrprarbrns`r3r^r^gror5c|jdxsi}|jdxsi}|jdxsg}|S)z4 Get the request guardrails from the kwargs metadatarequester_metadata guardrailsget)kwargsrsrtapplied_guardrailss r3get_request_guardrailsrzvsHjj,2H ( -A B Hb 2 6 6| D J r5ct|}g}t}tjD]W}|t ||s|j  |j dus|j |vs=|j|j Y|S)z\ - Add 'default_on' guardrails to the list - Add request guardrails to the list T)rzr^rp callbacks isinstanceguardrail_name default_onrI)rxrequest_guardrailsryr[rcs r3get_applied_guardrailsrs !7v > "9;O%%G  Jx$I&&2&&$.(2I2IM_2_&--h.E.EF G r5cttjtd}|j d}|rFt j r5|j|}|jD]\}}||vs |||<yyy)zI Updates kwargs with the credentials if credential_name in kwarg rlitellm_credential_nameN) getattrsysmodules__name__rwrpcredential_listget_credential_valuesitems)rxrcredential_namecredential_accessorkeyvalues r3load_credentials_from_listrs~ !( H(=?S T#ZZ(ABO722;c;M;c;cds;t-335 $JC& #s  $3r5chtjj}|r|j||Sr;)rpr|copyextend)dynamic_callbacksreturned_callbackss r3get_dynamic_callbacksrs1!( 1 1 6 6 8!!"34 r5ct|dvr|dvr|duxrd|jvSy|duxrd|jvS)zJ Check if the target model is a Gemini or Vertex AI Gemini model. )gemini vertex_aivertex_ai_betarrNrTlowermodelcustom_llm_providers r3_is_gemini_modelrsQGG "A A$BU[[])B B   :U[[]!::r5c8||vr|j|ddS|S)z7 Remove thought signature from a tool call ID. rrsplit) tool_call_id separators r3!_remove_thought_signature_from_idrs*L !!)Q/22 r5c|jd}|jd}|dk(rt|trg}|D]}t|dr|j }n3t|t r|j }n|j|St|jdtr||dvrt|d||d<|j|||d<|S)zT Process assistant message to remove thought signatures from tool call IDs. role tool_calls assistant model_dumpid) rwr}listhasattrrdictrrIstrr)msg_copythought_signature_separatorrrnew_tool_callstctc_dicts r3%_process_assistant_message_tool_callsrs,,v&D \2J {z*d; " +Br<(--/B%'')%%b)'++d+S1.'$-?$EgdmUp$qGDM  ! !' *! +""0 Or5c|jddk(r8t|jdtr||dvrt|d||d<|S)zM Process tool message to remove thought signature from tool_call_id. rtoolr)rwr}rr)rrs r3_process_tool_message_idrsV||Fv%*X\\.5QSV*W &(>*B B'H(*E(H^ $ Or5c g}|D]{}t|dr|j}n3t|tr|j }n|j |St ||}t||}|j |}|S)zG Remove thought signatures from tool call IDs in all messages. r)rrr}rrrIrr)messagesrprocessed_messagesmsgmsg_dicts r3(_remove_thought_signatures_from_messagesrs!#, 3 %~~'H T "xxzH  % %c * 9C^_,H6QR!!(+#,& r5c2t|dd}||yy)aCall logging_obj._restore_correlation_context() if it's actually there. Some call sites (tests, narrow unit paths) inject a minimal stand-in object as litellm_logging_obj instead of a real Logging instance - this method is new plumbing specific to request_correlation_in_logs, not part of any pre-existing stand-in's expected interface. `object` (not `Any`) is deliberate: the getattr() below is exactly how this stays type-safe while still tolerating a stand-in that lacks the method. _restore_correlation_contextN)r) logging_objrestores r3)_restore_correlation_context_if_supportedr s#[*H$OG r5c&ddlm}t||S)aCTrue if `result` is a lazy stream wrapper rather than an already-complete response. Only wrapper_async() consults this - it must NOT restore the originating Task's trace_id/session_id as soon as a streaming call returns this: the caller is about to iterate it over however many subsequent lines of their own code, and those log lines should still show this call's ids, not the pre-call ones. This is safe specifically because each async call already runs in its own asyncio Task with its own copy of the contextvars, so leaving it "open" can only affect that one Task, never a different, unrelated future request - Tasks, unlike a thread pool's worker threads, are never recycled across requests. The corresponding terminal handler (async_success_handler, dispatched once the full stream is actually assembled) is what restores it once streaming genuinely finishes. wrapper() (the sync path) does NOT consult this at all: sync calls pass supports_correlation_logging=False into function_setup()/Logging(), so they never stamp trace_id/session_id in the first place - a plain OS thread has no per-call isolation the way an asyncio Task does, and a thread pool's worker threads *are* recycled across unrelated requests, so stamping ids there without a safe restore mechanism could permanently misattribute a later, unrelated request's logs. Full sync support is deferred to a follow-up PR with its own restore mechanism; see Logging.__init__'s supports_correlation_logging parameter. Genuinely circular otherwise: utils.py -> streaming_handler.py -> redact_messages.py -> llms/vertex_ai/common_utils.py -> utils.py, which needs names (supports_response_schema, etc.) this module hasn't finished defining yet at that point in its own top-to-bottom execution. rr),litellm.litellm_core_utils.streaming_handlerrr})resultrs r3&_is_streaming_response_for_correlationrs<Q f1 22r5T) is_async_callc1tjdurtjdd} t t |}d|vr|dnd}t j} | } |jdd} t| } t| dkDr}| D][1t1trQtjjj1dd11"t!1fdtj"Dre1tj$vrtj$j'11tj(vrtj*j-11tj.vrtj*j111tj"vrtj*j311tj4vs=tj*j71^t9d tj"ttj$dkDs8ttj(dkDsttj.dkDrtt:dk(rxt=t?tj$tj(ztj.zatAtBjDtFd } | t:| ttj$dkDrg}tItj$D]G\}1| jK1stjLj'1|j'|ItO|D]!}tj$j|#ttj(dkDrg}tItj(D]\}1| jK1r1tj*j31|j'|H1d k(s1d k(r1tj*j31|j'|1tjPvst1tstS1dtO|D]!}tj(j|#ttj.dkDrg}tItj.D]w\}1| jK1r1tj*j71|j'|H1tjPvs[t1tsltS1dytO|D]!}tj.j|#d}d}d}d}|jUddt|dt<rg}tI|dD]Y\}1| jK1s 1d k(s1dk(s"|"t|t<r|j'1n1g}|j'|[tO|D]}|dj||jd}|jUdd$t|dt<r|jd}tVrf ddl,m-}||}tj^r6|jdd|jdd|jddtWdd|dd|vr|da0t|dkDr|dn|jUdd}|}|tbjdjfk(s;|tbjhjfk(s|tbjjjfk(rd}t|dkDr|d}n|jUddr|d}t jl}|jnrt|t<rt|dkDrt|dtprpd|dvrits}|D]9}|jUdd}|t|ts)|ju|;|jw|jy| t|t<rt|dkDr dd!l=m>}dd"l?m@}|jUd#}|s|r |||$\} }} } t||sTtjd%t||}!d|vr|!|d<n)t|dkDrt=|}"|!|"d<t|"}n|tbjjfk(s|tbjjfk(r't|dkDr|dn|jUdd}n|tbjjfk(s|tbjjfk(rt|dkDr|dn|d}nb|tbjjfk(s|tbjjfk(rt|dkDr|dn|d}n|tbjjfk(s|tbjjfk(rt|dkDr|dn|d}n|tbjjfk(s|tbjjfk(r|jUd'}nm|tbjjfk(s|tbjjfk(rLt|dkDr|dn|d(}$t}%|%j|$)}&d*|vr |&|d*d+<nd+|&i|d*<|&}n|tbjjfk(s|tbjjfk(r|jUdd,}n|tbjjfk(s|tbjjfk(r:t|dkDr|dn$|jUdxs|jUdd-}n%|tbjjfk(sW|tbjjfk(s:|tbjjfk(s|tbjjfk(r dd.l[m\}'dd/l]m^}(t|dkDr|dn|jUd0})t|dkDr|dn|jUdd}*|)rI|'}+|+j|*|)|jUd12},|,jUdg}-|(|-xsd-}nd-}n |tvrg}nd-}d4}.t||5rd}.tAtBjDtFd6}/|/|||.|d7|jUd8|xsd|||||||||9}d:di}0d*|vr|d*|0d*<d;|vrBt|d;tpr/|d;|0d;<|0jUd*s|d;j|0d*<|j|di|0|jUd|#d}#~#wwxYw)?NTze`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs.rr|)rrrPc3JK|]}t|tuywr;ri)r@rkrcs r3rBz!function_setup..gs#/79RDN2/rmz8Initialized litellm callbacks, Async Success Callbacks: r)r3 function_iddynamodb openmeterrSrTr[s3ra)safe_deep_copyrinputpromptzlitellm.llm_callzKeyword Args: r=)categorymessagelevel logger_fnrrcontentrr)r)THOUGHT_SIGNATURE_SEPARATORrrzCRemoving thought signatures from tool call IDs for non-Gemini modelz8Error removing thought signatures from tool call IDs: %squeryfile)file_objrs file_checksumspeechzdefault-message-value)GoogleGenAIAdapter)get_last_user_messagecontentsconfig)rrrz2Error extracting messages from Google contents: %sFrx call_typerlitellm_call_idlitellm_trace_id)rrstreamrrrr start_timedynamic_success_callbacksdynamic_failure_callbacksdynamic_async_success_callbacksdynamic_async_failure_callbacksrxrysupports_correlation_loggingapi_baselitellm_metadatastream_options)ruseroptional_paramslitellm_paramsrzKlitellm.utils.py::function_setup() - [Non-Blocking] Error in function_setup)frpr?rwarningrLr litellm_utilsrpoprlenr}rrmlitellm_loggingrOrFr]input_callbackrIr[rXrYrar_rZrbr`rDr3rsetrrrr enumerateis_async_callable_async_input_callbackreversedrVrfrwadd_breadcrumb'litellm.litellm_core_utils.core_helpersrrUturn_off_message_logginguser_logger_fnr{ completionr acompletionanthropic_messagesrhas_pre_call_rulesrr writepre_call_rulesgetvalue1litellm.litellm_core_utils.get_llm_provider_logicr3litellm.litellm_core_utils.prompt_templates.factoryrrr<rtuple embedding aembeddingimage_generationaimage_generation moderation amoderationatext_completiontext_completionrerankarerankatranscription transcriptionrqget_audio_file_content_hashaspeechr aresponses responsesgenerate_contentagenerate_contentgenerate_content_streamagenerate_content_stream,litellm.google_genai.adapters.transformationr8litellm.litellm_core_utils.prompt_templates.common_utilsr(translate_generate_content_to_completionr)_is_streaming_requestrupdate_environment_variablesrrV)2original_function rules_objrrrPrxrryrget_coroutine_checker_fncoroutine_checkerr all_callbacksrremoved_async_itemsindexrrrrrdetails_to_logrrrrbuffermrrrr_r args_liste _file_objrnrrrcontents_param model_paramadapter transformedtransformed_messagesrrrrcs2 @r3function_setupr:<s d" s 04KO %;6$B:>T +8*M*M #;#=PVzzZegkOl4GXY }  !) bh,&99IInn -1#' o H  '3/=D=\=\/,!7#9#99**11(;7#;#;;44QQRZ[7#;#;;44QQRZ[7#B#BB44WWX`a7#B#BB44WWX`a+ b, TU\UtUtTuv w && '! +s73K3K/Lq/PTWX_XpXpTqtuTu- A% W%;%;g>V>V%VY`YqYq%q!rsM'.s{{8/DFY'Z    m U w%% & *"$ #,W-C-C#D 6x$66x@1188B'..u5 6 ""56 2&&**51 2 w'' (1 ,"$ #,W-E-E#F Wx$66x@44WWX`a'..u5+x;/F44WWX`a'..u5!R!RRWabjloWpA(IV W""56 4((,,U3 4 w'' (1 ,"$ #,W-E-E#F Wx$66x@44WWX`a'..u5!R!RRWabjloWpA(IV  W""56 4((,,U3 4QU!VZ'PT!]a' ::($ / ; 6RdKegk@l"$ #,V4F-G#H 6x$66x@HPZDZ^fjn^n6Bz7H8>>xH;C*7'..u5 6""56 6)*..u5 6(. 3E(F % ::($ / ; 6RdKegk@l(. 3E(F %  (R!/!7//"":t4""7D1""8T2 +((89  & #K0N"%d)a-tAwVZZ5N, --33 3I11777I88>>>H4y1}7J-!*-(..E(((*x.MA%x{D1!, ( !.AeeIr2G*z'3/G W-. (( //+)(D)c(ma.?)j +1**5J*K'/5!;K&+4G<8A2Aq,E3FG&,,-rs5]$&A5* &/1CF:. Y]/3DzI+=IaL#(#3D )--33 3yIDXDXD^D^7^"%d)a-tAwVZZ5NH )44:: :i9KfKfKlKl>l"%d)a-tAwVH5EH )..44 4 YEZEZE`E`8`"%d)a-tAwVG_H )44:: :i9KdKdKjKj>j"%d)a-tAwVH5EH )**00 0IARARAXAX4Xzz'*H )2288 8II`I`IfIf>DDD 347t9q=QfjjQ[F\58Y]$q' SZ\^H_ !%7%9G)0)Y)Y)!/%zz(3*Z*K 3>//*b2Q(45IJeNeH6H2 2H.H  F+23;;x3HJe+f!1/1"#45#ZZ(:;#)r!&?&?,K,K1)6 &5?3C  )/ );N: &  'Jv>P7QSW,X178J1KN- . "%%j1-34F-G-L-L-Nz*00)!::&6= 1 F""W (!' (N )! !*!j"**+eghiijt 3$$%Y[\]2 3l    " 5k B  !no sF!z*E*z*:D-z*(z*9Cz*z*Bz*5B+z*!x,/Fz*8z* Az*!yx>A y/M(z*B"y;:C1z*, x;7z*:x;;z*> y y y  y y8y3-z*3y88z*; z'z"z*"z''z** {3${{cK|durWtd|jddlm}|j |j||||j |||yyw)NFz>Async Wrapper: Completed Call, calling async_success_handler: r)GLOBAL_LOGGING_WORKERrrend_time)async_coroutine)rDasync_success_handler)litellm.litellm_core_utils.logging_workerr<ensure_initialized_and_enqueue-handle_sync_success_callbacks_for_async_calls)rrrr>is_completion_with_fallbacksr<s r3_client_async_logging_helperrEs} %-L[MnMnLo p T<<'==VXbmu=v =  AA! B # .sAAc&|jdd}|tj}|jddrXttj t d}tj}|||jd}||d<||}||fS)zm Get the number of retries from the kwargs and the retry policy. Used for the wrapper functions. num_retriesN retry_policyrrVrH) rwrprGrrrrrr)rxrVrGrrretry_policy_num_retriess r3_get_wrapper_num_retriesrKs **]D1K))  zz.$'NU KK !#FO )%2$D$D6WN37  "4!5~ # /2K  r5c|tttztjzdz|j dd}|S)zM Get the timeout from the kwargs Used for the wrapper functions. Ntimeout)rfloatinthttpxrrw)rxrVrMs r3_get_wrapper_timeoutrQs3 %#+ 5YZG Nr5cNtj}|j|Sr;)rrr)rrs r3check_coroutinerSs"#0#F#F " 4 4U ;;r5cK t|}|j}t}tj D].}t ||s|j||d{}|-|}0|S#t$rd}YjwxYw7w)z Allow modifying the request just before it's sent to the deployment. Use this instead of 'async_pre_call_hook' when you need to modify the request AFTER a deployment is selected, but BEFORE the request is sent. N)r{ ValueErrorrr4rpr|r}async_pre_call_deployment_hook)rxrtyped_call_typemodified_kwargsr0rcrs r3rVrVs #I.kkmO35L%%) h -#BB?TcddF!"( )  es?A? A,9A? A?A=A?&A?, A:7A?9A::A?cK t|}t}tjD]>}t ||s|j |tt||d{}|<|cS|S#t$rd}YjwxYw7w)z` Allow modifying / reviewing the response just after it's received from the deployment. N) r{rUr4rpr|r}'async_post_call_success_deployment_hookrr) request_dataresponserrWr0rcrs r3rZrZs #I.45L%% h -#KKd#3X>F!  O  s>A? A,)A?$A?A=A?%A?, A:7A?9A::A?cK t|}|jd}t|tr|nd}t |j Dcic]\}}|dk7s ||c}}}t|} t} tjD]]} t| | s tt| r| j|| ||d{n| j|| |d{_y#t$rd}YwxYwcc}}w7:7 #t$r4} tj dt| j"| Yd} ~ d} ~ wwxYww)a8 Notify CustomLogger callbacks that a deployment attempt failed. Unlike its pre-call/post-success siblings, this wraps each callback call in its own try/except: it runs on the wrapper's exception path, so a broken callback must never replace the real exception that's about to be re-raised to the caller. Reads ``fallback_depth`` off ``request_data`` (set by ``Router`` on each fallback hop) and passes it through to the callback; ``None`` when missing or not an int, since a bare SDK call has no fallback chain. Callbacks receive a same-class snapshot of ``exception``, not the live object that's about to be re-raised, so a callback setting an attribute on it (e.g. ``status_code``) cannot change what the real caller sees. ``request_data`` omits ``attempted_targets``: unlike the rest of this attempt's own kwargs, it's the *same* object shared by reference across every hop of the live fallback walk, so a callback calling ``.record()`` on it would make the router skip a deployment it hasn't actually tried. Nr9attempted_targets)r9z7async_post_call_failure_deployment_hook error in %s: %s)r{rUrwr}rOrrrXr4rpr|rJrLrDrUrr<r) r[rVrrW_raw_fallback_depthr9kvsafe_request_datasafe_exceptionr0rccallback_errors r3rDrDsV.#I.".!1!12B!C3=>QSV3W/]aN/,BTBTBV0s$!QZ[_rZrA0st8CN35L%% h - :4>J"JJ)>?[iK#JJ)>?  1t $$MN++" sE C-=E C> C> !0 3 34S T%)4$;#F!8%,%J%J; >E!)(7(C,=,P,;O.P$P$34E$Fv$NR_$_$5IZ9[$[$.$34E$FGX$Y$(%"%9OL]<^$^$34E$FG[$\`d$d!( : : O O _ _+:;L+MN_+`-;!`!"%e%_ %"%\%`%Q%Y%I!<[6IkC ;cb(1!)$(!), sJH2AH2BH2-BH# B H2# H/,H2.H//H22 I;H==Ictj}|tfd}tfd}tj}|j }|r|S|S)Nc j}t|rtjN|j dxsij dd}|'tjt |kr t d|i|}t||rZd|vrT|ddurMg}t|D]\}}|j|tj||j ddS|S|St||tjj}d}|j d d} d |vr ttj |d <t |d kDr|d n|j d d} | t#j|g|d di|\} }| Jdt%|| |d <t'} | ||} | | _tj*rJtj,tj*kDr)t/tj,tj*tjN|j dxsij dd}|'tjt |kr t dt1d|j dddtj2d|j dij dd|j ddtj2|j dddur|j dij dddur|j dddur|j dddur|j dddur|j dddur|j dddur|j dddurr|j dddur^t5j6d| j9| xsd | ||||!} | j:!t5j6d"| j:S|j d#d| tj<dur|t>j@jBk(s:|t>jDjBk(s|t>jFjBk(r | }|j d$dd%|j d$}d}t |d&kDr|d&}n|j ddr|d}|j d#}tI| |||dd'}||d#<|i|}tjj}t||rd|vrT|ddurMg}t|D]\}}|j|tj||j ddStKtLjNtd)}||| | |||*|Sd|vr|ddusAd|vr|ddus6d|vr|ddus+d|vr|ddus d+|vr|d+dustQjR|r|StU|| xsd|,| jW|||-t5jXd.t[j\}tKtLjNtd/}|j^|j`| jb|||tKtLjNtd)}||| | |||*|S#t $r}t1d(|Yd}~d}~wwxYw#t $rH}j}|t>jDjBk(r|j d0dxstjdxsd}|j d1drTtKtLjNtd2}tfjh}|||j d13}||d1<dt_2|j d4i}d5|j dxsiv}|rp|sntk|tljns5tk|tljpstk|tljrr||d0<tjt|i|cYd}~Stk|tjvjxrv|rs| |vrn|skt |d kDr || |d <n|| |d <|i|cYd}~S|t>jzjBk(r!|j d0dxstjdxsd}|j d1drTtKtLjNtd2}tfjh}|||j d13}||d1<dt_2d5|j dxsiv}|ro|smtk|tljns4tk|tljpstk|tljrr||d0<tj||i|cYd}~Stj}tjj}| r| j|||||d}~wwxYw)6Nrsprevious_modelszMax retries per request hit!rcomplete_responseTrrlitellm_logging_objrrrrF3logging_obj should not be None after function_setupr'request_kwargsr current_cost max_budgetzSYNC kwargs[caching]: caching; litellm.cache: z#; kwargs.get('cache')['no-cache']: cachezno-cacherrraimg_generationrr _arealtimezINSIDE CHECKING SYNC CACHErrr'rrrrxrPz Cache hit! max_tokens hf_model_name huggingface/rr base_modelruser_max_tokens buffer_num buffer_perc&Error while checking max token limit: rrrrrxrr>rrurrr'r()rrPrxz0Wrapper: Completed Call, calling success_handlerrrGrHrrIcontext_window_fallback_dict model_group)Br_is_async_requestrpnum_retries_per_requestrwrrUr%rrIstream_chunk_builderprint_args_passed_to_litellmdatetimenowrruuid4r:rri_llm_caching_handlerr _current_costrrDrrr<_sync_get_cache cached_result modify_paramsr{rrrrrrrrasyncior r{sync_set_cacher= contextvars copy_contextsubmitrunsuccess_handlerrGrrr}openair rr completion_with_retries exceptionsrrresponses_with_retries traceback format_excfailure_handler)rPrxrr~rchunksidxchunkrrrrgrcaching_handler_responserrrmodified_max_tokensr3r>rctxrrGrrr_is_litellm_router_calltraceback_exceptionr'r(s r3wrapperzclient..wrappers &.. V $..:#)::j#9#?R"D"DEVX\"]".66#o:NN'(FGG'77F$#'&0VSUY3Z  F *(+DJJL(9F$ %.1$i!m47GUYAZD "&4%.. :'HL'\a'ek'# V * a,a a* 'v .,7F( )'F'H =N"3%%> 0DK ,!!((7+=+==-%,%:%:#*#5#5 ..:#)::j#9#?R"D"DEVX\"]".66#o:NN'(FGG (Iu)E(FFWX_XeXeWfgJKQKUKUV]_aKbKfKfgqsxKyJz{  It4<AZ!::i74? 7B/33JFdRJJ|U34?JJ159EJJ}e4D@JJ0%8DJJ/7tCJJy%0<JJ|U34?$$%ABJ^JnJn+2&7 +)'!KoK(,99E"((63AAA <.:%))T1!6!6!$>> I$@$@$F$FFP!&Jzz/48D'3FJJ4O3P%Q #H4y1}#'7J5#)*#5-3ZZ -EO1K1I1K##-!)(7#'$( 2',?F<('77F((,,.H$#'&0V*&*#6>OOOq'"4"4"O"OP4!==34y1}">u"EQ*Fu*Mw,d=f==i11777$jj=\ATAT\X\ ::nd38? H-/R95*7)I)I&"C"#%+ZZ%?#K.@-AF>*&*#*7JJz*0b+' (?#1foo6%a8%a)B)BC0;}-&==tNvNN)2)=)=)? ((,,.H++*JGYL s1J%\A=\B[-B \!0\A \ C \- \6\ \ \\ i$Ei4i$:Aii$D9ii$A ii$c  !Kt"||tjj!d ttj t d}|jddt}|"|!}"j }d|vr ttj|d<t|dkDr|dn|jdd}|jddu|jddtj}d} t"j #!g|i|\}Jd t!||d{} | | }|jd } | j"| k7r| _|d<t%||_t(j*rJt(j,t(j*kDr)t/t(j,t(j* t1rBt3d |jd ddt(j4d|jdd|j7|xsd"!|||d{} | l| j8/| j:#| j8t= s t?SS| j@dur#| j:t= s t?SS|jdd|t(jBdur|tDjFjHk(s:|tDjJjHk(s|tDjLjHk(r |} |jddd|jd} d} t|dkDr|d} n|jddr|d} |jd}tO|| | |dd}||d< "|i|d{ tjjtW||rd|vrk|ddurdg}tY D]\}}|j[|t)j\||jddt= s t?SS| ||! |j_ |!t= s t?SS|tDj`jHk(r t= s t?SStc ||"#"tdj|}|tg| |#d{ |ji "||$d{|sBtd%drd4 !fd& }|_5n#tmjntq !'js !(tu tvr:| 8| j:,|jy| !)t= s t?SS| ||!  t= s t?SS77#tP$r}t3d|Yd}~_d}~wwxYw7[#tP$rK}tjj} tS|||d{7#tT$rYwxYwd}~wwxYw7p7V#tP$r4}t{j|}||ntjjr]|s[ j||!n#tP$r}|d}~wwxYw j||!d{7n#tP$r}|d}~wwxYw"j }t||*\}}|tDjFjHk(rL|jd+i}d,|jd-xsiv}|r|s dt(_B||d.<"|d/<tu|tjrd0|d1<ntu|tjrd2|d1<t)j|i|d{7 cYd}~t= s t?SS#tP$rYnZwxYwtu|t(jjr1|r.||vr)|s&t|dkDr |||d<n|||d<"|i|d{7 cYd}~t= s t?SS|tDjjHk(rd,|jd-xsiv}|r|s dt(_B||d.<"|d/<tu|tjrd0|d1<ntu|tjrd2|d1<t)j|i|d{7 cYd}~t= s t?SS#tP$rYnwxYw|jd.}| t|d.|t||*}t|d3||d}~wwxYw#t= s t?wwxYww)5Nrrrrrr fallbacks_is_litellm_internal_callrrrzASYNC kwargs[caching]: rFrz; kwargs.get('cache'): rrrTrrrrrrr)r[rVrrrrr)rrr)r[r\r)rr'rxrP_defer_async_loggingc Ltjty)Nrrrr>rD)r create_taskrE)r>rDrrrsr3_enqueue_deferred_loggingz@client..wrapper_async.._enqueue_deferred_loggings(++8,7'-+5)1=Y r5rr=)_caching_handler_responseembedding_responserr>)rxrVrrrsrGr'exponential_backoff_retryretry_strategyconstant_retryrMreturnNone)Mrrrrrrrrwrirrrrrrr:rVrrrrprrrrrDr_async_get_cacherfinal_embedding_cached_responserr embedding_all_elements_cache_hitrr{rrrrrrUrD BaseExceptionr%rrIrwrap_streaming_result_for_cache arealtimer{rrZasync_set_cacherrrrErCr}r2_combine_cached_embedding_response_with_api_resultrrrasync_failure_handlerrKrGrrr acompletion_with_retriesrrraresponses_with_retriessetattrrQ)$rPrx_update_response_metadatargrrrr_deployment_call_end_timerX _hook_streamrrrrrr3deployment_errorrrr_call_type_enumrrrGrrdeployment_num_retriesrMr>rDrrrr'r(s$ @@@@@r3 wrapper_asynczclient..wrapper_asyncs $%6fE$--113 +23;;x3HJd+e!39::>SUY3Z #B#D9J/!!:  &.. F *(+DJJL(9F$ %.1$i!m47GUYAZ.4jj.ET.Q$ .5+;+?+?+A!  "s G"&45F5O5OQZ\f&xim&xqw&x# V* a,a a*+I&R[+\%\O*(#)**X"6L'K,>,>,,N%1 ",7F( ) &v ./CK ,!!((7+=+==-%,%:%:#*#5#5  !-fjjE.J-KK\]d]j]j\klCDJDNDNOVX\D]C^_NbMrMrkr"3'%#NsNH %)4-;;G1QQY4BBD:&A9+FBA/OOSWW4TT~:&A9+FBw <.:%))T1!6!6!$>> I$@$@$F$FFP!&Jzz/48D'3FJJ4O3P%Q #H4y1}#'7J5#)*#5-3ZZ -EO1K1I1K##-!)(7#'$( 2',?F<( 0$A&AA ((,,.H$#'&0VF  =VK9''4(3#)'1%-9U   E E%! F 6#45-9-MMY+^^.G'-)% _b:&A9+FBU &'%!  B:&A9+FBW &]4Hd!P!$J1#"NOOP B ,4,=,=,A,A,C)A%+"2"+% ` DR )2)=)=)? 5N4Y0_g_p_p_t_t_vH#<//. H!G%;;A?RT^`hiii G*22I":&TU"V KI117776)>?7RF#34'6??;7GF#34'.'G'G'XQW'X!X!X & n:&A9+FBu% q'"4"4"O"OP4!==34y1}">u"EQ*Fu*Mw#4d#Ef#EEEF!MV:&A9+FBUi22888*7JJz*0b+'  (? &.2+0;}-6G23%a)>)>?7RF#34'6??;7GF#34'.'F'F'WPV'W!W!W & ,:&A9+FB3% -3JJ},E "%1=*@A11MG Ay' *GeR @:&A9+FBsODg+8ZXD ZX)Z?g+Z0g+A=ZBX  X1X.X1BZg+2 Zg+*Zg+ 8ZZZ5Z 6BZg+Z+g+ZZ X+X& Z&X++Z.X11 Z:ZY1*Y-+Y10Z1 Y=:Z<Y==ZZZ Z g :g[('g( [81[33[88g<\\\g \*#\%%\**A*gA,`%``%gg  g g+% `2.g1`22Agbgg gg+38g,A,e<ee<gg  g$g+< fgf?gg  gg((g+)rrrrr)r'rrrrrvr(s` @r3clientrs &&EwI mm^  LGLG\ $1#F#F/1CCDUVLr5c|y|jdddus|jdddus|jdddus|jdddus||jdddush|jdddusT|jd ddus@|jd ddus,|jd ddus|jd ddus|duryy) aj Returns True if the call type is an internal async request. eg. litellm.acompletion, litellm.aimage_generation, litellm.acreate_batch, litellm._arealtime Args: kwargs (dict): The kwargs passed to the litellm function is_pass_through (bool): Whether the call is a pass-through call. By default all pass through calls are async. FrTrrrrrrr acreate_batchacreate_fine_tuning_jobrv)rxis_pass_throughs r3rr,s~ =%(D0 ::lE *d 2 ::' /4 7 ::mU +t 3 ::(% 0D 8 ::& .$ 6 ::i '4 / ::lE *d 2 ::ou - 5 ::/ 74 ? d " r5_STREAMING_CALL_TYPESc*d|vr|ddury|tvS)a$ Returns True if the call type is a streaming request. Returns True if: - if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages) - if call_type is generate_content_stream or agenerate_content_stream (litellm google genai) rT)rrs r3r%r%Vs'6fX.$6 - --r5cP|t|d|d|d}|St|S)N identifierrevision auth_token)rrrr)create_pretrained_tokenizer_select_tokenizer_helper)rcustom_tokenizer _tokenizers r3_select_tokenizerres=#7' 5%j1' 5  #% 00r5ctjdur t|S t|}||S t|S#t$r*}t j d|Yd}~t|Sd}~wwxYw)NTzError selecting tokenizer: %s)rpdisable_hf_tokenizer_download_return_openai_tokenizer_return_huggingface_tokenizerrUrr<)rrr3s r3rrps},,4'..A5e<  M  $E ** A"+";";NOO '** *z/F"+"4"4_"E/>NOO ekkm #{ekkm'C--.ST /iHH ekkm #--.HI /iHHr5rc|xs t|}t|dtr|dj|d}n|dj|}t |dr |j S|S)a Encodes the given text using the specified model. Args: model (str): The name of the model to use for tokenization. custom_tokenizer (Optional[dict]): A custom tokenizer created with the `create_pretrained_tokenizer` or `create_tokenizer` method. Must be a dictionary with a string value for `type` and Tokenizer for `tokenizer`. Default is None. text (str): The text to be encoded. Returns: enc: The encoded text. rr)disallowed_specialids)rr}rencoderr)rtextrtokenizer_jsonencs r3rrsl-N0A0NN.-x8[)00"0M[)006sEww Jr5c|xs t|}|ddk(r)|rt|d|}|dj||}|S|dj|}|S)a Decodes token ids using the selected tokenizer. Args: skip_special_tokens: For HuggingFace tokenizers, keep the historical LiteLLM round-trip behavior by omitting special tokens by default. Set to False to inspect decoded BOS/EOS tokens. rrLrr)skip_special_tokens)r$_strip_huggingface_special_token_idsdecode)rtokensrr r decs r3rrsq-N0A0NNf!88 9.:UW]^F[)00M`0a  % , ,V 4C Jr5c  |j}|jDchc]\}}t|dds|}}}|s|S|Dcgc] }||vs| c}S#t$r|cYSwxYwcc}}wcc}w)NspecialF)get_added_tokens_decoderrUrr)rradded_tokens_decodertoken_id added_tokenspecial_token_idstokens r3rrs&/&H&H&J /C.H.H.J *X{gVaclnsNt    % He6G)GE HH    Is'AA,A, A2A2 A)(A)c tj|||}d|dS#t$r7}tjd|tj||}Yd}~@d}~wwxYw)a Creates a tokenizer from an existing file on a HuggingFace repository to be used with `token_counter`. Args: identifier (str): The identifier of a Model on the Hugging Face Hub, that contains a tokenizer.json file revision (str, defaults to main): A branch or commit id auth_token (str, optional, defaults to None): An optional auth token used to access private repositories on the Hugging Face Hub Returns: dict: A dictionary with the tokenizer and its type. )rrzTError creating pretrained tokenizer: %s. Defaulting to version without 'auth_token'.)rNrr)rrrUrr>)rrrrr3s r3rrsfM-- ! ,) DD Msuvw--j8L Ms A-AAc6tj|}d|dS)z Creates a tokenizer from a valid JSON string for use with `token_counter`. Args: json (str): A valid JSON string representing a previously serialized tokenizer Returns: dict: A dictionary with the tokenizer and its type. rr)rr)jsonrs r3create_tokenizerrs!))$/I+) DDr5rOc Xtjduryt||||||||| S)zh The same as `litellm.litellm_core_utils.token_counter`. Kept for backwards compatibility. Tr)rpdisable_token_counterr) rrr rcount_response_tokenstools tool_choiceuse_default_image_token_countdefault_token_counts r3 token_counterr%sC.$$, # ! #   %  r5cgd}||vryy)zU Helper function to know if a provider implementation supports httpx timeout )razurebedrockTFr)rsupported_providerss r3supports_httpx_timeoutr*& s"@11 r5ct||dS)a Check if the given model supports system messages and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. Returns: bool: True if the model supports system messages, False otherwise. Raises: Exception: If the given model is not found in model_prices_and_context_window.json. supports_system_messagesrrr_supports_factoryrs r3r,r,2 s / & r5ct||dS)a Check if the given model supports web search and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. Returns: bool: True if the model supports web search, False otherwise. Raises: Exception: If the given model is not found in model_prices_and_context_window.json. supports_web_searchr-r.rs r3r1r1G s / ! r5ct||dS)a Check if the given model supports URL context and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. Returns: bool: True if the model supports URL context, False otherwise. Raises: Exception: If the given model is not found in model_prices_and_context_window.json. supports_url_contextr-r.rs r3r3r3\ s / " r5c tj||\}}}}t||}|jdd}|d}|S#t$r"}t j d|||Yd}~yd}~wwxYw)a Check if the given model supports native streaming and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. Returns: bool: True if the model supports native streaming, False otherwise. Raises: Exception: If the given model is not found in model_prices_and_context_window.json. rsupports_native_streamingTNz~Model not found or error in checking supports_native_streaming support. You passed model=%s, custom_llm_provider=%s. Error: %sF)rpr_get_model_info_helperrwrUrr<)rrr1 model_infor5r3s r3r5r5q s+2+C+C-@, ("Aq3Tgh $.NN3NPT$U! $ ,(, %((  M    sAA A. A))A.c tj}|||\}}}}t j jt j jt j jt j jt j jg}||vryt||dS#t$r"}tjd|||Yd}~yd}~wwxYw)ak Check if the given model + provider supports 'response_schema' as a param. Parameters: model (str): The model name to be checked. custom_llm_provider (str): The provider to be checked. Returns: bool: True if the model supports response_schema, False otherwise. Does not raise error. Defaults to 'False'. Outputs logging.error. rztModel not found or error in checking response schema support. You passed model=%s, custom_llm_provider=%s. Error: %sNFTsupports_response_schemar-) rrrUrr<rpr PREDIBASE FIREWORKS_AI LM_STUDIONEBIUS DATABRICKSr/)rrrr1r3*PROVIDERS_GLOBALLY_SUPPORT_RESPONSE_SCHEMAs r3r9r9 s "/"@"@+;%]p+q("Aq &&))&&##'' 9.HH / & )  C    sB33 C<CCct||dS)z[ Check if the given model supports parallel tool calls and return a boolean value. "supports_parallel_function_callingr-r.rs r3rArA s / 0 r5ct||dS) Check if the given model supports function calling and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (Optional[str]): The provider to be checked. Returns: bool: True if the model supports function calling, False otherwise. Raises: Exception: If the given model is not found or there's an error in retrieval. supports_function_callingr-r.rs r3rDrD s / ' r5ct||dS)zU Check if the given model supports `tool_choice` and return a boolean value. supports_tool_choicer-r.rs r3rFrF s 5>QWm nnr5cLt||}||j|dduryy)zf Check if the given model supports a provider specific model info and return a boolean value. rNFT)get_provider_inforw)rrr provider_infos r3_supports_provider_info_factoryrJ s1 -5NabM ]%6%6sE%Bd%J r5c*ddlm} |||}||j|d}|}ntj||\}}}}t ||}|j |ddury|j |Vt|}|8tjj |xsi}|j |dduryt|||} | | Sy#t$r9} tjd|||| t|||} | | cYd} ~ SYd} ~ yd} ~ wwxYw) rCr declared_authenticating_providerN/rFTzgModel not found or error in checking %s support. You passed model=%s, custom_llm_provider=%s. Error: %s) r rM removeprefixrprr6rw_get_model_cost_key model_costrJrUrr<) rrrrMdeclaredr1r7bare_model_key bare_entrysupported_by_providerr3s r3r/r/ sHc-:5BUV  &&*AE#+ /6/G/G1D0 ,E&13Tgh >>#u % - ^^C ( %8$>N)$+$6$6$:$:>$J$Pb >>#u-5$CEK^`c$d !$0,,   u     !@GZ\_ ` ,( ( s+AC(AC>C D)D D Dc tj||}|d}|d}t||}|j|}t |t r|St |}|tjj|nd} t | tr#| j|} t | t r| Sy#t$r#} tjd|||| Yd} ~ yd} ~ wwxYw)aReturn a string value the model map declares for *key*, or ``None`` when it says nothing. The string-valued sibling of :func:`_supports_factory` and :func:`_is_explicitly_disabled_factory`, public where those two are not because it is read from the provider configs rather than from this module, sharing their ``get_llm_provider`` -> ``_get_model_info_helper`` chain and their unprefixed-twin fallback (#20885), so a provider-prefixed entry that omits the key still answers from the bare entry that carries it. ``None`` means "the map does not say", never "the map says no" - callers decide what an unknown declaration implies, and for a capability gate that decision must be the conservative one. rrrNz^Model not found or error in reading %s. You passed model=%s, custom_llm_provider=%s. Error: %s) rprr6rwr}rrPrQrrUrr<) rrrresolvedresolved_modelresolved_providerr7rRrSrT bare_declaredr3s r3declared_value_factoryr[2 s!22Tgh ( #+A;2]no $..- h $O 3N CFTF`G..22>Bfj j$ '#->>##6M--$$  l     sAB5A B55 C!>CC!cn tj||\}}}}t||}|j|}|dury|Dt |}|7tj j|xsi}|j|duryy#t $r#}tjd||||Yd}~yd}~wwxYw)ubReturn True only when the model map explicitly sets *key* to ``False``. This is the opt-out mirror of :func:`_supports_factory`. Where ``_supports_factory`` requires an explicit ``True`` to return ``True``, this function requires an explicit ``False``. A missing key (``None``) is treated as *not* disabled so that unknown or newly-added models are allowed through without any model-map entry. Uses the same ``get_llm_provider`` → ``_get_model_info_helper`` chain as ``_supports_factory`` so caching, fallback, and normalisation improvements apply here automatically. rFTNznModel not found or error in checking %s disabled state. You passed model=%s, custom_llm_provider=%s. Error: %s) rprr6rwrPrQrUrr<) rrrr1r7valrSrTr3s r3_is_explicitly_disabled_factoryr^Z s+2+C+C-@, ("Aq3Tgh ^^C( %< ;$7$>N)$+$6$6$:$:>$J$Pb >>#&%/  |     s>BAB B4B//B4ct||dS)zECheck if a given model supports audio input in a chat completion callsupports_audio_inputr-r.rs r3r`r`  5>QWm nnr5ct||dS)zCCheck if a given model supports pdf input in a chat completion callsupports_pdf_inputr-r.rs r3rcrc s 5>QWk llr5ct||dS)zFCheck if a given model supports audio output in a chat completion callr`r-r.rs r3supports_audio_outputre rar5ct||dS)a Check if the given model supports prompt caching and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (Optional[str]): The provider to be checked. Returns: bool: True if the model supports prompt caching, False otherwise. Raises: Exception: If the given model is not found or there's an error in retrieval. supports_prompt_cachingr-r.rs r3rgrg s / % r5ct||dS)N supports_prompt_cache_breakpointr-r.rs r3riri s / . r5ct||dS)a Check if the given model supports computer use and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (Optional[str]): The provider to be checked. Returns: bool: True if the model supports computer use, False otherwise. Raises: Exception: If the given model is not found or there's an error in retrieval. supports_computer_user-r.rs r3rkrk s / # r5ct||dS)a Check if the given model supports vision and return a boolean value. Parameters: model (str): The model name to be checked. custom_llm_provider (Optional[str]): The provider to be checked. Returns: bool: True if the model supports vision, False otherwise. supports_visionr-r.rs r3rmrm s /  r5ct||dS)zQ Check if the given model supports reasoning and return a boolean value. supports_reasoningr-r.rs r3roro s 5>QWk llr5ct||dS)za Check if the given model supports native structured outputs and return a boolean value. !supports_native_structured_outputr-r.rs r3rqrq s / / r5cj tj||\}}}}t||}|jd}|ytjj|i}|jdd}|yt |t r|Sy#t$r"}tjd|||Yd}~yd}~wwxYw)z Get a list of supported regions for a given model and provider. Parameters: model (str): The model name to be checked. custom_llm_provider (Optional[str]): The provider to be checked. rrNsupported_regionszvModel not found or error in checking supported_regions support. You passed model=%s, custom_llm_provider=%s. Error: %s) rprr6rwrQr}rrUrr<)rrr1r7 model_keymodel_cost_datarsr3s r3get_supported_regionsrv s +2+C+C-@, ("Aq3Tgh &>>%0  !(!3!3!7!7 2!F#2#6#67JD#Q  $ ' .$ $  E    s"|j |}t|tr|j ||||<p|||<v|||<||Sr;)rr}r_convert_stringified_numbersrrwrO) existing_dictnew_dictr`raexisting_nested_dicts r3_update_dictionaryr~ s %1 =!S!#?#B a At$'4'8'8';$2D9(//2';M!$'(M!$#$ a  % r5ct|tr- d|jvsd|vr t|St |S|S#t t f$r|cYSwxYw)zYConvert stringified numbers (including scientific notation) to appropriate numeric types.r3.)r}rrrNrOrUrt)rs r3rzrz0 s]% ekkm#se|U|#5z! LI& L s A AAA)zus.zeu.zapac.zjp.zau.zus-gov.zglobal.zap-northeast-1._BEDROCK_REGION_PREFIXES)cache_creation_input_token_cost)cache_creation_input_token_cost_above_1hr1cache_creation_input_token_cost_above_200k_tokenscache_read_input_token_cost-cache_read_input_token_cost_above_200k_tokens_CACHE_PRICING_FIELDScNg}|jd}d}|t|dz krP||tvrE|dz }|jdj ||d|t|dz kr ||tvrE|r|dn|}t D]1}|j |s|j|t|d3|r#t||}||k7r|j||D]B}tjj|} | %| jd7t| cSy)aBest-effort lookup of a built-in ``model_cost`` entry for a custom key whose shape ``get_model_info`` cannot resolve (repeated provider prefixes like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases). Returns a copy of the matching entry so the caller can inherit its defaults (most importantly cache pricing) without mutating the shared built-in. Returns ``None`` when no safe match exists. rNrrNrlitellm_provider) rrrrIr r startswith_strip_model_namerprQrwr) rrF candidatessegmentsrbase region_prefixstripped candidateentrys r3!_resolve_builtin_model_cost_entryrT s0$&JiinH C H ! !hsm&F q#((8CD>23 H ! !hsm&F%/*R.CD1: ??= )   d3}#5#78 9:+$HU t    h ' ""&&y1  +=!>!J;  r5c t|}|dtjvr|St |d|Sy#t$rYywxYw)aResolve ``model`` to its built-in cost-map entry for registration merging. Returns ``None`` when the lookup raises or when it resolved via a fallback-generalization capability rule, detected as the resolved key missing ``litellm.model_cost`` while matching a capability rule. A rule-derived entry carries no pricing, so treating it as a hit would skip the built-in cache-pricing inheritance for prefix-mangled keys. rNr)get_model_inforUrprQr-)rr=s r3(_get_builtin_model_info_for_registrationrv sU$51 E{g((( 'U 4<  s 6 AAz#Final[dict[str, dict[str, object]]]_runtime_registered_model_costc eZdZUdZdZded<y)_LiveDeploymentReplayzSingle-slot holder for the callback that rebuilds live router deployments. A class attribute rather than a module global so there is one writer and one reader, and neither needs a ``global`` statement. NzCallable[[], None] | Nonerc)r __module__ __qualname____doc__rc__annotations__rr5r3rr s +/H'.r5rc|t_y)ajInstall the callback that re-asserts live router deployments after a refresh. ``litellm.router`` installs this at import time. The seam exists because the deployment metadata a refresh has to restore belongs to whichever Router objects are alive at that moment, which this module cannot see, and importing the router here would be circular. N)rrc)replays r3set_live_deployment_replayr s &,"r5ctjtjtrtt tyy)aRe-apply runtime model metadata on top of a freshly adopted cost map. Adopting a new catalog replaces ``litellm.model_cost`` wholesale, which on its own discards everything registered at runtime: the deployment ``model_info`` the Router registers from ``model_list``, and pricing overrides passed to ``register_model``. Both are re-applied here so a price data reload only updates pricing rather than erasing operator-supplied model metadata. The two are restored differently, and the difference is what keeps this bounded. Deployment metadata is re-derived from the routers that are alive right now, so a deployment that has been deleted or repointed, and a router that has been discarded, are simply not part of the rebuild; nothing has to withdraw them and nothing accumulates. Only ``register_model`` calls that have no such owner are recorded and replayed, and a registration describing a single request opts out of even that. N)rQ)rrcrregister_modelrrr5r3(reapply_runtime_model_cost_registrationsr s2$%%1&&(%$'E"FG&r5)persist_across_reloadswarning_display_nameci}t|tr|}n&t|trtj|}|r,|}|j D]\}}t|t |<t}|j D]\}} | jdd} t|| |vstfd|Dr#tjj|i} |} nt} | tt| } | d} ni} |} t| }|8tD].}| j||j|'||| |<0no| jd ^| jd M| jd <| jd | jd tj d|xs|| jd| j#ddtjj| }|tjj|}|i}dD]}||vs|| vs | j#|d t%| | }tjj'| ij)|t+tj,d| | | jddk(r5|tj.vsKtj.j1|l| jddk(r5|tj2vstj2j1|| jddk(r5|tj4vstj4j1|| jddk(r5|tj6vs&tj6j1|G| jddk(rM|j9dd}|dtj:vstj:j1|d| jddk(r5|tj<vstj<j1|| jddk(r5|tj>vstj>j1|:| jddk(r5|tj@vsbtj@j1|| jddk(r5|tjBvstjBj1|| jddk(r5|tjDvstjDj1|| jddk(r5|tjFvs=tjFj1|^| jddk(r5|tjHvstjHj1|| jdd k(r5|tjJvstjJj1|| jdd!k(r5|tjLvstjLj1|9| jdd"k(sO|tjNvsctjNj1||S)#a Register new / Override existing models (and their pricing) to specific providers. Provide EITHER a model cost dictionary or a url to a hosted json blob Example usage: model_cost_dict = { "gpt-4": { "max_tokens": 8192, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "litellm_provider": "openai", "mode": "chat" }, } ``persist_across_reloads`` controls whether the registration is replayed when the cost map is refreshed. It defaults to True because a caller registering a model is declaring durable intent. Pass False for a registration that only describes one request, so it is dropped rather than re-asserted over every future catalog. ``warning_display_name`` names the model in the missing-cache-pricing warning instead of the registered key, for callers that register under an opaque key (e.g. the router's hashed deployment ids). urlrrc3FK|]}j|dyw)rNN)r)r@rA_key_strs r3rBz!register_model.. s'= -.H  1#Q (= s!rNr)rrFrrtiered_pricinginput_cost_per_tokenoutput_cost_per_tokena'register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info)rrz0added/updated model=%s in litellm.model_cost: %srtext-completion-openaicohere anthropic openrouterrNrrvercel_ai_gatewayzvertex_ai-text-modelszvertex_ai-code-text-modelszvertex_ai-chat-modelszvertex_ai-code-chat-modelsai21 nlp_cloud aleph_alphar(novita)(r}rrrpget_model_cost_maprrr+rwrFrQrrrrrrrr~ setdefaultrO$_invalidate_model_cost_lowercase_mapr<open_ai_chat_completion_modelsaddopen_ai_text_completion_modelsrrropenrouter_modelsvercel_ai_gateway_modelsvertex_text_modelsvertex_code_text_modelsvertex_chat_modelsvertex_code_chat_models ai21_modelsnlp_cloud_modelsaleph_alpha_modelsbedrock_models novita_models)rQrrloaded_model_cost_registrations_registered_key_registered_value_skip_get_model_info_providersrrrFexisting_modelmodel_cost_keybuiltin_model_info builtin_entryr _raw_entry _cost_fieldupdated_dictionary split_stringrs @r3rr s>*d#& J $#66:FDU2@2F2F2H V .O.>BCT>U *? ; V-Y"'--/n/ U99/4s8 5 5= 2P= : %//33C!?!/!6!#!$ AhYa b  ,!6I 99U+3 8I8I%8P8\4A%4HN51III?@H "?@H "23; "89ESjIkIw#**B,3   0 1 9   14 8''++N;   ++//4J  JL 6K*,E1I"";5 60F%%nb9@@AST -.OQ_aop 99' (H 4'@@@66::3? YY) *.F F'@@@66::3? YY) *h 6'///%%))#. YY) *k 9'222((,,S1 YY) *l :99S!,LBw'@'@@))--l2.>? YY) *.A A':::0044S9 YY) *.E E'444**..s3 YY) *.J J'999//33C8 YY) *.E E'444**..s3 YY) *.J J'999//33C8 YY) *f 4'---##'', YY) *k 9'222((,,S1 YY) *m ;'444**..s3 YY) *i 7'000&&**3/ YY) *h 6'///%%))#.]n/^ r5c2|t|tr||vryyNTF)r}rr`additional_drop_paramss r3_should_drop_paramrY s")j9OQU.V[\`v[v r5ci}|jD](\}}||vs |||k7st||dus$|||<*|S)NrF)rr) passed_paramsdefault_paramsrnon_default_paramsr`ras r3_get_non_default_paramsr` s] "##%&1  ^A&&"Q?UVZ__$% q ! & r5c Fddlm} t} | jd| jd| jd| jd} | j D] \} } | | | < dddddd}| j D cic]\} } | |vs | || k7s| | c} } i}fd}d} t j |t }d k(sd k(r}nd k(rWtjj}|| tjj||nd}n6|4|j|}|| |j||nd}t|| | |jdd}|Scc} } w)NrOPENAI_TRANSCRIPTION_PARAMSrr drop_paramsrx)languagerrg temperaturetimestamp_granularitiesctjdkDrdtj}|D]D}dustjdur||vrj |d0||vs5t dddSy)NrTz1Setting user/encoding format is not supported by =. To drop it from the call, set `litellm.drop_params = True`. status_coderrkeysrrprrr)supported_paramsrr`rrrs r3_check_valid_argz;get_optional_params_transcription.._check_valid_arg s !&&( )A -16689D 4'7+>+>$+F//&**1d3..0$'"STgShif!g & % .r5rrFrr'groqrFrrrrrrrrr openai_paramsr)litellm.constantsrlocalsrrProviderConfigManager'get_provider_audio_transcription_configrrp GroqSTTConfigget_supported_openai_params_sttmap_openai_params_sttrmap_openai_params/add_provider_specific_params_to_optional_paramsrw)rrrrrgrrrrxrrspecial_paramsr`rarrrprovider_configrrs ` ` @r3!get_optional_params_transcriptionrm s>"8M34'++,AB##M2K2?2C2CH2MN$$&1 a#' N,9+>+>+@u41aQ.EX]^bpqrbs]s!Q$uO &<@O&/WW!"56X  h&*=*H,  &"002RRT*:;!//1GG1+'2'> E H   $*FFUFS*:;);;1+'2'> E < F'#/1%zz*BDI O ovs FF&Fc<|ydddddd}|j|dS)z3Map OpenAI size parameter to Vertex AI aspectRatio.z1:1z16:9z9:16)256x256512x512 1024x1024 1792x1024 1024x1792rv)sizesize_to_aspect_ratios r3*_map_openai_size_to_vertex_ai_aspect_ratior  s8 |  # # #D% 00r5c   tjddjdjdd} jdd jdd} jd} | jD]L\}}|jdr dk7rd k7s |d k(rd k7r,|jd r d k7rd k7rH||<Ndddddddddd }t || i} fd}| =| j xsd}||| j |xsd  nd}ndk(sdk(stjvr}ndk(rOtjj}|j }|||j i}n3d k(r.ddg} |||t||d<|t||d<t|j}| !| j xsd}t|}t|xsd|| }|jDcic]0\}}| t!|t"tfrt%|dkDs.||2}}}|Scc}}w)Nrrrrrrxaws_r( sagemakerrvertex_rr) nqualityrgrstyler imageConfigr!web_search_options)rrrc 6tjdkDr|tj}|D]\}tjdusdur)||vr%j |dj |dB||vsGt dd|dddSy) NrTrz Setting `z` is not supported by , rrr)rrr`rrrrrs r3rz7get_optional_params_image_gen.._check_valid_arg s !&&( )A -16689D ''4/;$3F//&**1d3!%%a...0$'"+A3.DEXDYY[\a[bc`!a & % .r5rrrFrrr'rrrr sampleCount aspectRatiorr)rrrrrrrrpopenai_compatible_providersBedrockImageGenerationget_config_classrOr rrrr}rr)rrrrgrrrrrrrrrxrr`rarrrr config_classrrrs` ` ` @@r3get_optional_params_image_genr s "8M   gt ,E'++,AB#''(94@O##M48K*../GN2?2C2CH2MN$$& 1 LL $ 16I[6XO##{2  LL #(;{(JObfvOv  a " N!8#%5! *,O&& "*FFU[VXFY*:;);;1++2'2'> E <  x' ' ) '"E"E E,  )%<<MMTYMZ 'CC%CP*:;&88L^pr8s  +=  *:; =-0VOM *  -WX\-]OM *#N$7$7$9:M"*FFU[VXFY-. E'#/52#5 O)..0AAM:VWZ^`dYeKfjmnojpstjt1O s J$J6Jc &ttjtd} t } | j dd| j d} | j dd| j dd}| j ddxsg}| j ddd;fd } t j| | |d} i}@tjjvr tjt } | a| j }| | | jind }|r|D]#}|tvr || vs| |||vs| |||<%nĉdk(rLFdvrBdvr>d|xsgvr6t j"dusdurj ddn t%dd}nsdk(rA| d}| | t!j&jind }n-dk(r?| xsddd}| | t!j(ji}ndk(r<| xsddd}| | t j*ji|}ndk(sdk(rdj-dd k(rj d| dd}| | t!j.ji|\}}n:d!k(rRt!j0j}| | t!j0ji}nd"k(rd#vrt!j2}nd$vrt!j4}nd%vrt!j6}nnd&vrt!j8}nUd'vsd(vrt!j:}n8d)j=vrt!j>}ng}| | i|}|S|j}| | |ji}nd*k(r;| d*d}| | t!j@ji}nd+k(rA| d+d}| | t!jBjind }nqd,k(r| d,d}| | t jDjGr,t!jDjind }nvt jHjKr,t!jHjind }n+t!jLjind }i||}|Sd-k(rA| d-d}| | t!jNjind }nId.k(rG| d.d}| | t!jPjind }i||}|Sd/k(r<| d/d}| | t!jRji0}nd1k(rA| d1d}| | t!jTjind }nvd2k(rA| d2d}| | t!jVjind }n0d3k(rdvrj d|d<tYj[d4kDrt j"dusdur3t]j[}|D]}j |dnt%dd5d6d7dk7rd8k7rt j^vr|tYj[d4kDr]t j"dusdur3t]j[}|D]}j |dnt%dd5d6d7}n}ta|| t]tcjZ|j-dd9}d:|vr#tY|d:d4k(r|j d:d|S)._check_valid_argn s  # $&# >A(((:1(="1% > ""d*{/F;Z^K^, #233QRdQeeqrwqxycd r5)rrrrrrrrFrrztext-embedding-3 dimensionsTrzSetting dimensions is not supported for OpenAI `text-embedding-3` and later models. To drop it from the call, set `litellm.drop_params = True`.rtriton embeddings)rr request_type databricksrr nvidia_nim)rrrxrrencoding_formatrN lm_studior(zamazon.titan-embed-text-v1zamazon.titan-embed-image-v1zamazon.titan-embed-text-v2:0z cohere.embed twelvelabsmarengonovamistraljina_aivoyagesapinfinity fireworks_ai)rrr sambanovaovhcloudollamarzSetting z is not supported by rr'r extra_body)r list | None)2rrrrrrPreProcessNonDefaultParams(embedding_pre_process_non_default_paramsr _member_map_rGrget_provider_embedding_configrrr*rprrTritonEmbeddingConfigDatabricksEmbeddingConfignvidiaNimEmbeddingConfigrwVertexAITextEmbeddingConfigLmStudioEmbeddingConfigrrrrrrrMistralEmbeddingConfigJinaAIEmbeddingConfigVoyageContextualEmbeddingConfigis_contextualized_embeddingsVoyageMultimodalEmbeddingConfigis_multimodal_embeddingsVoyageEmbeddingConfigGenAIHubEmbeddingConfigInfinityEmbeddingConfigFireworksAIEmbeddingConfigSambaNovaEmbeddingConfigOVHCloudEmbeddingConfigrrrrrr)rrr*r$rrrrrxrrrrrrrparamobject final_paramsrr`rs` `` @r3get_optional_params_embeddingsrRT s5 *1X1FHe)f"8M'++,A4H)--h7N##M48K*../GN)--.EtLRPR3T: !; c c#%/5 !d!37OO&+>,B[B[BbBbBd+d/MM!"56N  "(7(S(SZ_(S(`*:;);;1'2'> E <  ) B33M)mE.B.NSX`oSo-:5-AOE*  B  (  "%/ 22%:%@bA ""d*kT.A"&&|T:, #n-  (6 3%  *:;!779KK1'2'> E L   ,6+2 ,%  *:;!;;=OO12P   ,6+2 ,%  *:;!::LL12fM   +/Bh/N  ! !"3 4 ?  " "#4 56 +%  *:;  / / 1 C C12f D     +"::<XXZ*:;!99;MM12N   ) '5 0++- +e 3CCEF +u 4002F u $99;F U "i5&8==?F u{{} $668F!  .> ?%f:L !==?*:; 22FXjl2m  )6 )%  *:;!88:LL12M   )6 )%  *:;!779KK1'2'> E L   (6 (%  *:;  2 2 O OPU V%EEGYY#5 "+6+BK ZO  4 4 M Me T%EEGYY#5 "+6+BK ZO&;;=OO#5 "+6+BK PO 5/4V4   %6 %%  *:;!99;MM1'2'> E N   *6 *%  *:;!99;MM1'2'> E N 5/4V4   .6 .%  *:;!<<>PP12UQ   +6 +%  *:;!::<NN1'2'> E O   *6 *%  *:;!99;MM1'2'> E N   ( - -,>,B,B<,POL ) !&&( )A -""d*kT.A.33564A&**1d34- #&'9&::OPcOdebc x' 7 * w'J'J J !&&( )A -""d*kT.A.33564A&**1d34- #&'9&::OPcOdebc 1O,B'#/9>>@A%zz*BDI L|#L,F(G1(Lt, r5ct|tr3d|vr |ddur|d=|jD]\}}t||St|tr|D] }t||S)a clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240 Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088 additionalPropertiesF)r}rr_remove_additional_propertiesrrjrritems r3rUrUs &$ !V +7M0NRW0W-.!,,. 1JC )% 0 1 M FD ! 0D )$ / 0 Mr5ct|tr,d|vr|d=|jD]\}}t||St|tr|D] }t||S)zy Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088 strict)r}rr_remove_strict_from_schemarrVs r3rZrZsr&$ v x !,,. .JC &u - . M FD ! -D &t , - Mr5c|dkr|St|trM|jdd|jdd|jD]\}}t ||dz |St|t r|D]}t ||dz |S)a Remove JSON schema reference fields like '$id' and '$schema' that can cause issues with some providers. These fields are used for schema validation but can cause problems when the schema references are not accessible to the provider's validation system. Args: schema: The schema object to clean (dict, list, or other) max_depth: Maximum recursion depth to prevent infinite loops (default: 10) Relevant Issues: Mistral API grammar validation fails when schema contains $id and $schema references rz$idNz$schemar)r}rrr_remove_json_schema_refsr)rj max_depthrrrWs r3r\r\sA~ &$ 5$ 9d#!,,. ;JC $UIM : ; M FD ! :D $T9q= 9 : Mr5c~g}|iS|D]}||vs|j||D]}|j|d|S)z; Remove unsupported params from non_default_params N)rIr)rsupported_openai_params remove_keysrOrs r3_remove_unsupported_paramsrasaK& #& / /   u %&*sD)* r5ch|jDcic]\}}|tvs||c}}Scc}}w)a  Filter out LiteLLM internal parameters from kwargs dict. Returns a new dict containing only non-LiteLLM parameters that should be passed to external provider APIs. Args: kwargs: Dictionary that may contain LiteLLM internal parameters Returns: Dictionary with LiteLLM internal parameters filtered out Example: >>> kwargs = {"query": "test", "shared_session": session_obj, "metadata": {}} >>> filtered = filter_out_litellm_params(kwargs) >>> # filtered = {"query": "test"} )rr)rxrrs r3filter_out_litellm_paramsrcs.&*0 Y:33FX;XCJ YY Ys..c|dvry t|}tjd|}t t |ddS#t$rYywxYw)NrTFrrsupports_vertex_params)rrUrget_provider_chat_configboolr)rrFrs r3 _provider_supports_vertex_paramsrh)s`==&':;3KKRT_gKhO )A5I JJ s ? A  A cpeZdZe ddZe d ddZy)r:c|jD]^\}}|dk(r |jdr|dk7r|jdr&|dk(r|dk7s|jdr t|sZ|||<`|jDcic]E\}}|dk7r;|dk7r6|d k7r1|d k7r,|d k7r'|d k7r"||vr||vr|||k7rt|| dur||G}}}|Scc}}w)Naws_bedrock_project_idr r(r rrrr api_versionrrrrF)rrrhr) rrrrdefault_param_values#additional_endpoint_specific_paramsr`rars r3#base_pre_process_non_default_paramsz>PreProcessNonDefaultParams.base_pre_process_non_default_params5s$#((* !DAq,, V$(I5>Q>\>\]h>i'';6<< *89LM M!  !&&++-% 1W ..&&0011@@---a00&CYZ^cc qD% % ""!#% sA Cc ntj||||tDcic]}|dc}dg}|Scc}w)Nrrrrrrmrn)r:ror*) rrrrrremove_sensitive_keysadd_provider_specific_paramsr`rs r3r;zCPreProcessNonDefaultParams.embedding_pre_process_non_default_paramsbsL%?$b$b') 3#93J!Ka!T'!K18 %c% "! "Ls 2N)rrrrrrrr2rmrrn list[str]rr)FF)rrrrrrrr2rrrrrgrsrgrr)rrr staticmethodror;rr5r3r:r:4s*"*"*"!*"!1 *" # *" .7 *" *"*"X',-2"""!"!1 "  " $ "'+" ""r5r:c tj||||tdg}d|vr-||j|d|d<nt |d|d<d|vrzt |t rj|d} | D]`} | jdi} | jdd} | *tj| } d | vr| d d ur| jd d| | d<b|r+t|||t tj| }|r t|}|S) zA Pre-process non-default params to a standardized format rrqrgNrir!functionrErTFr)r:ror$get_json_schema_from_pydantic_objectrr}rrwrdeepcopyrrrremove_sensitive_keys_from_dict)rrrrrrrrsrrr!r tool_functionrEnew_parameterss r3pre_process_non_default_paramsr}xsa4WW#%/5A-7L X..  &4C4h4h 23D E5i5 0 15R 23D E5 0 1$$D**'2 =D HHZ4M&**<>J%!%z!:)^;Oe@fjo@o"&&'=tD.< l+ =$L.' 3CHHJK#9  <=OP r5cgd}g}|D](tfd|Ds|j*|D]|j|S)z1 Remove sensitive keys from a dictionary )rsecretaccess credentialc3BK|]}|jvywr;r)r@phrasers r3rBz2remove_sensitive_keys_from_dict..sIv$Is)rFrIr)dsensitive_key_phrasesr`rs @r3rzrzs^$MK$ I3HI I   s #$ c  Hr5c i}tj}||dvr |dk(r&tjj||}n|dk(r&tjj||}nZ|dk(s|dk(r&tj j||}n*|dk(r%tj j||}d|vs d |vsd |vr6|d k(r0|d k7r*|dk7r$|dk7r|d k7r|dk7r|dk7r |dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|dk7r|tjvr|d k(rVd |d!<d"t_d |vr(|jd |d#<|jd$d%|Sd|vr|jd|d#<|Stjr'|jd |jdd%|d#<|Std&d'|d()|S)*z-For .completion(), preprocess optional params providersr'rr(rrwatsonx functions function_callr!r7ranyscale together_airr)cerebrasxai ai21_chat volcenginedeepseek codestralr/r cohere_chatr ollama_chatrrnebiuswandbrformatTfunctions_unsupported_modelr"Nrz%Function calling is not supported by rr) rp!common_cloud_provider_auth_paramsAzureOpenAIConfigmap_special_auth_paramsAmazonBedrockGlobalConfigVertexAIConfigIBMWatsonXAIConfigradd_function_to_promptrr)rrrrcommon_auth_dicts r3pre_process_optional_paramsrsO%GG.{;;  ' )%779QQ#0/RO!I -%??AYY#0/ZO!K /3FJZ3Z%446NN#0/OO!I -%88:RR#0/SO ((O?Q,QU\`rUr 8 +#'??#w.#{2#z1#}4#v-#|3#z1#u,#{2#|3#z1#{2#y0#{2#}4#x/#y0#}4#|3#'::#x/#w.#7+N+NN"h.,2)15.00EWE[E[\cEdO$AB&**=$? !$66EWE[E[\gEhO$AB //ASAWAW/33KFB =>  - #CDWCXXYZ r5c) $;<tj<vstj@vr=tjBj!;|-t#t$rnd }- nndk(r=tjDj!;|-t#t$rnd }- n,dk(s dk(rAdvr=tjBj!;|-t#t$rnd }- ntjFjI r=tjFj!;|-t#t$rnd }- ndk(rItjJvr~dvr=tjLj!;|-t#t$rnd }- n*tjNj!;|-t#t$rnd }- ntjPvr=tjRj!;|-t#t$rnd }- n|++|+j!;|-t#t$rnd }- nqtjTj!;|-t#t$rnd }- n4dk(r=tjVj!;|-t#t$rnd }- ndk(rttjtd}1|1jX}2|1jZ}3|2d k(s|2d!k(r=tj\j!;|-t#t$rnd }- n`|2d k(r=tj^j!;|-t#t$rnd }- nd |3vr|2d"k(r|3tj`jcvr=tj`j!;|-t#t$rnd }- ntjdj!;|-t#t$rnd }- n{|+ x|+j!;|-t#t$rnd }-|2d#k( rI|1jf<|-$}- n3d%k(r=tjhj!;|-t#t$rnd }-nd&k(r=tjjj!;|-t#t$rnd }-nd'k(r=tjlj!;|-t#t$rnd }-nmd(k(r=tjnj!;|-t#t$rnd }-n+d)k(r=tjpj!;|-t#t$rnd }-nd*k(r=tjrj!;|-t#t$rnd }-nd+k(r-|++|+j!;|-t#t$rnd }-nud,k(sdk(r=tjNj!;|-t#t$rnd }-n.d-k(r=tjLj!;|-t#t$rnd }-nd.k(r=tjtj!;|-t#t$rnd }-nd/k(r=tjvj!;|-t#t$rnd }-nhd0k(r=tjxj!;|-t#t$rnd }-n&d1k(r=tjzj!;|-t#t$rnd }-nd2k(r(tj|j!;|-3}-nd4k(sd5k(r=tj~j!;|-t#t$rnd }-npd6k(r=tjj!;|-t#t$rnd }-n.d7k(r=tjj!;|-t#t$rnd }-nd8k(r=tjj!;|-t#t$rnd }-nd9k(r=tjj!;|-t#t$rnd }-nhd:k(r=tjj!;|-t#t$rnd }-n&d;k(r=tjj!;|-t#t$rnd }-ndk(r=tjj!;|-t#t$rnd }-nd?k(rutjj!;|-t#t$rnd }-._check_valid_argZsP MuVijJMZVXjk$&# >A((338Q(:1(="1%  > ""d*{/F;Z^K^+4A&**1d34- #233QRVWiWnWnWpRqQrr~@EFF|}ABTBYBYB[}\|]]no r5r)rrrrrrrF)rrrranthropic_textrrrr%maritalkr predibase huggingfacerrrrrr r(rconverse converse_likeinvokeclaude_platform)rr cloudflarer7rrpetals deepinfra perplexityr/text-completion-codestraltext-completion-inceptionr(r)rr)rrrrrr4r hosted_vllmvllmrbedrock_mantlertencentrrzLiteLLM now defaults to Watsonx's `/text/chat` endpoint. Please use the `watsonx_text` provider instead, to call the `/text/generation` endpoint. Param: watsonx_textrr'rzoAzure optional params - api_version: api_version={}, litellm.api_version={}, os.environ['AZURE_API_VERSION']={}AZURE_API_VERSION)rrrrlrrz Final returned optional params: )rrrrr)rrt)arrrrrrrfr}rrrrrrrpAnthropicConfigrr}rgAnthropicTextConfigCohereChatConfig TritonConfigMaritalkConfigReplicateConfigPredibaseConfigHuggingFaceChatConfigTogetherAIChatConfigrrrrvertex_language_modelsvertex_vision_modelsVertexGeminiConfigGoogleAIStudioGeminiConfigVertexAIAnthropicConfigis_supported_modelvertex_mistral_modelsCodestralTextCompletionConfig MistralConfigvertex_ai_ai21_modelsVertexAIAi21ConfigVertexAILlama3ConfigSagemakerConfigget_bedrock_routeget_base_modelAmazonConverseConfigAmazonBedrockOpenAIConfigAmazonAnthropicConfig get_legacy_anthropic_model_namesAmazonAnthropicClaudeConfigmap_claude_platform_auth_paramsCloudflareChatConfig OllamaConfigOllamaChatConfigNLPCloudConfig PetalsConfigDeepInfraConfigInceptionTextCompletionConfigDatabricksConfigNvidiaNimConfigCerebrasConfig XAIChatConfigAI21ChatConfigFireworksAIConfigVolcEngineConfigHostedVLLMChatConfig VLLMConfigGroqChatConfigBedrockMantleChatConfigDeepSeekChatConfigTencentChatConfigOpenrouterConfigIBMWatsonXChatConfigris_watsonx_text_paramrU OpenAIConfig NebiusConfigAzureOpenAIO1Configis_o_series_modelAzureOpenAIGPT5Configis_model_gpt_5_modelrr<rrlrAZURE_DEFAULT_API_VERSIONrOpenAILikeChatConfigrrrrrD_apply_openai_param_overrides)=rrrrtop_prrrstoprmax_completion_tokens modalities predictionaudiopresence_penaltyfrequency_penalty logit_biasrrrgseedr!r" max_retrieslogprobs top_logprobs extra_headersrlparallel_tool_callsrrreasoning_effort verbosityrrthinkingrsafety_identifierstoreprompt_cache_keyrrxrrrFrrrrr bedrock_routebedrock_base_modelrO_azure_detection_modelrrrA nested_pathspathrrs=` ` ` ` @@r3get_optional_paramsrs\"8==?M)--h7NlD))-O&+>bnBoV^8>>Bo+o/HH!"56!I  !?#%/5' !2#-/O :*1X1FHe)f2)<6UX`a'-217R12)/Rk)!113EE1+(3(?J{\`Dagl F   0 0!557II1+(3(?J{\`Dagl J  "557II1+(3(?J{\`Dagl J   -1D1P!224FF1+(3(?J{\`Dagl G   (!..0BB1+'2'> E C   *!002DD1+(3(?J{\`Dagl E   +!113EE1+(3(?J{\`Dagl F   +!113EE1+(3(?J{\`Dagl F   -!779KK1+(3(?J{\`Dagl L   -!668JJ1+(3(?J{\`Dagl K   + +++ G33 3 G.. . G33 3 G22 2 G00 0!446HH1+(3(?J{\`Dagl I   (!<<>PP1+(3(?J{\`Dagl Q   0 05HK5W\dhm\m!446HH1+(3(?J{\`Dagl I   ( ( ; ;%]p ; q!99;MM1+(3(?J{\`Dagl N   + G11 1e#")"G"G"I"["['9$30;0GJWbdhLiot #\##*"7"7"9"K"K'9$30;0GJWbdhLiot #L# g33 3%88:LL#5 /,7,C S^`dHe[kp MO  (-??#5 /,7,C S^`dHe[kp @O&::<NN#5 /,7,C S^`dHe[kp OO  +!113EE1+(3(?J{\`Dagl F   )")#++h*?AS"TA/AA%H $C$4$C$CE$J J &-?*J%::<NN#5 /,7,C S^`dHe[kp OO h &%??ASS#5 /,7,C S^`dHe[kp TO . .=H3L!W%B%B%c%c%ee")"?"?"A"S"S'9$30;0GJWbdhLiot #T##*"E"E"G"Y"Y'9$30;0GJWbdhLiot #Z#  (-??#5 /,7,C S^`dHe[kp @O  11"R"2"R"R"/#  ,!668JJ1+(3(?J{\`Dagl K   (!..0BB1+(3(?J{\`Dagl C   -!224FF1+(3(?J{\`Dagl G   +!002DD1+(3(?J{\`Dagl E   (!..0BB1+(3(?J{\`Dagl C   +!113EE1+(3(?J{\`Dagl F   ,1L);;1+(3(?J{\`Dagl <   )-@K-O!//1CC1+(3(?J{\`Dagl D   ; ;!??ASS1+(3(?J{\`Dagl T   ; ;!??ASS1+(3(?J{\`Dagl T   ,!224FF1+(3(?J{\`Dagl G   ,!113EE1+(3(?J{\`Dagl F   *!002DD1+(3(?J{\`Dagl E   %!//1CC1+D   +/Bf/L!002DD1+(3(?J{\`Dagl E   .!335GG1+(3(?J{\`Dagl H   ,!224FF1+(3(?J{\`Dagl G   -!668JJ1+(3(?J{\`Dagl K   &!,,.@@1+(3(?J{\`Dagl A   &!002DD1+(3(?J{\`Dagl E   0 0!99;MM1+(3(?J{\`Dagl N   *!446HH1+(3(?J{\`Dagl I   )!335GG1+(3(?J{\`Dagl H   ,!224FF1+(3(?J{\`Dagl G   )!668JJ1+(3(?J{\`Dagl K # E))+AA%H pqvpwx   .!446HH1+(3(?J{\`Dagl I   (!..0BB1+(3(?J{\`Dagl C   (!..0BB1+(3(?J{\`Dagl C   '(2(;e  & & ( : :AW : X%99;MM#5 /,,7,C S^`dHe[kp NO  * * ? ?F\ ? ]%;;=OO#5 /,,7,C S^`dHe[kp PO  BII!4!4jAT6U  5&&512544  &779KK#5 /,',7,C S^`dHe[kp LO  $);;1+(3(?J{\`Dagl < "668JJ1+(3(?J{\`Dagl K F'#/?DDFG5 O4_4EFG3'-3O ' H(=?O P%,S[[-BDY%Z*@VQNSTDUqV V  ID1/4HO I wCpnWsAI&I AI+IAI+c2|gdtjzvrtd|durt|j ddxsi}|D]}||vs||||||<t |j dtsi|d<i|d|}|)|jDcic] \}}||vs ||} }}n|} ttjtd} | | |d<|S|D]#}||vs||t||r||||<%|Scc}}w)z9 Add provider specific params to optional_params )rr'rr8rFNr)r8) rprrrrr}rwrrrrr) rrrrrr8r`initial_extra_bodyraprocessed_extra_bodyrs r3rrdskKgNqNqqq  E[ \`e e $]%6%6|T%J%Pb QJ" 5M)mA.>.J$1!$4JqM 5o11,?F02 -)!,/))  &19K9Q9Q9S'wAWX`vWv1'w$'w'9$07 H8MOk0l &,FRf,gOL )   6A %-*:*F%BXY%21%5"  6 (xs " D/DcT|r%|D] }||vr||vr |j|||<"|S)a_ If user passes in allowed_openai_params, apply them to optional_params These params will get passed as is to the LLM API since the user opted in to passing them in the request Only params the caller actually sent are forwarded. Previously this function unconditionally wrote `None` for any allowed param missing from the request, which then reached the provider SDK as a top-level kwarg it did not recognize (e.g. openai SDK raising `AsyncCompletions.create() got an unexpected keyword argument 'enable_thinking'`). See https://github.com/BerriAI/litellm/issues/25697 )r)rrrrOs r3rrsL* CE'..%7%;%;E%BOE "  C r5>rrrrrcNt|}tt|tz S)aThe params a provider can actually be rejected for, i.e. the ones _check_valid_arg compares against its supported list. Anything outside this set never reaches that comparison. Endpoint and transport controls such as base_url, timeout, default_headers, organization and deployment_id are not chat completion params at all, so a caller filtering on "is this an OpenAI param" would discard configuration the request needs while never touching what the provider would have rejected. )r frozensetget_non_default_paramsr)rrIs r3provider_rejectable_paramsrs%'F +F3 47R RRr5c|jDcic]&\}}|dk7r|dk7r|tvr|t|k7r||(}}}|Scc}}w)Nrr)rr)rr`rars r3rrsg"'') ! Aq L**999!<< 1 ! !  !s+AcT||S||S|t|dz dz}|dk(rd}|S||Sy)a  Returns the max parallel requests to send to a deployment. Used in semaphore for async requests on router. Parameters: - max_parallel_requests - Optional[int] - max_parallel_requests allowed for that deployment - rpm - Optional[int] - requests per minute allowed for that deployment - tpm - Optional[int] - tokens per minute allowed for that deployment - default_max_parallel_requests - Optional[int] - default_max_parallel_requests allowed for any deployment Returns: - int or None (if all params are None) Order: max_parallel_requests > rpm > tpm / 6 (azure formula) > default max_parallel_requests Azure RPM formula: 6 rpm per 1000 TPM https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits Nrr)rO)max_parallel_requestsrpmtpmdefault_max_parallel_requestscalculated_rpms r3calculate_max_parallel_requestsr#sS:($$  S4Z!^, Q N & 2,, r5c|jdijd}|!|jdijd}|S)z Returns the routing order for a deployment. Checks litellm_params first (static config), then model_info (dynamic/team models added via API where order lives in model_info, not litellm_params). rorderr7rv) deploymentr%s r3_get_deployment_orderr'sE NN+R 0 4 4W =E }|R044W= Lr5c|$|Dcgc]}t||k(s|}}|r|S|S|Dcgc]}t|}||}}}|r t|nd}| |Dcgc]}t||k(s|}}|S|Scc}wcc}}wcc}wr;)r'min) healthy_deployments target_orderrfilteredr&o _valid_orders min_orderfiltered_deploymentss r3_get_order_filtered_deploymentsr1s&9f=RST=UYe=e1ff O""0':OPZ:[A`a`m'M':G3}#5DI)<' %@UV`@aen@nJ' ' $# %g' ' s A9A9A>A>B/Bc|s|St|}|Dcgc]+}|jdxsijd|vs*|-c}Scc}w)a= Filter out deployments whose `model_info.id` appears in `excluded_deployment_ids`. Used by weighted-routing failover so a single logical request can re-pick across the remaining deployments in the same model group after one of them has failed. If the filter would leave no deployments, an empty list is returned so the caller raises its usual no-deployments error and the weighted-failover helper falls through to the cross-group fallback path. Returning the original unfiltered list here would re-include the just-failed deployment. r7r)rrw)r*excluded_deployment_ids excluded_setrs r3"_get_excluded_filtered_deploymentsr5sM #""56L* h!quu\/B/Hb.M.Md.S[g.gA hh hs +AAc@|dk(rX|jxs*tjxstdxs td}|t|tr|S|jS|dk(r|j }||S|jS|dk(r|j }||S|jS)z= Return the region for a model, for a given provider rVERTEXAI_LOCATIONVERTEX_LOCATIONr(r)vertex_locationrprr}raws_region_namewatsonx_region_name region_name)rrvertex_ai_locationr:r;s r3_get_model_regionr>*sk)  * * -&& --. -+,   )j9KS.Q% %  % %%  )!/!?!?  &" "  % %%  )%3%G%G  *& &  % %%r5cttj|j|\}}}}t||}|!t j d|jy|dk(rEtj j}tj j}n|dk(rEtjj}tjj}n|dk(rEtjj}tjj}nN|dk(rEtjj}tjj}ng}g}|D]}||jvsy |D]}||jvsy y) z Infer if a model is in the EU or US region Returns: - str (region) - "eu" or "us" - None (if region not found) )rr)rrNz'Cannot infer model region for model: %sr'rr(reuus) rprrr>rr<rget_eu_regionsget_us_regionsrrrr)rrrr1 model_region eu_regions us_regionsregions r3_infer_model_regionrHCs(/'?'?"">($E 1,@SdrsLFH\H\]g%..0??A ..0??A  +++-<<> ++-<<>  )668GGI 668GGI  )//1@@B //1@@B    \'') ) \'') ) r5cL|jdk(ryt|}||dk(ryy)8 Return true/false if a deployment is in the EU r@TrFr<rHrrDs r3 _is_region_eurNn4!!T).^LLLD$8 r5cL|jdk(ryt|}||dk(ryy)z8 Return true/false if a deployment is in the US rATrKFrLrMs r3 _is_region_usrQ|rOr5c$|j|k(ryy)rJTF)r<)rallowed_model_regions r3is_region_allowedrTs!!%99 r5c d|jvrt|jtrt|jtr|jj dd}t jj||j|j|jxst jd|xsd}|jdd}|Sy) zM Pass the litellm params for an azure model, and get back the region r'zazure/r chat)rapi_keyrrlrMmodez x-ms-regionN) rr}rXrrreplacerpAzureChatCompletion get_headersrlrrw)rrY_modelr\rGs r3get_model_regionr^s >''' ~--s 3 ~.. 4&,,44XrB ' ; ; = I I"**#,,&22Wg6W6W !J! %-LL$E r5cd |jd}t|dd}|S#t$rYywxYw)Nrdr)rwrrU)rx _messagess r3get_first_chars_messagesrbs=JJz*  N4C(  s # //cHdjd|D}t|S)Nrc3BK|]}|jr|ywr;)isspace)r@chars r3rBz$_count_characters..s"OD 4"Os)r r)r  filtered_texts r3_count_charactersrhs!77"OD"OOM } r5ct|drt|drt|dd}|rt|dr|j}g}|D]p}t|dr:|jD]*}t|ds|j |j ,It|dsV|j |j rdj |St|drBt|dr6t|dd}d|vsd |vr!t|dd}t|tr|SdS|j}g} |D]} t| trF| jj*| j t| jjYt| tsj| jj| j t| jjdj | S) NrLr\outputrr rdeltaz text.deltazoutput_text.delta)rrrjrrIr r r}rrnr~rrrk) response_objresponses_api_response output_listresponse_output_parts output_item content_part event_typerk_choicesresponse_partschoices r3get_response_stringrvs|V$z)J)0 j$(O !g.Dh&O!7!>!>K79 !* C ; 2(3(;(;L "<81889J9JKL[&1)001A1AB C7701 1|V$w)G#L&"= : %)< )J"<"=E&uc25 : :?K>R>RH(*NA fg &~~%%1%%c&..*@*@&AB  0 1||##/%%c&,,*>*>&?@ A 77> ""r5c"|xstj}|dk(s|dk(r#|xstjxs td}|S|dk(s|dk(r#|xstjxs td}|S|dk(r#|xstj xs td}|S|d k(r#|xstj xs td }|S|d k(r#|xstjxs td }|S|d k(s|dk(r#|xstjxs td}|S|dk(r#|xstjxs td}|S|dk(r#|xstjxs td}|S|dk(r#|xstjxs td}|S|dk(r0|xs*tjxstdxs td}|S|dk(r#|xstjxs td}|S|dk(r!|xstjxs td}|S)NrrOPENAI_API_KEYrrANTHROPIC_API_KEYr AI21_API_KEYrALEPH_ALPHA_API_KEYbasetenBASETEN_API_KEYrrCOHERE_API_KEYrHUGGINGFACE_API_KEYrNLP_CLOUD_API_KEYrREPLICATE_API_KEYrTOGETHERAI_API_KEYTOGETHER_AI_TOKENrNEBIUS_API_KEYr WANDB_API_KEY)rprX openai_keyr anthropic_keyai21_keyaleph_alpha_key baseten_key cohere_keyhuggingface_key nlp_cloud_key replicate_keytogetherai_api_key nebius_key wandb_key) llm_providerdynamic_api_keyrXs r3 get_api_keyrsC0Gx<3K#KOW//O:>N3OH NE  $ 8H(HUW22UjAT6UB N?  KW--KN1K< N9  &YW44Y CX8Y6 N3  "QW00QJ?P4Q0 N-  !\]%BOW//O:>N3O* N'  &YW44Y CX8Y$ N!  $UW22UjAT6U N  $UW22UjAT6U N  &  xw11 xZ@T5U xYcdwYx  N  !OW//O:>N3O N  MW..M*_2M Nr5cddl}ddlm}t|dr|j|jS|jS)NrrUTC)rrrrutcnow)dtrs r3get_utc_datetimers6!r5x||BFF##x  r5cTd} |tjvrVdtj|vrtj|dSdtj|vrtj|dStj}||\}}}}|dk(r ||}|S|tjvrWdtj|vrtj|dSdtj|vrtj|dSyt #t$rt d|d wxYw) aY Get the maximum number of output tokens allowed for a given model. Parameters: model (str): The name of the model. Returns: int: The maximum number of tokens allowed for the given model. Raises: Exception: If the model is not mapped yet. Example: >>> get_max_tokens("gpt-4") 8192 cd|d} tjj|t}|j |j }|jd}||Sy#t $rYywxYwNzhttps://huggingface.co/z/raw/main/config.json)rMmax_position_embeddingsrpmodule_level_clientrwr#raise_for_statusrrU model_name config_urlr\ config_jsonrs r3_get_max_position_embeddingsz4get_max_tokens.._get_max_position_embeddings/s|5j\AVW  %99==jRq=rH  % % '5=MMOK-8__=V-W #&2..  AA## A/.A/max_output_tokensrrrrNzModel zr isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)rprQrrrU)rrrrr1rs r3get_max_tokensrsD$& G&& &"g&8&8&??))%01DEE!3!3E!::))%0>>"/"@"@+;%+H("Aq - / < NJ  G&& &"g&8&8&??))%01DEE!3!3E!::))%0>>+   UGN O   s) "meta.llama3-2-11b-instruct-v1" AND "meta.llama3-2-11b-instruct-v1:0" -> "meta.llama3-2-11b-instruct-v1" rr)!litellm.llms.bedrock.common_utilsrr)rrs r3_get_base_bedrock_modelr_sC *  * *: 66r5c0tjdd|S)a Strips the organization, custom suffix, and ID from an OpenAI fine-tuned model name. input: ft:gpt-3.5-turbo:my-org:custom_suffix:id output: ft:gpt-3.5-turbo Args: model_name (str): The full model name Returns: str: The stripped model name z (:[^:]+){3}$rrrs r3!_strip_openai_finetune_model_namerks 66/2z 22r5c|r|dvrt|}|S|r |dk(s |dk(s|r|dk(rt|}|Sd|vrt|}|S|S)Nr(bedrock_converserrrr(zft:)rrr)rrstripped_bedrock_model strip_versionstrip_finetunes r3rr{si26UU(?5(Q%% K /3F(3R  L 0;uM  % AU S r5dict[str, str] | None_model_cost_lowercase_map_model_cost_mutation_generationctSr;)rrr5r3"get_model_cost_mutation_generationrs **r5cjdatdz atjtjy)zInvalidate the case-insensitive lookup map for model_cost. Call this whenever litellm.model_cost is modified to ensure the map is rebuilt. Also clears related LRU caches that depend on model_cost data. Nr)rr_cached_get_model_info cache_clear_cached_get_model_info_helperrr5r3rrs/!%#q(#&&(!--/r5crtjDcic]}|j|c}atScc}w)zRebuild the case-insensitive lookup map from the current model_cost. Returns: The rebuilt map (guaranteed to be not None). )rprQrr)r`s r3!_rebuild_model_cost_lowercase_maprs18?7I7I J!A J $$!Ks4cntatj|}||tjvr|Sy)z Handle stale _model_cost_lowercase_map entry (key was popped). Rebuilds the map and retries the lookup. Returns: The matched key if found after rebuild, None otherwise. N)rrrwrprQ)potential_key_lower matched_keys r3_handle_stale_map_entry_rebuildrs;!B C2667JKK;'2D2D#D r5cptjD]#}|j|k(sta|cSy)z Handle new key added to model_cost without invalidating _model_cost_lowercase_map. Scans model_cost for case-insensitive match and rebuilds the map if found. Returns: The matched key if found, None otherwise. N)rprQrrr)rrs r3_handle_new_key_with_scanrs;!! 99;- -(I(K %J r5c|tjvr|St ta|j }tj |}||tjvr|S|t |}||Sy)a Get the actual key from model_cost, with case-insensitive fallback. WARNING: Only O(1) lookup operations are acceptable. O(n) lookups will cause severe CPU overhead. This function is called frequently during router operations. ALLOWED HELPER FUNCTIONS (conditionally called, O(n) operations are acceptable): - _rebuild_model_cost_lowercase_map: Rebuilds the lookup map (only when map is None) - _handle_stale_map_entry_rebuild: Rebuilds map when stale entry detected (rare case) If you need to add a new helper function with O(n) operations that is conditionally called and confirmed not to cause performance issues, add it to the allowed_helpers list in: tests/code_coverage_tests/check_get_model_cost_key_performance.py N)rprQrrrrwr) potential_keyrrs r3rPrPs$***!($E$G!!.!4!4!6+//0CDK;'2D2D#D56IJ  "  r5c(tj|Sr;)rprQrs r3_get_model_info_from_model_costrs   c ""r5c|r|jdz|d|k7rr|dk(r|djds>|dk(r|djds%|jdr|djdry|dk(ry|dk(r|ddvry|d k(ryy y) a Check if the model info provider matches the custom provider. A missing ``litellm_provider`` key and a ``litellm_provider`` set to ``None`` both mean "no specific provider constraint" and are treated as a wildcard match. ``register_model`` may persist ``None`` here via ``get_model_info`` when a deployment is registered without a provider, so normalising the two cases keeps custom pricing applied consistently. rrr4r(T litellm_proxyazure_ai)r'rgithubF)rwrr7rs r3_check_provider_matchr s)*6:FX;Y]p;p ; .-.99+F"n4-.99.I"--i8-.99)D ? 2 J .:>P3QV 4  H , r5)ReadOnly TypedDictcJeZdZUded<ded<ded<ded<ded<ded<y ) 'PotentialModelNamesAndCustomLLMProviderr split_modelcombined_model_namestripped_model_namecombined_stripped_model_namez ReadOnly[str]provider_prefixed_model_namerN)rrrrrr5r3rr7s&"%%"//r5rc|d||d|d|d|df}td|Dry|D]#}t|}||||fcS|i|d|ifcSy) aLResolve an unmapped model via the declarative capability generalization rules. Tries the same name candidates as the exact lookups, in the same order, and returns ``(matched_name, model_info)`` for the first candidate matched by at least one capability rule, with ``litellm_provider`` backfilled from the provider the caller requested. Rules lose to exact entries: if ANY candidate is an exact ``litellm.model_cost`` key (necessarily provider-mismatched, or the exact lookups would have returned it), the model is known rather than unmapped, and resolving it from rules would hand an unpriced rule-derived entry to callers whose fallback ladder (e.g. the cost calculator's model-name variants) still had a priced exact name to try. O(number of rules); only call after the exact lookups have missed. rrrrrc36K|]}t|duywr;)rP)r@rs r3rBz6_get_model_info_from_generalization..Zs R) y ) 5 RsNr)rFr-)rpotential_model_namesrrrgeneralized_infos r3#_get_model_info_from_generalizationr@s& 34 m,<=34<= J Rz RRX ;IF  #   &.. .W-W/ACVWWW X r5c |3 tj}||\}}}}|}t||}|}|}nh|rG|j |dzr3|j ddd}|}t||}|d|}|d|}n|}|d|}t||}|d|}|}|dvrddlm} | |}t|||||tt|S#t$r|}YwxYw) NrrrNrrr)strip_bedrock_routing_prefix)rrrrrr) rrrUrrrrrrrr) rrrrr1rrrrrs r3_get_potential_model_namesrfsJ" &3&D&D 5EE5R 2K,a$/eQde':$',$ !1!1c!"kk#q)!, #/kWjk*=)>a@S?T'U$*=)>aw'G$ !4 5Qug>/eQde*=)>a@S?T'U$':$==R2;? 2//%A%A &9:  5 K sC CCcd|d} tjj|t}|j |j }|jd}||Sy#t $rYywxYwrrrs r3rrs~1*=RSJ!5599*Nm9n!!#19  *59R)S " .* * rct|||S)z_ _get_model_info_helper wrapped with lru_cache Speed Optimization to hit high RPS rrr)r6rs r3rrs "/ r5cd}|r(|tvr tj|t|}d}|r|j |}|S)Nrr)rrget_provider_model_inforrH)rrrr7s r3rHrHsX04O2oE/GG,/B"CH 48J$66U6C r5cBtd|jDS)zT Check if the potential model name is in the model cost (case-insensitive). c3HK|]}tt|duywr;)rPr)r@potential_model_names r3rBz9_is_potential_model_name_in_model_cost..s(  C 456dBs ")rFrG)rs r3&_is_potential_model_name_in_model_costrs' $9$@$@$B r5z_above_\d+k?_tokens$_ABOVE_THRESHOLD_COST_KEYc ddlm} itjtj}||vr||}||dk(rd}|O|dk(rJd|ztj vrd|z}n/|dztj vs|dztjvr|dz}t||xs||}tjd ||d }|d }|d } |d } |d} |d}|} d} |r(|tvr tj|t|} | )t| dd}t!|r ||||}||S |dk(rHt'|}t)did|d|ddddddddddddddd dd!dd"dd#dd$dd%dd&dd'dS d}d}t+|}|+|}t-t/t0|(}t3|| )sd}|8t+|}|+|}t-t/t0|(}t3|| )sd}|8t+| }|+|}t-t/t0|(}t3|| )sd}|8t+| }|+|}t-t/t0|(}t3|| )sd}|8t+|}|+|}t-t/t0|(}t3|| )sd}|8t+| }|+|}t-t/t0|(}t3|| )sd}|t5|||*}||\}}|| t7d+|j9d}|tjd,||d}|j9d}|tjd-||d}t)did|d|j9ddd|j9ddd|j9ddd|d.|j9d.dd/|j9d/dd0|j9d0dd1|j9d1dd2|j9d2dd3|j9d3dd4|j9d4dd5|j9d5dd6|j9d6dd7|j9d7dd8|j9d8dd9|j9d9dd:|j9d:dd;|j9d;dd<|j9d|j9d>dd?|j9d?dd@|j9d@ddA|j9dAddB|j9dBddC|j9dCddD|j9dDddE|j9dEddF|j9dFddG|j9dGddH|j9dHddI|j9dIddJ|j9dJddK|j9dKddL|j9dLddM|j9dMddN|j9dNddO|j9dOddP|j9dPddQ|j9dQddR|j9dRddS|j9dSddT|j9dTddU|j9dUdV|j9dVd|dW|j9dWddX|j9dXddY|j9dYddZ|j9dZdd[|j9d[dd\|j9d\dd]|j9d]dd^|j9d^dd_|j9d_dd`|j9d`dda|j9daddb|j9dbddc|j9dcddd|j9dddde|j9deddf|j9dfddg|j9dgddh|j9dhddi|j9diddj|j9djddk|j9dkddl|j9dlddm|j9dmddn|j9dnddo|j9doddp|j9dpddq|j9dqddr|j9drdds|j9dsddt|j9dtdd|j9d|d|j9ddu|j9dudd|j9ddd |j9d ddv|j9dvdd!|j9d!ddw|j9dwdd"|j9d"dd#|j9d#dd$|j9d$dd%|j9d%ddx|j9dxddy|j9dydd'|j9d'ddz|j9dzdd{|j9d{dd||j9d|dd}|j9d}dd~|j9d~dd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd&|j9d&dd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9ddd|j9dd}|j;D]%\}}||vs t<j?|!|||<'|S#t"$r#}tj$d|||Yd}~ d}~wwxYw#t"$r-}tjd|t#d|d|dd}~wwxYw)zy Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's rrLNrrzmeta/z@latestrz8checking potential_model_names in litellm.model_cost: %srrrrrrrr)rrrXzcCould not get dynamic model info for model=%s, provider=%s; falling back to the static cost map: %srrrrmax_input_tokensrrrrrYrWr,r9rDrFsupports_assistant_prefillrgrirkrcrr)rrrz|This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.jsonz`model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.zamodel=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.input_cost_per_token_flexinput_cost_per_token_priorityinput_cost_per_token_ultrafastrr1cache_creation_input_token_cost_above_272k_tokens:cache_creation_input_token_cost_above_272k_tokens_priority6cache_creation_input_token_cost_above_272k_tokens_flex$cache_creation_input_token_cost_flex(cache_creation_input_token_cost_priority)cache_creation_input_token_cost_ultrafastrprompt_cache_min_tokensr6cache_read_input_token_cost_above_200k_tokens_priority-cache_read_input_token_cost_above_272k_tokens6cache_read_input_token_cost_above_272k_tokens_priority2cache_read_input_token_cost_above_272k_tokens_flex-cache_read_input_token_cost_above_512k_tokens cache_read_input_token_cost_flex$cache_read_input_token_cost_priority%cache_read_input_token_cost_ultrafastrinput_cost_per_character&input_cost_per_token_above_128k_tokens&input_cost_per_token_above_200k_tokens/input_cost_per_token_above_200k_tokens_priority&input_cost_per_token_above_272k_tokens/input_cost_per_token_above_272k_tokens_priority+input_cost_per_token_above_272k_tokens_flex&input_cost_per_token_above_512k_tokensinput_cost_per_queryinput_cost_per_secondinput_cost_per_audio_tokeninput_cost_per_image_tokeninput_cost_per_video_tokeninput_cost_per_imageinput_cost_per_audio_per_secondinput_cost_per_video_per_secondinput_cost_per_token_batchesoutput_cost_per_token_batchesoutput_cost_per_token_flexoutput_cost_per_token_priorityoutput_cost_per_token_ultrafast(regional_processing_uplift_multiplier_eu(regional_processing_uplift_multiplier_us#regional_endpoint_uplift_multiplieroutput_cost_per_audio_tokenoutput_cost_per_characteroutput_cost_per_reasoning_token$output_cost_per_reasoning_token_flex(output_cost_per_reasoning_token_priority'output_cost_per_token_above_128k_tokens+output_cost_per_character_above_128k_tokens'output_cost_per_token_above_200k_tokens0output_cost_per_token_above_200k_tokens_priority'output_cost_per_token_above_272k_tokens0output_cost_per_token_above_272k_tokens_priority,output_cost_per_token_above_272k_tokens_flex'output_cost_per_token_above_512k_tokensoutput_cost_per_secondoutput_cost_per_second_1080poutput_cost_per_second_480poutput_cost_per_second_4k output_cost_per_video_per_secondoutput_cost_per_imageoutput_cost_per_image_tokenoutput_cost_per_video_tokenoutput_vector_sizecitation_cost_per_tokenrsupported_endpointsrmrAr`rerxr5rqr1r3rosupports_adaptive_thinkingsupports_legacy_thinkingthinking_always_onsupports_tool_search supports_mid_conversation_systemsupports_none_reasoning_effort!supports_minimal_reasoning_effortsupports_low_reasoning_effortsupports_xhigh_reasoning_effortsupports_max_reasoning_effortreasoning_effort_levelsdefault_reasoning_effort$bedrock_output_config_effort_ceiling&bedrock_converse_supports_strict_toolssearch_context_cost_per_queryweb_search_billing_unit$google_maps_grounding_cost_per_queryr rocr_cost_per_pageocr_cost_per_creditannotation_cost_per_pageprovider_specific_entryuses_embed_contentsupports_image_sizezError getting model info: %sz#This model isn't mapped yet. model=z, custom_llm_provider=zb. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json.r) r rMrp azure_llmsazure_embedding_modelsvertex_llama3_modelsrrrrr<rrrrrcallablerUrrrrPrrrrrrUrwrrsearch)rrrrXrMrNrrrrrrmodel_cost_custom_llm_providerrprovider_get_model_infoprovider_model_infor3r _model_infor _matched_keygeneralization_input_cost_per_token_output_cost_per_tokenreturned_model_infocost_key cost_values r3r6r6s cu Tw11TW5S5ST J u%E  */BFV/V"-   */Bk/Q'">">>% !W%B%BBeiFW[b[x[xFx )'A-@-kDdejDk(  WYno%:;P%Q%:;P%Q.CDb.c$.CDb.c$2=A 34IJ0C&37 #6/#I3KKl3F&GLO  &-4_FVX\-] #/01H#!) '2' +6227 - / < NJ %"&#'  &'  '( "/*.*.+/&*,0)-26 '+!"$(# ( 26K"C//BCL'"=$sC.Q ,*(F#'K"259 +&C"Ad3PSn"UK0#.,J'+ "2;? +&C"Ad3PSn"UK0#.,J'+ "23OP +&C"Ad3PSn"UK0#.,J'+ "23FG +&C"Ad3PSn"UK0#.,J'+ "23OP +&C"Ad3PSn"UK0#.,J'+ "(K*?(;) "-'5$C"ck S3>//BX2Y !$,$$v' )*%3>??CZ3[ "%-$$w' *+&)6s*s*&??<>s*"-1CT!Js*#.//2Et"L s* &; s* +6//:UW[*\ s*/:oo>]_c.ds*0;?_ae/fs*1<@acg0hs*CN//GCs*CN//GCs* LW??PRVL!s*&HSLdH's*,6A__Ekmq5r-s*.:E>:/s*4;F//?;5s*:-8OO?JooCT??s*DHSLdHEs*J?JooCT?Ks*PHSLdHQs*VDO??H$DWs*\?JooCT?]s*b2=Acei1jcs*d6A__Ekmq5res*f7BooFmos6tgs*h;F//?;is*n*59SUY)Zos*p8CGoqu7vqs*r8CGoqu7vss*tALEtAus*z8CGoqu7v{s*|ALEtA}s*B=HOOA4=Cs*H8CGoqu7vIs*J&1__5KT%RKs*L'2oo6Mt&TMs*N,7??;WY]+^Os*P,7??;WY]+^Qs*R,7??;WY]+^Ss*T&1__5KT%RUs*V1<@acg0hWs*X1<@acg0hYs*Z.9__=[-\[s*\/:oo>].^]s*^'=_s*`,7??;WY]+^as*b0;?_ae/fcs*d1<@acg0hes*f:E>:gs*l:E>:ms*r5@OODiko4pss*t-8OO:}s*B9D=t9Cs*H=HOOA4=Is*N9D=t9Os*TBMFBUs*Z9D=t9[s*`BMFBas*f>I__BD>gs*l9D=t9ms*r(37OQU'Vss*t.9__=[]a-bus*v-8OO//Begk2lks*l%0OO4I4$Pms*n&1__5KT%Ros*p$/??3G#Nqs*r,7??;WY]+^ss*t*59SUY)Zus*v$/??3G#Nws*x&1__5KT%Rys*z2=Acei1j{s*|0;?_ae/f}s*~3>//Begk2ls*@/:oo>]_c.dAs*B1<@acg0hCs*D/:oo>]_c.dEs*F)48QSW(XGs*H*59SUY)ZIs*J6A__Ekmq5rKs*L8CGoqu7vMs*N'2oo6Mt&TOs*P/:oo>]_c.dQs*R)48QSW(XSs*T6A__Ekmq5rUs*V OOE40Ws*X OOE40Ys*Z#.//2Et"L[s*\%0OO4I4$P]s*^*59SUY)Z_s*`)48QSW(Xas*b$/??3G#Ncs*d%0OO4I4$Pes* h)4(9(9(; ?$*#66;T;[;[\d;e;q4>'1 ?' &I !"**B+ J  ;Q?1%8NObNcdF G   sVDs!r0A s>lsss s (sss  s t(s??tctj||}t||||}t||}|r |j D] \}}| |||<t di|d|iS)NrrrrrXr_r)rprr6rHrr) rrrrXr_rVrIrrs r3_build_model_infor`]s &AAcvw// K-5NabM'--/ )JC #( C  )  T{ TQ\d eer5c<|t||||St|||S)a Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token for a given model. Parameters: - model (str): The name of the model. - custom_llm_provider (str | null): the provider used for the model. If provided, used to check if the litellm model info is for that provider. Returns: dict: A dictionary containing the following information: key: Required[str] # the key in litellm.model_cost which is returned max_tokens: Required[Optional[int]] max_input_tokens: Required[Optional[int]] max_output_tokens: Required[Optional[int]] input_cost_per_token: Required[float] input_cost_per_character: Optional[float] # only for vertex ai models input_cost_per_token_above_128k_tokens: Optional[float] # only for vertex ai models input_cost_per_character_above_128k_tokens: Optional[ float ] # only for vertex ai models input_cost_per_query: Optional[float] # only for rerank models input_cost_per_image: Optional[float] # only for vertex ai models input_cost_per_audio_token: Optional[float] input_cost_per_audio_per_second: Optional[float] # only for vertex ai models input_cost_per_video_per_second: Optional[float] # only for vertex ai models output_cost_per_token: Required[float] output_cost_per_audio_token: Optional[float] output_cost_per_character: Optional[float] # only for vertex ai models output_cost_per_token_above_128k_tokens: Optional[ float ] # only for vertex ai models output_cost_per_character_above_128k_tokens: Optional[ float ] # only for vertex ai models output_cost_per_image: Optional[float] output_vector_size: Optional[int] output_cost_per_video_per_second: Optional[float] # only for vertex ai models output_cost_per_audio_per_second: Optional[float] # only for vertex ai models litellm_provider: Required[str] mode: Required[ Literal[ "completion", "embedding", "image_generation", "chat", "audio_transcription" ] ] supported_openai_params: Required[Optional[List[str]]] supports_system_messages: Optional[bool] supports_response_schema: Optional[bool] supports_vision: Optional[bool] supports_function_calling: Optional[bool] supports_tool_choice: Optional[bool] supports_prompt_caching: Optional[bool] supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] supports_web_search: Optional[bool] supports_url_context: Optional[bool] supports_reasoning: Optional[bool] Raises: Exception: If the model is not mapped yet. Example: >>> get_model_info("gpt-4") { "max_tokens": 8192, "input_cost_per_token": 0.00003, "output_cost_per_token": 0.00006, "litellm_provider": "openai", "mode": "chat", "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"] } )r`rr_s r3rrs-^ (;XwOO !%) response_timer3s r3load_test_modelrRs(KJ   KPQ[K\]aK@A]H] J    3'  99; :- #0   ^$  99; :- #0    s" A89A== B0B+%B0+B0c|dk(r"tjjS|dk(r"tjjS|dk(r"tjjSgS)z,Return the fields required for each providerr(r7r)rprget_required_paramsrAzureAIStudioConfigrs r3get_provider_fieldsr}skl*'')==??  (##%99;;  ***,@@BB r5cFtjDcic]\}}||dn t|}}}i}i}tjdd}|r|j dng}|jD]h\}}|/t j||<t j||<7t j|||<t j|||<j|D]0}t j||<t j||<2||fScc}}w)NrNO_PROXYrv)proxy) rrrosgetenvrrP HTTPTransportAsyncHTTPTransport)rrproxiessync_proxy_mountsasync_proxy_mountsno_proxy no_proxy_urlsrs r3!create_proxy_transport_and_mountsrs%SjSlSrSrStuxsCc3;4EcNBuGu! "ii D1H2:8>>#.MmmoL U =%*%8%8%: c "&+&>&>&@ s #%*%8%8u%E c "&+&>&>U&K s # L=!&!4!4!6#"'":":"<3= 0 00+vsDcd}g}|||dS tj}||\}}}}| r||dk(r)dtjvrd} n|j d n|dk(rOd tjvr(d tjvrd tjvrd} n|j gd  nn|d k(r;dtjvsdtjvrd} nA|j d n.|dk(r)dtjvrd} n|j d n|dk(r)dtjvrd} n|j d n|dk(r)dtjvrd} n|j d n|dk(r)dtjvrd} n|j d nv|dk(r)dtjvrd} n[|j d nH|dk(r=dtjvrdtjvrd} n|j ddg n|dk(r)dtjvrd} n|j d n|dk(r)d tjvrd} n|j d  n|d!k(r)d"tjvrd} n|j d" n||d#k(r)d$tjvrd} na|j d$ nN|d%k(r)d&tjvrd} n3|j d& n |d'k(r)d(tjvrd} n|j d( n|d)k(s|d*k(rd+tjvrd,tjvsZd-tjvsHd.tjvs6d/tjvs$d0tjvsd1tjvrd} nf|j d+|j d, nB|d2vr)d3tjvrd} n(|j d3 n|d4k(r)d5tjvrd} n|j d5 n|d6k(r)d7tjvrd} n|j d7 n|d8k(r)d9tjvrd} n|j d9 n|d:k(rLd;tjvsdtjvrd} n|j d> n |d?k(r)d@tjvrd}n|j d@n|dAk(r)dBtjvrd}n|j dBn|d%k(r)d&tjvrd}n|j d&n|dCk(r)dDtjvrd}ng|j dDnT|dEk(r)d tjvrd}n9|j d n&|dFk(r)dGtjvrd}n |j dGn|dHk(s|dIk(r)dJtjvrd}n|j dJn|dKk(s|dLk(r)dMtjvrd}n|j dMn|dNk(r)dOtjvrd}nw|j dOnd|dPk(r)dQtjvrd}nI|j dQn6|dRk(r)dStjvrd}n|j dSn|dTk(r)dUtjvrd}n|j dUn|dVk(r)dWtjvrd}n|j dWn|dXk(r)dYtjvrd}n|j dYn~|dZk(r)d[tjvrd}nc|j d[nP|d\k(r_d]tjvs6d^tjvs$d_tjvsd`tjvrd}n|j d]n|dak(r^dbtjvr(dctjvsddtjvrd}n|j db|j ddn|dek(r)dftjvrd}nn|j dfn[|dgk(r)dhtjvrd}n@|j dhn-|dik(r)djtjvrd}n|j djn|dkk(r)dltjvrd}n|j dln|dmk(r)dntjvrd}n|j dnn|dok(rdptjvrd}n|j dpnt|tjvsG|tjvs5|tjvs#|tjvs|jdqr)dtjvrd}n|j dn|tjvr;dtjvsdtjvrd}n|j dn|tjvr)dtjvrd}n}|j dnj|tjvr)dtjvrd}nB|j dn/|tj vr)dtjvrd}n|j dn|tj"vr)dtjvrd}n|j dn|tj$vr)dtjvrd}n|j dn~|tj&vs'|tj(vs|tj*dvr=dtjvrdtjvrd}n|j ddgn|tj,vr)dtjvrd}n|j dn|tj.vr)d tjvrd}n|j d n|tj0vr)d"tjvrd}nj|j d"nW|tj2vr)d$tjvrd}n/|j d$n|tj4vr'd&tjvrd}n|j d&n|tj6vr'd(tjvrd}n|j d(n|tj8vr'dftjvrd}n|j dfnq|tj:vr'dhtjvrd}nJ|j dhn8|tj<vr&djtjvrd}n|j djdwdr} | | |ds}| | |dt}| | |du}t?|dvk(rd}||dS#t$rd}Y;wxYw)xa Checks if the environment variables are valid for the given model. Args: model (Optional[str]): The name of the model. Defaults to None. api_key (Optional[str]): If the user passed in an api key, of their own. Returns: dict: A dictionary containing the following keys: - keys_in_environment (bool): True if all the required keys are present in the environment, False otherwise. - missing_keys (List[str]): A list of missing keys in the environment. FN)keys_in_environment missing_keysrrrxTr'AZURE_API_BASEr AZURE_API_KEY)rrrrryANTHROPIC_AUTH_TOKENrr~rrrOPENROUTER_API_KEYrVERCEL_AI_GATEWAY_API_KEY datarobotDATAROBOT_API_TOKENrVERTEXAI_PROJECTr7rrrrzrrrr{r|r}rrr(r AWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEY AWS_ROLE_ARN AWS_PROFILEAWS_WEB_IDENTITY_TOKEN_FILE&AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"AWS_CONTAINER_CREDENTIALS_FULL_URI)r7rOLLAMA_API_BASErANYSCALE_API_KEYrDEEPINFRA_API_KEYfeatherless_aiFEATHERLESS_AI_API_KEYrGOOGLE_API_KEYGEMINI_API_KEYr GROQ_API_KEYr)NVIDIA_NIM_API_KEYrCEREBRAS_API_KEYr XAI_API_KEYrrVOLCENGINE_API_KEYrrCODESTRAL_API_KEY inceptionrINCEPTION_API_KEYrDEEPSEEK_API_KEYrTENCENT_API_KEYr/MISTRAL_API_KEYpalm PALM_API_KEYrPERPLEXITYAI_API_KEYr1VOYAGE_API_KEYr3INFINITY_API_KEYr4FIREWORKS_AI_API_KEYFIREWORKS_API_KEYFIREWORKSAI_API_KEYFIREWORKS_AI_TOKENrCLOUDFLARE_API_KEYCLOUDFLARE_ACCOUNT_IDCLOUDFLARE_API_BASErNOVITA_API_KEYrrrr dashscopeDASHSCOPE_API_KEY modelscopeMODELSCOPE_API_KEYmoonshotMOONSHOT_API_KEYz gpt-imagecR|Dcgc]}||jvs|c}Scc}w)zDFilter out keys that contain the exclude_pattern (case insensitive).r)rexclude_patternrs r3filter_missing_keysz1validate_environment..filter_missing_keys)s##Jciik'IJJJs$$rXrrlr)rrtrrrrt) rrrUrenvironrIrrprropen_ai_embedding_modelsopenai_image_generation_modelsrrrreplicate_modelsrrdatarobot_modelsrrmodels_by_providerhuggingface_modelsrtogether_ai_modelsrbaseten_modelsrr nebius_models wandb_modelsr) rrXrrlrrrr1rrs r3validate_environmentrs $  L }#6(  #"/"@"@'7e'D$ 1 ( *2::-&*###$45 G +2::-2E2SXgkmkukuXu&*###$\] K /"bjj04Jbjj4X&*###$78 H ,2::-&*###$45 K /"bjj0&*###$78 L 0#rzz1&*###$89 $7 7*bjj8&*###$?@ K /$ 2&*###$9: K /!RZZ/4G2::4U&*###%79L$MN M 1$ 2&*###$9: F *+&*###N3 M 1#rzz1&*###$89 M 1$ 2&*###$9: I - BJJ.&*###$56 K /"bjj0&*###$78 I -1D 1S#rzz16MQSQ[Q[6["**, BJJ.0BJJ>;rzzI72::E&*###$78##$;< $= = BJJ.&*###$56 J .!RZZ/&*###$67 K /"bjj0&*###$78 $4 4'2::5&*###$<= H , BJJ.4D 4R&*###$45##$45 F *+&*###N3 L 0#rzz1&*###$89 J .!RZZ/&*###$67 I - BJJ.&*###$56 E ) *&*###M2 K /+&*###N3 L 0#rzz1&*###$89 K /3FJe3e"bjj0&*###$78 K /3FJe3e"bjj0&*###$78 J .!RZZ/&*###$67 I - BJJ.&*###$56 I - BJJ.&*###$56 F *+&*###N3 L 0%3&*###$:; H ,2::-&*###$45 J .!RZZ/&*###$67 N 2&"**4&"**4(BJJ6'2::5&*###$:; L 0#rzz1'2::59NRTR\R\9\&*###$89##$9: H ,2::-&*###$45 H ,2::-&*###$45 G +"**,&*###O4 K /"bjj0&*###$78 L 0#rzz1&*###$89 J .!RZZ/&*###$67 W;; ;>>>888>>> ,2::-&*###$45 g.. ."bjj04Jbjj4X&*###$78 g++ +2::-&*###$45 g.. ."bjj0&*###$78 g// /#rzz1&*###$89 g66 6*bjj8&*###$?@ g.. .$ 2&*###$9: W// /22222;??!RZZ/4G2::4U&*###%79L$MN g00 0$ 2&*###$9: g)) )+&*###N3 g00 0#rzz1&*###$89 g00 0$ 2&*###$9: g,, , BJJ.&*###$56 g.. ."bjj0&*###$78 g++ +2::-&*###$45 g++ +2::-&*###$45 g** *"**,&*###O4K*<C *<D *<G  <A"#6 UUk #"#sy y'&y'c,tj|i|Sr;)rpr)rPrxs r3acreater<s    / //r5c |tjvs|tjvr tjj |ydddg}tj ||y#t$rtd|dwxYw)Nrz Hello Worldrrrr)rrr) rprrrmodelsretrieverrUrrs r3 valid_modelr@srH G:: :ewGmGm>m MM " "5 )(.=IJH   UX > HbBGGHsAA$A$$A<cvdddg} tj|||dy#t$rYyt$rYywxYw)aW Checks if a given API key is valid for a specific model by making a litellm.completion call with max_tokens=10 Args: model (str): The name of the model to check the API key against. api_key (str): The API key to be checked. Returns: bool: True if the API key is valid for the model, False otherwise. rzHey, how's it going?rrV)rrrXrTF)rprr rU)rrXrs r3check_valid_keyr LsL!'3IJKH7WYZ  s" 888c4|dk(ry|dk(ry|dk(ry|dk\ryy)a Retries on 408, 409, 429 and 500 errors. Any client error in the 400-499 range that isn't explicitly handled (such as 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, etc.) would not trigger a retry. Reimplementation of openai's should retry logic, since that one can't be imported. https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L639 iTiirFr)rs r3 _should_retryr as9cccc r5cN ddl}||jd} t|}|Sd}|S#t$r_|jj |}|d}n;|jj |}t|tjz }Y|SwxYw#t$rd}YywxYw)z Reimplementation of openai's calculate retry after, since that one can't be imported. https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L631 rNz retry-afterr)emailrwrOrUro parsedate_tz mktime_tzr)response_headersr retry_header retry_afterretry_date_tuple retry_dates r3&_get_retry_after_from_exception_headerr~s  ''7';';M'JL @!,/ K @*/++*B*B<*P #+"$K(- (=(=>N(OJ"%j499;&>"?K @  s7B +BBA$BBBB B$#B$ct|}ttjz}|d|cxkr dkr||zS||z }tt d|z}t ||}t |t}||zS)Nr<g@)rr%randomr$powmaxr)r&)remaining_retriesrr min_timeoutrjitterrG sleep_secondss r3_calculate_retry_afterr s @@PQKV]]_,F1{#8b#8V##$9%'88K'#c;*??M {3M 7M 6 !!r5c@|}|g} tj}||d}|j||r-|D]}|ddtj |<tj S|D]}|||dtj |<tj S#t$rYhwxYw)a Register a prompt template to follow your custom format for a given model Args: model (str): The name of the model. roles (dict): A dictionary mapping roles to their respective prompt values. initial_prompt_value (str, optional): The initial prompt value. Defaults to "". final_prompt_value (str, optional): The final prompt value. Defaults to "". Returns: dict: The updated custom prompt dictionary. Example usage: ``` import litellm litellm.register_prompt_template( model="llama-2", initial_prompt_value="You are a good assistant" # [OPTIONAL] roles={ "system": { "pre_message": "[INST] <> ", # [OPTIONAL] "post_message": " <> [/INST] " # [OPTIONAL] }, "user": { "pre_message": "[INST] ", # [OPTIONAL] "post_message": " [/INST]" # [OPTIONAL] }, "assistant": { "pre_message": " " # [OPTIONAL] "post_message": " " # [OPTIONAL] } } final_prompt_value="Now answer as best you can:" # [OPTIONAL] ) ``` rrrS)rr)rolesinitial_prompt_valuefinal_prompt_value)rrrIrUrpknown_tokenizer_configcustom_prompt_dict) rr"r#r$tokenizer_configcomplete_modelpotential_modelsrr0s r3register_prompt_templater*sT"N-. "/"@"@ u-a0&! A-#1G * *1 -   % %%" A(<&8-G & &q )   % %%!    s-B BBc@eZdZ d d dZdZdZd dZdZdZy) TextCompletionStreamWrapperNc<||_||_||_||_yr;)completion_streamrrr)selfr.rrrs r3__init__z$TextCompletionStreamWrapper.__init__s#"3 ,#6 r5c|Sr;rr/s r3__iter__z$TextCompletionStreamWrapper.__iter__  r5c|Sr;rr2s r3 __aiter__z%TextCompletionStreamWrapper.__aiter__r4r5c8 t}|jdd|d<d|d<|jdd|d<|jdd|d<t}t|trt |ddd}|d |d <|jd |d <|ddd |d <|ddd |d <|g|d<|j r3|j jdddur|jdd|d<|S#t $r}t d|d|d}~wwxYw)NrrrPcreatedrrnrrkrr reasoning_contentr- finish_reason include_usageFTusagez=Error occurred converting to text completion object - chunk: z ; Error: )rrwrr}r~rUr)r/rr\ text_choicesrkr3s r3!convert_to_text_completion_objectz=TextCompletionStreamWrapper.convert_to_text_completion_objectsU q46H"YYtT2HTN!2HX "'))It"'>PU'VZ^'^$)IIgt$<!O q[\a[bbklmknop p qsC7C:: DDDcTt |jD]$}|dk(s|t|j|}|cSt#t$rtt$rK}t t jtd}||j|jxsd|iid}~wwxYw)Nrrrr)rroriginal_exceptioncompletion_kwargs extra_kwargs) rr.rUr> StopIterationrrrrrr)r/rprocessed_chunkr3rs r3__next__z$TextCompletionStreamWrapper.__next__*s  // 'F?em#O"&"H"Hu"H"U&&  '     $+CKK,ACS$TN jj$($<$<$B#$"$   s1AAB'AB""B'cK |j23d{}|dk(s|t|j|}|cS7'6t#t$rtwxYww)Nrr@)r.rUr>rDStopAsyncIteration)r/rrEs r3 __anext__z%TextCompletionStreamWrapper.__anext__@sn %#55 ' 'eF?em#O"&"H"Hu"H"U&&  '5   %$ $ %s5A A><>#AA>AAANN)r dict | Noner str | None)rr) rrrr0r3r6r>rFrIrr5r3r,r,s= '+*. 7$ 7 ( 7q2, %r5r,c #Kt|tjr|t|tr|yt dt |dD]}t d|||dz}|||jd_npg}t |D]Y}tjj|tjj d|||dz}|j|[||_|ywNrrr)r-rk r}rprrrrrrnrkrorrI rw mock_responserricompletion_obj _all_choicesj_streaming_choices r3mock_completion_streaming_objrXLs-!6!67-!45 1c-(! , Kq1q59QR 9. r5c0|g|z}t||}tjd||||kr|Sd|vr`tjdt|||}t ||||rtjd|g|zStjd|S)Nrrz'temp_message_tokens: %s, max_tokens: %srz*attempting to shorten message to fit limitz=can add message, returning [updated_message] + final_messagesz,cannot add message, returning final_messages)rdrr<recan_add_message)rlrrmrr temp_messagestemp_message_tokensupdated_messages r3rnrns#9~5M!0-u!UBDWYcdj(  'IJ!=gGWY^!_ ?NJ N  !` a#$~5 5  !O P r5c,t||gz||kryyr)rd)rrrrs r3rsrssx7)+U3zA r5ct||S)Nr)r%rrs r3rdrds ux 88r5c | d|vr|dkr|S|d}d}tjd||tkrtjd|t|g|}tjd||||krno||z }t t ||zd z }t d|}|d z} |d| } || d} | d z| z} | |d<tjd | | }|d z }|tkr|tk\r|rtd |dtd|S)a| Shorten a message to fit within a token limit by removing characters from the middle. Args: message: The message to shorten tokens_needed: The maximum number of tokens allowed model: The model being used (optional) raise_error_on_max_limit: If True, raises an error when max attempts reached. If False, returns final trimmed content. NgptrVrrz content: %sz#getting token count for message: %sz#total_tokens: %s, tokens_needed: %srz..ztrimmed_content: %sz%Failed to trim message to fit within z tokens after z attempts)rr<r'rdrOrrrU) r tokens_neededrraise_error_on_max_limitrattempts total_tokensratio new_length half_length left_half right_halftrimmed_contents r3reresM Ue^ 0Ci GH0 0 0BGL&y%8 BLR_` = ( ,.W-.2 J'  Ao L[) k\]+ #d*Z7, 2OD!A + 0 0...3K3M?.QlPmmv w   Nr5cL|}tj|} |Z|tjvrFtj|j dtj|d}t ||z}n|Sd}|D]}|ddk(s ||rdndz }||dz }g} t |D]}|dd k7rn| j|| jt| r|dt|  }t|xsd| } td | d || |kr|| zStd |d| d|d} |r1t|||\} }|dk(r| gS|Dcgc] }|ddk7s |}}tjd| t|||} tjd| | r| g| z} t| dkDr| j!| tjd| ||r|t#| |z } | | fS| Scc}w#t$$r"}tj&d||cYd}~Sd}~wwxYw)a Trim a list of messages to fit within a model's token limit. Args: messages: Input messages to be trimmed. Each message is a dictionary with 'role' and 'content'. model: The LiteLLM model being used (determines the token limit). trim_ratio: Target ratio of tokens to use after trimming. Default is 0.75, meaning it will trim messages so they use about 75% of the model's token limit. return_response_tokens: If True, also return the number of tokens left available for the response after trimming. max_tokens: Instead of specifying a model or trim_ratio, you can specify this directly. Returns: Trimmed messages and optionally the number of tokens available for response. NrrrrrcrqrrrzCurrent tokens: z, max tokens: zNeed to trim input messages: z, current_tokensz, max_tokens: )rfrrrzProcessed system message: %s)rrrzProcessed messages: %sz.Final messages: %s, return_response_tokens: %sz'Got exception while token trimming - %s)rryrprQrwrOrrIreverserr%rDrjrr<rprrdrUrV)rr trim_ratioreturn_response_tokensroriginal_messagesmax_tokens_for_modelrfr tool_messagescurrent_tokensrgrlresponse_tokensr3s r3 trim_messagesr s, (}}X&HJ!  ***.5.@.@.G.K.K&(:(:5(A,(O/$!!5 !BC   5Gv(*.$b@')"44 5 " ) *Gv&(   ) *  }  53}#5"56H -EKR( S((8zlST J &m+ + +H:5EnEUUcdnco p -1 /E-*E0 , *Q,--08WG76?h;VWHW;=QR)8 Z_`5~F 23nDN }  !  ! !- 0M~_uv !%//.RW2X%XO!?2 2#X$ !  !JAN  !sVAG85G8BG80G8 G8 G3G3"BG81G83G88 H#HH#H#) InMemoryCachecreZdZddfd Zd dZd dZ d dZ d d dZ ddZxZ S)AvailableModelsCachec4t|||d|_yr;)superr0 _env_hash)r/ ttl_secondsmax_size __class__s r3r0zAvailableModelsCache.__init__vs h/%)r5ctjjDcic]\}}|jds||}}}t t t |jScc}}w)z/Create a hash of relevant environment variables)OPENAI ANTHROPICAZUREAWS)rrrrrhashr)r/r`raenv_varss r3 _get_env_hashz"AvailableModelsCache._get_env_hashzsX,.JJ,<,<,>xDAq!,,OvBw1a4xx4 (.."23455ys A/A/ch|j}|j||_y||jk7S)z+Check if environment variables have changedT)rr)r/ current_hashs r3_check_env_changedz'AvailableModelsCache._check_env_changeds3"002 >> !)DNt~~--r5cd}||j}||d|}tj|jj S)Nr:)model_dump_jsonhashlibsha256r hexdigest)r/rr valid_strs r3_get_cache_keyz#AvailableModelsCache._get_cache_keysU   %&668I  *./q ' >. > >+/04'.  &C C.C$ Cr5r _model_cachec(g}tjj}tjD]^}|r||k7r |j dd}|}|j d}|j d}||vs||vsN|j|`|S)Nr1r_API_KEY)rrrrprHrZupperrI)rvalid_providers environ_keysrFenv_provider_1env_provider_2expected_provider_key_1expected_provider_key_2s r3#_infer_valid_provider_from_env_varsrs)+O**//+L))- 8/B#B "))#r2!&4%9%9%;$0J^,,PT3  **+>PVW ?C s#A$AA$$ B-B B BcV ddlm}| |d}|||_|||_|xstj }g}g}|r|g}n t |}|D]}tjdt|} |r||k7r+|dk(r|jdB| !|r||jt| ||etjtjj!|g} |j| |S#t"$r"} t%j&d | gcYd} ~ Sd} ~ wwxYw) a Returns a list of valid LLMs based on the set environment variables Args: check_provider_endpoint: If True, will check the provider's endpoint for valid models. custom_llm_provider: If provided, will only check the provider's endpoint for valid models. api_key: If provided, will use the API key to get valid models. api_base: If provided, will use the API base to get valid models. Returns: A list of valid LLMs rrNrrrr'z Azure-LLMr)litellm.types.routerrrXrrpcheck_provider_endpointrrrrrIrrrryrrwrUrr) rrrrXrrr valid_modelsrFrmodels_for_providerr3s r3get_valid_modelsrsU&4 8  !+"5N  %,N "  &.N ##:"\W=\=\&()+  23OABUVO' 9H3KK%h/LO #x3F'F7"##K0 ,1HXMa##7' &'+mmG4N4N4R4RS[]_4`&a###$78+ 9. ?C sC:C== D(D#D(#D(c tsy d|vr|ddur|jdk(s4d|vr|ddur|jdk(sd|vr|ddur|jdk(rydjtt|}djd |j D}t d t d |r"|r t d |jd |d|dnW|rt d |jd |dn8|rt d |jd |dnt d |jdt d y#t$rYywxYw)NrTrrrrimg_generationrc30K|]\}}|d|yw)=Nr)r@rrs r3rBz/print_args_passed_to_litellm..Cs%ZZS%Qui&8%ZsrqzRequest to litellm:z litellm.(z)z())rrr mapreprrrDrU)r'rPrxargs_str kwargs_strs r3rr.su  $  V #}%-!**l:v%|$,!**k9!V+,-5%..2BB ))CdO4 II%Z6<<>%ZZ     0  F ,->-G-G,H(SUV`Uaaij k  ,->-G-G,H(S[\ ]  ,->-G-G,H*U]^ _ ,->-G-G,H R Sd    sAD>C!D>> E  E cx d|jdzdz|jdz}|S#t$rYywxYw)Nztime-z %H-%M-%S-%fr1r)strftimerwrU)rrl response_ids r3get_logging_idrXsK$z':':='IICOR^RbRbcgRhh  s *- 99c\|y|jdi}||jdd}||S|jdxsi}ttjtd}||}||S|jdi}ttjtd}||Sy)Nrrrsr)rsr)rwrrrr)model_call_detailsr _base_modelrsrbase_model_from_metadatars r3_get_base_model_from_metadatar`s!.223CRHN!+// dC  " (,,Z8>BPW KK !#OQ 2+U^f*g # /+ +#1"4"45G"L5< KK !#O6 2:CSTT r5c.eZdZdddZdZdZdZdZy) ModelResponseIteratorc|durXt}|jdjj|jdj_||_d|_y||_d|_y)NTrF)rrnrrrkrwis_done)r/rwconvert_to_delta_stream_responses r3r0zModelResponseIterator.__init__{sf t #&9&; 8F8N8Nq8Q8Y8Y8a8a  $ $Q ' - - 5GWD  #1D  r5c|Sr;rr2s r3r3zModelResponseIterator.__iter__r4r5cL|jrtd|_|jSNT)rrDrwr2s r3rFzModelResponseIterator.__next__s" <<  """r5c|Sr;rr2s r3r6zModelResponseIterator.__aiter__r4r5cTK|jrtd|_|jSwr)rrHrwr2s r3rIzModelResponseIterator.__anext__s& <<$ $ """s&(NF)rwrrrgrrrr0r3rFr6rIrr5r3rrzs##r5rc.eZdZdddZdZdZdZdZy) ModelResponseListIteratorNc.||_d|_||_yNr)model_responsesr-delay)r/rrs r3r0z"ModelResponseListIterator.__init__s.  r5c|Sr;rr2s r3r3z"ModelResponseListIterator.__iter__r4r5c|jt|jk\rt|j|j}|xjdz c_|jrt j |j|SNr)r-rrrDrrsleepr/rws r3rFz"ModelResponseListIterator.__next__s] ::T112 2  $ 4 4TZZ @ a :: JJtzz "r5c|Sr;rr2s r3r6z#ModelResponseListIterator.__aiter__r4r5c$K|jt|jk\rt|j|j}|xjdz c_|jr't j |jd{|S7wr)r-rrrHrrrrs r3rIz#ModelResponseListIterator.__anext__sl ::T112 2$ $ $ 4 4TZZ @ a ::-- + + + ,sBBBBr;)r float | Nonerrr5r3rrs r5rc eZdZdfd ZxZS)CustomModelResponseIteratorc"t|yr;)rr0)r/rs r3r0z$CustomModelResponseIterator.__init__s r5r)rrrr0rrs@r3rrs r5rctjdury|jd}|%t|tr|jddk(ryd|vry|d}t|t sy|D]c}t|ts|jd}|jddk(s:|=t|tsN|jddk(scyy)a+ Returns true, if message is marked as needing to be cached. Used for anthropic/gemini context caching. Follows the anthropic format {"cache_control": {"type": "ephemeral"}} Can be disabled globally by setting litellm.disable_anthropic_gemini_context_caching_transform = True TF cache_controlrL ephemeralrr )rp2disable_anthropic_gemini_context_caching_transformrwr}rr)rmessage_level_cache_controlr content_itemrs r3is_cached_messagersAATI*1_)E#/ 2D 9 ' + +F 3{ BY'G gt $  ,- $((9   V $ .)=$/!!&)[8  r5c |jdsy|jdd}tj|d}tj|j d|k(S#t $rYywxYw)Nzdata:FrvrT)validater)rrbase64 b64decode b64encoderrU)r decoded_bytess r3is_base64_encodedr su||   GGCLO &//DA  .55g>!CC sA&AA&& A21A2c6d|vr|jddS|S)z3 s: b64str OR data:image/png;base64,b64str rvrr)rs r3get_base64_strrs" axwws|A Hr5c8|D]}|jdyy)zm Returns true, if messages has tool call blocks. Used for anthropic/bedrock message validation. rTFrv)rrs r3has_tool_call_blocksr s*  ;;| $ 0 r5c|D]G}|jddk(s|jd}|,t|drt|dkDsGyy)a Returns true if ANY assistant message has thinking_blocks. This is used to prevent dropping the thinking param when some messages in the conversation already contain thinking blocks. Dropping thinking when thinking blocks exist causes Anthropic error: "When thinking is disabled, an assistant message cannot contain thinking" Related issue: https://github.com/BerriAI/litellm/issues/18926 rrthinking_blocks__len__rTFrwrr)rrrs r3)any_assistant_message_has_thinking_blocksrsY ;;v + -%kk*;QRi j r5c p|3t|tr#tjst |dkDr|dd}|S)a Validates and fixes OpenAI optional parameters. Args: stop: Stop sequences (string or list of strings) **kwargs: Additional optional parameters Returns: Validated stop parameter (truncated to 4 elements if needed) N)r}rrpdisable_stop_sequence_limitr)rrxs r3validate_openai_optional_paramsrJs6 JtT27;^;^ t9q=8D Kr5c tjdjd}tj|j S#t $ricYSwxYw)Nrpz+model_prices_and_context_window_backup.json)rfilesjoinpathrloads read_textrU)model_cost_paths r3_get_bundled_model_cost_maprQ"sM!*!;!D!DEr!szz/33566  sAA AAc||jd|f}|D]'}tjj|}|%|cSt }|D]}|j|}||cSiS)NrN)rrprQrwrQ)rrFcandidate_keysrtr7bundled_model_costs r3)_get_model_cost_entry_for_provider_configrU+s#x~~&6aw$?@N# ''++I6  !  !< =# '++I6  !  Ir5cBeZdZUdZded<ed+dZed,d-dZed.dZed.dZ ed.dZ ed.d Z ed/d Z ed/d Z e d, d0d Ze d1d Ze d2dZe d3dZeee d3dZe d4dZe d, d5dZe d, d6dZe d7dZe d8dZe d9dZe d:dZe d;dZe ddZe d?dZ e d, d@dZ!e dAdZ"e dBd Z#e dCd!Z$e dDd"Z%e dEd#Z&e dFd$Z'e dGd%Z(e dHd&Z)e dId'Z*e dJd(Z+e dKd)Z,e dLd*Z-y)MrNz0dict[LlmProviders, tuple[Callable, bool]] | None_PROVIDER_CONFIG_MAPcitjddftjddftjddftjddftj ddftj ddftjd dftjd dftjd dftjd dftjd dftjddftjddftjddftjddftj ddftj"ddfitj$ddftj&ddftj(ddftj*ddftj,ddftj.ddftj0ddftj2ddftj4ddftj6ddftj8ddftj:ddftj<d dftj>d!dftj@d"dftjBd#dftjDd$dfitjFd%dftjHd&dftjJd'dftjLd(dftjNd)dftjPd*dftjRd+dftjTd,dftjVd-dftjXd.dftjZd/dftj\d0dftj^d1dftj`d2dftjbd3dftjdd4dftjfd5dfitjhd6dftjjd7dftjld8dftjnd9dftjpd:dftjrd;dftjtd<dftjvd=dftjxd>dftjzd?dftj|d@dftj~dAdftjdBdftjdCdftjdDdftjdEdftjdFdfitjdGdftjdHdftjdIdftjdJdftjdKdftjdLdftjdMdftjdNdftjdOdftjdPdftjdQdftjdRdftjdSdftjdTdftjdUdftjdVdftjdWdfitjdXdftjdYdftjdZdftjd[dftjd\dftjd]dftjd^dftjd_dftjd`dftjdadftjdbdftjdcdftjdddftjdedftjdfdftjdgdfS)hzBuild the provider-to-config mapping dictionary. Returns a dict mapping provider to (factory_function, needs_model_parameter). This avoids expensive inspect.signature() calls at runtime. c*tjSr;)rpOpenAIGPTConfigrr5r3zBProviderConfigManager._build_provider_config_map..Ms'*A*A*Cr5Fc*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..NW-D-D-Fr5c,tj|Sr;)r_get_azure_ai_configrs r3r[zBProviderConfigManager._build_provider_config_map..Rs3HHOr5Tc,tj|Sr;)r_get_vertex_ai_configrs r3r[zBProviderConfigManager._build_provider_config_map..Vs3II%Pr5c,tj|Sr;)r_get_bedrock_configrs r3r[zBProviderConfigManager._build_provider_config_map..Zs3GGNr5c,tj|Sr;r_get_cohere_configrs r3r[zBProviderConfigManager._build_provider_config_map..^3FFuMr5c,tj|Sr;rers r3r[zBProviderConfigManager._build_provider_config_map..brgr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..fG,F,F,Hr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..g7+D+D+Fr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..h(>(>(@r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..j779r5c*tjSr;)rp A2AConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..msw'8'8':r5c*tjSr;)rpBytezChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..n)@)@)Br5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..og.F.F.Hr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..pw'<'<'>r5c*tjSr;)rp ZAIChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..qryr5c*tjSr;)rpLambdaAIChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rW-G-G-Ir5c*tjSr;)rpInceptionChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..sW-H-H-Jr5c*tjSr;)rpLlamaAPIConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..ts)?)?)Ar5c*tjSr;)rpOpenAITextCompletionConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..vs::<r5c*tjSr;)rpSnowflakeConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..yr]r5c*tjSr;)rpClarifaiConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..zG,B,B,Dr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..{'2M2M2Or5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..|s'2L2L2Nr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..}g.J.J.Lr5c*tjSr;)rpSagemakerChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..~rr5c*tjSr;)rpSagemakerNovaConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s0I0I0Kr5c*tjSr;)rpFriendliaiChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s7+G+G+Ir5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s0J0J0Lr5c*tjSr;)rpEmpowerChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rlr5c*tjSr;)rpMinimaxChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rlr5c*tjSr;)rpGithubChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..'*B*B*Dr5c*tjSr;)rpCompactifAIChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..w/L/L/Nr5c*tjSr;)rpGithubCopilotConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp ChatGPTConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..7+@+@+Br5c*tjSr;)rpGigaChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp RAGFlowConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s'*F*F*Hr5c*tjSr;rrr5r3r[zBProviderConfigManager._build_provider_config_map..s1M1M1Or5c*tjSr;rrr5r3r[zBProviderConfigManager._build_provider_config_map..w/K/K/Mr5c*tjSr;)rpAiohttpOpenAIChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rpr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpLlamafileChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpLMStudioChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..r~r5c*tjSr;)rpGaladrielChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rwr5c*tjSr;)rpVercelAIGatewayConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..s557r5c*tjSr;)rpCometAPIConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpDataRobotConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s'*L*L*Nr5c*tjSr;rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rnr5c*tjSr;rrr5r3r[zBProviderConfigManager._build_provider_config_map..W-C-C-Er5c*tjSr;)rpAzureOpenAITextConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..sg.K.K.Mr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpOobaboogaConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..w/G/G/Ir5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rpPerplexityChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;rrr5r3r[zBProviderConfigManager._build_provider_config_map..sW-B-B-Dr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..sg.E.E.Gr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp BasetenConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rwr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..==?r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpSambanovaConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..s(:(:(<r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..'*>*>*@r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..r]r5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpGenAIHubOrchestrationConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..;;=r5c*tjSr;)rpFeatherlessAIConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp NovitaConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rprrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp WandbConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..s)<)<)>r5c*tjSr;)rpDashScopeChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpModelScopeChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpMoonshotChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rjr5c*tjSr;)rpDockerModelRunnerChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp V0ChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..sg&:&:&<r5c*tjSr;)rpMorphChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rur5c*tjSr;)rpLiteLLMProxyChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..s668r5c*tjSr;)rpGradientAIConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp NscaleConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpHerokuChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rp OCIChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..ryr5c*tjSr;)rpHyperbolicChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)rpOVHCloudChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rjr5c*tjSr;)rpAmazonNovaChatConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..rr5c*tjSr;)r_get_langgraph_configrr5r3r[zBProviderConfigManager._build_provider_config_map..s-CCEr5c*tjSr;)r_get_langflow_configrr5r3r[zBProviderConfigManager._build_provider_config_map..s-BBDr5c*tjSr;)rpGDCGeminiConfigrr5r3r[zBProviderConfigManager._build_provider_config_map..s//1r5)frrrAZURE_AI VERTEX_AIBEDROCKCOHERE COHERE_CHATDEEPSEEKTENCENTGROQBEDROCK_MANTLEA2ABYTEZr>XAIZAI LAMBDA_AI INCEPTIONLLAMATEXT_COMPLETION_OPENAI SNOWFLAKECLARIFAIANTHROPIC_TEXTVERTEX_AI_BETA CLOUDFLARESAGEMAKER_CHATSAGEMAKER_NOVA SAGEMAKERr; FRIENDLIAIWATSONX WATSONX_TEXTEMPOWERMINIMAXGITHUB COMPACTIFAIGITHUB_COPILOTCHATGPTGIGACHATRAGFLOWCUSTOM CUSTOM_OPENAI OPENAI_LIKEAIOHTTP_OPENAI HOSTED_VLLM LLAMAFILEr< GALADRIEL REPLICATE HUGGINGFACE TOGETHER_AI OPENROUTERVERCEL_AI_GATEWAYCOMETAPI DATAROBOTGEMINIAI21 AI21_CHAT AZURE_TEXT NLP_CLOUD OOBABOOGA OLLAMA_CHAT DEEPINFRA PERPLEXITYMISTRAL CODESTRAL NVIDIA_NIMCEREBRASBASETEN VOLCENGINETEXT_COMPLETION_CODESTRALTEXT_COMPLETION_INCEPTION SAMBANOVAMARITALKVLLMOLLAMAr:TRITONPETALSSAP_GENERATIVE_AI_HUBFEATHERLESS_AINOVITAr=WANDB DASHSCOPE MODELSCOPEMOONSHOTDOCKER_MODEL_RUNNERV0MORPH LITELLM_PROXY GRADIENT_AINSCALEHEROKUOCI HYPERBOLICOVHCLOUD AMAZON_NOVA LANGGRAPHLANGFLOWGDCrr5r3_build_provider_config_mapz0ProviderConfigManager._build_provider_config_mapCs ^    "CU!K^   " "%F$N ^   ! !O$^   " "P%^   N#^ &   M"'^ .  $ $M'/^ 8  ! !$H%#P9^ :  #F"N;^ <    @%H=^ >  ' '9*?^ F   :EBG^ H   !BE JI^ J  # #&H%%PK^ L   >FM^ N   >FO^ P  " "%I5$QQ^ R  " "%JE$RS^ T   !A5 IU^ V  / /<2W^ ^  " "%F$N_^ `  ! !$De#La^ b  ' '*OQV)Wc^ d  ' '*NPU)Ve^ f  # #&Le%Tg^ h  ' '*OQV)Wi^ j  ' '*OQV)Wk^ l  " "%F$Nm^ n  % %(KU'So^ p  # #&Le%Tq^ r  #I5"Qs^ t  % %(Le'Tu^ v  #F"Nw^ x  #F"Ny^ z   "De!L{^ |  $ $'NPU&V}^ ~  ' '*OQV)W^ @  #BE"JA^ B  ! !$De#LC^ D  #BE"JE^ F   "H%!PG^ H  & &)OQV(WI^ J  $ $'Mu&UK^ L  ' '9*M^ T  $ $'Mu&UU^ V  " "%JE$RW^ X  " "%I5$QY^ Z  " "%JE$R[^ \  " "%F$N]^ ^  $ $'NPU&V_^ `  $ $'Mu&Ua^ b  # #&H%%Pc^ d  * *7-e^ l  ! !$De#Lm^ n  " "%F$No^ p   "NPU!Vq^ r    @%Hs^ t  " "%Eu$Mu^ v  # #&Mu%Uw^ x  " "%Eu$My^ z  " "%F$N{^ |  $ $'I5&Q}^ ~  " "%F$N^ @  # #&Le%TA^ B  #BE"JC^ D  " "%De$LE^ F  # #&G%OG^ H  ! !$De#LI^ J  #BE"JK^ L  # #&H%%PM^ N  2 2?5O^ V  2 2?5W^ ^  " "%F$N_^ `  ! !$De#La^ b     F{^ |  " "%JE$R}^ ~  # #&Le%T^ @  ! !$H%#PA^ B  , ,=/C^ J OOF]^ ^  # #&Le%T_^ `  ! !$H%#Pa^ b  $ $'Mu&Uc^ d  " "E%e^ l  ! !D$m^ t   1u^ ^ r5c|xs|}tjj|rtjStjj |rtjStj S)aGet Azure config based on model type. When *base_model* is provided (e.g. ``"azure/gpt-5.2"``), it is used for model-type detection instead of *model* (the deployment name). This allows non-standard deployment names like ``"azure/foo"`` to be routed through the correct config when the user specifies the true underlying model via ``base_model``. r)rprrrrr)rrdetection_models r3_get_azure_configz'ProviderConfigManager._get_azure_configsl",!4u  & & ( : : : Q..0 0  ( ( = =O = T002 2((**r5c0ddlm}|j|S)z(Get Azure AI config based on model type.rAzureFoundryModelInfo)"litellm.llms.azure_ai.common_utilsrx get_azure_ai_config_for_model)rrx s r3r_z*ProviderConfigManager._get_azure_ai_configs M$BB5IIr5ctd|vrtjSd|vrtjSd|vr ddlm}|S|tj vr,d|vrtj StjS|tjvrtjStjS)z)Get Vertex AI config based on model type.rclaudezgpt-ossr)VertexAIGPTOSSTransformationr) rprrFlitellm.llms.vertex_ai.vertex_ai_partner_models.gpt_oss.transformationr} rrrrrr)rr} s r3raz+ProviderConfigManager._get_vertex_ai_config s u --/ /  224 4 %  01 1 g33 3e#<<>>((* * g33 3--/ ///1 1r5c ddlm}||S)z"Get Bedrock config based on model.r)get_bedrock_chat_configr)rr )rr s r3rcz)ProviderConfigManager._get_bedrock_config s N&U33r5ctj}|j|}|dk(rtjStj S)z!Get Cohere config based on route.v2)rrget_cohere_routerpCohereV2ChatConfigr)rrroutes r3rfz(ProviderConfigManager._get_cohere_config sF"/!>!>777> D=--/ /''))r5cddlm}|S)zGet LangGraph config.rLangGraphConfig)*litellm.llms.langgraph.chat.transformationr r s r3r z+ProviderConfigManager._get_langgraph_config( s O  r5cddlm}|S)zGet LangFlow config.rLangFlowConfig))litellm.llms.langflow.chat.transformationr r s r3r z*ProviderConfigManager._get_langflow_config/ s Mr5c|tjk(rdtjj |rtjStj j |rtj S|tjk(rtj||Stjtjt_ tjj|}||\}}|r||S|Sddl m}ddlm}|j#|j$rC|j|j$}|t'd|j$d||Sy) al Returns the provider config for a given provider. Uses O(1) dictionary lookup for fast provider resolution. Python classes take priority over JSON (they have custom overrides). For Azure, *base_model* (when set) drives model-type detection so that non-standard deployment names still route to the correct config. r)rrNr)create_config_classJSONProviderRegistryz Provider z not found)rrrpopenaiOSeriesConfigis_model_o_series_modelOpenAIGPT5Configrrrru rWrr rw'litellm.llms.openai_like.dynamic_configr $litellm.llms.openai_like.json_loaderr existsrrU) rrFr config_entryconfig_factory needs_modelr r rs r3rfz.ProviderConfigManager.get_provider_chat_config6 sC |** ***BBBO222''<<5<I//11 |)) )(::S]:^ ^ ! 5 5 =9N9i9i9k ! 64HHLLXV  #*6 'NK%e,,%'' PM  & &x~~ 6%9%=%=hnn%MO& 9X^^,|k(r ddl m!}|StjjD|k(rtjFStjjH|k(rtjJStjjL|k(rtjNStjjP|k(rtjRStjjT|k(r ddl+m,}|StjjZ|k(r ddl.m/}|Stjj`|k(rtjbStjjd|k(rtjfStjjh|k(rdd l5m6} | jo|Stjjp|k(rtjrSy) Nr)OCIEmbedConfig)CohereEmbeddingConfig)rD)VolcEngineEmbeddingConfig)DashScopeEmbeddingConfig)OpenrouterEmbeddingConfig)VercelAIGatewayEmbeddingConfig)SagemakerEmbeddingConfig):rprVOYAGErErFrGrHrIr[ r>r. IBMWatsonXEmbeddingConfigr] rJINFINITYrKrW rMrk %litellm.llms.oci.embed.transformationr r r (litellm.llms.cohere.embed.transformationr JINA_AI-litellm.llms.jina_ai.embedding.transformationrDrT 0litellm.llms.volcengine.embedding.transformationr ra +litellm.llms.dashscope.embed.transformationr rm rNr% SnowflakeEmbeddingConfigrD CometAPIEmbeddingConfigr4 GithubCopilotEmbeddingConfigrB 0litellm.llms.openrouter.embedding.transformationr rC 7litellm.llms.vercel_ai_gateway.embedding.transformationr r6 GigaChatEmbeddingConfigr< HostedVLLMEmbeddingConfigr, /litellm.llms.sagemaker.embedding.transformationr get_model_configrN PerplexityEmbeddingConfig) rrFr r rDr r r r r s r3r=z3ProviderConfigManager.get_provider_embedding_configi sr  ' '8 377TTUZ[::< <  ' '8 377PPQVW::< <  ! ! ( (H 4002 2  ! ! ( (H 4002 2  ! ! ) )X 5446 6  ! ! 7 78 C224 4  ! ! * *h 6224 4  ! ! + +x 7335 5  ! ! % % 1 L!# #  ! ! ( (H 48L8L8X8X\d8d V(* *  ! ! ) )X 5 )* *  ! ! , , 8 -. .  ! ! + +x 7 ,- -  ! ! * *h 6224 4  ! ! + +x 7335 5  ! ! * *h 6224 4  ! ! 0 0H <779 9  ! ! , , 8 -. .  ! ! 3 3x ? 23 3  ! ! * *h 6224 4  ! ! - - 9446 6  ! ! + +x 7 ,<vrdd l m!} | | Sy) Nrrr| ),VertexAIPartnerModelsAnthropicMessagesConfig)AzureAnthropicMessagesConfig)MinimaxMessagesConfig)DeepSeekAnthropicMessagesConfig)TencentAnthropicMessagesConfig)$GithubCopilotAnthropicMessagesConfigr z /v1/messages)#JSONProviderAnthropicMessagesConfig)"rrprrAnthropicMessagesConfigr rr,get_bedrock_provider_config_for_messages_apir blitellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformationr r 7litellm.llms.azure_ai.anthropic.messages_transformationr r1 ,litellm.llms.minimax.messages.transformationr r -litellm.llms.deepseek.messages.transformationr r ,litellm.llms.tencent.messages.transformationr r4 3litellm.llms.github_copilot.messages.transformationr r r rwrr60litellm.llms.openai_like.messages.transformationr ) rrF model_lowerrr r r r r r r json_providerr s r3r zDProviderConfigManager._get_provider_anthropic_messages_config_cached s #[[]    ) )X 5224 4 ! ! ) )X 5 JP#PPQVW W  ! ! + +x 7;&DEE  ! ! * *h 6;&455  ! ! ) )X 5 )* *  ! ! * *h 6 34 4  ! ! ) )X 5 23 3  ! ! 0 0H <;&<==M377G  $=;\;\)\ 7}E Er5c<t||}tjj|k(r!|j ddk(r ddlm}|Stjj|k(rtjStjj|k(r ddl m }|Stjj|k(r,d|vrtjStjStjj|k(r ddlm}|Stjj$|k(r dd lm}|Stjj*|k(r dd lm}|Stjj0|k(r dd lm}|Stjj6|k(r dd lm} | Stjj<|k(r dd lm } | StjjB|k(r ddl"m#} | StjjH|k(r@|jKd} | jMdrd| vr ddl'm(} | Sddl)m*}|StjjV|k(r ddl,m-}|Sy)Nraudio_transcription_config azure_speechr)#AzureSpeechAudioTranscriptionConfig)"ElevenLabsAudioTranscriptionConfigzgpt-4o)"HostedVLLMAudioTranscriptionConfig)"IBMWatsonXAudioTranscriptionConfig) OVHCloudAudioTranscriptionConfig) ScalewayAudioTranscriptionConfig)MistralAudioTranscriptionConfig)"NvidiaRivaAudioTranscriptionConfig)SonioxAudioTranscriptionConfigz vertex_ai/r transcribe)$VertexGeminiAudioTranscriptionConfig) VertexAIAudioTranscriptionConfig)GeminiAudioTranscriptionConfig).rUrprrrw5litellm.llms.azure.audio_transcription.transformationr DEEPGRAM DeepgramAudioTranscriptionConfig ELEVENLABS:litellm.llms.elevenlabs.audio_transcription.transformationr r!OpenAIGPTAudioTranscriptionConfig%OpenAIWhisperAudioTranscriptionConfigr< 6litellm.llms.hosted_vllm.transcriptions.transformationr r. 7litellm.llms.watsonx.audio_transcription.transformationr rm 8litellm.llms.ovhcloud.audio_transcription.transformationr SCALEWAY8litellm.llms.scaleway.audio_transcription.transformationr rO 7litellm.llms.mistral.audio_transcription.transformationr NVIDIA_RIVA;litellm.llms.nvidia_riva.audio_transcription.transformationr SONIOX6litellm.llms.soniox.audio_transcription.transformationr r rOrKlitellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformationr 9litellm.llms.vertex_ai.audio_transcription.transformationr rF 6litellm.llms.gemini.audio_transcription.transformationr )rrFmodel_cost_entryr r r r r r r r r bare_vertex_modelr r r s r3rz=ProviderConfigManager.get_provider_audio_transcription_config7!sO #L#   & &( 2 $$%ABnT 78 8  ! ! * *h 6;;= =  ! ! , , 8 67 7  ! ! ( (H 45 @@BBDDFF  ! ! - - 9 67 7  ! ! ) )X 5 67 7  ! ! * *h 6 45 5  ! ! * *h 6 45 5  ! ! ) )X 5 34 4  ! ! - - 9 67 7  ! ! ( (H 4 23 3  ! ! + +x 7','9'9,'G  ++H5,J[:[<== 45 5  ! ! ( (H 4 23 3r5cddlm}ddlm}t |t r |j n t|}d}t |t r|}n t |}tj||}||S|j|r1|j|r |j|}| ||Sy#t$rYhwxYw)Nr)create_responses_config_classr )r r r r r}rrrrUr _get_python_responses_api_configr supports_responses_apirw)rFrr r provider_str provider_enumrrs r3!get_provider_responses_api_configz7ProviderConfigManager.get_provider_responses_api_config!s   N1;8\0RhnnX[\dXe .2 h -$M  ,X 6 .NN}^cd  M & &| 49M9d9deq9r%9%=%=l%KO*E4_EGG  s B22 B>=B>cr|ytjj|k(rtjStjj|k(rh|xrd|j v}|xr$d|j vxst |xr| }|rtjStjStjj|k(rtjStjj|k(r&ddl m }| ||rtjSytjj|k(rtj Stjj"|k(rtj$Stjj&|k(rtj(Stjj*|k(rtj,Stjj.|k(rtj0Stjj2|k(r)|r&d|j vrtj4Sytjj6|k(rtj8Stjj:|k(rtj<Stjj>|k(rPddl m!}m"}|r||tjFsytjH||tjFdk( Sy) zFCheck for Python-class-based responses API configs (custom overrides).Nrzo_seriesr)%github_copilot_supports_responses_apir)mantle_base_segmentmantle_supports_responsesz openai/v1)use_openai_path)%rprrOpenAIResponsesAPIConfigrrro$AzureOpenAIOSeriesResponsesAPIConfigAzureOpenAIResponsesAPIConfigr XAIResponsesAPIConfigr4 4litellm.llms.github_copilot.responses.transformationr GithubCopilotResponsesAPIConfigr5 ChatGPTResponsesAPIConfigrg LiteLLMProxyResponsesAPIConfigrT VolcEngineResponsesAPIConfigMANUSManusResponsesAPIConfigrN PerplexityResponsesConfigr>DatabricksResponsesAPIConfigrB OpenRouterResponsesAPIConfigr< HostedVLLMResponsesAPIConfigr (litellm.llms.bedrock_mantle.common_utilsr r rQBedrockMantleResponsesAPIConfig)rFr is_gpt_model is_o_seriesr r r s r3r z6ProviderConfigManager._get_python_responses_api_config!s      & &( 2335 5  ! ! ' '8 3#("BEU[[],BLuZ5;;=%@%tEWX]E^EsgscsKCCEE<<>>  ! ! % % 1002 2  ! ! 0 0H < } EE R>>@@  ! ! ) )X 5446 6  ! ! / /8 ;99; ;  ! ! , , 8779 9  ! ! ' '8 3224 4  ! ! , , 8446 6  ! ! , , 8%++-/;;==  ! ! , , 8779 9  ! ! - - 9779 9  ! ! 0 0H <   9%ASAS T:: 3E7;M;M NR] ] r5cftjj|k(rtjSy)z Get provider-specific Skills API configuration Args: provider: The LLM provider Returns: Provider-specific Skills API config or None N)rprrAnthropicSkillsConfig)rFs r3get_provider_skills_api_configz4ProviderConfigManager.get_provider_skills_api_config"s*    ) )X 5002 2r5cXtjj|k(r ddlm}|Sy)z Get provider-specific Evals API configuration Args: provider: The LLM provider Returns: Provider-specific Evals API config or None r)OpenAIEvalsConfigN)rprr(litellm.llms.openai.evals.transformationr* )rFr* s r3get_provider_evals_api_configz3ProviderConfigManager.get_provider_evals_api_config"s'    & &( 2 R$& &r5ctj|k(rtjStj|k(rtj Stj |k(rtjStjSr;) rr;rpFireworksAITextCompletionConfigrA TogetherAITextCompletionConfigrV rrrs r3#get_provider_text_completion_configz9ProviderConfigManager.get_provider_text_completion_config'"sj  $ $ 0::< <  % % 199; ;  3 3x ?88: :1133r5cRtj|k(rtjStj|k(rtj Stj |k(rtjStj|k(r ddl m }|Stj|k(rtjStj|k(rtjStj|k(rtj Stj"|k(rtj$Stj&|k(stj(|k(r ddlm}|Stj.|k(stj0|k(r ddlm}|Stj6|k(rtj8Stj:|k(rtj<Stj>|k(r ddl m!}|StjD|k(rddl#m$}||Sy)Nr)VertexAIModelInfo)OllamaModelInfo) VLLMModelInforrw r)%rr;rprrrZrF GeminiModelInfor #litellm.llms.vertex_ai.common_utilsr2 rg rTOPAZTopazModelInforAnthropicModelInfor XAIModelInforZ rL litellm.llms.ollama.common_utilsr3 rY r< litellm.llms.vllm.common_utilsr4 LEMONADELemonadeChatConfigr& rr rrr ry rx )rrFr2 r3 r4 rrx s r3rz-ProviderConfigManager.get_provider_model_info4"s  $ $ 0,,. .  H ,**, ,  H ,**, ,  # #x / M$& &  ' '8 3113 3   8 +))+ +  # #x /--/ /    )'') )  H , 0H0HH0T H"$ $   ( *l.F.F(.R !? "  " "h .--/ /  " "h .))+ +  ! !X - J#% %  " "h . P(u5 5r5cjtj|k(r ddlm}|Stj|k(r ddlm}|Stj|k(stj|k(r ddl m }|Stj|k(r ddl m }|Stj|k(r ddlm}|Sy)Nr)BedrockPassthroughConfig)BedrockMantlePassthroughConfig)VLLMPassthroughConfig)AzurePassthroughConfig)WatsonxPassthroughConfig)rr /litellm.llms.bedrock.passthrough.transformationr@ r 6litellm.llms.bedrock_mantle.passthrough.transformationrA rY r< ,litellm.llms.vllm.passthrough.transformationrB r-litellm.llms.azure.passthrough.transformationrC r. /litellm.llms.watsonx.passthrough.transformationrD )rrFr@ rA rB rC rD s r3get_provider_passthrough_configz5ProviderConfigManager.get_provider_passthrough_configd"s   8 + ,- -  ( (H 4 23 3   ( *l.F.F(.R )* *   8 + *+ +  ! !X - ,- -r5ctj|k(rtjStj|k(rtj Syr;)rrrpOpenAIImageVariationConfigr7 TopazImageVariationConfigrs r3#get_provider_image_variation_configz9ProviderConfigManager.get_provider_image_variation_config"sA   ( *557 7   8 +446 6r5cDtj|k(r ddlm}|Stj|k(r ddlm}|Stj|k(r ddlm }|Stj|k(r ddl m }|Stj|k(r ddlm}|Sy)Nr)GoogleAIStudioFilesHandler)VertexAIFilesConfig)BedrockFilesConfig)ManusFilesConfig)AnthropicFilesConfig)rrF (litellm.llms.gemini.files.transformationrP r +litellm.llms.vertex_ai.files.transformationrQ r )litellm.llms.bedrock.files.transformationrR r 'litellm.llms.manus.files.transformationrS r+litellm.llms.anthropic.files.transformationrT )rrFrP rQ rR rS rT s r3get_provider_files_configz/ProviderConfigManager.get_provider_files_config"s   ( * ./ /  # #x / W&( (  ! !X - T%' '   8 + P#% %  # #x / X') )r5cDtj|k(r ddlm}|Sy)Nr)BedrockBatchesConfig)rr +litellm.llms.bedrock.batches.transformationr\ )rrFr\ s r3get_provider_batches_configz1ProviderConfigManager.get_provider_batches_config"s!   8 + X') )r5cVddlm}tj|k(r|j Sy)Nr)BedrockVectorStore)Clitellm.integrations.vector_store_integrations.bedrock_vector_storer` rr get_initialized_custom_logger)rFr` s r3 get_provider_vector_store_configz6ProviderConfigManager.get_provider_vector_store_config"s)    8 +%CCE Er5ctjj|k(r ddlm}|Stjj |k(r ddlm}|Stjj|k(r'|dk(s| ddl m }|S|dk(r ddl m }|Sytjj|k(r dd lm}|Stjj |k(r dd lm}|Stjj&|k(r dd lm}|Stjj,|k(r dd lm} | Stjj2|k(r dd lm} | Stjj8|k(r ddlm} | Stjj>|k(r ddl m!} | StjjD|k(r ddl#m$} | Sy)zT v2 vector store config, use this for new vector store integrations r)OpenAIVectorStoreConfig)AzureOpenAIVectorStoreConfigrag_apiN)VertexVectorStoreConfig search_api) VertexSearchAPIVectorStoreConfig)BedrockVectorStoreConfig)PGVectorStoreConfig)AzureAIVectorStoreConfig)MilvusVectorStoreConfig)GeminiVectorStoreConfig)RAGFlowVectorStoreConfig)S3VectorsVectorStoreConfig)ValkeyVectorStoreConfig)%rprr0litellm.llms.openai.vector_stores.transformationre r/litellm.llms.azure.vector_stores.transformationrf r ;litellm.llms.vertex_ai.vector_stores.rag_api.transformationrh >litellm.llms.vertex_ai.vector_stores.search_api.transformationrj r 1litellm.llms.bedrock.vector_stores.transformationrk PG_VECTOR3litellm.llms.pg_vector.vector_stores.transformationrl r 2litellm.llms.azure_ai.vector_stores.transformationrm MILVUS0litellm.llms.milvus.vector_stores.transformationrn rF 0litellm.llms.gemini.vector_stores.transformationro r7 1litellm.llms.ragflow.vector_stores.transformationrp S3_VECTORS4litellm.llms.s3_vectors.vector_stores.transformationrq VALKEY0litellm.llms.valkey.vector_stores.transformationrr )rFapi_typere rf rh rj rk rl rm rn ro rp rq rr s r3!get_provider_vector_stores_configz7ProviderConfigManager.get_provider_vector_stores_config"s    & &( 2 +, ,  ! ! ' '8 3 01 1  ! ! + +x 79$(8/00\)899ba ! ! ) )X 5 ,- -  ! ! + +x 7 '( (  ! ! * *h 6 ,- -  ! ! ( (H 4 +, ,  ! ! ( (H 4 +, ,  ! ! ) )X 5 ,- -  ! ! , , 8 ./ /  ! ! ( (H 4 +, ,r5cXtjj|k(r ddlm}|Sy)Nr)OpenAIVectorStoreFilesConfig)rprr5litellm.llms.openai.vector_store_files.transformationr )rFr s r3&get_provider_vector_store_files_configz|k(rdd l m!} | |StjD|k(rdd l#m$} | |StjJ|k(rddl&m'}||StjP|k(rddl)m*}||StjV|k(rddl,m-}||Stj\|k(rddl/m0}||Stjb|k(rddl2m3}||Sy)Nr)"get_openai_image_generation_config)!get_azure_image_generation_config)$get_azure_ai_image_generation_config)&get_xinference_image_generation_config)#get_recraft_image_generation_config) get_aiml_image_generation_config)$get_cometapi_image_generation_config)"get_gemini_image_generation_config)!LiteLLMProxyImageGenerationConfig)"get_fal_ai_image_generation_config)%get_stability_image_generation_config)$get_runwayml_image_generation_config)-get_black_forest_labs_image_generation_config)%get_vertex_ai_image_generation_config)&get_openrouter_image_generation_config)%get_dashscope_image_generation_config)&get_modelscope_image_generation_config)4rr$litellm.llms.openai.image_generationr r#litellm.llms.azure.image_generationr r &litellm.llms.azure_ai.image_generationr XINFERENCE(litellm.llms.xinference.image_generationr RECRAFT%litellm.llms.recraft.image_generationr AIML"litellm.llms.aiml.image_generationr rD &litellm.llms.cometapi.image_generationr rF $litellm.llms.gemini.image_generationr rg :litellm.llms.litellm_proxy.image_generation.transformationr FAL_AI$litellm.llms.fal_ai.image_generationr STABILITY'litellm.llms.stability.image_generationr RUNWAYML&litellm.llms.runwayml.image_generationr BLACK_FOREST_LABS/litellm.llms.black_forest_labs.image_generationr r 'litellm.llms.vertex_ai.image_generationr rB (litellm.llms.openrouter.image_generationr ra 'litellm.llms.dashscope.image_generationr rb (litellm.llms.modelscope.image_generationr )rrFr r r r r r r r r r r r r r r r r s r3$get_provider_image_generation_configz:ProviderConfigManager.get_provider_image_generation_config'#s&   ( * 6e< <   8 + 5U; ;  " "h . 8> >  $ $ 0 :%@ @  ! !X - 7u= =   ( * 4E: :  " "h . 8> >  H , 6e< <  ' '8 3 56 6  H , 6e< <  # #x / 9? ?  " "h . 8> >  + +x 7 AG G  # #x / 9? ?  $ $ 0 :%@ @  # #x / 9? ?  $ $ 0 :%@ @r5ctj|k(r ddlm}|Stj|k(r ddlm}|Stj|k(r ddlm }|Stj|k(r ddl m }|Stj|k(r ddlm}|Stj |k(rddlm}||Sy)Nr)OpenAIVideoConfig)AzureVideoConfig)GeminiVideoConfig)VertexAIVideoConfig)RunwayMLVideoConfig)get_hosted_vllm_video_config)rr)litellm.llms.openai.videos.transformationr r(litellm.llms.azure.videos.transformationr rF )litellm.llms.gemini.videos.transformationr r ,litellm.llms.vertex_ai.videos.transformationr r +litellm.llms.runwayml.videos.transformationr r< litellm.llms.hosted_vllm.videosr )rrFr r r r r r s r3get_provider_video_configz/ProviderConfigManager.get_provider_video_config#s   ( * S$& &   8 + Q#% %  H , S$& &  # #x / X&( (  " "h . W&( (  % % 1 T/6 6r5ctj|k(r ddlm}|S|tjtj fvr ddlm}|Sy)Nr)OpenAIContainerConfig)AzureContainerConfig)rr-litellm.llms.openai.containers.transformationr rrI ,litellm.llms.azure.containers.transformationr )rFr r s r3get_provider_container_configz3ProviderConfigManager.get_provider_container_config#sN   ( * )* *  **L,C,CD D () )r5cDtj|k(r ddlm}|Sy)Nr)GeminiRealtimeConfig)rrF +litellm.llms.gemini.realtime.transformationr )rrFr s r3get_provider_realtime_configz2ProviderConfigManager.get_provider_realtime_config#s!   ( * X') )r5ctj|k(r ddlm}|Stj|k(r ddlm}|Sy)z Return the HTTP transformation config for realtime HTTP endpoints (POST /realtime/client_secrets and POST /realtime/calls). r)OpenAIRealtimeHTTPConfig)AzureRealtimeHTTPConfigN)rr0litellm.llms.openai.realtime.http_transformationr r/litellm.llms.azure.realtime.http_transformationr )rrFr r s r3!get_provider_realtime_http_configz7ProviderConfigManager.get_provider_realtime_http_config#sC   ( * ,- -    ) +, ,r5ctj|k(rddlm}||Stj|k(r ddlm}|Stj|k(r ddlm }|Stj|k(r ddl m }|Stj|k(rddlm}||Stj |k(rddlm}||Stj&|k(r dd lm}|Stj,|k(rdd lm} | |Stj2|k(rdd lm} | |Stj8|k(rdd lm} | |Stj>|k(rdd l m!} | |Sy)Nr)get_openai_image_edit_configr)AzureImageEditConfig)RecraftImageEditConfig)BlackForestLabsImageEditConfig)get_azure_ai_image_edit_config)get_gemini_image_edit_config)LiteLLMProxyImageEditConfig)get_vertex_ai_image_edit_config)get_stability_image_edit_config)'get_bedrock_image_edit_config_for_model) get_openrouter_image_edit_config)"rrlitellm.llms.openai.image_editr r,litellm.llms.azure.image_edit.transformationr r .litellm.llms.recraft.image_edit.transformationr r 8litellm.llms.black_forest_labs.image_edit.transformationr r litellm.llms.azure_ai.image_editr rF litellm.llms.gemini.image_editr rg 4litellm.llms.litellm_proxy.image_edit.transformationr r !litellm.llms.vertex_ai.image_editr r !litellm.llms.stability.image_editr r Llitellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformationr rB "litellm.llms.openrouter.image_editr ) rrFr r r r r r r r r r r s r3get_provider_image_edit_configz4ProviderConfigManager.get_provider_image_edit_config#sS   ( * S/e< <   8 + () )  ! !X - *+ +  + +x 7 23 3  " "h . W1%8 8  H , S/6 6  ' '8 3 /0 0  # #x / 359 9  # #x / 359 9  ! !X - ;5A A  $ $ 0 4E: :r5cddlm}|tjjk(rddlm}||S|tjjk(rddlm }||S|tjjk(r!ddl m }m }|dk(r|S|dk(r|Sy tj}tjj |i}|j#|d } | y | S) z= Get OCR configuration for a given provider. r)VertexAIOCRConfig)get_azure_ai_ocr_configr)get_vertex_ai_ocr_config)ReductoParseLegacyConfigReductoParseV3Configzparse-v3z parse-legacyN))litellm.llms.vertex_ai.ocr.transformationr rprr &litellm.llms.azure_ai.ocr.common_utilsr r 'litellm.llms.vertex_ai.ocr.common_utilsr REDUCTO'litellm.llms.reducto.ocr.transformationr r rrrO rw) rrFr r r r r rPROVIDER_TO_CONFIG_MAPrs r3get_provider_ocr_configz-ProviderConfigManager.get_provider_ocr_config+$s P w++44 4 V*7 7 w++55 5 X+%8 8 w++33 3   "+--&/11"/"@"@  ( (*:) 5884H  ~r5cddlm}ddlm}ddlm}ddlm}ddlm }ddl m }ddl m }dd lm}dd lm} dd lm} dd lm} dd lm} ddlm} ddlm}ddlm}ddlm}ddl m!}ddl"m#}ddl$m%}ddl&m'}itPjR|tPjT|tPjV| tPjX|tPjZ|tPj\| tPj^|tPj`| tPjb|tPjd|tPjf| tPjh|tPjj|tPjl|tPjn|tPjp|tPjr|tPjt|tPjv| tPjx|i}|j{|d}|y|S)z@ Get Search configuration for a given provider. r)APISerpentSearchConfig)BingGroundingSearchConfig)AgentCoreSearchConfig)BraveSearchConfig)DataForSEOSearchConfig)DuckDuckGoSearchConfig)ExaAISearchConfig)FastCRWSearchConfig)FirecrawlSearchConfig)GooglePSESearchConfig)LinkupSearchConfig)NimbleSearchConfig)ParallelAISearchConfig)PerplexitySearchConfig)SearchAPIConfig)SearXNGSearchConfig)SerperSearchConfig)TavilySearchConfig)TinyfishSearchConfig)YouComSearchConfigN)>-litellm.llms.apiserpent.search.transformationr (litellm.llms.azure.search.transformationr *litellm.llms.bedrock.search.transformationr (litellm.llms.brave.search.transformationr -litellm.llms.dataforseo.search.transformationr -litellm.llms.duckduckgo.search.transformationr )litellm.llms.exa_ai.search.transformationr *litellm.llms.fastcrw.search.transformationr ,litellm.llms.firecrawl.search.transformationr -litellm.llms.google_pse.search.transformationr )litellm.llms.linkup.search.transformationr )litellm.llms.nimble.search.transformationr .litellm.llms.parallel_ai.search.transformationr -litellm.llms.perplexity.search.transformationr ,litellm.llms.searchapi.search.transformationr *litellm.llms.searxng.search.transformationr )litellm.llms.serper.search.transformationr )litellm.llms.tavily.search.transformationr +litellm.llms.tinyfish.search.transformationr *litellm.llms.you_com.search.transformationr rrN TAVILY PARALLEL_AIEXA_AIBRAVE GOOGLE_PSE DATAFORSEO FIRECRAWLFASTCRWSEARXNGLINKUP DUCKDUCKGO SEARCHAPISERPERYOU_COM APISERPENTTINYFISH AGENTCORENIMBLEBING_GROUNDINGrw)rFr r r r r r r r r r r r r r r r r r r r r rs r3get_provider_search_configz0ProviderConfigManager.get_provider_search_configU$s  WTNXXORVWPP  YPRPPTQ)  & &(>)  " "$6)   ' ')?)   " "$5 )  ! !#4 )  & &(= )   & &(>)   % %'<)   # #%8)   # #%8)   " "$6)   & &(>)   % %)   " "$6)   # #%7)  & &(>!) "  $ $&:#) $  % %'<  " "$6  * *,E)) ,5884H  ~r5cddlm}ddlm}|tj k(r|S|tj k(r|Sy)zR Get sandbox (code execution) configuration for a given provider. r)E2BSandboxConfig)OpenSandboxSandboxConfigN)'litellm.llms.e2b.sandbox.transformationr5 /litellm.llms.opensandbox.sandbox.transformationr6 rE2B OPENSANDBOX)rFr5 r6 s r3get_provider_sandbox_configz1ProviderConfigManager.get_provider_sandbox_config$sB M  '++ +#% % '33 3+- -r5c,ddlm}tjj|k(r|j dr ddlm}|Sy tjj|k(r ddl m }|Stjj|k(r ddl m }|Stjj|k(r ddlm}|Stjj"|k(r ddlm}|Stjj(|k(r dd lm}|Sy ) zH Get text-to-speech configuration for a given provider. rrzspeech/)AzureAVATextToSpeechConfig)ElevenLabsTextToSpeechConfig)RunwayMLTextToSpeechConfig)VertexAITextToSpeechConfig)MinimaxTextToSpeechConfig)AWSPollyTextToSpeechConfigN)3litellm.llms.base_llm.text_to_speech.transformationrrprrr0litellm.llms.azure.text_to_speech.transformationr= r 5litellm.llms.elevenlabs.text_to_speech.transformationr> r 3litellm.llms.runwayml.text_to_speech.transformationr? r 4litellm.llms.vertex_ai.text_to_speech.transformationr@ r1 2litellm.llms.minimax.text_to_speech.transformationrA AWS_POLLY4litellm.llms.aws_polly.text_to_speech.transformationrB ) rrFrr= r> r? r@ rA rB s r3"get_provider_text_to_speech_configz8ProviderConfigManager.get_provider_text_to_speech_config$s     % % 1 *233>= ! ! , , 8 01 1  ! ! * *h 6 ./ /  ! ! + +x 7 ./ /  ! ! ) )X 5 -. .  ! ! + +x 7 ./ /r5ctjj|k(r ddlm}|Stjj |k(r%ddlm}ddlm }|j|ry|Sy)Nr)GoogleGenAIConfig)VertexAIGoogleGenAIConfig)VertexAIPartnerModels) rprrF /litellm.llms.gemini.google_genai.transformationrM r 2litellm.llms.vertex_ai.google_genai.transformationrN 4litellm.llms.vertex_ai.vertex_ai_partner_models.mainrO is_vertex_partner_model)rrFrM rN rO s r31get_provider_google_genai_generate_content_configzGProviderConfigManager.get_provider_google_genai_generate_content_config$sg    & &( 2 %& &  ! ! + +x 7  %<>> ,>3>@[[[ -[[z!"$"" '""H!F%FF 'FFP #  "$ 4 4 4 " 4 4--- !--^""" &""H ) 8 "      $PPP &PPd   *  jjj *jjX < #" # '.@@@ $@@D''' ''R8!8 88t" !"222 '22h 5r5rcDttjtd}t t |t |}t t dz|jdxs|jd}tjry|dk(rtjdury|S)z Used for enforcing `disable_end_user_cost_tracking` param. service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking. rrKNuser_api_key_end_user_id prometheusT) rrrrrrrrwrpdisable_end_user_cost_tracking-enable_end_user_cost_tracking_prometheus_only)r service_typer _metadata end_user_ids r3!get_end_user_id_for_cost_trackingr` $s/6ckk(6KMo.p$D"B4WeCf"ghI d 56c)--Hb:cK-- |#  @ @ L r5cn|syd|vxrd|v}|jdxs|xr|jd S)NFmax_chunks_per_docmax_tokens_per_docz /v1/rerankz /v2/rerank)endswith)rr uses_v1_paramss r3r r %sP 15KK$::   \ * f~/ehFWFWXdFeBefr5cttS t|jd}|tS|S#t$r tcYSwxYw)a Returns the smallest prefix `model` will actually cache. Resolution order is an explicitly configured `MINIMUM_PROMPT_CACHE_TOKEN_COUNT`, then the model's `prompt_cache_min_tokens` in the cost map, then the provider-agnostic default. The cost map is the source of truth because the real minimum is per-model and per-platform: Anthropic's ranges from 512 to 4096 and moves in both directions across releases, and the same model can differ by platform. Never raises. An unresolvable model falls back to the default rather than propagating, so a caller cannot mistake "no entry for this model" for "this prompt is not cacheable". rr)r(rrwrUr)r min_tokenss r3get_prompt_cache_min_tokensrh %sV1<888*7;;)rrr!rmin_token_count token_countr3s r3is_prompt_caching_valid_promptrl 8%s"     *53C3CDW3X'#-5E**.   "9FOo-- JANsA|r t |SdS)a Extracts the total duration (in seconds) from SRT or VTT content. Args: srt_or_vtt_content (str): The content of an SRT or VTT file as a string. Returns: Optional[float]: The total duration in seconds, or None if no timestamps are found. z"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})Nirg@@)rfindallrrOrIr) srt_or_vtt_contenttimestamp_pattern timestamps durationsmatchhoursminutesseconds milliseconds total_secondss r3 extract_duration_from_srt_or_vttry \%s E=?ZZHY[m=nJ I(03C-w w|3g= v@UU '( '3y>0D0r5cXtj|}|ji}|jj d}|j d}|j dDcgc]}|s| }}|j dDcgc]}|s| }}g} tt|D]#} || d|dt|| z k(s|d| |z} n||z} ddj| z} |j| } t| j|jScc}wcc}w)z Adds an ending path to an API base URL while preventing duplicate path segments. Args: api_base: Base URL string ending_path: Path to append to the base URL Returns: Modified URL string with proper path handling )rIrNNr) rPrrrrstriplstriprrrr rrI) r ending_pathrbase_url base_pathend_pathr base_segments end_segmentsfinal_segmentsrS modified_pathrs r3_add_path_to_api_baser y%s< ))H-L",,B,7H#((//4I!((-H(1s';A!qAAMA&.nnS&9?Q1?L?N 3}% &6   -Es=/AA/E F F*2A.=N 6 '5.!99M",,-,@L |%%\-@-@%A BB%B?s'D"/D"D'D'c|jDcic]\}}|tjvs||| c}}Scc}}wr;)rrpOPENAI_CHAT_COMPLETION_PARAMS)rIr`ras r3get_standard_openai_paramsr %s7#\\^ lTQqG4Y4Y/Y^_^kAqD ll ls ;;;ctj}|tz}|jDcic] \}}||vs ||}}}|Scc}}wr;)rpr rr)rxrrr`rars r3!get_non_default_completion_paramsr %sU"@@M),>>N!A1N+B1!!  !s AAcd|vr|dSd|vr|dS|jd}t|trd|vr|dSd|vr|dSy)zRead AI-SDK-style reasoning summary from optional_params or nested extra_body. Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped. reasoningSummaryreasoning_summaryr8N)rwr}r)rr8s r3peek_reasoning_summary_aliasesr %sq _,122o-233'++L9J*d#  +01 1 * ,12 2 r5cZt|}|jdd}|jdd}||}|jd}t|trTt|}|jdd}|jdd}||}||}|r ||d<||fS|jdd||fS)zTCopy optional_params; remove reasoningSummary aliases from top-level and extra_body.r Nr r8)rrrwr})roprs_val snake_rs_valeb eb_rs_valeb_snake_rs_vals r34strip_reasoning_summary_aliases_from_optional_paramsr %s_%B VV& -F&&!4d;L ~  B"d "X66"4d; !#(;T!B >F~( !B|  v: FF< & v:r5cddlm}|tz}|jDcic] \}}||vs ||}}}|Scc}}w)Nrr)rrrr)rxrrr`rars r3$get_non_default_transcription_paramsr %sE=7:LLN28,,. \$!QA^D[A \ \ !]s ::c0|y|jDcic](\}}|dk7s t|tst||*}}}t|dkDr-i}d}|jD]\}}|dkr|||<|dz }|}|j Scc}}w)a\ Add metadata to openai optional parameters, excluding hidden params. OpenAI 'metadata' only supports string values. Args: params (dict): Dictionary of API parameters metadata (dict, optional): Metadata to include in the request Returns: dict: Updated parameters dictionary with visible metadata only N hidden_paramsrr)rr}rrr)rsr`ravisible_metadatafiltered_metadatars r3add_openai_metadatar %s'nn.(a!2F:VWY\K]A ((  r!#%$**, DAqRx'(!!$ 1HC -  ""(s BBBc|sy|jd}t|trt|}|r|St|}|r|Sy)Nrt)rwr}rr )rsrtcleaned_metadatas r3get_requester_metadatar &sO  ( -A B$d+./AB # #*84 r5c hddlm}ddlm}|ddddgdd d |jd d }t t |j }d } |di||dd|jjd}|rtt|St|S#t$r}t|}Yd}~Rd}~wwxYw)z Return the json str of the request This is currently in BETA, and tested for `/chat/completions` -> `litellm.completion` calls. rr)Loggingz gpt-3.5-turborhirFr1234T)rrrrrrrlog_raw_request_responserzmy-fake-api-key)rrXNraw_request_typed_dict)r>r) rrUr rrrprrUrrrwrr) endpointrxrr rllm_api_endpointreceived_exceptionr3r s r3return_raw_requestr &s "B!(!d348<<>!% "&gx~~>$  3% %8$J$J$N$NOg$h')?@@"$   $ V$s B B1 B,,B1cg}|D]g}t|tr|jd}n t|tr|j }t|tsW|j |i|S)zn Fixes https://github.com/BerriAI/litellm/issues/9321 Where user passes in a pydantic base model Tr$)r}rrrrrIr4s r3r0r0?&sg $&I# dI &???5D d #99;D dD !   T " # r5ctdddS)Nr) prompt_tokenscompletion_tokensr)rrr5r3get_empty_usager P&s  r5c|s|s|ryyrr)rRmock_tool_calls mock_timeouts r3should_run_mock_completionr X&s < r5chddlm}|}||vr ||}||Stdtd|)zSLazy import handler for utils module with cached registry for improved performance.r)_get_lazy_import_registryzmodule z has no attribute )litellm._lazy_importsr AttributeErrorr)r}r registry handler_funcs r3 __getattr__r b&sG@/1H x&tn D!! 78,.@I JJr5)rHrLrrg)rVrUrrU)Fr9)rBrgrCz!Literal['DEBUG', 'INFO', 'ERROR'])rcrrdzLiteral['success', 'failure']rr)rer0rrg)rxdict[str, Any]rrt)rxdict[str, object]rrt)rxr)rz*list[str | Callable | CustomLogger] | Nonerr)rrLrrLrrg)rrrrrr)rrrrrr)rrrrrr)rrPrr)rrPrrg)r'rr(rrzdatetime.datetimerPrrrgrxrrz+tuple[LiteLLMLoggingObject, dict[str, Any]])rrrDrg)rxr rVrUrz!tuple[int | None, dict[str, Any]])rxr rVrUrz"float | int | httpx.Timeout | Noner)rxr rr)r[rr\rPrzCallTypes | Nonerz Any | None)r[Mapping[str, object]rVrUrrrr)rrKr)rxrKrrgrrg)rxr rzCallTypes | strrrgr;)rrrz!CustomHuggingfaceTokenizer | None)rrrr)rrrzSelectTokenizerResponse | None)rrN)rrK)rrNT)r Sequence[int]rrKr rg)rrrr rr )mainN)rrrrL)rr) rNNNFNNFN)rz%dict | SelectTokenizerResponse | Noner str | list[str] | NonerzSequence | Noner  bool | Noner!$list[ChatCompletionToolParam] | Noner"z)ChatCompletionNamedToolChoiceParam | Noner#r r$ int | NonerrO)rrrrg)rrrrLrrg)rrrrLrrrzLiteral[True] | None)rrrrLrrrrg)rrrrLrrrrL)rrrrLrr2)r{rr|rrr)rrrFrrzdict[str, object] | None)rrrzModelInfo | None)rzCallable[[], None]rrr)rQz str | dictrrgrrL)rrrrrr9rr)NNNNNN)rrrrrrLrrLrgrLrr rz'list[Literal['word', 'segment']] | Nonerr )rrLrr) NNNNNNNNNNNN)rrLrr rrLrgrLrrLrrLrrLrrKrrLrr9rrW rr )NNNrNNN)rrrrLr*rLr$r rr rr2rr2)rV)rrr_r2rr)rxrrr)FFN)rrrrrrrr2rrrrrgrsrgrrU rr)rrrr)rrrrrrrr)(NNNNNFNNNNNNNNNNNrNNNNNNNNNNNNNNNNNNNNNN)rrrr2rlist[AllMessageValues] | NonerAnthropicThinkingParam | NonerzOpenAIWebSearchOptions | Noner rLr r r rLrrL) rrrrrrrrtrr9rr)rrrrrr)rr rzfrozenset[str])rrrr) rr rr r r r!r rr )r&z dict | Anyrr )r* list[dict]r+r rr)r*r r3zIterable[str] | Nonerr)rrrrrrL)rrrzAllowedModelRegion | None)rrrrg)rrrSrrrg)rrrYrLrrL)rxrrr)r rrrO)rlz#ModelResponse | ModelResponseStreamrr)rrrrL)rrrr r)rrrr)rrrrLrr)rrO)rzdict[str, str])rrrrL)rrrrL)rrrr)r7rrrLrrg)rrrrrrLrztuple[str, dict] | None)rrrrLrr)rrrr )rrrrLrrLrr)rrrrLrz ProviderSpecificModelInfo | None)rrrrg)NNN) rrrrLrrLrXrLrr) rrrrLrrLrXrLrrrJ)rrrrLrrLrr)rkr)rr)rrrrr) rrrrrrrrrrOrrO)rrrzlist[ProviderField])NNNN) rrLrXrLrrLrlrLrr)rrrXr)rrO)rhttpx.Headers | Noner) rrOrrOrr rrOrz float | int) rrr"rr#rr$rr'r)rr )rRz)str | MockException | ModelResponseStreamrr )rrLr}rg)rrLrrNrrg)rrLrrt)rrrrrrrrt)NNNNN) rr rrLrrrXrLrrLrrt)rrrrrg)rrrrg)rrrr)rlist[AllMessageValues]rrg)rrrzlist[ChatCompletionToolParam])rzBaseModel | dictrr)rr)rr)r!r9rzlist[dict] | None)rz$AnthropicThinkingParam | bool | Nonerr )rrr)rr )r"dict | str | Nonerr )rr rr )rr )rrrFrrr )r)rrr] z(Literal['litellm_logging', 'prometheus']rrL)rrLr rt)rrrrO) rrrr r!r rrLrj r rrg)ro rrr)rrr} rrr)rIr rr)rxr rr)rrr object | None)rrrztuple[dict, object | None])rszMapping[str, object] | Nonerr)rsr )r r{rxrrr)r!zSequence[object]rr )rr)rRr r r r r rrg)r}rrr(dr __future__rrrr binasciirrrrr=io itertoolsrloggingrrrstruct subprocessrtextwrap threadingrr dataclassesrr functoolsrr importlibrr r os.pathr r r typesrdotenvrPrtiktokenr httpx._utilsr openai.librr*openai.types.chat.completion_create_paramsrpydanticrr tokenizersrrplitellm.litellm_core_utils/litellm.litellm_core_utils.json_validation_rulelitellm._internal_contextrr rrr litellm._uuidrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,3litellm.litellm_core_utils.fallback_generalizationsr-rbrhr\r1r4rJrXr^rdrirkrqlitellm.types.llms.openairrrsrtrurvrwrxrylitellm.types.utilsrzr{r|r}r~rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrLrMr]fr^ json_data ImportErrorr rt open_textdumpsrimportlib.metadatacollections.abcrrrrtypingrrrrrrrrror litellm.llms.base_llm.base_utilsrrrcrargr2r0)litellm.litellm_core_utils.cached_importsrrrrrrr.litellm.litellm_core_utils.credential_accessorr0litellm.litellm_core_utils.dot_notation_indexingrr2litellm.litellm_core_utils.exception_mapping_utilsr-litellm.litellm_core_utils.get_litellm_paramsrrr rr6litellm.litellm_core_utils.get_supported_openai_paramsr,litellm.litellm_core_utils.llm_request_utilsrFlitellm.litellm_core_utils.llm_response_utils.convert_dict_to_responserrrrr:litellm.litellm_core_utils.llm_response_utils.get_api_baserBlitellm.litellm_core_utils.llm_response_utils.get_formatted_promptr9litellm.litellm_core_utils.llm_response_utils.get_headersr?litellm.litellm_core_utils.llm_response_utils.response_metadatarrr#r*litellm.litellm_core_utils.redact_messagesrr litellm.litellm_core_utils.rulesrrr/litellm.litellm_core_utils.thread_pool_executorr7litellm.llms.base_llm.anthropic_messages.transformationr8litellm.llms.base_llm.audio_transcription.transformationr,litellm.llms.base_llm.batches.transformationr/litellm.llms.base_llm.containers.transformationr.litellm.llms.base_llm.embedding.transformationr*litellm.llms.base_llm.files.transformationr1litellm.llms.base_llm.google_genai.transformationr/litellm.llms.base_llm.image_edit.transformationr5litellm.llms.base_llm.image_generation.transformationr5litellm.llms.base_llm.image_variations.transformationr(litellm.llms.base_llm.ocr.transformationr0litellm.llms.base_llm.passthrough.transformationr2litellm.llms.base_llm.realtime.http_transformationr-litellm.llms.base_llm.realtime.transformationr+litellm.llms.base_llm.rerank.transformationr,litellm.llms.base_llm.sandbox.transformationr+litellm.llms.base_llm.search.transformationrrC r1litellm.llms.base_llm.vector_store.transformationr7litellm.llms.base_llm.vector_store_files.transformationr+litellm.llms.base_llm.videos.transformationrrr5litellm.llms.bedrock.embed.amazon_nova_transformationr9litellm.llms.bedrock.embed.amazon_titan_g1_transformationrAlitellm.llms.bedrock.embed.amazon_titan_multimodal_transformationr9litellm.llms.bedrock.embed.amazon_titan_v2_transformationr0litellm.llms.bedrock.embed.cohere_transformationrrHrNrQrTr^rbrhrvrrrrrrrrrrrrrrrPrrtyping_extensionsrrrrrrrrHrcompilerr6r`rrr cache_informrrrrrrrrr r rr r*r,rXrZrarjrprnrsrdrerlitellm.caching.in_memory_cacherrrrrrrrrrrrrr rrrrrrrrr r!r"r'r.r2r6r<r)r1rFrJrQrUrr` r rh rl ry r r r r r r r r r r0r r r )cts0r3r. s0M#        (&**" 0*E !76 &   4oo2  #      #########J 8  3R ]]]"]  ]  ]  ]1]@  % #'  @2< ."(5E.2&23<2IL2 2jT!TnJ ^"  @ )))**))//**00  u .  . .  .1 -. +/ +N*2 $( $  " 0 IE2 E >B#' $).26=A16&*$;$ !$ $ ' $ 0 $;$$/$$$ $N ***D)X*o =@%P$No m o **$m(V&  #%  uD(GI CH//,H6$('+ \\!\% \~   "&"GK#V VVV  V  V  VEVVr1$"&#&**.8<#z zzz z  z  z zz$z(z6zz@"&!#/3.2` ` ` `  ``-`,`F .*B Z,KA"A"T#().)-::::- :  :  :#':': :z  Kd       .2.2.28<$(#'!WP  P B,CP J,KP L,MP N6OP P"QP R SP T!UP VWP p+/ &&&& & ( &  &R.&//b%cUc S  (%( ( ($. (  (V 259i#i1i i.&2(V  0 +#\(V!; |, 7 3 (4807 ()(+ 0%&&&R#(V21i# #B#$# #L'T. -.  # /"$ B   $.2::.E#F5F '+ @ @ #@ @  @  @ J '+ U U#UU U  U6 -.'+f f#ff f/f'+ QH QH#QHQH QH  QHh4??2==G4Vr " ( ( ( (   (  (  ( V  14" SV SV SVSV SV  SVl 0 H*<.2*J.2 """+" "  ": "  @& @& @&@& @&  @&FK%K%\4 < 8 " D6( 922t*#( b! b!b!! b!J:9:C=:C|+, e,'+#6-1%* .,0&*,0 G(G#G*G G  G  GT' T4##>B( .b&  $ *j$j j@*d$I& 626"62C&R"4& 1 $zz~->O::g637&*"& ! !+! 0!$ !  !  !H1:#CLm$2 #) # #F"( V" "   KMk$F!! ^Y/!   DF` a!efDIIaL !!!!sH1{ 5{{{{{|(;| |(|! |('|(