tPjOLUdZddlZddlZddlZddlZddlmZmZmZm Z ddl m Z ddl m Z mZddlmZddlmZmZmZmZmZmZmZddlmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#dd l$m%Z%ddl&m'Z'ddl(m)Z)m*Z*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ddlm?Z?m@Z@ddlAmBZBmCZCmDZDmEZEmFZFmGZGer ddlHmIZJddlKmLZMddlNmOZOddlZkeeQd?<d@ZleeQdA<dBZmeeQdC<dDZneeQdE<eoepzeqzerzZseeQdF<eesdzZteeQdG<eueqevfeozepzeqzerzZweeQdH<GdIdJedKLZxGdMdNeZyGdOdPeZzGdQdReZ{GdSdTeZ|GdUdVeZ}GdWdXe=Z~GdYdZeZGd[d\eZGd]d^eZGd_d`eZGdadbeZeeeqeeoeefZeeQdc<GdddeeZe dfgGdhdiZe djdkZee edzeQdl<dmedzfdnZdmefdoZdpeudmefdqZdreqdzdmedzfdsZdtevdmeqdzfduZdvevdmeodzfdwZGdxdye#Zy)zz This is a rate limiter implementation based on a similar one by Envoy proxy. This is currently in development and not yet ready for production. N)CallableMappingSequenceSet) ContextVar) dataclassfield)datetime) TYPE_CHECKINGAnyFinalLiteralProtocol TypeAlias TypedDict) NotRequiredReadOnly) DualCache)verbose_proxy_logger)-DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE!INTERNAL_CALL_ORIGIN_METADATA_KEY) CustomLogger)get_str_from_messages)UserAPIKeyAuth)ESTIMATED_OUTPUT_TOKENS_FIELDget_estimated_output_tokensget_key_tag_rpm_limit"get_model_rate_limit_from_metadata)throttled_limit)get_tags_from_request_body)ProxyRateLimitErrormap_v3_rate_limit_type)BATCH_ENQUEUED_REFUND_STATUSESBatchEnqueuedTokenReservationBatchEnqueuedTokenStorebatch_response_viewcanonical_provider_batch_id)#resolve_llm_provider_for_rate_limit)RedisPipelineIncrementOperation)BaseLiteLLMOpenAIResponseObjectResponseAPIUsage) CallTypesEmbeddingResponse ModelResponseRerankResponseTextCompletionResponseUsage)Span)InternalUsageCache) AgentResponsea. local results = {} local now = tonumber(ARGV[1]) local window_size = tonumber(ARGV[2]) -- Process each window/counter pair for i = 1, #KEYS, 2 do local window_key = KEYS[i] local counter_key = KEYS[i + 1] local increment_value = 1 -- Check if window exists and is valid local window_start = redis.call('GET', window_key) if not window_start or (now - tonumber(window_start)) >= window_size then -- Reset window and counter redis.call('SET', window_key, tostring(now)) redis.call('SET', counter_key, increment_value) redis.call('EXPIRE', window_key, window_size) redis.call('EXPIRE', counter_key, window_size) table.insert(results, tostring(now)) -- window_start table.insert(results, increment_value) -- counter else local counter = redis.call('INCR', counter_key) -- This happens when window_key exists but counter_key doesn't (e.g., tokens key -- created after requests key when both share the same window_key) local current_ttl = redis.call('TTL', counter_key) if current_ttl == -1 then redis.call('EXPIRE', counter_key, window_size) end table.insert(results, window_start) -- window_start table.insert(results, counter) -- counter end end return results BATCH_RATE_LIMITER_SCRIPTa- -- Atomic check-and-increment-by-N across one or more descriptors. -- All-or-nothing: if any descriptor would exceed its limit, no counter is -- modified. -- -- Uses Redis server time (`redis.call('TIME')`) instead of a client-supplied -- timestamp so that window resets are deterministic across replicas with -- skewed wall-clocks. This prevents a clock-skew-induced reopening of the -- TOCTOU window across multi-replica deployments. -- -- KEYS layout: pairs of (window_key, counter_key), one pair per descriptor. -- ARGV layout: per-descriptor 4-tuple, starting at ARGV[1]: -- ARGV[(i-1)*4 + 1] = limit -- ARGV[(i-1)*4 + 2] = increment -- ARGV[(i-1)*4 + 3] = ttl_seconds (counter TTL when window resets) -- ARGV[(i-1)*4 + 4] = window_size_seconds (sliding-window length) -- -- Return on success: -- { 0, new_counter_1, window_start_1, new_counter_2, window_start_2, ... } -- Return on over-limit: { 1, descriptor_index, current_counter, limit } local time_reply = redis.call('TIME') local now = tonumber(time_reply[1]) local descriptor_count = #KEYS / 2 -- Pass 1: read state, validate. Abort without writing if any over limit. local descriptor_state = {} for i = 1, descriptor_count do local window_key = KEYS[(i - 1) * 2 + 1] local counter_key = KEYS[(i - 1) * 2 + 2] local arg_base = (i - 1) * 4 + 1 local limit = tonumber(ARGV[arg_base]) local increment = tonumber(ARGV[arg_base + 1]) local window_size = tonumber(ARGV[arg_base + 3]) local window_start = redis.call('GET', window_key) local window_expired = (not window_start) or ((now - tonumber(window_start)) >= window_size) local current_counter if window_expired then current_counter = 0 else current_counter = tonumber(redis.call('GET', counter_key) or 0) end local blocked if increment > 0 then blocked = current_counter + increment > limit else blocked = current_counter >= limit end if blocked then return { 1, i, current_counter, limit } end descriptor_state[i] = { window_expired, current_counter, window_start } end -- Pass 2: all checks passed. Apply increments. local results = { 0 } for i = 1, descriptor_count do local window_key = KEYS[(i - 1) * 2 + 1] local counter_key = KEYS[(i - 1) * 2 + 2] local arg_base = (i - 1) * 4 + 1 local increment = tonumber(ARGV[arg_base + 1]) local ttl = tonumber(ARGV[arg_base + 2]) local window_size = tonumber(ARGV[arg_base + 3]) local window_expired = descriptor_state[i][1] local active_window_start if window_expired then active_window_start = now redis.call('SET', window_key, tostring(now)) redis.call('SET', counter_key, increment) redis.call('EXPIRE', window_key, window_size) if ttl > 0 then redis.call('EXPIRE', counter_key, ttl) end table.insert(results, increment) else active_window_start = tonumber(descriptor_state[i][3]) local new_counter = redis.call('INCRBY', counter_key, increment) local current_ttl = redis.call('TTL', counter_key) if current_ttl == -1 and ttl > 0 then redis.call('EXPIRE', counter_key, ttl) end table.insert(results, new_counter) end table.insert(results, active_window_start) end return results CHECK_AND_INCREMENT_BY_N_SCRIPTa local results = {} for i = 1, #KEYS, 2 do local window_key = KEYS[i] local counter_key = KEYS[i + 1] local arg_base = ((i - 1) / 2) * 3 + 1 local expected_window_start = ARGV[arg_base] local increment = tonumber(ARGV[arg_base + 1]) local ttl = tonumber(ARGV[arg_base + 2]) local active_window_start = redis.call('GET', window_key) if active_window_start and active_window_start == expected_window_start then local new_counter = redis.call('INCRBY', counter_key, increment) local current_ttl = redis.call('TTL', counter_key) if current_ttl == -1 and ttl > 0 then redis.call('EXPIRE', counter_key, ttl) end table.insert(results, 1) table.insert(results, new_counter) else table.insert(results, 0) table.insert(results, tonumber(redis.call('GET', counter_key) or 0)) end end return results %WINDOW_GUARDED_TOKEN_INCREMENT_SCRIPTa -- Atomic check-and-acquire for the max_parallel_requests concurrency gauge. -- Each gauge key is a sorted set of per-request slot ids scored by acquire -- time (Redis server clock). In-flight requests are counted by ZCARD after -- pruning slots older than the slot TTL, so unlike the windowed RPM/TPM -- counters the gauge is never reset while requests are in flight, a -- rejected request never occupies a slot, and a slot leaked by a crashed -- worker self-heals after the slot TTL even under continuous traffic. -- -- KEYS: one gauge zset key per descriptor. -- ARGV: per-key triples (limit, slot_ttl_seconds, slot_id). -- Success: { 0, in_flight_1, ... }. Over-limit: { 1, key_index, in_flight, limit }. local time_reply = redis.call('TIME') local now = tonumber(time_reply[1]) for i = 1, #KEYS do local limit = tonumber(ARGV[(i - 1) * 3 + 1]) local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - slot_ttl) local in_flight = redis.call('ZCARD', KEYS[i]) if in_flight + 1 > limit then return { 1, i, in_flight, limit } end end local results = { 0 } for i = 1, #KEYS do local slot_ttl = tonumber(ARGV[(i - 1) * 3 + 2]) local slot_id = ARGV[(i - 1) * 3 + 3] redis.call('ZADD', KEYS[i], now, slot_id) redis.call('EXPIRE', KEYS[i], slot_ttl) table.insert(results, redis.call('ZCARD', KEYS[i])) end return results PARALLEL_ACQUIRE_SCRIPTa -- Release one slot per gauge key by removing this request's slot id. -- ZREM of an absent member (or key) is a no-op, so a release without a -- matching acquire (proxy-side rejection, double-fired callback, slot -- already expired) can never free a slot owned by another request. -- KEYS: gauge zset keys. ARGV: per-key slot_id. -- Returns the remaining in-flight count per key. local results = {} for i = 1, #KEYS do redis.call('ZREM', KEYS[i], ARGV[i]) table.insert(results, redis.call('ZCARD', KEYS[i])) end return results PARALLEL_RELEASE_SCRIPTa -- Read the current in-flight count per gauge key (prunes expired slots -- first so leaked slots do not inflate the reading). -- KEYS: gauge zset keys. ARGV: per-key slot_ttl_seconds. local time_reply = redis.call('TIME') local now = tonumber(time_reply[1]) local results = {} for i = 1, #KEYS do redis.call('ZREMRANGEBYSCORE', KEYS[i], '-inf', now - tonumber(ARGV[i])) table.insert(results, redis.call('ZCARD', KEYS[i])) end return results PARALLEL_COUNT_SCRIPTa local results = {} -- Process each key/increment_value/ttl triplet for i = 1, #KEYS do local key = KEYS[i] local increment_value = tonumber(ARGV[i * 2 - 1]) local ttl_seconds = tonumber(ARGV[i * 2]) -- Increment the value local new_value = redis.call('INCRBYFLOAT', key, increment_value) -- Handle TTL: only set expire if ttl_seconds > 0 and key has no current TTL -- ttl_seconds can be 0 (no TTL) or positive (set TTL) if ttl_seconds and ttl_seconds > 0 then local current_ttl = redis.call('TTL', key) if current_ttl == -1 then redis.call('EXPIRE', key, ttl_seconds) end end table.insert(results, new_value) end return results TOKEN_INCREMENT_SCRIPTi@REDIS_CLUSTER_SLOTSall_keysREDIS_NODE_HASHTAG_NAMEiDEFAULT_MAX_TOKENS_ESTIMATEDEFAULT_CHARS_PER_TOKEN_TPM_FLOOR_FRACTION) aresponses responsesRESPONSES_API_CALL_TYPES) aembedding embeddingEMBEDDING_API_CALL_TYPES)atext_completiontext_completionTEXT_COMPLETION_API_CALL_TYPESRERANK_API_CALL_TYPESGOOGLE_GENAI_NATIVE_CALL_TYPESRESPONSES_API_MIN_OUTPUT_TOKENSi,DEFAULT_AUDIO_TOKEN_ESTIMATEi@_AUDIO_BYTES_PER_TOKENmodel_per_project_itpmPROJECT_ITPM_DESCRIPTOR_KEYmodel_per_project_otpmPROJECT_OTPM_DESCRIPTOR_KEYi!PARALLEL_REQUEST_SLOT_TTL_SECONDSCacheCounterValueCacheCounterValuesParallelGaugeCacheValuecNeZdZUedzed<edzed<edzed<edzed<y)"RateLimitDescriptorRateLimitObjectNrequests_per_unittokens_per_unitmax_parallel_requests window_size)__name__ __module__ __qualname__int__annotations__t/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/proxy/hooks/parallel_request_limiter_v3.pyr[r[s*Tz!4Z:%trfr[F)totalc2eZdZUeed<eed<edzed<y)RateLimitDescriptorkeyvalueN rate_limit)r`rarbstrrdr[rerfrgrjrjs H J2T99rfrjc,eZdZUeed<eed<eed<y)ParallelRequestGauge counter_keylimitdescriptor_keyN)r`rarbrnrdrcrerfrgrprps Jrfrpc(eZdZUeed<eeed<y)ParallelSlotAcquisitionslot_id counter_keysN)r`rarbrnrdlistrerfrgrurus Ls)rfruc\eZdZUeed<eed<eed<eded<eed<eeeed<y) RateLimitStatuscode current_limitlimit_remainingrequeststokensr^rate_limit_typersdescriptor_valueN) r`rarbrnrdrcrrrrerfrgrzrzs6 IJKK "(3-00rfrzcVeZdZUeed<eeed<eee e eee dfed<y)RateLimitResponse overall_codestatusesredislocalreservation_windowsN) r`rarbrnrdrxrzrr frozensettuplerrerfrgrrs:?##$Xic3P`Ha>a8b.c%deerfrcDeZdZUeeed<eeed<eeded<y)"ReservationAwareIncrementOperation window_keyexpected_window_startrreservation_backendN)r`rarbrrnrdrrerfrgrrs)C  &s++$W-=%>??rfrc(eZdZUeeed<eed<y) RateLimitResponseWithDescriptors descriptorsresponseN)r`rarbrxrjrdrrerfrgrrs)**rfrceZdZdeddfdZy)_RateLimitDescriptorSink descriptorreturnNcyNre)selfrs rgappendz_RateLimitDescriptorSink.appendsrf)r`rarbrjrrerfrgrrsE!4EDErfrcBeZdZUedzed<edzed<eed<eed<y)WindowKeyMetadataNrequests_limit tokens_limitr_rs)r`rarbrcrdrnrerfrgrrs"$J*rfrcteZdZUeed<eeed<eed<eded<eed<eed<eed<eed <eed <y ) AtomicCounterMetarsrr|rrrrrq incrementttlr_N)r`rarbrnrdrrcrrerfrgrrs?sm#122ON Hrfrc2eZdZUeed<eed<eeed<y)AtomicCounterStatewindow_expiredcurrent window_startN)r`rarbboolrdrcrrnrerfrgrrs L3-rfrDescriptorAtomicGroupc NeZdZdededeeefdedeezeeefzdzf dZ y)CallTypeRateLimiteruser_api_key_dictcachedata call_typerNc Kywrre)rrrrrs rgasync_pre_call_hookz'CallTypeRateLimiter.async_pre_call_hooks 69s) r`rarbrrdictrnobject ExceptionrrerfrgrrsW9)993; 9  9 S4V , ,t 3 9rfrT)slotsceZdZUdZdZedzed<dZedzed<dZ e dzed<dZ e ed<dZ edzed<ee Zeeeefed <dZe ed <ee Zeeeefed <ee Zeeeeed fed<dZe ed<ee Zeeeefed<ee Zeeeeed fed<dZedzed<dZeed<y)RequestRateLimiterStashab Per-request bookkeeping the pre-call hook hands to the success/failure/ disconnect callbacks. Lives on a ContextVar instead of the request body so it never reaches provider-facing ``metadata`` channels. A single mutable instance is shared by every context forked from the request task (the SDK call, streaming generators, and the logging worker's captured context all see the same object), which is what makes the ``reservation_released`` flag and ``parallel_slot`` clearing effective across sibling callbacks: the first release wins, later callbacks observe the cleared state. Because the stash is context-inherited, nested LiteLLM calls made inside the request (LLM-judge guardrails, silent experiments) would also see it from their own logging callbacks. ``owner_litellm_call_id`` pins the stash to the proxy request's ``litellm_call_id`` so those callbacks can tell the owning request's events apart from a nested call's: router retries and fallbacks reuse the request's call id and keep access, while nested calls mint fresh ids and are ignored. Nowner_litellm_call_idrate_limit_response parallel_slotrreserved_tokensreserved_model)default_factoryreserved_scopesitpm_reserved_tokensitpm_reserved_scopesritpm_reserved_window_identitiesotpm_reserved_tokensotpm_reserved_scopesotpm_reserved_window_identitiesbatch_enqueued_reservationFreservation_released)r`rarb__doc__rrnrdrrrrurrcrr rrrrrrrrrrrr$rrrerfrgrrs9*)-3:,48*T1848M*T18OS!%NC$J%27 2ROYuS#X/R !#!7e7d'e'dzddfd?Z8 dd,ee-d@eee dAe fd'e'dzde#fdBZ9dCe-dDee dAe fde:fdEZ; ddFee:d'e'dzde#fdGZdJee?dKee=de#fdLZ@ ddKee=d'e'dzde#fdMZA dd,ee-dNe d'e'dzde#fdOZBeCdfdPe)eeefdQe dReCeeee dSfd'e'dzddf dTZD dd,e)e-dUe dVe d'e'dzdee#e e ff dWZEdXeFdzdYedzdUe dVe ddf dZZG ddXeFdYedzdee-fd[ZHdXeFdYedzd,ee-ddfd\ZIdXeFd ed,ee-ddfd]ZJdXeFd^edzd,ee-ddfd_ZKdXeFd^edzd,ee-ddfd`ZLdaedzdbedefdcZMdde dzdaedzdbede dzfdeZNdfedzdgedzdefdhZOdieddjfdkZPdXeFd ededzfdlZQd ededzfdmZRdied edee-fdnZS ddXeFd edfedzdgedzdbed edzdee-fdoZT dded'e'dzdefdpZUd ededzfdqZVdXeFdYedzd,ee-ddfdrZWdXeFdYedzd,ee-ddfdsZXdXeFdYedzd,eYddfdtZZ ddue#d,ee-dYedzddfdvZ[e dwe de fdxZ\edye de fdzZ]e dye de fd{Z^e d e de)e fd|Z_e de de dzfd}Z`e d eee fdefd~Zadd e dedzd edzde fdZb dd,e)e-d e dYedzdXeFde)eeefde d edzddfdZcdXeFdedd ed efdZededede dde dedf dZfdegdzde dde fdZhe deiejzeee fzdzde fdZke de deiejzeee fzdzfdZldedddfdZm ddedd'e'dzddfdZn dde)eod'e'dzddfdZp dde)eod'e'dzddfdZq dde)eod'e'dzddfdZrde dfdZse deee fdee1deee ffdZte de dee e efdzfdZude dee e effdZvde de de)ewfdZxdeeegfdegdedzdeeeeffdZyde)eeefdezeeefde de deewf dZ{deeefdezeeefde de deCeeee dSfdeodzf dZ|de)eeefdezeeefde de deCeeee dSfdeeodff dZ}degdegde ddeewfdZ~dZdede d edeee ffdZde de ddfdZdZdXeFddfdZd edXeFfdZdXeFdue ddfdZ ddededXeFdedzddf dZy)$_PROXY_MaxParallelRequestsHandler_v3Ninternal_usage_cache time_providerc||_|xstj|_|jjj |jjj j t|_|jjj j t|_ |jjj j t|_ |jjj j t|_|jjj j t|_|jjj j t"|_|jjj j t&|_n1d|_d|_ d|_ d|_d|_d|_d|_t+t-j.dd|_t-j.ddj3dk(|_d|_t9||_t=j>|_ y)NLITELLM_RATE_LIMIT_WINDOW_SIZE<%LITELLM_TPM_TOKEN_RESERVATION_ENABLEDtrue)r)!rr now_time_provider dual_cache redis_cacheasync_register_scriptr5batch_rate_limiter_scriptr;token_increment_scriptr6check_and_increment_by_n_scriptr7%window_guarded_token_increment_scriptr8parallel_acquire_scriptr9parallel_release_scriptr:parallel_count_scriptrcosgetenvr_lowertpm_reservation_enabled_batch_rate_limiterr%batch_enqueued_token_storeasyncioLock_check_and_increment_lock)rrrs rg__init__z-_PROXY_MaxParallelRequestsHandler_v3.__init__Ds %9!+;x||  $ $ / / ; ; G-1-F-F-Q-Q-]-]-s-s).D *+/*C*C*N*N*Z*Z*p*p&+D '))44@@VVWvw  0))44@@VV9  6 ,0+D+D+O+O+[+[+q+q',D (,0+D+D+O+O+[+[+q+q',D (*.)B)B)M)M)Y)Y)o)o%*D &.2D **.D '37D 09=D 6+/D (+/D ()-D &ryy)I2NO (*yy1XZ`'a'g'g'ims's$@D *AWk*l' *1&rfrc|j, ddlm}||j||_|jS|jS#t$r+}t j d|Yd}~|jSd}~wwxYw)z(Get or lazy-load the batch rate limiter.Nr)_PROXY_BatchRateLimiter)rparallel_request_limiterz%Could not load batch rate limiter: %s)r&litellm.proxy.hooks.batch_rate_limiterr rrrdebug)rr es rg_get_batch_rate_limiterz<_PROXY_MaxParallelRequestsHandler_v3._get_batch_rate_limiters  # # + W,C)-)B)B-1,( '''t''' W$**+RTUVV''' WsA A9A44A9c"|jS)z7Return the current time for rate limiting calculations.)r)rs rg_get_current_timez6_PROXY_MaxParallelRequestsHandler_v3._get_current_times""$$rfmin_configured_tpm_limitc^ttz}||St|td|tzS)a Output-budget floor used when the request omits max_tokens. Capped at a fraction of the smallest configured TPM limit so a small per-tenant cap can't be tripped by the floor alone. Returns the baseline floor when no limit is provided. )r?rBminmax)rbaselines rgno_max_tokens_output_floorz?_PROXY_MaxParallelRequestsHandler_v3.no_max_tokens_output_floors569LL # +O8S$<@S$STUUrfrrct|tvry|tvry|ryt|tsy|j dduS)NTFinput)rHrErrrrrs rg_is_embedding_requestz:_PROXY_MaxParallelRequestsHandler_v3._is_embedding_requestsA 0 0 0 0 $%xx ,,rfc Tt|tr|jdnd}t|tr|tvst|ttfsyddlm}d|vr|jdn|jd}|jt|jdtr|jdnd|t|tr|nd|jd|jd |jd |jd |jd  S)Ncontentsr)GoogleGenAIAdapterconfiggenerationConfigmodelsystemInstructionsystem_instructiontools toolConfig tool_config)r!rrr#r$r%r&r') rrrrMrx,litellm.google_genai.adapters.transformationr(translate_generate_content_to_completionrn)rrrrrs rg&_translate_google_genai_native_requestzK_PROXY_MaxParallelRequestsHandler_v3._translate_google_genai_native_requests 3=T42H$((:.d4& >>ht 5S.6$.>*DHHM_D`!#LL'1$((72CS'I$((7#r'564"hh':;#xx(<=((7#xx -/M  rfcttsy|tvrGdvrjdnjdt fddD}t |dS|t vr-tjd}|yt t|S|tvry|rdnd}t fd |D}t |dS) Nrr c3zK|]2}ttr tj|}||4ywr)rrrr).0r parsedrs rg zP_PROXY_MaxParallelRequestsHandler_v3._get_explicit_output_cap..s<-fd+6vzz%7HIF% -s8;)maxOutputTokensmax_output_tokensrr1) max_tokensmax_completion_tokens)r2r3r1c3\K|]#}tj|}| |%ywr)rr)r-r r.rs rgr/zP_PROXY_MaxParallelRequestsHandler_v3._get_explicit_output_cap..s/) 6MdhhW\o6^6djdvF) s!,,) rrrMrrrrErrOrH)rrgoogle_cap_values responses_capfieldsoutput_cap_valuesrs` @rg_get_explicit_output_capz=_PROXY_MaxParallelRequestsHandler_v3._get_explicit_output_caps$% 6 62:d2BDHHX.QcHdF',-E-( ($7 7 0 0#:488DW;X#YM$6 F F 0 0 4M  $)) &) $ $d33rfc*|j||duS)zWhether the caller explicitly set an output-token cap. Checked via ``is not None`` (not truthiness) so an explicit 0 -- a legitimate zero-output request -- counts as explicit. N)r9)clsrrs rg_has_explicit_output_capz=_PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_caps++D)BT\`B`(fxL HH\2F~FD!9&|&%}&\" 2;?W2W.]i "hhy1  =<#?+DO$@rfr!cp|j||||\}}||z}tjd||||S)a Estimate total tokens this request will consume so we can reserve them upfront (input + output budget): estimated = input_tokens + max_tokens. Supports chat (messages), completions (prompt), embeddings (input), and the Responses API (also `input`, disambiguated from embeddings via ``call_type``). ``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among the TPM-bearing descriptors this request will be charged against. When provided, the no-``max_tokens`` output-budget floor is capped at a fraction of that limit so small TPM caps remain usable. Omit to preserve the unconstrained floor. ``configured_output_tokens`` is the operator-declared estimate resolved from key or team metadata. When provided it replaces the heuristic floor entirely, so the reservation reflects what this tenant's model actually emits rather than one constant shared by every tenant. )rrrrFz;TPM reservation estimate: input=%s, max_tokens=%s, total=%s)!_estimate_input_and_output_tokensrr ) rrr!rrrFestimated_input_tokensmax_tokens_estimatetotal_estimateds rg_estimate_tokens_for_requestzA_PROXY_MaxParallelRequestsHandler_v3._estimate_tokens_for_requestCs\87;6\6\%=%= 7]7 3 3 "8:M!M"" I "    rfct|tsy|j||}||n|}|tvs|tvrdd|j dfnc|t vrd|j ddfnG|r|j dddfn1|j d|j d|j df}|\}} } t|tr|rtt|n{t| tr t| n`t| trtd| Dn>t| tr t| n#t| trtd| Dnd} | dkDrtd | tznd} |j||} |j||}|j!|}|tvrt|t"n|}|s | | dk(r|dn| | n||n t| |}| ||j%||zfS) a Estimate input tokens and output (max_tokens) budget separately, so callers needing independent ITPM/OTPM reservations (rather than one combined TPM reservation) can use each half on its own. ``min_configured_tpm_limit`` is the smallest ``tokens_per_unit`` among the TPM-bearing descriptors this request will be charged against. When provided, the no-``max_tokens`` output-budget floor is capped at a fraction of that limit so small TPM caps remain usable. Omit to preserve the unconstrained floor. ``call_type`` disambiguates embeddings from the Responses API: both put their prompt in ``data["input"]``, but only embeddings have no output tokens. Unset (the default) preserves the historical "any `input` means zero output" behavior for callers that don't have a call type to pass. ``configured_output_tokens`` is the operator-declared estimate resolved from key or team metadata. When provided it replaces the heuristic floor entirely, so the reservation reflects what this tenant's model actually emits rather than one constant shared by every tenant. )rrNrpromptmessagesc3DK|]}tt|ywrlenrnr-items rgr/zY_PROXY_MaxParallelRequestsHandler_v3._estimate_input_and_output_tokens..s7ST^7 c3DK|]}tt|ywrr[r]s rgr/zY_PROXY_MaxParallelRequestsHandler_v3._estimate_input_and_output_tokens..s;ST^;r_rr)rrr*rErHrrKrxr\rrnsumrrAr9rrrOrD)rrrrrFtranslated_dataestimable_dataselected_fieldsrYrX input_text total_charsrSexplicit_max_tokensrKbase_output_floor output_floorrTs rgrRzF_PROXY_MaxParallelRequestsHandler_v3._estimate_input_and_output_tokenspsH:$%!%!L!LTS\!]3B3NTX44 E]8]4++G4 5::**84d;!$$Z0$="":.""8,""7+ (7$&*(D)h %h/ 0&#&V&$'777*c*Z*d+; ;; [fhiZiA{>U/U(Vop%)%B%B4%S"88yI #'#B#BC[#\44 !#B C"  3 ; q@PUmUu #.%(3*+\: &':T=\=\]acl=m'mmmrfcddlm}|jjjduxr*t |jjj|S)z Check if the dual cache is using Redis cluster. Returns: bool: True if using Redis cluster, False otherwise. r)RedisClusterCacheN)#litellm.caching.redis_cluster_cacherkrrrr)rrks rg_is_redis_clusterz6_PROXY_MaxParallelRequestsHandler_v3._is_redis_clustersM J((33??tK PZ  % % 0 0 < <>OQ  rfkeysnow_intr_cKg}tdt|dD]R}||}||dz}d}|jj|ddd{} | |t | z |k\r|jj |t ||ddd{|jj |||ddd{|jt ||j||jj|ddd{} | t | nd|z} |jj || |ddd{|j| |j| U|S7$777l74w)z Implement sliding window rate limiting logic using in-memory cache operations. This follows the same logic as the Redis Lua script but uses async cache operations. rrNTrklitellm_parent_otel_span local_onlyrkrlrrsrt)ranger\rasync_get_cachercasync_set_cachernr) rrnror_resultsirrqincrement_valuercurrent_counternew_counter_values rgin_memory_cache_sliding_windowzC_PROXY_MaxParallelRequestsHandler_v3.in_memory_cache_sliding_windows:<q#d)Q'/ 2AaJq1u+KO<@;T;T;d;d)-M=XS%9^_cr$r!//??#+#-1# @|,01_/ 2bW6= s\A E>E3AE>E6'E>;E8 E:9E>E<,E>6E>8E>:E><E>rkrlrr~cd|d|d|}|S)zI Create the rate limit keys for the given key and value. {:z}:re)rrkrlrrqs rgcreate_rate_limit_keysz;_PROXY_MaxParallelRequestsHandler_v3.create_rate_limit_keyss# "#awc/1BC rf keys_to_fetch cache_values key_metadatac g}d}tdt|dD]}d}||}||dz} ||dz} ||d} ||d} d} d}| jdr| } d }n| jd r| } d }| |^| t| | kDrd }d }| | t| z n| }|j || ||||d dt ||S)z? Check if the cache values are over the limit. OKrrqrrrNz :requestsrz:tokensr OVER_LIMITrsr{r|r}rrsrr)rvr\endswithrcrr)rrrrrrrz item_coderrq counter_valuerrr|rr}s rgis_cache_list_over_limitz=_PROXY_MaxParallelRequestsHandler_v3.is_cache_list_over_limits124 q#l+Q/$ AI&q)J'A.K(Q/M)*56FGN' 3NCL)-M]aO##K0 . ",%%i0 , "*$(?(S-?--O+ ( ERD]mc-.@@cpO OO%%2'6'6&2:&>?O&P  9$ L!lXNNrfc|jd}|dk7r*|jd|dz}|dk7r||dzk7r||dz|}tj|jdd}|tzS)a Compute the Redis Cluster slot for a given key. Simple implementation of `HASH_SLOT = CRC16(key) mod 16384` Read more about hash slots here: https://medium.com/@linz07m/how-hash-slots-power-data-distribution-in-redis-cluster-bc5b7e74ca7d Args: key (str): The Redis key. Returns: int: The slot number (0-16383). r}rzutf-8r)findbinasciicrc_hqxencoder<)rrkstartendcrcs rgkeyslot_for_redis_clusterz>_PROXY_MaxParallelRequestsHandler_v3.keyslot_for_redis_clusterSsw"xx} B;#uqy1CbySEAI-%!)c*%%cjj&91=(((rfci}|jr<|D]5}|j|}d|}||vrg||<||j|7|S||t<|S)a  Group keys by their Redis hash tag to ensure cluster compatibility. For Redis clusters, uses slot calculation to group keys that belong to the same slot. For regular Redis, no grouping is needed - all keys can be processed together. slot_)rmrrr>)rrngroupsrkslotslot_keys rg_group_keys_by_hash_tagz<_PROXY_MaxParallelRequestsHandler_v3._group_keys_by_hash_tagns/1  ! ! # -55c:"4&>6)')F8$x '',  - /3F* + rfparent_otel_spanrtcZK|jj|||d{S7w)z@Typed view over the DualCache batch read of window/counter keys.rnrrtNrasync_batch_get_cache)rrnrrts rg_batch_get_counter_valuesz>_PROXY_MaxParallelRequestsHandler_v3._batch_get_counter_valuess:..DD-!E    "+)+cZK|jj||dd{S7w)zDTyped view over the DualCache batch read of parallel-request gauges.TrNr)rrnrs rg_batch_get_gauge_valuesz<_PROXY_MaxParallelRequestsHandler_v3._batch_get_gauge_valuess: ..DD-E    rcK|jgS|j|}g}|jD]>\}} |j|||jgd{}|j |@|S7#t $rY}t jd|||j|||jd{7}|j |Yd}~d}~wwxYww)a Execute Redis operations grouped by hash tag for cluster compatibility. Args: keys_to_fetch: List[str] - List of keys to fetch now_int: int - Current timestamp Returns: List of cache values Nrnargsz+Redis Lua script failed for hash tag %s: %srnror_) rritemsr_extendrrwarningr~) rrro key_groupsall_cache_valueshash_tag group_keysgroup_cache_valuesr s rg(_execute_redis_batch_rate_limiter_scriptzM_PROXY_MaxParallelRequestsHandler_v3._execute_redis_batch_rate_limiter_scripts  ) ) 1I 88G BD$.$4$4$6 < Hj 46%* J'.N)'2 |,T0R0T (^^,?@N#14z~~FW7XL*4..9P*Q '$..7K4;K;KKn-Q/?.@ JJ*6 ($($?$?*,<>U%""=>'5 #N)55nFVXbc$$j'%:;!%'55nFVX`a$$j'%:;!%!;I:T3~#6Z^5A5ML 1SW";/"0 (L $K* VlF22rfgauge in_flightr{c Nt||dtd|d|z d|dS)Nrrrr^rsr)rzr)rrrr{s rg _gauge_statusz2_PROXY_MaxParallelRequestsHandler_v3._gauge_statuss8.5>I#=>3 !12   rfrc|yt|trG|jjtz t fd|j DStdt|S)z In-flight count from a cached gauge value: a dict of slot_id -> acquire timestamp when the in-memory registry is authoritative, or the mirrored integer count from the last Redis script result. rc3\K|]#}t|ttfs|k\s d%yw)rN)rrcr)r-tscutoffs rgr/zY_PROXY_MaxParallelRequestsHandler_v3._gauge_in_flight_from_cache_value..s'gRBe 9UZ\`fZfqgs ,,,) rrrrrVravaluesrrc)rrrs @rg!_gauge_in_flight_from_cache_valuezF_PROXY_MaxParallelRequestsHandler_v3._gauge_in_flight_from_cache_values`   i & 224>>@CddFg9#3#3#5gg g1c)n%%rfrrvc tK|Dcgc]}|d }}|r|jQ |j||Dcgc]}t c}d{}|D cgc]} tdt| } } n|j||d{} g} d} t|| D]:\}}||dk\rdnd}|dk(rd} | j|j|||<t| | S|j||d{}t||D].\}}||dk\std|j||dg cS|j |j||Dcgc]}|dt|fD]}|c}}d{}tdd k(r?|t|d d z }td|j|t|d dg Sg} t||d dD]h\}}|jj!|dt|t|d d{| j|j|t|djtd| S|j4d{|j|||d{cdddd{Scc}wcc}w7acc} w#t$r<} t j d| |j||d{7} Yd} ~ gd} ~ wwxYw7t7cc}}w7#t$r} t j d | |j4d{7|j|||d{7cdddd{7cYd} ~ S#1d{7swYnxYwYd} ~ d} ~ wwxYw7t7(77#1d{7swYyxYww)a Enforce max_parallel_requests as a concurrency gauge over a per-slot registry: each admitted request registers ``slot_id`` with its acquire time, and admission requires in_flight + 1 <= limit over the unexpired slots. Unlike the windowed RPM/TPM counters, the gauge is never reset while requests are in flight, a rejected request never occupies a slot, and a slot leaked by a crashed worker is pruned after PARALLEL_REQUEST_SLOT_TTL_SECONDS even under continuous traffic. Releases remove exactly this request's slot id, so a double-fired or unmatched release can never free another request's slot. rqNrrz4parallel_count_script failed, using local mirror: %srrrrrzCparallel_acquire_script failed, falling back to in-memory gauge: %srrqTru)rrVrrcrrr_read_local_gauge_countsziprrrrr!_acquire_parallel_slots_in_memoryrrx)rrrvrrr gauge_keys_ raw_countsrlcountsr rrrr{ local_countsargraws rgrzC_PROXY_MaxParallelRequestsHandler_v3._check_parallel_request_gaugess&@FFeU=1F F ))5_GKGaGa'IOPA?PHbHBJ?IIUc!SZ0IFI $<N8/N8 L L 8 L L L BN8#N$AN82N3N86N# NN# N8N  N8(J50J55 K:>+K5)K,*K5/N85K::N8N8L N&N;L><NM4MM4 N(M+)N.N/N84N :M=;N N N8NN8N8N# N8#N5)N, *N51N8rcK|j||d{}||Dcgc]}dc}S|Dcgc]}|j|c}S75cc}wcc}ww)N)rnrr)rr)rrrrrrls rgrz=_PROXY_MaxParallelRequestsHandler_v3._read_local_gauge_countssi #::-;   >)*!A* *KQR%66u=RR  +Rs*AA A AAA A Ac pK|jj}|tz }g}|D]}|jj |d|dd{}t |t rY|jD cic].\} } t | ttfs| |k\s#| t| 0} } } t| xsi} n|i} d} nd} tdt|} | dz|dkDr!td|j|| dg cS|j| | fg} t||D]k\}\} } | i| ||in| dz}|jj!|d|t|d d{| j|j|| dzd mtd | S7Vcc} } w7Aw) a All-or-nothing in-memory slot-registry acquire. Caller holds the lock. A cached dict is the authoritative in-memory registry. A cached integer is the count mirrored from the last successful Redis script call: when Redis fails over to this path, that mirror still counts the slots in flight on the Redis side, so it is carried forward as an integer counter (not discarded as an empty registry, which would briefly double the admitted concurrency during a Redis outage). rqTrrNrrrrrrrur)rrrVrrwrrrrcrr\rrrrrrx)rrrvrrrstatesrrrkrregistryrr new_values rgrzF_PROXY_MaxParallelRequestsHandler_v3._acquire_parallel_slots_in_memorys ++-779??CE 1E>B>W>W>g>g-()9?h?9I )T*2;//2C5'.sBzRTWZ\aVbGchjnthtCrN55 B/ " 3y>2 1}uW~-(!-"00 <PQ MM8Y/ 0- 10,/,? L (E(HiNVNb0J80JWc0JhqtuhuI++;;-(5)9 <   OOD..ui!mTJ K L!dXFFE9 5( s=AF6F+'F6=F.F."F.0CF62F439F6.F6 acquisitionc K|d}|d}|r|sy|j~ |j||Dcgc]}|c}d{}t||D]D\}}|jj|t dt |t |dd{Fy|j4d{|D]}|jj||d d{} t| tr/|| vr>| jD cic]\} } | |k7s | | } } } n| kt dt | d z } |jj|| t |dd{dddd{ycc}w777#t$r!} tjd| Yd} ~ d} ~ wwxYw77cc} } w7X7H#1d{7swYyxYww) a Release the max_parallel_requests slots acquired at pre-call by removing this request's slot id from every gauge it was registered under. Removing an absent slot id is a no-op, so a release without a matching acquire or a double-fired release can never free another request's slot. The in-memory fallback decrements integer mirror values (floored at 0) because the mirror carries no per-slot ids. rwrvNrrTruzEparallel_release_script failed, falling back to in-memory release: %srrr)rrrrxrrcrVrrrrrwrrr)rrrrwrvrrrq remainingr rrkrrs rg_release_parallel_request_slotszD_PROXY_MaxParallelRequestsHandler_v3._release_parallel_request_slots.s*.9 $Y/7   ' ' 3 y<@N>N   02B1C9M !#!#/1Mc# Cj (nn-@A  !2!6!6z1!E!JK (nn->?  !2!6!6x!C!HqI "j1n55nFVX[\K k*K"%k"2  KK[1 2 KK[):{DWX Y KK&4(8%(%5'*",#.!+  ) BT4rfrc K|s tdgSg}g}t|D]\}\}}} |j||d{} |j |}|ddk(r|j|d{|cSt|d k(r|cS|j||j|d td|t! S7#t$r} t j dt | j| t||j|j|d{7|D cgc]\} } } | D]}|ncc}} } } w}} } } }|j4d{7|j||d{7cdddd{7cYd} ~ cS#1d{7swYnxYwYd} ~ yd} ~ wwxYw7Sw) a3 Run Lua check-and-increment one descriptor at a time so each call's keys co-locate on a single Redis Cluster slot. On OVER_LIMIT for descriptor i, refund descriptors 0..i-1's increments. On Lua failure mid-loop, refund applied increments and fall back to in-memory. rrrNuatomic_check_and_increment_by_n: Redis Lua execution failed (%s: %s). Refunding %s prior descriptors and falling back to in-memory enforcement — counters will diverge from Redis until window expires (window_size=%ss).rrrrrrrr)r enumeraterrrerrortyper`r\r_!_refund_applied_descriptor_groupsrr_build_atomic_responserrr)rrrappliedr_idxrnrrrr _k_arrrrs rgrz?_PROXY_MaxParallelRequestsHandler_v3._atomic_lua_per_descriptors!$! 9;13)22C(D! 2 $D$4t  @@A0223=H'<7<.vsF*  E4 'CEAI ./0*s03r)rrcrzrrrr) rrr status_codedescriptor_indexr|rrrrr new_counters ` rgrz;_PROXY_MaxParallelRequestsHandler_v3._build_atomic_response>se $$D D Q[ ! &)#a&kAo %(Q[Os1v;E#$45D$)#)&+(+Au/F(G(,->(?'+,<'=)-.@)A    24$%56 KE4a%!)m,K OO"&"7$'4+@3{CS+S$T$():$;#'(8#9%)*<%=   ! )* $--=#> *!  rfc Kt|jj}g}|D]}|d}|jj |d|dd{}|duxs|t|z |k\}|rdn(|jj |d|dd{} |rdnt| xsd} |ddkDr| |dz|d kDn| |d k\} | r;t d t d |d td|d | z |d |d |d gcS|j|| t|r|n t|dg} t||D]\}} | dr|dn | d|dz}| dr7|jj|dt||d|dd{|jj|d||d|dd{| jt d|d td|d |z |d |d |d t d| tdt||DS7777rw)u`In-memory all-or-nothing check-and-increment. Caller holds lock. Reads/writes the LOCAL DualCache (`local_only=True`) — note this is a different store from Redis. When this fallback fires after a Lua failure, in-memory counters will diverge from Redis until each key's window expires (TTL bounds divergence). r_rTrrNrqrrr|rrrsrrr)rrrrrrurrc38K|]\}}|d|ddfyw)rqrrNre)r-rstates rgr/z]_PROXY_MaxParallelRequestsHandler_v3._atomic_check_and_increment_in_memory..s-*D%m$eN&;WE*r) rcrrrrwrrzrrrnrrxr)rrrrodescriptor_staterr_rr raw_counterr| over_limitrrrs rgrzJ_PROXY_MaxParallelRequestsHandler_v3._atomic_check_and_increment_in_memorysT335??AB=?$+ D}-K;?;T;T;d;d&)9sB$C$CDU$V!"|!4!: ? ? N  =  = $$D D3BM -.X' (M M 99'!-:   ' = M  s)C0BC$&C0; C)C0C. C0scopesamountrrc K|dks|sy|s8|j|j|t|d||d{y|j|t|d||}|j ||d{y7@7w)ap Directly decrement previously-reserved token counters for ``scopes`` by ``amount``. Used to roll back a reservation that already succeeded once a *different* bucket in the same request turns out to be over its limit (e.g. ITPM reserved fine, OTPM then hits its limit -- the ITPM reservation must not be left inflated). rNtargetsr actual_tokensrpipeline_operationsrr)rr*rreservation_window_identities),async_increment_tokens_with_ttl_preservation _build_reservation_aware_tpm_opsr_build_project_reservation_ops(async_increment_reservation_aware_tokens)rr%r&rrr,s rg_refund_reserved_tokensz<_PROXY_MaxParallelRequestsHandler_v3._refund_reserved_tokenss Q;f "CC$($I$I"$-f$5"#$* %J% "2D   %)%H%H%f-"*= &I& ;; 3-<   # " s"=BB:B:B;BBrSestimated_output_tokensc ZK|Dcgc]}|dtk(s|}}|Dcgc]}|dtk(s|}}|s|stdgddfS|r,|j||Dcgc]}d|ic}|d{nd} | | dd k(r| ddfS| |nd} |r|j||Dcgc]}d|ic}|d{} | dd k(rU| dkDrK|j |Dcgc] }|d|d fc}| | j d t | d{| ddfS| g| d | d n| d } td| | | j d t n t | j d t z| |fS| J| | dfScc}wcc}wcc}w7"cc}w7cc}w7w)a] Reserve ``estimated_input_tokens`` against project ITPM descriptors and ``estimated_output_tokens`` against project OTPM descriptors. ITPM and OTPM are reserved from different-sized estimates, so unlike same-size TPM descriptors they can't share a single ``atomic_check_and_increment_by_n`` call -- each bucket gets its own all-or-nothing atomic call. If the OTPM reservation is over limit after ITPM already succeeded, the ITPM reservation this call made is rolled back before returning, so a partial reservation never leaks. Returns ``(response, itpm_reserved, otpm_reserved)`` -- the latter two are the amounts actually reserved (0 if that bucket wasn't configured, or if the reservation failed), for the caller to stash for post-call reconciliation. rkrrrrr!Nrrrlr)r%r&rrrr)rSrUrrr3rr) rrrSr4rr"itpm_descriptorsotpm_descriptorsr itpm_response itpm_reserved otpm_responsers rgreserve_io_tokensz6_PROXY_MaxParallelRequestsHandler_v3.reserve_io_tokens8s0## ah2M&MA# # ## ah2M&MA# #  (8$$DaJ J 66,.56"2 7     $~)F,)V !Q& &9F9R5XY )-)M)M,.67"2 *N*$M^, < 1$66SU^U`,a)9 7%a** !, ":.":. #:. "!%% -8*--.CY[Q!*'++,A9;O P ' "(((mQ..Q# #  $$ sxF+F F F+FF(F+! F , F+8F9-F+& F 1 F+=F"> F+F$/&F+F)BF+F+$F+rrequested_modelcK|yg}|j||||sy|j||||jd{\}}}|ddk(r|j|||yy7%w)a]Reserve one WebSocket ``response.create`` frame's tokens against the caller's project ITPM/OTPM quota. The Responses WebSocket connection-level pre-call hook only runs once per connection, but a connection accepts many ``response.create`` frames over its lifetime. Without this, a project caller could send unlimited high-token generations after a single minimal reservation. There is no per-frame post-call hook to reconcile against, so -- like the batch rate limiter -- this charges the estimate immediately and never refunds it. Nrr<rrrSr4rrr)9add_project_io_token_rate_limit_descriptors_from_metadatar;r_handle_rate_limit_error) rrr<rSr4rr_itpm_reserved_otpm_reserveds rg(enforce_project_io_token_quota_for_framezM_PROXY_MaxParallelRequestsHandler_v3.enforce_project_io_token_quota_for_frames$  $ 8:  FF/+# G  9=9O9O##9$;.?? :P: 4 0.. N #| 3  ) )(K Q 4 4 s?A)A'&A)c g}|ja|j |jI|jt d|j|j|j|j dt |dd t |dd t |ddxsi}t |ddxsi}d}||vs||vrd}|rPd}d}||vr||}||vr||}|jt d |jd ||||j d|S) N organizationr\r]r_r organization_metadatamodel_rpm_limitmodel_tpm_limitFTmodel_per_organizationr)org_idorganization_rpm_limitorganization_tpm_limitrrjr_r rrr<r_tpm_limit_for_team_model_rpm_limit_for_team_modelshould_check_rate_limitmodel_specific_tpm_limitmodel_specific_rpm_limits rg)create_organization_rate_limit_descriptorzN_PROXY_MaxParallelRequestsHandler_v3.create_organization_rate_limit_descriptorsz9;   # # /  4 4 @DUDlDlDx   #&+22->-U-U+<+S+S'+'7'7   //@BY[l m 12CE\^op33DF]_pqwuw &33DF]_pqwuw &', #";;Rk?k*.'&+/(+/("&??/H/Y,"&??/H/Y,""'4!2!9!9 :!O;LM1I/G+/+;+;$ rfc 6ddlm}m}|sy|||}|||}||y|xsi}|xsi}||vxs||v}|sy|j|} |j|} |j t d|j d|| | |jdy) a& Add model-specific rate limit descriptor for API key if applicable. Args: user_api_key_dict: User API key authentication dictionary requested_model: The model being requested descriptors: List of rate limit descriptors to append to r)get_key_model_rpm_limitget_key_model_tpm_limitN model_name model_per_keyrrGr )litellm.proxy.auth.auth_utilsrWrXrrrjapi_keyr_) rrr<rrWrX_tpm_limit_for_key_model_rpm_limit_for_key_modelrRrSrTs rg(_add_model_per_key_rate_limit_descriptorzM_PROXY_MaxParallelRequestsHandler_v3._add_model_per_key_rate_limit_descriptors  #:;LYh#i #:;LYh#i # +0H0P #;#Ar #;#Ar  7 7 f?Nf;f ' 7O6R6RSb6c 6N6R6RSb6c  #*2231_4EF)A'?#'#3#3  rfc $|jsyt|xsi}|sytjt |D]P}|j |}||j td|jd||d|jdRy)aZ Add per-request-tag rpm limit descriptors for the API key. Each tag carried on the request that has a configured limit gets its own ``{api_key}:{tag}`` counter, so a burst on one tag/group never consumes another's budget. Tags without a configured limit fall through to the key-level descriptor. N tag_per_keyrrGr ) r]rrfromkeysr rrrjr_)rrrr tag_rpm_limittag rpm_limits rg&_add_tag_per_key_rate_limit_descriptorzK_PROXY_MaxParallelRequestsHandler_v3._add_tag_per_key_rate_limit_descriptor1 s!(( 45FGM2  ==!;D!AB C%))#.I    #%.667q>-6+/'+'7'7    rfmcp_server_namec ddlm}|r |jsy||}|sy|j|}|y|j t d|jd||d|j dy)a$ Add a per-MCP-server rpm descriptor for the API key, if a limit is configured for the server being called. MCP tool calls have no token usage, so only requests_per_unit is set; tokens_per_unit stays None so the TPM reservation path is never engaged. r)get_key_mcp_rpm_limitN mcp_per_keyrrGr )r\rjr]rrrjr_)rrrhrrj mcp_rpm_limitserver_rpm_limits rg&_add_mcp_per_key_rate_limit_descriptorzK_PROXY_MaxParallelRequestsHandler_v3._add_mcp_per_key_rate_limit_descriptorV s H&7&?&? 45FG  "/"3"3O"D  #  !*2231_4EF)9'+#'#3#3  rfc ddlm}|syg}|jr#|j|j||f|jxsij D]\}}|j||f|D]N\}} |r| s | j |} | |jtd|d|| d|jdPy)z Add a per-MCP-server rpm descriptor for the team, if a limit is configured for the server being called. r)get_team_mcp_rpm_limitN mcp_per_teamrrGr ) r\rpteam_idrmcp_source_team_rpm_limitsrrrjr_) rrrhrrp team_limitssource_team_id source_limitrrrlrms rg'_add_mcp_per_team_rate_limit_descriptorzL_PROXY_MaxParallelRequestsHandler_v3._add_mcp_per_team_rate_limit_descriptor| s I NP  $ $    1 9 9;QRc;de f->-Y-Y-_]_,f,f,h ? (NL    = > ?'2  "G]-,00A '   #&$IQ&78-=+/'+'7'7    rf limit_typemodel_has_failuresc|dk(r|Sy)a Determine if rate limit should be enforced based on limit type and model health. Args: limit_type: Type of rate limit ("dynamic", "guaranteed_throughput", "best_effort_throughput", or None) model_has_failures: Whether the model has recent failures Returns: True if rate limit should be enforced, False otherwise dynamicTre)rrxrys rg_should_enforce_rate_limitz?_PROXY_MaxParallelRequestsHandler_v3._should_enforce_rate_limit s  "% %rfrc4|y|j||r|Sy)a Get the rate limit value to enforce based on limit type and model health. Args: limit_value: The configured limit value limit_type: Type of rate limit ("dynamic", "guaranteed_throughput", "best_effort_throughput", or None) model_has_failures: Whether the model has recent failures Returns: The limit value if it should be enforced, None otherwise N)rxry)r|)rrrxrys rg_get_enforced_limitz8_PROXY_MaxParallelRequestsHandler_v3._get_enforced_limit s1"    * *!1 +  rfrpm_limit_typetpm_limit_typec|dk(xs|dk(S)a Check if dynamic rate limiting is enabled for either RPM or TPM. Args: rpm_limit_type: RPM rate limit type tpm_limit_type: TPM rate limit type Returns: True if dynamic mode is enabled for either limit type r{re)rrrs rg!_is_dynamic_rate_limiting_enabledzF_PROXY_MaxParallelRequestsHandler_v3._is_dynamic_rate_limiting_enabled s*In .IIrfagent_idzAgentResponse | Nonec2ddlm}|j|S)z3Look up an agent from the in-memory registry by ID.r)global_agent_registry)r),litellm.proxy.agent_endpoints.agent_registryrget_agent_by_id)rrrs rg_get_agent_from_registryz=_PROXY_MaxParallelRequestsHandler_v3._get_agent_from_registry sV$44h4GGrfcpt|dd}|r|S|jdxsi}|jdS)z Resolve the agent_id from either the API key or request metadata. Key-level agent_id takes precedence over metadata/header-supplied agent_id. rNmetadata)getattrr)rrr key_agent_idrs rg_get_resolved_agent_idz;_PROXY_MaxParallelRequestsHandler_v3._get_resolved_agent_id s> &&7TJ  ((:.4"||J''rfc |jd}|r t|S|jdxsi}|jd}|r t|S|jdxsi}|jd}|r t|Sy)z?Extract session_id from request metadata or litellm_session_id.litellm_session_idr session_idlitellm_metadataN)rrn)rrrrrs rg_get_session_id_from_dataz>_PROXY_MaxParallelRequestsHandler_v3._get_session_id_from_data sXX23 z? "((:.4"\\,/ z? ""&((+=">"D"%)),7 z? "rfc g}|j|}||St|dd}t|dd}||+|jtd||||jdt|dd}t|dd}||C|j |} | 0|jtd |d | |||jd|S) a Create rate limit descriptors for agent-level and session-level limits. Agent-level: caps total RPM/TPM across all sessions for a given agent. Session-level: caps RPM/TPM within a single session (identified by session_id). Nrf tpm_limitagentrGr session_rpm_limitsession_tpm_limit agent_sessionr)rrrrjr_r) rrrrr agent_rpm agent_tpm session_rpm session_tpmrs rg$_create_agent_rate_limit_descriptorszI_PROXY_MaxParallelRequestsHandler_v3._create_agent_rate_limit_descriptors s9; 44X> = "5+t< "5+t<  I$9   #"-6+4'+'7'7  %U,?F $U,?F  "k&= $ > >t DJ%""'+!) !J<81d|vr:|j1dd} |j=|| | |j?|| | ||||v||xsi}||xsi}d}| |vs| |vrd}|rPd}d}| |vr|| }| |vr|| }| jtd|j d | |||jd |jA||}|r"| jC|jE||| S)z Create all rate limit descriptors for the request. Returns list of descriptors for API key, user, team, team member, end user, model-specific, agent, and agent-session limits. r)get_team_model_rpm_limitget_team_model_tpm_limitNr])rrxry)r\r]r^r_r userrGteamr team_memberend_userr!r>)rrr server_idrh)rrhrFTmodel_per_team)rr)#r\rrr]rfrr^budget_throttle_pctrrjr~rr_user_iduser_rpm_limituser_tpm_limitrrteam_rpm_limitteam_tpm_limitteam_member_rpm_limitteam_member_tpm_limit end_user_idend_user_rpm_limitend_user_tpm_limitrr`rgr, call_mcp_toolrlrnrwrrr)rrrrrryrrrr throttle_pctteam_member_valuer<rhrPrQrRrSrTresolved_agent_ids rg_create_rate_limit_descriptorszC_PROXY_MaxParallelRequestsHandler_v3._create_rate_limit_descriptors> s^    $ $  ' ' 3 **6 66B"3"G"GL   #!+33-1-E-E(78I8S8SUa(b'5/A.F. ,0+C+C(78I8S8SUa(b'5/A,D, 2C1X1X'+'7'7  ,  $ $  , , 8-M-M+<+K+K'+'7'7    $ $  , , 8-M-M+<+K+K'+'7'7    $ $  3 3 ?CTCjCjCv*;*C*C)DAFWF_F_E`'a    #%+->-T-T+<+R+R'+'7'7    ( (  0 0 <@Q@d@d@p   #"+77->-Q-Q+<+O+O'+'7'7  "&'4!8 55/+# 6  33/# 4   //55 5+T:Q%)XX.?%FO  7 7"3 /' 8   8 8"3 /' 9  %%6 7 C'(9:F/GHY/Z/`^` %/GHY/Z/`^` %&+ #";;Rk?k*.'&+/(+/("&??/H/Y,"&??/H/Y,""',!2!:!: ;1_#>?PRV#W    99.: rfcKddlm}ddlm}|y |j |}|sy|D]S}|j dij d}|s'||| }|t kDs;tjd |||y tjd |y#t$r } tjd | Yd} ~ y d} ~ wwxYww)z Check if any deployment for this model has recent failures by using the router's existing failure tracking. Returns True if any deployment has failures in the current minute. r) llm_router)*get_deployment_failures_for_current_minuteNFrY model_infoid)litellm_router_instance deployment_idzi[Dynamic Rate Limit] Deployment %s has %s failures in current minute - enforcing rate limits for model %sTzS[Dynamic Rate Limit] No failures detected for model %s - allowing dynamic exceedingzEError checking model failure status: %s, defaulting to enforce limits) litellm.proxy.proxy_serverr>litellm.router_utils.router_callbacks.track_deployment_metricsrget_model_listrrrr r) rr!rrr model_list deploymentr failure_countr s rg _check_model_has_recent_failureszE_PROXY_MaxParallelRequestsHandler_v3._check_model_has_recent_failures s :   #  * 9 9U 9 KJ) *|R @ D DT J $!K,6"/! !#PP(..D%%   % ( ! & &egl   & &'npq r sJCBCC?BC C B;6C;CCc2|dk(r|j}|Sy)z'Get the rate limiter for the call type. acreate_batchN)r)rr batch_limiters rgget_rate_limiter_for_call_typezC_PROXY_MaxParallelRequestsHandler_v3.get_rate_limiter_for_call_type1 s!  '#'#?#?#AM rfc Vt|dd t|ddt|ddxsi}t|ddxsi}||vxs||v}|r`|]|j|}|j|}|jtd|jd||||j dyyyy) zFAdd team model rate limit descriptor from team_metadata if applicable. team_metadatarINrJrrrGr )rrrrjrrr_rOs rg3_add_team_model_rate_limit_descriptor_from_metadatazX_PROXY_MaxParallelRequestsHandler_v3._add_team_model_rate_limit_descriptor_from_metadata8 s //@/Sd e q12C_Vght33DoWhiomo &33DoWhiomo & #<<lSl@l $'?+F2K2O2OP_2`(2K2O2OP_2`(""',!2!:!: ;1_a?PQ1I/G+/+;+;$ ,G&xrfc | |jyt|ddxsi}t|ddxsi}|j|}|j|}||y|jd|}|/|jt t |d||j d|0|jt t|d||j dyy)aAdd project-scoped ITPM/OTPM descriptors from project_metadata. Enforced independently of, and alongside, the combined ``model_per_project`` TPM descriptor above -- these give Bedrock Mantle-style separate input/output token quotas at the project level. Nrmodel_itpm_limitmodel_otpm_limitrrGr )rrrrrjrSr_rU) rrr<ritpm_limit_for_project_modelotpm_limit_for_project_modelrrrs rgr@z^_PROXY_MaxParallelRequestsHandler_v3.add_project_io_token_rate_limit_descriptors_from_metadata s  "&7&B&B&J  //@BTVh i  % //@BTVh i  % #?"B"B?"S">"B"B?"S  #(8(@ %6%A%A$B!OCT"U  '   #3*-1+;'+'7'7    '   #3*-1+;'+'7'7   (rfrc |dD]}|ddk(s |dtfd|Dd}||dnd}|jj}||jz}t j |j d } td |d } |d } |d } dd|d| d| d| d| } t|\}}t| t|jt|d | dt|d ||y)zKHandle rate limit exceeded by raising :class:`ProxyRateLimitError` (a 429).rr{rrsc34K|]}|dk(s |ywrkNre)r-descrss rgr/zP_PROXY_MaxParallelRequestsHandler_v3._handle_rate_limit_error.. sSdT%[N5RTSs Nrlunknownz%Y-%m-%d %H:%M:%S UTCrr}rr|zRate limit exceeded for z: z. Limit type: z. Current limit: z , Remaining: z. Limit resets at: )z retry-afterrreset_at)detailheadersrr! llm_provider) nextrrr_r fromtimestampstrftimerr(r!rnr")rrrr<statusmatching_descriptorrr reset_timereset_time_formattedremaining_displayrr|rresolved_modelrrss @rgrAz=_PROXY_MaxParallelRequestsHandler_v3._handle_rate_limit_error smz*# Ff~-!'(8!9&*SkS'#DWCb#6w#?hq ,,.88: 4#3#33 '/'='=j'I'R'RSj'k$$'62C+D$E!"():"; & 7 /~.>bAQ@RS##2"34&&3_MBSATU((<'=?0SSb/c, )!'*4+;+;'<+.v6G/H+I$8 %;6BS;T$U(!-  3# rfblockct|tstS|jd}t|tr|jdnd}|r8t|tr(t |dzdz}t |tztStS)a Token estimate for one ``input_audio`` content block. When the block carries a base64 ``data`` payload, the estimate comes from the decoded byte count (``len(b64) * 3 // 4 // _AUDIO_BYTES_PER_TOKEN``), assuming the lowest reasonable audio bitrate so we never under-reserve for higher-quality recordings of the same duration. When no payload is present (reference-only block or missing ``data``), falls back to ``DEFAULT_AUDIO_TOKEN_ESTIMATE``. input_audiorNrr@)rrrPrrnr\rrQ)rrb64_data decoded_bytess rg_estimate_audio_block_tokenszA_PROXY_MaxParallelRequestsHandler_v3._estimate_audio_block_tokens su%&/ /"YY}5 5? T5R+//&1X\  8S1#&x=1#4#9M}(>>@\] ]++rfrYcNt|tsytfd|DS)z Sum of per-block audio token estimates across all ``messages``. Returns 0 when there are no ``input_audio`` blocks, which the caller uses to skip the (relatively expensive) strip pass. rc3K|]q}t|tr_|jd}t|tr>|D]9}t|tr'|jddk(rj |;syw)contentrrN)rrrrxr)r-messagerrr;s rgr/zV_PROXY_MaxParallelRequestsHandler_v3._estimate_audio_content_tokens.. su '4(#KK 2'4(  %&599V+< +M  , ,U 3 3 sA7A:)rrxra)r;rYs` rg_estimate_audio_content_tokenszC_PROXY_MaxParallelRequestsHandler_v3._estimate_audio_content_tokens s,(D) #   rfct|ts|Sg}|D]}t|ts|j|%|j d}t|ts|j|X|Dcgc])}t|tr|j ddk(r(|+}}|ji|d|i|Scc}w)a Drop ``input_audio`` content blocks before passing ``messages`` to ``token_counter``, which raises ``ValueError`` on them (no per-type handling, unlike images). The audio contribution is added back separately via ``DEFAULT_AUDIO_TOKEN_ESTIMATE`` so the rest of the message (text/images/tools) still gets counted accurately instead of the whole call falling back to the cheap char-count estimate. rrr)rrxrrr)rY sanitizedrrrfiltered_contents rg_strip_audio_content_blocksz@_PROXY_MaxParallelRequestsHandler_v3._strip_audio_content_blocks s(D)O)+  Ggt,  )kk),Ggt,  )#* :eT3JuyyY_O`dqOq      878I'78    s 3)B>B>c|ddlm}t|tsy|j |j dxsd|S)a Convert a Responses API ``input`` (string or list of input items) into chat-completion-style messages via the standard LiteLLM transformation (the same one guardrails use, e.g. ``purview_dlp.py``), so multimodal ``input_image``/``input_text`` content blocks get counted by ``token_counter``'s ``messages`` path instead of silently contributing zero tokens via its ``text`` path, which only joins plain strings. r) LiteLLMCompletionResponsesConfigrerr")rresponses_api_request)Blitellm.responses.litellm_completion_transformation.transformationrrr)transform_responses_api_input_to_messagesr)rrs rg!_responses_input_to_chat_messageszF_PROXY_MaxParallelRequestsHandler_v3._responses_input_to_chat_messages* sC $%/YY((7#)r"&Z  rfct|tsytd|Dr t|Std|Drt d|DSy)Nc3<K|]}t|tywrrrcr-tokens rgr/z[_PROXY_MaxParallelRequestsHandler_v3._count_pretokenized_embedding_input..C s9%z%%9c3dK|](}t|txrtd|D*yw)c3<K|]}t|tywrrrs rgr/ze_PROXY_MaxParallelRequestsHandler_v3._count_pretokenized_embedding_input...F s/^5 5#0F/^rN)rrxallr- token_idss rgr/z[_PROXY_MaxParallelRequestsHandler_v3._count_pretokenized_embedding_input..E s0 clJy$ ' ^C/^T]/^,^ ^ s.0c32K|]}t|ywr)r\rs rgr/z[_PROXY_MaxParallelRequestsHandler_v3._count_pretokenized_embedding_input..H s=)s9~=s)rrxrr\ra)rls rg#_count_pretokenized_embedding_inputzH_PROXY_MaxParallelRequestsHandler_v3._count_pretokenized_embedding_input? sR%& 959 9u:   pu  =u== =rfc|jd}t|tr|nd}|jdg|}djd|DS)N documentsrequery c3^K|]%}t|ttfr t|'ywr)rrnr)r-parts rgr/zM_PROXY_MaxParallelRequestsHandler_v3._rerank_input_to_text..S s* $d , I s+-)rrrxjoin)rr document_items input_partss rg_rerank_input_to_textz:_PROXY_MaxParallelRequestsHandler_v3._rerank_input_to_textK s]88K0 ?I)UY?Z)`b HHW 2 2 yy #   rfc ddlm}t|tsy|tv}|rdn|j ||}|j ||}|r|jdnd}|r|j|nd} | | S|jd} | | n|jd} |r3|j|d|jd|jdfn|3|jdd|jd|jdfn|r$d||jd|jdfn|tvr3d|j||jd|jdfn^|tvr$d| |jd|jdfn2|jd| |jd|jdf} | \} }}}|j| }|dkDr|j| n| } tdt!||xsd ||||d }||zS#t"$rT|tvr.t|t$rtdt'|t(zcYS|j+|| \}}||zcYSwxYw) a Model-aware input token estimate for the project ITPM reservation, using ``litellm.token_counter`` -- the same approach the deployment-level itpm/otpm check uses in ``io_token_rate_limit_check.py``. Unlike the cheap char-count estimate the combined-TPM path uses, this accounts for image/tool content and derives per-``input_audio``-block estimates from the base64 payload size (assuming the lowest reasonable bitrate so longer recordings always reserve proportionally more), so a burst of multimodal, tool-heavy, or audio-heavy requests can't each reserve only the one-token floor and blow past ITPM before post-call reconciliation catches up. For the Responses API, ``input`` is converted to chat messages first (via ``_responses_input_to_chat_messages``) so its own multimodal content blocks are counted the same way; ``token_counter``'s ``text`` argument can only see plain strings in a list, not content blocks. Falls back to the cheap char-count estimate if ``token_counter`` can't resolve a tokenizer for this model (e.g. an unrecognized custom model name) or otherwise raises -- the audio add-on still applies on top of the fallback. r) token_counterNrrXr% tool_choicerYr"T)r!rYtextr%ruse_default_image_token_countr)litellmrrrrEr*rrr rrLrrKrrrrcrrnr\rArR)rrr!rris_responses_requesttranslated_requestis_embedding_requestembedding_textpretokenized_input_tokensrX fallback_textselected_inputsrY selected_textcountable_toolscountable_tool_choiceaudio_token_estimatecountable_messagesestimaterSrs rg_estimate_precise_input_tokenszC_PROXY_MaxParallelRequestsHandler_v3._estimate_precise_input_tokensY s0 *$%&/3K&K(Dd.Y.YZ^`i.j '+&@&@y&Q5I 1tH\D 4 4^ Dbf " % 0, ,*)/);v'AR $ 3 3D 94'ARTXT\T\]jTk l"- #&&z2"&&w/"&&}5 $(9488M;RS112248$((7:KTXXVcMde:: 1488M3JK((:& txx7H$((S`Jab! $KZG-2G&*&I&I(&SRfijRjD$D$DX$Npx A!!#kr!3*-$96:   H22 2 A11jPS6T1c-04KKLL(,(N(NTXdm(N(n % "A),@@ @  As7)H!!=I> I>=I>tpm_reservation_scopestpm_reservation_amountcKt|tsyt|}|D cgc]} | dttfvs| } } | sy| D cgc]?} | dtk(r1| j dxsij d} | t | A} } } | r t| nd} |j|| |\}}|j|||}t|d}|j||r|n t|d}|j|| ||j| |||j d{\}}}|d d k(r|d kDr-|j|||j d{d|_|j"}|,|j%||jd{d|_|j'||||d kDrXt)d| D}||_t-||_t-d|j dt-D|_|d kDrXt)d| D}||_t-||_t-d|j dt-D|_|j8"|j8dj;|dn |dr||_t=j>d|||ycc} wcc} } w77u7>w)a Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style separate input/output token buckets), independently of -- and, when both are configured, in addition to -- the combined-TPM reservation the caller already made. Raises (via ``_handle_rate_limit_error``) on an over-limit reservation, first rolling back the combined-TPM reservation named by ``tpm_reservation_scopes``/``tpm_reservation_amount`` if one was made, so a partial reservation never leaks. Nrkrmr])rrr)rr!rr)rrErr?rrr)r%r&rTrrrrr<c3JK|]}|dtk(s|d|dfywrkrlN)rSr-r"s rgr/z[_PROXY_MaxParallelRequestsHandler_v3._reserve_project_io_tokens_or_raise.. .'+,%ToHo51W:&'##c36K|]\}}}d|vr|||fyw)rRNrer-rqrbackends rgr/z[_PROXY_MaxParallelRequestsHandler_v3._reserve_project_io_tokens_or_raise.. ->6Kw+{:lG4>rc3JK|]}|dtk(s|d|dfywr0)rUr1s rgr/z[_PROXY_MaxParallelRequestsHandler_v3._reserve_project_io_tokens_or_raise.. r2r3c36K|]\}}}d|vr|||fyw)rTNrer5s rgr/z[_PROXY_MaxParallelRequestsHandler_v3._reserve_project_io_tokens_or_raise.." r7r8rz8ITPM/OTPM tokens reserved: itpm=%s, otpm=%s for model %s) rrrrSrUrrcrrRr)rr<rPr;rr3rrrrArrrrrrrrrrrr )rrrr<rr*r+rrr"io_token_descriptorsvconfigured_otpm_limitsmin_configured_otpm_limitrraw_estimated_output_tokensraw_estimated_input_tokensrSr4 io_responser9 otpm_reservedr itpm_scopes otpm_scopess rg#_reserve_project_io_tokens_or_raisezH_PROXY_MaxParallelRequestsHandler_v3._reserve_project_io_tokens_or_raise s{&$% 3D9"' ah3NPk2l&lA' ' $ * ) x66|$*//% } F )  ) Ka3/E+Ffj!)-)O)O%>*P* & & -1,O,O_ -P- "),,F(J,,T9= (0!4  ''!: ( ;?:P:P,#9$;.?? ;Q; 5 1 ]M ~ &, 6&)2211%6%G%G3 .2*!&!4!4K&:: +%6%G%G;'+#  ) )$' / *  1 !&'0D'"K*7E &)2;)?E &4=>:E//J_ajal:m>5E 1 1 !&'0D'"K*7E &)2;)?E &4=>:E//J_ajal:m>5E 1  $ $ 0  % %j 1 8 8Z9P Q  $(3E %"" F     Q' ) H5 sS!KK K  KAK BK+K,6K"K#9KKD8KKKrc Ktjdt|}|j|}|r|j ||||d{S|j xsi}|j d}|j d} d} |j dd} |j|| r'| r%|j| |j d{} |j|||| | | } |j|| | |j|| | |j|| | | j|j|| | r| D cgc]?} | j d xsij d|j!| d| ddA}} |rt#j$j&nd}|j(r| nt+d| D}|j-||j|j(|d{}|ddk(r|j/|| | n||_|t3|||_| D cgc]D} | dt6t8fvr1| j d xsij d}| t;|F}} }t=|}d}d}|rt|j(rgt?|}tA|| }|jC||||tE|jG|| |||d}|#||kDrtjd|| tH|||jK| ||jd{}|ddk(rO|j4}|,|jM||j d{d|_|j/|| | nx||_'| |_(tSd!| D|_*t+|jT}|}|j0}||d"j|d"tjd#|| |jW| || ||||$d{yy77mcc} w7Acc}} w7 77w)%z Pre-call hook to check rate limits before making the API call. Supports dynamic rate limiting based on deployment health. zInside Rate Limit Pre-Call Hook)r)rrrrNrrFr!)rr)r!r)rrrrryrr>rmr^rkrlc3DK|]}|dttfvs|ywr)rSrUr1s rgr/zK_PROXY_MaxParallelRequestsHandler_v3.async_pre_call_hook.. s'ah?Z\w>x.xAs  )rrrrrrr.)rvrwr]rer)rrZ)rrErrF)rr!rrrFrzReserving %s tokens for model %s (declared %s=%s plus the input estimate) exceeds the smallest TPM limit this request is charged against (%s), so it cannot be admitted even against an empty window. Lower the declared estimate or raise the TPM limit.)rrrr-c3K|]D}|dttfvr1|jdxsijd |d|dfFyw)rkrmr]Nrl)rSrUrr1s rgr/zK_PROXY_MaxParallelRequestsHandler_v3.async_pre_call_hook..sa6U8,GId+eeUU<06B;;- $$51W:.6sA A rz$TPM tokens reserved: %s for model %s)rrr<rr*r+r),rr rrrrrrrrrrrr@rrUrrrrrrrrArrurrSrUrcrrrrPrrVrr$rrrrrrE)rrrrrrcall_type_specific_rate_limiterrrrryr<rr"parallel_counter_keysrfirst_pass_descriptorsrr<configured_tpm_limitshas_tpm_limitsr*r+rrFr tpm_responserstored_responses rgrz8_PROXY_MaxParallelRequestsHandler_v3.async_pre_call_hook4 sz ""#DE3D9 261T1T_h1T1i' *8LL"3# M ,44: ( -= > ( -= >#!%'4!8  2 2-- 3  '+'L'L%!2!C!C(M("  "@@/))1 A  @@/+# A  CC/+# D FF/+# G  4IIJ[]lmn "%,EE,'-2223JKW++AeHajBYZ, !, ;Pdjjl&6&6UY //* #%)$:$:2!2!C!C#;;!1 %;%H'<7--% +$3. -5)#/*A 0%:+E' %,U8$?A\#]]55.4"99:KLA= A, !,%))>$?NAC "%& "$">">256K2L(2M&7.3(//)A'-E 0+.55!-1I"+1I 6 + ,7-O-OC/3+11!-$/(72-=E)+:E(,56!,6-E).3--.*.>*.3-F-FO&2' 3::< ;ST(..>@PRa::' /"3'='=#;  e {."j,<,F'V sA QP9A?QP<A?QAP?A0QQAQA QB5QQ :Q>Q?B3Q2Q3Q<Q?QQQQ total_tokensr)cg}|j||d}|jt|||j|S)z? Create pipeline operations for TPM increments r)rkrlrrkr{r)rrr)r_)rrkrlrrPr,rqs rg_create_pipeline_operationsz@_PROXY_MaxParallelRequestsHandler_v3._create_pipeline_operationsBsYMO!88$9 "" + ,$$  #"rfusageoutputrrhcd}d}|r+t|try|dk(r|jxsd}n+|dk(r|jxsd}n|dk(r|jxsd}|dvrt |dr|j t|j ddxsd}nt|tr|dk(r|jddxsd}n7|dk(r|jd dxsd}n|dk(r|jd dxsd}|dvr;|jdxsi}t|tr|jddxsd}|dkDrtd||z }|S) a6 Get total tokens from response usage for rate limiting. For 'input' and 'total' rate limit types, cached tokens are excluded because providers like AWS Bedrock don't count cached tokens toward rate limits. This aligns LiteLLM's TPM calculation with provider behavior. rrVrrh)rrhprompt_tokens_details cached_tokenscompletion_tokens prompt_tokensrP) rr1rZr[rPhasattrrXrrrr)rrTrrPrYprompt_detailss rg_get_total_tokens_from_usagezA_PROXY_MaxParallelRequestsHandler_v3._get_total_tokens_from_usage\sd  %'"h.#(#:#:#?aL$/#(#6#6#;!L$/#(#5#5#:L#&88u&=>5C^C^Cj(/0K0K_^_(`(ede E4("h.#(99-@!#D#IL$/#(99_a#@#EAL$/#(99^Q#?#D1L#&88,1II6M,N,TRTN!.$7(6(:(:?A(N(SRS  1 q,">?Lrfc|yt|tr/|jxsd|jxsd|jxsdfnt|t r/|j xsd|jxsd|jxsdfnc|jdxs|jdxsd|jdxs|jdxsd|jdxsdf}|\}}}|s|st|tsy|S)aTotal for usage that carries no input/output split, else 0. A source that can only report one number for the whole request (a pass-through target pricing its own multi-model call) charges that number under every ``token_rate_limit_type``. Splitting it is impossible, and reading 0 out of it would leave the window uncharged, which is how pass-through traffic slips past a TPM limit it is supposed to share. rr[ input_tokensrZ output_tokensrP) rr1r[rZrPr+r`rarrc)rT token_countsr[rZrPs rg_aggregate_only_total_tokenszA_PROXY_MaxParallelRequestsHandler_v3._aggregate_only_total_tokenss =%' %Au'>'>'C!UEWEWE\[\ ]%!12$$)5+>+>+C!UEWEWE\[\] /*Leii.GL1 -.Q%))O2LQPQ .).Q :F6 (, -Z c5Rrf response_objcLt|ttfr|St|ttt t fr,t|dd}t|tttfr|SdSt|tr0|jd}t|tttfr|S|Sy)NrT) rr1r+r.r-r0r*rrr)rdrT nested_usages rg_response_usagez4_PROXY_MaxParallelRequestsHandler_v3._response_usages lU,<$= >    -/EGf g #<$?E&uu6F.MN5 XTX X lD )"."2"27";L,0@$(GH## rfr,cK|jy|Dcgc]}|d }}|j|}|jD]\}}|Dcgc] }|d|vs |}}g}g} |D]V}|d|dnd} tjd|d|d| |j |d| j |d| gX|j|| d{ycc}wcc}w7w)z_ Execute token increment script grouped by hash tag for cluster compatibility. NrkrrzCExecuting TTL-preserving increment for key=%s, increment=%s, ttl=%sr{r)rrrrr rr) rr,opoperation_keysr _hash_tagrgroup_operationsrnr ttl_values rg_execute_token_increment_scriptzD_PROXY_MaxParallelRequestsHandler_v3._execute_token_increment_scripts0  & & . 6I IrE I I 88H %/%5%5%7  !Iz-@\rBuIQ[D[\ \DD& @)+E)>BuIA $**YuI()   BuI& R 12I>? @--.  ) !J ]$ s.C) C-C) C"C"A7C)C'C)cK|sy|jEtjd|jjj ||d{y |j |d{tjdt|y7?7&#t$rP}tjd||jjj ||d{7Yd}~yd}~wwxYww)z Increment token counters using Lua script to preserve existing TTL. This prevents TTL reset on every token increment. Nz=TTL preservation script not available, using regular pipelineincrement_listrsz:Successfully executed TTL-preserving increment for %s keysz=TTL preservation failed, falling back to regular pipeline: %s) rrr rrasync_increment_cache_pipelinernr\rr)rr,rr s rgr/zQ_PROXY_MaxParallelRequestsHandler_v3.async_increment_tokens_with_ttl_preservations#   & & . & &'f g++66UU2)9V    667JK K K & &LcReNf   L   ( ()hjk l++66UU2)9V   s[AC1BC1B,B-#BC1B C.AC)C!C)$C1)C..C1 operationsc 2K|j4d{|D]}|jd}|jd}||*|jj||dd{}|t ||k7ra|jj|d|dd{xsd}|jj |dt ||dz|d|d d{dddd{y777\77 #1d{7swYyxYww) NrrTrrrkrr{rru)rrrrwrnrxr)rrsr operationrractive_window_startr|s rg,_apply_local_window_guarded_token_incrementszQ_PROXY_MaxParallelRequestsHandler_v3._apply_local_window_guarded_token_incrementss] 11  '  &]]<8 (1 6M(N%%)>)F,0,E,E,U,U"-=#-V-'# '.#6I2JNc2c33CC%e,1A#'D   //??!%(09=N3OO!%(-=# @)     '+    s|DC8DAD%C:&9DC< AD C>!D' D2D3D:D<D>DDDD DDcK|D]}|jd}|jd}||*|j/ |j||dg||d|dxsdgd{e|ddkDsn|j j |d|d||d d{y7A#t$r$}tjd|d|Yd}~fd}~wwxYw79w) Nrrrkr{rrrz1Window-guarded token adjustment failed for %s: %srkrlrsr)rrrrrrasync_increment_cache)rrsrrurrr s rg,_apply_redis_window_guarded_token_incrementszQ_PROXY_MaxParallelRequestsHandler_v3._apply_redis_window_guarded_token_increments)s5 $ I"|4J$-MM2I$J !!%:%B99EDD&%e, 2%&78%e,1 E   *+a///EE!%(#$56-=!%( F5  !(00K!%(sR9C(B($B&%B() C3+CCC&B(( C1C CCCcK|D]U}|jd|jd&|jj|d|d||dd{Wtd|D}td|D}|r|j || d{|r|j || d{yy7f7%7 w) Nrrrkr{rryc3K|]<}|jd)|jd|jddk(r|>ywrNrrrrr-rus rgr/z`_PROXY_MaxParallelRequestsHandler_v3.async_increment_reservation_aware_tokens..\I0 }}\*6 56B 34? 0 AAc3K|]<}|jd)|jd|jddk7r|>ywr~rrs rgr/z`_PROXY_MaxParallelRequestsHandler_v3.async_increment_reservation_aware_tokens..crr)rsr)rrrzrrwr{)rr,rrulocal_guarded_operationsredis_guarded_operationss rgr2zM_PROXY_MaxParallelRequestsHandler_v3.async_increment_reservation_aware_tokensOs - I}}\*2immD[6\6d//EE!%(#$56-=!%( F +00 00 +  +00 00 +   $CC3!1D   $CC3!1D   $3*   s;(C+CB=ACB?C6C7C?CCc@ddlm}|jdd}|dvry|S)Nr)general_settingstoken_rate_limit_typerhrU)rrr)rrspecified_rate_limit_types rgget_rate_limit_typez8_PROXY_MaxParallelRequestsHandler_v3.get_rate_limit_typeus2?+;+?+?@WY`+a! $-  ((rfadditional_headersrczt|}|D]*}d|d}|d||d|d<|d||d|d<,|S)z Return ``additional_headers`` extended with ``x-ratelimit-{descriptor_key}-{remaining|limit}-{rate_limit_type}`` entries. Non-mutating so callers pick their own target dict. z x-ratelimit-rsr}z -remaining-rr|z-limit-)r)rrmergedrprefixs rg1_merge_ratelimit_statuses_into_additional_headerszV_PROXY_MaxParallelRequestsHandler_v3._merge_ratelimit_statuses_into_additional_headerss},00B+C \F#F+;$<#=>FHNO`HaFfX[0A)B(CD EDJ?D[FfXWV,=%>$?@ A \ rfct|tr |jy|jjd}|G|jdxsd}|jdxsd}|s|rt d|t d|dfS|jjd}|&|jdxsd}|rt d|ddfSy)Nrr`rraT billed_unitsrP)rr/rrr)rd rerank_tokensr`rarrPs rg_resolve_rerank_token_usagez@_PROXY_MaxParallelRequestsHandler_v3._resolve_rerank_token_usages,7<;L;L;T+0044X>  $"/"3"3N"C"HqL#0#4#4_#E#JM}1l+SM-BDHH*//33NC  #"."2"2>"B"GaL1l+Q44rfcX|j|}||S|j|}t|trf|jxsd}|j xsd}|j t|j ddxsdnd}|dk(r|dk(rytd||z |dfSt|tre|jxsd}|jxsd}|j|jjxsdnd} |dk(r|dk(rytd|| z |dfSt|tr|jdxs|jdxsd} |jdxs|jdxsd} t| t r| nd} t| t r| nd} |jd xs|jd }t|t"r|jddndxs|jd xsd}t|t r|nd}| dk(r| dk(rytd| |z | dfSy) a` Resolve ``(billable_input_tokens, completion_tokens, usage_resolved)`` for ITPM/OTPM reconciliation. Cache-read tokens are excluded from billable input -- Bedrock Mantle doesn't count them toward ITPM -- but they're untouched everywhere else (cost/usage logging still sees the full prompt token count). rrY)rrFTr[r`rZrarXinput_tokens_detailscache_read_input_tokens)rrgrr1r[rZrXrrr+r`rarrYrrrcr)rrd rerank_usagerTr[rZrYresponse_input_tokensresponse_output_tokensresponse_cached_tokensraw_prompt_tokensraw_completion_tokensmapped_prompt_tokensmapped_completion_tokensr]raw_cached_tokensmapped_cached_tokenss rg!_resolve_io_token_reconcile_usagezF_PROXY_MaxParallelRequestsHandler_v3._resolve_io_token_reconcile_usagesa#>>|L  # ++L9 eU ##(#6#6#;!M','>'>'C! ..:33_aHMA  !&71&<"q--78:KTQ Q e- .+0+=+=+B !,1,?,?,D1 "AFA[A[Ag**88=Amn #%).D.I"q/2HHIKacgg g eW %',yy'A'cUYY~E^'cbc +0995H+I+lUYYWfMg+lkl !?IJ[]`?a*;gh GQRgilGm.Cst $$)II.E$F$k%))TjJkN;EnVZ;[##OQ7ab9967  @JJ[]`?a*;gh #q(-E-J"q.1EEFH`bff frfrcRt|tsytt|}|y|j}|j }|dkr|dkry|j |}|j |jd}|j|j|xs/|j|j|jd}|ds|ds|dkr |jsy|dr|n|dr|n|dkDr||dfn||df} | \} } } |js|js|js|jt|j|jr t!n |j| |jrdn||jt|j"|jr t!n |j"| |jr dzS|zS|dkDrS|j%t|j|jr t!n |j| ||jnd} |dkDrS|j%t|j"|jr t!n |j"| ||jnd}tg| |S) a Reconcile project ITPM/OTPM reservations to actual usage on success: ITPM to billable input tokens, OTPM to actual completion tokens. Reuses ``_build_reservation_aware_tpm_ops``'s delta pattern -- ITPM/OTPM are stored in the same ":tokens" cache bucket as combined TPM, just under distinct scope keys, so the reservation-aware increment math is identical; only the usage fields being reconciled against differ. rercombined_usage_objectrqTFr(r-)rrrrrrrrrcrgrrrr0rrrrr1)rrrdrr9rBresponse_usagecombined_usageaggregate_totalresolved_usagebillable_inputrZritpm_opsotpm_opss rg_build_io_token_reservation_opszD_PROXY_MaxParallelRequestsHandler_v3._build_io_token_reservation_opss&$'12OPV2WX =$99 $99 A -1"4 $ F F| T $ F FvzzRiGj k!%!B!B   ." "j  . .t/C/CFJJOfDg/h i a ):RS?S\a\v\va  a  ""?D96 0>,)1  % %55e>c>c88e889/4/I/I uOiOi,%*%?%?] 9 55e889/4/I/I uOiOi/%*%?%? 6  FS 6  (q   / /e889/4/I/I uOiOi, -.3.S.S 0  &q   / /e889/4/I/I uOiOi/ -.3.S.S 0  +x+(+,,rfstandard_logging_metadata model_groupc|jd}|jd}|jd}|jd}|jd}t|tr|jdndxs|jd} |jd } |jd xs|jd } g} |r| jd |f|r| jd|f|r| jd |f|r|r| jd|d|f| r| jd| f|r| jd|f|rh|r| jd|d|f|r| jd|d|f|r| jd|d|f|r| jd|d|f| r-| jd| f| r| jd| d| f| S)uf Enumerate every (scope_key, scope_value) pair that *might* carry a TPM counter for this request — independent of whether each scope had a configured TPM limit at pre-call. Reservation awareness happens at the emitter; this helper just lists the candidate scopes so callers can split reserved-vs-unreserved. user_api_key_hashuser_api_key_user_iduser_api_key_team_iduser_api_key_org_iduser_api_key_project_idrNuser_api_key_end_user_idrrtrace_idr]rrrrrFr[rrKrrr)rrrr) rrrr user_api_keyrruser_api_key_organization_idrrrrr)s rg_collect_tpm_scope_targetsz?_PROXY_MaxParallelRequestsHandler_v3._collect_tpm_scope_targets0s!8;;;-1PQR# 05I4J!K=2YZ[+078+G '+231[MB  NNGX. /H:Qzl1KLMrfr)rr*rc g}|D]Q\}}||f|vr||z }n|}|dk(r|jt|j||d||jS|S)u Emit per-scope TPM increment ops with reservation awareness. - Reserved scope (counter already at +reserved from pre-call): reconcile to actual via ``actual - reserved``. - Unreserved scope (counter never touched at pre-call): charge the full ``actual``. Same primitive serves success reconciliation, over-reservation release, and failure refund — pass ``actual_tokens=0`` for the pure refund case (reserved scopes get -reserved, unreserved get 0/skip). rrrR)rr)rr_) rr)rr*rops scope_key scope_valuers rgr0zE_PROXY_MaxParallelRequestsHandler_v3._build_reservation_aware_tpm_opsns|&=?&-  "I{;'?:)O; ) A~ JJ/33I{HU$-((   rfscoper.c   |\}}||v}|r||z n|} | dk(ry|j||d t fd|Dd} |r| t | |jSt | |jd|d|d| d| d S) Nrrc38K|]\}}}|k(r||fywrre)r-identity_counter_keyrr6rqs rgr/zU_PROXY_MaxParallelRequestsHandler_v3._build_project_reservation_op..s- ?(,';6w' srRrrrr)rkr{rrrr)rrrr_) rrrr*rr.rris_reserved_scoperwindow_identityrqs @rg_build_project_reservation_opzB_PROXY_MaxParallelRequestsHandler_v3._build_project_reservation_ops"' ;#(O#;>O=?:Ub >!88KQYZ !% C`   " !O$;5 )$$  2%  I;a }I>"1!"4 / 2   rf.c@tfd|DS)Nc3TK|]}j|x !yw))rrr*rr.N)r)r-rr*rur.rrrs rgr/zV_PROXY_MaxParallelRequestsHandler_v3._build_project_reservation_ops..sM  !??$3"/$32O @     s%()r)rr)rr*rr.rus` ````@rgr1zC_PROXY_MaxParallelRequestsHandler_v3._build_project_reservation_opss$       rfcddlm}ddlm}|j dxsi}||}|j t rgS|j dxsi}||} d} t |ttttfrt|dd} n#|j d} t | tr| } |j| | } | dk(r|j| } tt!|} | | j"nd}| | j$nd}| | j&n t)}|xs| }g}|j+||| }|dkDr | |krt-j.d || || z |j1|j3||| | |S)zGBuild Redis pipeline increment ops for TPM / parallel-request counters.r) get_litellm_metadata_from_kwargs)#get_model_group_from_litellm_kwargsstandard_logging_objectrNrTr)rTr)rT)rrrzJReleasing unused TPM budget on success: reserved=%s, actual=%s, release=%sr()'litellm.litellm_core_utils.core_helpersr)litellm.proxy.common_utils.callback_utilsrrrrr.r-r0r*rr1r^rcrrrrrrrrr rr0)rrrdrrrrrequest_metadatarr_usage_combined_usagerPrrrrreconcile_modelr,r)s rg(_build_success_event_pipeline_operationszM_PROXY_MaxParallelRequestsHandler_v3._build_success_event_pipeline_operationss ] *04M)N)TRT"B6"J    A BI+B+F+Fz+R+XVX!@H '+  !&/   \7D9F%+ZZ0G%HO/51(88vWf8g 1 <<6<JL12OPV2WX:?:K!6!6QR8=8I 4 4tV[VgU=R=Rmvmx "0!>;LN88&?'9  Q %>Y]=P=\-j9bd )/4M)N -t 4377HMmT2 " $(()=>H262h2h/9(D/I8r!3i3M. /8E #O 4!(7G!N ot ,&**+?@H484j4j/9(D/I8r!5k5O0 1 -rfcKddlm} ||}g}tt|}| |jnd} |$| "|j | |d{d|_| |j rdn"|j|j|jf\} } } |\| dkDrWtjd| |j|jt|j|jd| |N| dkDrI|j r=|j#t%|j&|j&d| |j n:|7| dkDr2|jt%|j&|j&d| nd } |N| dkDrI|j(r=|j#t%|j*|j*d| |j(n:|7| dkDr2|jt%|j*|j*d| nd }|r/|j,j.j1|| d{| |fD]`}t3|tr0|j,j.j1|| d{C|sF|j5|| d{b|| dkDs | dkDs| dkDrd |_yyy7j77E7(#t6$r }tj8d |Yd}~yd}~wwxYww)a" On failure: decrement max_parallel_requests and refund the upfront TPM reservation only against the scopes the reservation actually charged. Unreserved scopes were never incremented at pre-call, so refunding them would drive their counter negative. rrNr-)rrrz,Releasing reserved TPM tokens on failure: %sr(r-rerpr+Tz%Error in rate limit failure event: %s)rrrrrrrrrrrr rr0rxrrr1rrrrrrrrrr2rr)rrrdrrrrsr,rrrr9rBitpm_operationsotpm_operationsproject_operationsr s rgasync_log_failure_eventz<_PROXY_MaxParallelRequestsHandler_v3.async_log_failure_eventsw d W;\]c;d $PR 56STZ6[\E8=8I!4!4tK [%<:: +%=;'+#=E$>$>++U-G-GIcIcd :O]M  _q%8$**+Y[jk $**99 $U%:%: ;(-(=(=&'(7 :($):u?d?d33!%"<"<=$)$>$>"#$1272W2W 4$): ::!%"<"<=$)$>$>"#$1 ;! 6$):u?d?d33!%"<"<=$)$>$>"#$1272W2W 4$): ::!%"<"<=$)$>$>"#$1 ;! &#//::YY#6-EZ(7&H "0$733>>]]'91I^(GG,>)AH  o&9]Q=NR_bcRc-1*Sd sT    W * *+RTU V V WsK1AKJ<G"K5J?6AK;K<KKKK9K1<K?KKK K.K)$K1)K..K1cKt}| |jy|j|jdd{d|_y7 w)a Release the api-key ``max_parallel_requests`` slot that ``async_pre_call_hook`` acquired, for a request that ended without either logging callback firing. The slot is normally released by ``async_log_success_event`` (natural stream completion) or ``async_log_failure_event`` (LLM error). When a client cancels a stream mid-flight, the cancellation surfaces as ``asyncio.CancelledError`` / ``GeneratorExit`` and neither callback runs, so without this the slot leaks per cancelled stream until its TTL prunes it. The stashed acquisition's presence (not the key object's current max_parallel_requests configuration, which can change mid-request) decides whether there is anything to release. Nr-)rrr)rrrs rg1async_release_max_parallel_requests_on_disconnectzV_PROXY_MaxParallelRequestsHandler_v3.async_release_max_parallel_requests_on_disconnect s[$)* =E//7 22++!3   #  s:A A  A cbK ddlm}t}| |jnd}|t |dr t |d}nd}|ut ||st |trYt ||r|j}|j|jdixsi|d}t|di|d|i |j|| d{y#t$r } tjd| Yd} ~ Ad} ~ wwxYw71#t$r } tjd | Yd} ~ yd} ~ wwxYww) zN Post-call hook to update rate limit headers in the response. r) BaseModelNrrrrz&Error in rate limit post-call hook: %s)rrz0Error in batch enqueued-token post-call hook: %s)pydanticrrrr\rrr model_dumprrsetattrrrr _handle_batch_enqueued_post_call) rrrrrr!litellm_proxy_rate_limit_responser_additional_headersr s rgasync_post_call_success_hookzA_PROXY_MaxParallelRequestsHandler_v3.async_post_call_success_hook(sN X *,.ETYTeu7P7Pko -0<8%56%,X7G%HN%)N!-~y9ZX\=]!.)<)7)B)B)D151g1g+9+=+=>RTV+W+][]!B:!N2h2'  (U>U+?ATU b77J[fn7o o o  X * *+SUV W W X p b * *+]_` a a bseD/B4C9DDDD/ C>C94D/9C>>D/D D, D'"D/'D,,D/cKt|}|y|j}t}|V|jJ|jj t |j|j|d{d|_|jjtvra|jjt |j|d{}|&|jj||d{yyy7707 w)N)batch_id reservationrs)rrsrrs) r&rrrrsave_reservationr'rrrr#pop_reservationrefund)rrrviewspanrpoppeds rgrzE_PROXY_MaxParallelRequestsHandler_v3._handle_batch_enqueued_post_callRs )(3 < '88(*  !A!A!M11BB4TWW=!<<)-C   04E , ;;   "@ @"&"A"A"Q"Q4TWW=)-#R#F!55<<jn<ooo" A   ps7A1D3D4ADD&D9D:DDD request_dataoriginal_exception traceback_strczK t}|y|j6|j|j|jd{d|_|j@|j j |j|jd{d|_|jry|j}|j}|j}|dkr |dkr|dkry|dkDr2|jt|j|jd|nd} |dkDrI|jr=|jt|j |j d||jn8|dkDr2|jt|j |j d|nd} |dkDrI|j"r=|jt|j$|j$d||j"n8|dkDr2|jt|j$|j$d|nd} | s| s| rt'j(d|||| r9|j*j,j/| |j d{| | fD]t} t1| t2r:|j*j,j/| |j d{M| sP|j5| |j d{vd |_y77s77?7#t6$r } t'j8d | Yd} ~ yd} ~ wwxYww) um Release the parallel-request slot and any TPM/ITPM/OTPM reservation when the request is rejected after the pre-call hook acquired them but before the LLM call ran (e.g. a downstream guardrail/auth hook raised). Without this, those resources are stranded — async_log_failure_event is a litellm completion-level callback and never fires for proxy-side rejections, so a leaked slot would occupy the gauge for the full PARALLEL_REQUEST_SLOT_TTL_SECONDS. Idempotent: the slot release clears the stashed acquisition (and slot removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM refund is guarded by the stash's ``reservation_released`` flag — if both this hook and async_log_failure_event end up running in the same flow, only the first release/refund applies. Nr-rrr(rer-zLReleasing reserved tokens on proxy-level rejection: tpm=%s, itpm=%s, otpm=%srpr+Tz8Error releasing TPM reservation on post-call failure: %s)rrrrrrrrrrrr0rrrr1rrrrr rrrrrrxr2rr)rrrrrrrr9rB combined_opsrr project_opsr s rgasync_post_call_failure_hookzA_PROXY_MaxParallelRequestsHandler_v3.async_post_call_failure_hookgs,a j,.E}"".:: % 3 3%6%G%G;'+#//;55<< % @ @->-O-O=480))%*%:%:O#(#=#=M#(#=#=M!# (:}PQ?Q#Q& 55!%"7"78$)$9$9"#$3 6 $!1$)N)N33!%"<"<=$)$>$>"#$1272W2W 4!1$ ::!%"<"<=$)$>$>"#$1 ;! 4!1$)N)N33!%"<"<=$)$>$>"#$1272W2W 4!1$ ::!%"<"<=$)$>$>"#$1 ;! $x8$**b#!!  //::YY#/->-O-OZ!)(3  k4033>>]]'21B1S1S^!GG,7):)K)KH *.E & {H    j * *+egh i i jsL; LL;6LLA LLL+L;,3LL; FL;L 3i>> >  >@    !!NO  2OCy2O)2O3 112 2O  2Oh)S)S)6DI$sDI~:N0  3i  +    d "    3i  +   )D0 1D 8  & Cy& &   & V)-$'+ | 12| +|  |  | * |  | |83128383 tCy$s$556=Q8RR S 83t #7 C s Wf  &;RUY;Y &^a &")- Rc)*RcRc+ Rc  Rc  Rcn)- SI S+ S c S")- 6G)*6G6G+ 6G  6Gv)-8,8+8  8|)- D-.Dg&:;S@ABD+ D  DL7 '7  (< =s BC7   7 x)-;  56; +;   ; zd,-. 4@ # $@ 01@   @ J)-c 01c +c   c R)- * -.* * + *  * `V_U`(, % sCx)% % 'uS#w?O7P-P'QR % + %  % X)- _/12_/!$_/"% _/ + _/  #s* + _/B#R)D0#Rt#R!$ #R "% #R  #RLPT:!/:BE*: ! ":x5 )5 t5 -. 5  5 n#)##-. #  #J$ )$ t$ -. $  $ L+)+t+-. +  +Z$J!  *4Z$J!  t 8Jd Jd J  J"HH9OH ( (d (WZ]aWa ( d sTz 111 ! " 1t!%y)yyd y d y ! y:y ! "y|)-66+6  6p@SVZ@Z")"t"-. "  "H")"t"-. "  "H6)6t6. 6  6x'+ *#*-.*t *  *X,F,s,,* f   $f8  8F;K  ( 6 cDj    GCK$8  S    TA6TA#*TAY\_cYcTAorTA|!%C 12C C t C * C !)sCx 9 C !$C :C  C JL)LL L  L\# ##!!NO #  # / 0 #4.4Z.29:T2U. .`E4D,DwsTZ{G[,[^b,bgj8 ! !GCK$8 84 ?$%!"CD% %T)-"!"CD"+"  "N)- ?@ +   J)-$?@$+$  $R)-$%&HI$+$  $L )W-G%H ) f-' c6k  &U3T>=RUY=Y$77 sC~ 7rP-P-P- 1 2 P-d<#'S><<4Z < eCHo  <|"%S/*"U38_-" "  " - . "H# S#X# U38_-#  #  # (1sCIYAZ7Z1['\ #  ,d 2# J %S/* U38_-     (1sCIYAZ7Z1['\   136 7 .\#\#\#!!;< \# - . \#|2Wh  tV|  (%%%  %NoWb#)# #8(bt(bP^(bTppbhpmqp4%) xx&x* x Tz x  xrfr)rrrrrcollections.abcrrrr contextvarsr dataclassesrr r typingr r r rrrrtyping_extensionsrrrrlitellm._loggingrlitellm.constantsrr"litellm.integrations.custom_loggerr8litellm.litellm_core_utils.prompt_templates.common_utilsrlitellm.proxy._typesrr\rrrr"litellm.proxy.auth.budget_throttler-litellm.proxy.common_utils.http_parsing_utilsr 1litellm.proxy.common_utils.proxy_rate_limit_errorr!r")litellm.proxy.hooks.batch_enqueued_tokensr#r$r%r&r'&litellm.proxy.hooks.rate_limiter_utilsr(litellm.types.cachingr)litellm.types.llms.openair*r+litellm.types.utilsr,r-r.r/r0r1opentelemetry.tracer2_Spanlitellm.proxy.utilsr3_InternalUsageCachelitellm.types.agentsr4r5rdr6r7r8r9r:r;r<r>r?rArBrErHrKrerankrlarerankrLgenerate_contentagenerate_contentgenerate_content_streamagenerate_content_streamrMrOrPrQrSrUrVrcrrnbytesrWrXrrrYr[rjrprurzrrrrrrrrrxrrrrrrrrrrrrerfrgr%sA  <<"(41n;0 ?TWAW1M2E 3;D, D#$5#J]*]~0%u6 " D "  u !8#U"!++ &*U)!"" U #>%="=%=(OO ) 0 0 6 6 8I8I8O8OPuP $$ %% %%++ &&,, ) *,+'*e) !%$&>U=%=U= ,0!5/ #U{S05898 ():T)A BIB%)#v+%6%