uPjK UddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z m Z ddl mZmZmZmZmZddlZddlmZmZmZmZmZmZddlmZddlZddlmZmZddl m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4dd l5m6Z6ddl7m8Z8ddl9m:Z:ddl;mZ>m?Z?m@Z@mAZAmBZBmCZCddlDmEZEddlFmGZGddlHmIZImJZJddlKmLZLdeMddfdZNeOgeIeJdZPeeOeQeRd< dde eQeSfde eQeSfdeTdeMeQeSffdZUdZVeZWeeRd <ed!eQzZXGd"d#eZYGd$d%eZZeWjd&d'gee8g(d)efd*Z\eWjd+d'gee8g(ee8ejd,-fd.e3d/eXfd0Z^deMfd1Z_d2Z` dd3eQd4ead5ead6ebd7eQdzd8eQdzf d9Zcd:eadeMfd;ZddeMd?eMd6ebd@eQdzfdAZf dd:ead4ead5ead6ebd@eQdzf dBZgeOe/jje/jjhZkeeRdC<d.e3deTfdDZldEeMdeMfdFZmd:eadGeQdzd8eQdzdendzfdHZodIeMdJendeMfdKZp ddLZqdGeQdzd8eQdzdeQdzfdMZreWjdNd'gee8g(ee8ejddO-ejddP-fdQed.e3dGeQdzd8eQdzfdRZseWjdSd'gee8g(ee8ejddT-ejddU-ejdVdWdXdYZejdd[d\fd.e3dGeQdzd]eQdzd^etd_etf d`ZueWjdad'gee8g(ee8fd.e3fdbZveWjdcd'gee8g(ee8fd.e3fddZwdeMeQefdzfdeZxdfeMeQefdeafdgZyeWjdhd'gee8g(ee8fd.e3fdiZzGdjdkeZ{dle jdma}e{eRdn<doZ~eeRdp<dqZeeRdr<de{fdsZde{fdtZeWjdud'gee8g(eWjdvd'gee8g(dwZdxZdyZeeRdz<deTfd{Z ddQedzdeMeQeffd|ZdeTfd}ZdeTfd~ZdeQdzfdZd)eddfdZdQedeQfdZeWjdd'gdQefdZeWjdd'gee8g(dQefdZeWjdd'gee8g(dZeWjdd'gd)efdZeWjdd'geWjdd'gdQefdZeWj#dd'gdZeWj#dd'geWj#dd'gdZeWj)dd'gee8g(ej*dd-ej*dd-ej*dd-ee8fd)ededdzdeMdeMd.e3f dZy)N)IterableMapping)datetime timedelta)AnyFinalLiteral TypedDictcast) APIRouterDepends HTTPExceptionRequestResponsestatus)ReadOnly)verbose_loggerverbose_proxy_logger)HEALTH_CHECK_TIMEOUT_SECONDS)MS_TEAMS_ALERT_HEADERSbuild_ms_teams_payloadget_ms_teams_webhook_url)CustomLoggerRegistry)AsyncHTTPHandler) AlertTypeCallInfoEnterpriseLicenseDataLitellm_EntityTypeLitellmUserRolesProxyErrorTypesProxyExceptionSpecialModelNamesUserAPIKeyAuth WebhookEvent)_BANNED_REQUEST_BODY_PARAMS)user_api_key_auth)PrismaDBExceptionHandler)count_live_proxy_workers) ADMIN_ONLY_HEALTH_DISPLAY_PARAMS_clean_endpoint_data'_update_litellm_params_for_health_check0health_check_filter_kwargs_from_general_settingsperform_health_checkrun_with_timeoutget_in_flight_requests)GracefulShutdownManager)._ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDEclientside_credential_keys)get_secret_boolparamsreturnct|tsy|g}t|h}|r|j}t|tr|j }nt|t r|}nG|D]}t|t r |jdrtdddit|tt fsJt||vsX|jt||j||ryy)a/ Validate that the provided params do not contain any ``os.environ/`` references. Values with that prefix are expected to come only from server-side configuration (already resolved before reaching here). If a request-supplied value still carries the prefix, raise ``HTTPException``. Nz os.environ/errorzHEnvironment variable references are not permitted in request parameters. status_codedetail) isinstancedictidpopvaluesliststr startswithraddappend)r5stackseensrcrAvalues u/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/proxy/health_endpoints/_health_endpoints.py_reject_os_environ_referencesrLBs fd #"(EZLD iik c4 '*zz|F T "F  $E%%%*:*:=*I# ##%op%$.2e9D3HE# U# $ litellm_credential_name_CONFIG_CONNECTION_FIELDS config_paramsrequest_paramsallow_client_side_credentialsc|r t|StfdtDs t|S|jDcic]\}}|tvs||c}}Scc}}w)aReturn the configured parameters to merge under a connection-test request. A request that sets its own connection fields describes a connection of its own, so the configuration's credentials are not carried into it: they belong to the endpoint the configuration names. Anything the request does not set still comes from the configuration, which is what lets a request name a configured model and test it as configured. ``litellm_credential_name`` is dropped alongside the literal credential fields: it names a stored credential that ``load_credentials_from_list`` resolves into the same secrets further down the call, so leaving it in place would reintroduce them by reference. ``general_settings.allow_client_side_credentials`` is the existing proxy-wide opt-in for callers supplying their own connection parameters. Where an admin has enabled it, a request may pair its own endpoint with the configured credentials, as it could before. c3&K|]}|v ywN).0paramrQs rK z0_config_base_for_health_check..sP5u&Ps)r>anyr%itemsrO)rPrQrRkeyrJs ` rK_config_base_for_health_checkr]lsZ.%M"" P4OP PM"")6)<)<)> g:3#MfBfCJ gg gs AAc4t|tr|St|dr|jr |jSt|drLt j |j }t|dr|j|vr |jS|r|dSt|S)a Get the callback identifier string, handling both strings and objects. This function extracts a string identifier from a callback, which can be: - A string (returned as-is) - An object with a callback_name attribute - An object registered in CustomLoggerRegistry - Falls back to callback_name() helper function Args: callback: The callback to identify (can be str or object) Returns: str: The callback identifier string callback_name __class__r)r=rChasattrr_r%get_all_callback_strs_from_class_typer`)callback callback_strss rKget_callback_identifierres (C x)h.D.D%%%x%3YYZbZlZlm 8_ -(2H2HM2Y)) )  # #  ""rMrouter)slack_budget_alertslangfuse langfuse_otelslackms_teams openmeterwebhookemail braintrustdatadogdatadog_llm_observability generic_apiarizegalileonewrelicsqsceZdZUeeed<y)_ServiceTestErrorDetailr9N__name__ __module__ __qualname__rrC__annotations__rVrMrKrxrxs C=rMrxc.eZdZUeeed<eeed<y)_ServiceTestSuccessResponsermessageNryrVrMrKrrs SM c]rMrz/testhealth)tags dependenciesrequestc*Kddlm}d||iSw)a [DEPRECATED] use `/health/liveliness` instead. A test endpoint that pings the proxy server to check if it's healthy. Parameters: request (Request): The incoming request. Returns: dict: A dictionary containing the route of the request URL. r)get_request_routeroute)litellm.proxy.auth.auth_utilsr)rrs rK test_endpointrs&@ &w/ 00sz/health/serviceszSpecify the service being hit.) descriptionuser_api_key_dictservicec LK ddlm}m}m}|t dddi|dvrt ddd |d t id }|t jvrd }n=t jD]*}t|d d|k(rd }nt|}||k(s(d }n|dk(s|dk(s |dk(s|r3|dk7r.t jddddgddd{}dd|ddS|dk(rgvr|jXjZ|jXjZD]}|jXj\||jXj\vr2d?}|t^j`k(rd@}n|t^jbk(rdA}np|t^jdk(rdB}nZ|t^jPk(rdC}nD|t^jfk(rdD}n.|t^jhk(rdE}n|t^jjk(rdF}ndC}|jm|dG|Hd{n*|jmdIdGt^jPHd{|Ztojp|jXjstojp|jXju|jXj\xsg}tw|}d|dJdKSt dLddM|dNi|dOk(r"dO|jWd>dPvrddM|dNi}t dL|ty}|ddQi}t dL|dRt^jPjzdSt}j~jdTdU} |jXjj|tttjt| Vd{}!|!jdk\r+ddW|!jdX|!ji}"t dY|"ddZd}#|#S|d[k(rtd\tLjNd]|jDxsd5d^dd|jFtjd_|jJ` }$|jXj|$ad{ddbdSy7:7 77n707|7777i77*#t$r}%tjdc|%tjtjt%|%trPtt|%ddde|%dftjt|%dgdht|%dit:jjt%|%tr|%tdkt|%ztjt|%dgdht:jjd}%~%wwxYww)lz Use this admin-only endpoint to check if the service is healthy. Example: ``` curl -L -X GET 'http://0.0.0.0:4000/health/services?service=datadog' -H 'Authorization: Bearer sk-1234' ``` r)general_settings prisma_clientproxy_logging_objNr8r9zService must be specified.r:)rgrnrhrirjrkrlrmrootelcustom_callback_api langsmithrpdatadog_metricsrqrrrsrtrurvz!Service must be in list. Service=z not in FTr_rlrorrrhz"openai/litellm-mock-response-modeluserzHey, how's it going?)rolecontentzlitellm:/health/serviceszThis is a mock response)modelmessagesr mock_responsesuccesszMock LLM request made - check .)rrrp) DataDogLoggerr unhealthy error_messagezDatadog is healthyr)DatadogMetricsLogger)"get_custom_logger_compatible_class)start_periodic_flushzDatadog Metrics is healthyrs) ArizeLoggerzArize is healthyrt)GalileoObservezGalileo is healthy)LangFuseLoggerz'Mock LLM request made - check langfuse.ruz7Only proxy admins can trigger the New Relic test event.)NewRelicLoggeru(New Relic is healthy — test event sentrm)tokenspend max_budgetuser_id key_aliasteam_id event_group user_budget)type user_inforv) SQSLoggerrjrgalertingzdefault test messagezLLM Exception test alertzLLM Too Slow test alertzLLM Requests Hanging test alertzBudget Alert test alertzDB Exception test alertz!Outage Alert Exception test alertzDaily Reports test alertLow)rlevel alert_typez"This is a test slack alert messagezBMock Slack Alert sent, verify Slack Alert Received on your channel)r alert_typesri"z=" not in proxy config: general_settings. Unable to test this.rkrVz2MS_TEAMS_WEBHOOK_URL not set. Unable to test this.z Alert type: `z` Level: `Low` Timestamp: `z%H:%M:%Sz1` Message: This is a test MS Teams alert message)urlheadersdataz!MS Teams webhook returned status z: zHMock MS Teams Alert sent, verify MS Teams Alert Received in your channelrn key_createdzTest Email AlertzNEmail Test key (This is only a test alert key. DO NOT USE THIS IN PRODUCTION.)TEST_EMAIL_ADDRESS) eventr event_messagerrrrr user_emailr) webhook_eventz2Mock Email Alert sent, verify Email Alert ReceivedzMlitellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %sr<zAuthentication Error()rXNoner;)rrrXcodezAuthentication Error, )Ylitellm.proxy.proxy_serverrrrrserviceslitellmsuccess_callbackgetattrre acompletion$litellm.integrations.datadog.datadograsync_health_check,litellm.integrations.datadog.datadog_metricsr*litellm.litellm_core_utils.litellm_loggingrr= litellm.integrations.arize.arizerlitellm.integrations.galileor&litellm.integrations.langfuse.langfuserLangfuse auth_check completion_is_proxy_adminrHTTP_403_FORBIDDEN&litellm.integrations.newrelic.newrelicrrrrrrrKEY budget_alertslitellm.integrations.sqsrgetslack_alerting_instancealert_to_webhook_urlrrllm_exceptions llm_too_slowllm_requests_hanging db_exceptions outage_alerts daily_reportsalerting_handlerasyncio create_tasksend_monthly_spend_reportsend_weekly_spend_reportrBrrJrnowstrftimeasync_http_handlerpostr>rjsondumpsrr;textr$osgetenv&send_key_created_or_user_invited_email Exceptionrr9debug traceback format_excr!r auth_errorHTTP_500_INTERNAL_SERVER_ERRORrC)&rrrrrservice_in_success_callbackscbcb_id_rdatadog_loggerresponserrdatadog_metrics_loggerr arize_loggerrgalileo_loggerrlangfuse_loggerrnewrelic_loggerrr sqs_loggerr test_messagernot_configured_detailms_teams_webhook_urlmissing_webhook_detailms_teams_test_messagems_teams_responsedelivery_failed_detailms_teams_successres& rKhealth_services_endpointrs $l   ?CB^8_` `   , #DWIXV^U_!`a  (-$ g.. .+/ (.. 25@370/3G#370  { ",&-',J1F)):#)6LMN/7 A$;G9AF  ! J$1ON+>>@@H"8,9A(9K{9ZH_5 `t ) )  &HHY%Z "%-)=SX)Y&46JK KK3FFHHH"8,19(1C{1RH_- Yu    D"--L)<<>>H"8,9A(9K{9ZH_5 `r  ! C$2$4N+>>@@H"8,9A(9K{9ZH_5 `t  " M%3%5O  $ $ / / 1"":#)6LMN/7 A$D  ""#45# & 9 9#%^_ N%3%5O,??AAH"8, )[8_-  D   i ''--3)11+55)11.22 I$11"#2    : ) J'::<>%<<QQ]&7&O&O&d&d .EEQQ] *2C2[2[2g2g g$'= %)A)AA+EL'9+A+AA+DL'9+I+II+LL'9+B+BB+DL'9+B+BB+DL'9+B+BB+NL'9+B+BB+EL+DL/@@$0*A5<,<< D##,#:#:= !,''(9(Q(Q(k(k(mn''(9(Q(Q(j(j(lm/GGSSYWY ";/ '#.c $ ##q 1n%op j !1!5!5j"!EEq )fgI%$?'||~66zBCDAA " .?-V-V-i-i-n-n(34ZZ 67L MN.o.(  !,,3@ARA^A^@__absbxbxayzJ&$ *G,s1v5 ++!Wf-66    s`$B\" ;\"\ \"`$%\"9\:\"`$\"`$A\"\\"1`$2\"5`$6%\"\ \"2`$3\"6`$7%\"\ \"3`$4\"7`$8A \"`$A \"\\"*`$+\".`$/A'\"\*\"\\"`$D4\"\,\"2\3B \"=`$>C5\"3\4A\"8`$9A<\"5\ 6\">`$\"\"\" \" \"\"\"\"\"\"\" \"" `!+C1``!!`$c |j|j|j|j|j|j |j |j|j|j|jr|jjnd|jr|jjd Sdd S)z9Convert health check database record to dictionary formatN) health_check_id model_namemodel_idr healthy_countunhealthy_countrresponse_time_msdetails checked_by checked_at created_at) rrrrrrrrrrr isoformatr)checks rK_convert_health_check_to_dictr!s!00&&NN,,,, 00,,!22==&&6;6F6Fe&&002D6;6F6Fe&&002  MQ  rMcPddlm}|ttjddi|S)zIHelper to check if prisma_client is available and raise appropriate errorrrr9zDatabase not initializedr:)rrrrrr s rK_check_prisma_clientr!3s28==78   rMrhealthy_endpointsunhealthy_endpoints start_timerrc dK |r(|djdrt|ddddnd}|j|||rdndt|t||t j|z dzd| d{y7#t $r!}t jd ||Yd}~yd}~wwxYww) z8Helper function to save health check results to databaserr9Nrhealthyr rrrrrrrrrz8Failed to save health check to database for model %s: %s)rrCsave_health_check_resultlentimerrwarning) rrr"r#r$rrrdb_errors rK_save_health_check_to_dbr.?sw#':1'='A'A''J #A&w/ 0# 6  44! 19{/0 34'"iikJ6$>5   w$$%_akmuvvwsAB0A7B;B<BB0B B- B(#B0(B--B0 model_listci}|D]}|jdi}|jd}|jd}|jdi}|jd}|s]|s`||vrg||<||j||d|S)a2 Build a mapping from model parameter to model info (model_name, model_id). Multiple models might share the same model parameter, so we use a list. Args: model_list: List of model configurations Returns: Dictionary mapping model parameter to list of model info dicts model_inforr?litellm_paramsr)rr)rrF)r/model_param_to_inforr1rrr2 model_params rK"_build_model_param_to_info_mappingr5as(*YY|R0 YY|, >>$'#3R8$((1 :"5535#K0  , 3 3", (  rMr3ci}|D]T}|jd}|s||vs||D]1}|d|df}||vr|d|ddddd||<||dxxdz cc<3V|D]}|jd}|jd }|s(||vs-||D]Q}|d|df}||vr|d|ddddd||<||d xxdz cc<||d r;|s>t|dd ||d <S|S) a Aggregate health check results per unique model. Uses (model_id, model_name) as key, or (None, model_name) if model_id is None. Args: model_param_to_info: Mapping from model parameter to model info healthy_endpoints: List of healthy endpoint results unhealthy_endpoints: List of unhealthy endpoint results Returns: Dictionary mapping (model_id, model_name) to aggregated health check results rrrrN)rrrrrrrr9rrr)rrC) r3r"r# model_resultsendpointr4r1r\rs rK_aggregate_health_check_resultsr9sz$M& 9ll7+ ;*==1+> 9 !*-z,/GHm+&0&>$.z$:)*+,)- *M#&c"?3q83 9 9 (Sll7+  W- ;*==1+> S !*-z,/GHm+&0&>$.z$:)*+,)- *M#&c"#45:5$S)/:}:=m:LTc:RM#&7 S S& rMr7latest_checks_maprcK|jD]}|ddkDrdnd}d}|dr|dn|d}||vrf||} | j|k(rR| jrFddlm} m} | j | j | jz j} | d krd }|stj|j|d|d||d|d |d tj|z d zd| yw)a| Save health check results to database, but only if status changed or >1 hour since last save. OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. - Stable systems: ~1 write/hour per model (instead of 12 writes/hour with 5-min intervals) - Status changes: Immediate write (no delay) - Result: ~92% reduction in DB writes for stable systems, while maintaining real-time updates on changes Args: prisma_client: Database client model_results: Dictionary of aggregated health check results per model latest_checks_map: Dictionary mapping model_id/model_name to latest health check start_time: Start time of health check for calculating response time checked_by: Identifier for who/what performed the check rrr&rTrr)rtimezoneiFrrr'Nr() rArrrr<rutc total_secondsrrr)r+) rr7r:r$rresult new_status should_save lookup_key last_checkrr<time_since_last_checks rK%_save_health_check_results_if_changedrEs 0 &&(!"("9A"=Y;  +1*+=VJ'6,CW * **:6J  J.((;-9X\\(,,-G*J_J_-_,n,n,p)-t3&+    66%l3#J/%"("9$*+<$="("9&*iikJ&>$%F )7  +!s BC7 AC7cpK|y t|}t|||}|jd{}i} |D]0} | jr | jn | j} | | vs,| | | <2t ||| ||d{y7S7#t $r } tjd| Yd} ~ yd} ~ wwxYww)a Save background health check results to database for each model. Maps health check endpoints back to their original models to get model_name and model_id. Aggregates results per unique model (by model_id if available, otherwise model_name). OPTIMIZATION: Only saves to database if the status has changed from the last saved check. This dramatically reduces database writes when health status remains stable. Nz7Failed to save background health checks to database: %s) r5r9get_all_latest_health_checksrrrErrr,) rr/r"r#r$rr3r7 latest_checksr:rr\r-s rK$_save_background_health_checks_to_dbrIs"j%G %S ?    &3%O%O%QQ #%" /E$)NN%..8H8HC++).!#&  /4          R  j$$%^`hiijsVB6+B B3B 'B BB B6B B B3B.)B6.B33B6_PROXY_ADMIN_ROLEScd|j}|yt|dr |jn|}|tvS)a, Return True if the caller has a proxy-admin role (full or view-only). user_role on UserAPIKeyAuth can be either a LitellmUserRoles enum or its string value depending on how the auth path constructed the object, so we compare against the raw value rather than the enum identity. FrJ) user_rolerarJrJ)rr role_values rKrr9s7$--D |&-dG&< $J + ++rMr?c Ft|}tt}dD]t}|j|}t |t s%|Dcgc]=}t |tr)|j Dcic] \}}||vs ||c}}n|?c}}}||<v|Scc}}wcc}}}w)ab Return a copy of the /health response with provider routing fields (``api_base``, ``api_version``) removed from each healthy/unhealthy endpoint entry. Used to hide those fields from non-admin callers while still showing them which deployments they own and whether each one is healthy. Proxy admins receive the unmodified result. )r"r#)r>setr)rr=rBr[)r?outdropr\epsepkvs rK+_strip_admin_only_fields_from_health_resultrVHsfC67D;xggcl c4 svwwmo:VXZ^K_288:G41a$!Q$GeggwCHx JHws(B 0 B=BB B rcV|s|syt}|D]}|jdxsijd}|s*|r||k(r|j|C|sF|jdxsijd}|jd|k(s||k(s|j||S)ul Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of deployment IDs the response should be scoped to. Mirrors the live-path semantics in ``perform_health_check()``: ``model`` matches either the deployment's ``model_name`` alias or its ``litellm_params.model`` provider string. ``model_id`` matches ``model_info.id``. Both query params are validated against the supplied ``model_list``. Callers pass an already-scoped list (filtered to the caller's allowed models for non-admins, full list for admins), so a ``model_id`` that isn't present resolves to an empty set rather than a single-element set — preventing a non-admin from reading another deployment's cached health entry by guessing its ID. Returns ``None`` when no targeting is requested — callers should treat that as "no filter." Nr1r?r2rr)rOrrE)r/rr target_idsm deployment_id litellm_models rK_resolve_targeted_model_idsr\Ys(  UJ  .|,277=    1 NN= )  UU#34:??HMuu\"e+}/E}- . rMresultsallowed_model_idscD|jdxsgDcgc]!}|jd|vst|#}}|jdxsgDcgc]!}|jd|vst|#}}||t|t|dScc}wcc}w)u5 Restrict a cached background health-check result dict to endpoints whose model_id is in ``allowed_model_ids``. Endpoints without a model_id (e.g. CLI-model entries that predate the model_id wiring) are dropped conservatively — we cannot prove they belong to the caller, so they are excluded rather than leaked. Each retained endpoint is shallow-copied before being returned, so any downstream transform (e.g. _strip_admin_only_fields_from_health_result) cannot accidentally mutate the shared ``health_check_results`` cache. r"rr#r"r#rr)rr>r*)r]r^rSr&rs rK)_filter_health_check_results_by_model_idsra~s$+;;/B#C#IrvBbffU_N`duNutBxvGv#KK(=>D"PZI[_pIpRI%(Wy>   wsB BB0 Bc Ktd||||||d| d{\} } } |-||n|} | %tjt|| | | |||| | t | t | dS7Pw)zDHelper function to perform health check and save results to database)r/ cli_modelrrmax_concurrencyrNrr`rV)r-rrr.r*)r/ target_modelrcrrr$rrrdperform_health_check_extrar"r#rmodel_name_for_dbs rK_perform_health_check_and_saveris7K7' 7 %71-*A 090E9<  (   (!%%'% /2./23  71sA+A)AA+cl|r|r|S|!ttjdd|di |j|}|!ttjdd|di|jS#t$r>}t j d||ttjdd|di|d}~wwxYw)zNMap ``model_id`` (without ``model``) to ``model_name`` for live health checks.Nr9zModel with ID z not foundr:rez,Error getting deployment for model_id %s: %s)rrHTTP_404_NOT_FOUNDget_deploymentrrr9r)rr llm_router deploymentrs rK*_health_endpoint_resolve_target_model_nameros u 11~hZzBC  &55x5H 11~hZzBC      ""#QS[]^_11~hZzBC  sA,, B359B..B3z/healthz!Specify the model name (optional)zSpecify the model ID (optional)rcKddl}ddlm}m}m}m}m} m} m} m } m } t|}|j}t||| }t|t|xs|dtdtffd } | N| /t!dgd| || ||j"d|d |d{}||St%t&j(dd i t+j,| }t/|j0}t2j4j6|vr!|j8t/|j:}t=|dkDxrt2j>j6|v}|r.tA|}|Dcgc]}|jCd |vs|}}| rtE|||}|r|Dchc]J}|jCd xsijCd r$|jCd xsijCd L}}||n|}tG||}|<|s:tIjJd|j"t/|j0dg|d<||S||tG||S||St!d||d|| ||j"||| d |d{}||S7cc}wcc}w7#tL$rD}tIjNd|tIjPtSjT|d}~wwxYww)u 🚨 USE `/health/liveliness` to health check the proxy 🚨 See more 👉 https://docs.litellm.ai/docs/proxy/health Check the health of all the endpoints in config.yaml To run health checks in the background, add this to config.yaml: ``` general_settings: # ... other settings background_health_checks: True ``` else, the health checks will be run on models when /health is called. To skip deployments that set ``model_info.disable_background_health_check: true`` on ``GET /health`` as well as in the background loop, set ``general_settings.health_check_skip_disabled_background_models: true``. rN) rhealth_check_concurrencyhealth_check_detailshealth_check_resultsllm_model_listrmruse_background_health_checks user_modelr?r6cr*|jdddk(rtj_r|Sdjd<t |S)Nrrz8api_base and api_version are admin-only on this endpointzLitellm-Health-Field-Notice)rrHTTP_503_SERVICE_UNAVAILABLEr;rrV)r?is_adminmodel_specific_requestrs rK _post_processz&health_endpoint.._post_processsL "fjj!&D&I#)#F#FH M:t67:6BBrM) r/rfrcrrr$rrrdr9zModel list not initializedr:rr1r?zhealth_endpoint: scoped key %s has accessible models %s but none of the matching deployments carry a model_info.id; background health-check cache will return an empty result.zSome accessible deployments are missing model_info.id and were excluded from this response. Ask a proxy admin to populate model_info.id for these models.warnings) r/rfrcrrr$rrrdrfzHlitellm.proxy.proxy_server.py::health_endpoint(): Exception occured - %srV)+r+rrrqrrrsrtrmrrurvr,rorboolr>rirrrrcopydeepcopyrBmodelsr"all_team_modelsrJr team_modelsr*all_proxy_modelsrOrr\rarr,rr9rrr) rrrrr+rrqrrrsrtrmrrurv _hc_filterr$rfr{ cli_result_llm_model_listaccessible_modelsrestrict_to_allowed_modelsallowed_modelsrY targeted_idsr^ filter_idsfiltered router_resultrryrzs ` @@rKhealth_endpointrsZ6   IIYZJ! JDUHV`aL%&78H$():($;CdCtC"o  !%*H +!!%(0"/)-55!$< +! + % %Z00"AA!=> --7"!!2!9!9:  , , 2 26G GL]LeLeLq $%6%B%B C  ! "Q & l+<+M+M+S+S[l+l # &$'(9$:N*9cQQUU<=PTb=bqcOc '#>ouV^"_L)-,l+1r66t<UU<(.B33D9,!,5A4LLRc "KL`bl"m'0A)00U*11.556 F,HZ( %X..'%%NOceq%rss !56 6)G **),+%)11! 8! * * $M!/ /O %Td,L $ ""#mopq""9#7#7#9:sA2K48&J$J J$*K4+CJ$,JJ J$ AJ/AJ$K4 J$K4J$$K4%#J$J" J$K4J$ J$$ K1-?K,,K11K4z/health/historyzFilter by specific model namez$Filter by status (healthy/unhealthy)dzNumber of records to returnrr')rgelezNumber of records to skip)rr status_filterlimitoffsetcFKt} |j||||d{}|Dcgc] }t|}}|t|||dS7,cc}w#t$r;} t j d| ttjdd| id} ~ wwxYww)zp Get health check history for models Returns historical health check data with optional filtering. )rrrrN) health_checks total_recordsrrz&Error getting health check history: %sr9z)Failed to retrieve health check history: r:) r!get_health_check_historyrr*rrr9rrr) rrrrrrhistoryr history_datars rKhealth_check_history_endpointrs01M ,EE' F  RYYrCr)rr license_dataverification_results rK_read_license_datar sL1B1knFkFkL  NM4 8 NL$ 7 )7)Z)Z)44*66*[* #d*-DD  S#X --    s7B BBrch|jd}t|tr t|S|gS|gS)Nallowed_features)rr=rB)rraw_allowed_featuress rK_read_allowed_featuresr$s?"."2"23E"F&-())# !!rMz/health/licensecKddlm}m}t}t t |dd}|rdnd}| ||dgddddS|j d }|j d }|j d }|||t|||ddSw) zNReturn metadata about the configured LiteLLM license without exposing the key.r)r premium_userrN enterprise community) max_users max_teams) has_license license_typeexpiration_daterlimitsrrr)rrrrr}rrr) rrrrrrrrrs rKhealth_license_endpointr-sH,.LgnmTJKK*6,KL&(# "!!  *--.?@O#'' 4I#'' 4I#$*2<@""  sBBc"eZdZUeed<eed<y) DBHealthCacher last_updatedN)rzr{r|rCr}rrVrMrKrrXs KrMrunknownrrdb_health_cacheg@"DB_READINESS_CHECK_TIMEOUT_SECONDSg@#DB_READINESS_PROBE_DEADLINE_SECONDScK tjttd{S7#tj$rdt ddcYSwxYww)Ntimeout disconnectedrr)rwait_for$_db_health_readiness_check_unboundedr TimeoutErrorrrVrMrK_db_health_readiness_checkrgsZ[%% 0 27       [(/.:YZZ[s1A&1/1A1AAAAcKddlm} tjtdz }tddk(r|t dkrtS|dtjd atSt j|jt d{dtjd atS7!#t$r}dtjd atj|r tjd |jd tt d{7t j|jt d{7tj ddtjd atcYd}~S#t$rtj"dYnwxYwtcYd}~Sd}~wwxYww)Nrr rr connected)secondsrrrzE_db_health_readiness_check: health_check failed, attempting reconnecthealth_readiness_check)reasontimeout_secondslock_timeout_secondsz/_db_health_readiness_check: reconnect succeededz,_db_health_readiness_check: reconnect failed)rrrrrrrr health_checkrrr'is_database_transport_errorrr,attempt_db_reconnectinfor9)r time_diffrs rKrrqs8"#<<>ON,KK 8 $ 3 IVXDY8Y" "  )7XO" "}99;Eghhh%0(,,.Q i %3X\\^T # ? ? B [$,,-tu#883$F)K9 &&!..0>%))*[\)$,LLN#'& [$**+YZ [-sG >B8G  B8'G (,B8B6 B85G 6B88 G,G/4F#D&$1FE6F G G F30G2F33G;G<G GG z /settingsz/active/callbacksc rKddlm}m}t|j d}t j Dcgc] }t|}}t jDcgc] }t|}}t jDcgc] }t|}}t jDcgc] }t|}}t jDcgc] }t|}}t jDcgc] }t|} }t jDcgc] }t|} }||z|z|z|z| z| z} |j} d} | rt| tr t!| } ||||||| | | t!| | t j"d Scc}wcc}wcc}wcc}wcc}wcc}wcc}ww)au Returns a list of litellm level settings This is useful for debugging and ensuring the proxy server is configured correctly. Response schema: ``` { "alerting": _alerting, "litellm.callbacks": litellm_callbacks, "litellm.input_callback": litellm_input_callbacks, "litellm.failure_callback": litellm_failure_callbacks, "litellm.success_callback": litellm_success_callbacks, "litellm._async_success_callback": litellm_async_success_callbacks, "litellm._async_failure_callback": litellm_async_failure_callbacks, "litellm._async_input_callback": litellm_async_input_callbacks, "all_litellm_callbacks": all_litellm_callbacks, "num_callbacks": len(all_litellm_callbacks), "num_alerting": _num_alerting, "litellm.request_timeout": litellm.request_timeout, } ``` r)rrr) rzlitellm.callbackszlitellm.input_callbackzlitellm.failure_callbackzlitellm.success_callbackzlitellm._async_success_callbackzlitellm._async_failure_callbackzlitellm._async_input_callbackall_litellm_callbacks num_callbacks num_alertingzlitellm.request_timeout)rrrrCrr callbacksinput_callbackfailure_callbackr_async_success_callback_async_failure_callback_async_input_callbackrr=rBr*request_timeout)rr _alertingxlitellm_callbackslitellm_input_callbackslitellm_failure_callbackslitellm_success_callbackslitellm_async_success_callbackslitellm_async_failure_callbackslitellm_async_input_callbacksrr _num_alertings rKactive_callbacksrsFO+// ;E>]>]-^c!f-^#-^>E>]>]-^c!f-^#-^">"J7?4G)-8 8A == 8sA A AAc 0Kddlm}m} g} tjDcgc] }t |}}d}tjzddlm }tjj}ttjj|r6 tjjjd{}||d}tj t#j$} t#j&tj(} t+d{} |ft-d{} |1| ddk7r)t/j0st2j4|_d | d|||t9j:| | | d Sd d |||t9j:| | | d Scc}w#t $rtj}YwxYw7#t$r}dt|z}Yd}~,d}~wwxYw77#t$r}t=d d |dd}~wwxYww)z@ Detailed health payload for authenticated diagnostics. r)rversionN)RedisSemanticCachezindex does not exist - error: )r index_inforrr&) rdbrlitellm_versionsuccess_callbacksuse_aiohttp_transport log_levelis_detailed_debugshow_no_redis_warning Not connectedizService Unhealthy (rr:)rrrrrr_rrlitellm.caching.cachingrrr= _index_inforrClogging getLevelNamergetEffectiveLevel isEnabledForDEBUGrrr'&should_allow_request_on_db_unavailablerrxr;r_should_use_aiohttp_transportr) rrrsuccess_callback_namesr cache_typerrrlog_level_namer r db_health_statuss rK_get_health_readiness_detailsrs BJP!# >AH@X@X%Y1mA&6%Y "%Y  == $ B ++J'----/ABK'.}}':':'F'F'H!HJ'1 K !( 4 4^5U5U5W X#1#>#>w}}#M-C-E'E  $,F,H&H $$X.+=0WWY'-'J'J$#&x0##*%;)9)W)W)Y+%6)>  $%##*%;)9)W)W)Y+%6)>  o&Z >%,%=%= " > "I K!ACF!JJK(F'IJ P6I!A4NOOPs HG4F(F#F(AG4 +G 7G8G ?E%% yy!78I rMct}|y|jjdxsd}tj||st t jdy)a! Reject /health/drain calls that don't carry the configured X-Drain-Token. When no token is configured the endpoint is treated as already opted-in (the ``enable_drain_endpoint`` flag is the only gate). Comparison uses ``secrets.compare_digest`` to avoid timing leaks. Nz x-drain-tokenrz Invalid or missing X-Drain-Tokenr:)r)rrsecretscompare_digestrrHTTP_401_UNAUTHORIZED)rexpectedsupplieds rK_authorize_drain_requestr0s\,-Hoo))/:@bH  ! !(H 5445   6rMcKddlm}|ytd{}|ddk7r)tjst j |_|dS7:w)z Return the db status string for the public probe and flip the response to 503 when a configured DB is unreachable. Mirrors the legacy values: "Not connected" (no DB configured), "connected", "disconnected". rr Nrrr)rrrr'rrrxr;)rrrs rK_resolve_public_readiness_dbr2sX 9$>$@@"k1(OOQ%BB H %% AsAA;Az/health/readiness)rcKtjrtj|_ddiSt rt |d{St|d{}d|dS77 w)un Public readiness probe. Returns a low-detail payload safe to expose to unauthenticated load balancers — `status` plus `db` so orchestrators and external probes can distinguish "healthy" from "DB unreachable" without a credential. Admins can opt into the legacy detailed payload with general_settings.allow_public_health_readiness_details. r shutting_downrNr&)rr)r1is_shutting_downrrxr;r!rr2)r db_statuss rKhealth_readinessr8sc//1%BB/**-/2HEEE98LLIy 11FLs$AA+ A' A+A) A+)A+z/health/readiness/detailsc6Kt|d{S7w)zN Authenticated readiness diagnostics with DB/cache/callback metadata. r5N)rr5s rKhealth_readiness_detailsr:s/A AA As z/health/backlogc"KdtiSw)u` Returns the number of HTTP requests currently in-flight on this uvicorn worker. Use this to measure per-pod queue depth. A high value means the worker is processing many concurrent requests — requests arriving now will have to wait for the event loop to get to them, adding latency before LiteLLM even starts its own timer. in_flight_requestsr/rVrMrKhealth_backlogr=s !"8": ;;s z /health/draincKtsttjdt |t j t jdd{}d|dS7 w)a Graceful-drain probe for Kubernetes ``preStop`` hooks. Disabled by default and returns 404 unless ``general_settings`` sets ``enable_drain_endpoint: true``. Calling it flips a process-wide shutting-down flag, so a successful call permanently takes the worker out of rotation until the pod restarts. Because the kubelet calls preStop hooks without proxy credentials, the endpoint does not require ``user_api_key_auth``. To prevent any pod-reachable caller from triggering shutdown, set ``general_settings.drain_endpoint_token`` (or the ``DRAIN_ENDPOINT_TOKEN`` env var) and supply the same value on the ``X-Drain-Token`` header from the preStop hook. Calls without the header (or with a wrong value) get a 401 and have no side effect. When enabled, it marks the worker as shutting down (so /health/readiness and /health/liveliness immediately start returning 503, removing the pod from service) and blocks until the in-flight request counter drains to zero or ``GRACEFUL_SHUTDOWN_TIMEOUT`` elapses. Unlike a fixed ``sleep``, this returns as soon as real in-flight work is done. Wire it up as: ```yaml lifecycle: preStop: httpGet: path: /health/drain port: 4000 httpHeaders: - name: X-Drain-Token value: ``` z Not Foundr:T) exclude_selfNdrained)rdrained_requests)r$rrrkr0r1start_shutdownwait_for_drain)rr@s rK health_drainrDsZP # $(A(A+VVW%**,2AAtTTGW ==UsAA, A*! A,z/health/livelinessz/health/livenesscfKtjrtj|_ddiSyw)z Unprotected endpoint for checking if worker is alive. Returns 503 once graceful shutdown has begun so Kubernetes stops counting the draining pod as live and terminates it on schedule. rr4z I'm alive!)r1r6rrxr;r5s rKhealth_livelinessrFs0//1%BB/** s/1c0Kdddd}t|dSw)z6 Options endpoint for health/readiness check. GET, OPTIONS*AllowzAccess-Control-Allow-MethodszAccess-Control-Allow-Headersrr;rresponse_headerss rKhealth_readiness_optionsrQ*s) (6(+ ,# >>c0Kdddd}t|dSw)z7 Options endpoint for health/liveliness check. rHrIrJrLrMrNrOs rKhealth_liveliness_optionsrT:s) (6(+ ,# >>rRz/health/test_connectionzXThe mode to test the model with. If not provided, auto-detected from model capabilities.zIParameters for litellm.completion, litellm.embedding for the health checkzModel info for the health checkmode) chatr embedding audio_speechaudio_transcriptionimage_generation image_editvideo_generationbatchrerankrealtime responsesocrr2r1c Kddlm}ddlm}ddlm}m}m} m} ddl m } m }  | #tdd|jji |xsi} t| |r t|| j!d }i}d}| |xsi}|j!d } d}|r|j#| }|9|j$j'd }|j(j'd }n|r|j+|}|rt-|dk(rD|j+d}|r0|D]+}|j!dij!d |k(s(|g}n|rMt-|dkDr?t/|dj!di}t/|dj!dxsi}it7|| |j!dd u| }||n|}t9|xsi|}|j;| d| d!i|||| | d{|xs|j=dd}t?tAjB||ddgtDd{}tGi||d }d|vrdnd|dS#t0$r!}t3j4d||Yd}~d}~wwxYw77O#t$r}|d}~wt0$r;}t3j4d|ttHjJdd |i d}~wwxYww)"a, Test a direct connection to a specific model. This endpoint allows you to verify if your proxy can successfully connect to a specific model. It's useful for troubleshooting model connectivity issues without going through the full proxy routing. Example: ```bash # If model is configured in proxy_config.yaml, you only need to specify the model name: curl -X POST 'http://localhost:4000/health/test_connection' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ -d '{ "litellm_params": { "model": "gpt-4o" }, "mode": "chat" }' # The endpoint will automatically use api_key, api_base, etc. from proxy_config.yaml # You can also override specific params or test with custom credentials: curl -X POST 'http://localhost:4000/health/test_connection' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ -d '{ "litellm_params": { "model": "azure/gpt-4o", "api_key": "os.environ/AZURE_OPENAI_API_KEY", "api_base": "os.environ/AZURE_OPENAI_ENDPOINT", "api_version": "2024-10-21" }, "mode": "chat" }' ``` Note: - If the model is configured in proxy_config.yaml, credentials (api_key, api_base, etc.) will be automatically loaded from the config (with resolved environment variables). - You can override specific params by including them in the request. - You can use `os.environ/VARIABLE_NAME` syntax to reference environment variables, which will be resolved automatically (same as in proxy_config.yaml). Returns: dict: A dictionary containing the health check result with either success information or error details. r)CommonProxyErrors)ModelManagementAuthChecks)rrmrr) DeploymentLiteLLM_ParamsNrr9r:rr?reT) exclude_none)rr2r1zKCould not find model %s in router: %s. Proceeding with request params only.rR)rR)r1r2 test_model)rr2r1) model_paramsrrrrUztest from litellm)rirUpromptinput)rr)rr?zOlitellm.proxy.health_endpoints.test_model_connection(): Exception occurred - %szFailed to test connection: rV)&litellm.proxy._typesrc=litellm.proxy.management_endpoints.model_management_endpointsrdrrrmrrlitellm.types.routerrerfrdb_not_connected_errorrJrLrrlr2 model_dumpr1get_model_listr*r>rrrr]r+can_user_make_model_callr@r.r ahealth_checkrr*rr)rrUr2r1rrcrdrrmrrrerfrequest_litellm_paramsrconfig_litellm_paramsloaded_model_inforequest_model_inforequest_model_iddeployment_by_id deploymentsall_deploymentsrnrresolved_model_infor?cleaned_results rKtest_model_connectionr~Nsd7 @y  !2!I!I!O!OP  )7(<" &&<=  )* 5266w? '))-  !)3(8b &8&<&>2BB#G#K#KG#TXb#b3=,K$)* #s;'7!';15[^5G5GHXZ\5]0^-,0Q1C1CL1Q1WUW,X) +%&.>.B.BCb.cgk.k %  ;L:W%6]g@*0b)  (@@#'-??. 0'%A  7~))&$7.  ! !+**+   )  !55Q5Q&5Q[_ `")F!2g $  _ $**acmop ,  $   ""#tvwx==:1#>?   s!K-A2JB2I' AI'A!JJAJJJ&K-' J0J J JJJ K*!J## K*/6K%%K**K-)FrU)NN)rr~rrrr+r+rcollections.abcrrrrtypingrrr r r rr r rrrrtyping_extensionsrrlitellm._loggingrrlitellm.constantsr+litellm.integrations.SlackAlerting.ms_teamsrrr1litellm.litellm_core_utils.custom_logger_registryr&litellm.llms.custom_httpx.http_handlerrrlrrrrrr r!r"r#r$rr%$litellm.proxy.auth.user_api_key_authr&"litellm.proxy.db.exception_handlerr''litellm.proxy.db.proxy_worker_heartbeatr(litellm.proxy.health_checkr)r*r+r,r-r.6litellm.proxy.middleware.in_flight_requests_middlewarer00litellm.proxy.shutdown.graceful_shutdown_managerr12litellm.router_utils.clientside_credential_handlerr2r3litellm.secret_managers.mainr4r>rL frozensetrOrCr}objectr}r]rerfrrxrrrQueryrrr!rBfloatr.r5r9rErI PROXY_ADMINrJPROXY_ADMIN_VIEW_ONLYrJrrVrOr\rarirorintrrrrrrrrrrrrrrr_rrrr!r$r)r0r2r8r:r=rDrFoptionsrQrTrBodyr~rVrMrKrs.    -(77PP&A: SC   CGLU9 $$$4$B4= 7 # "4530+0h3;'hCK(h$(h #v+ h<#:   $ %  .i)  +,-  11  1& +,-  )00A(B% 2RSy %y y  y x D$ & www w  w 4Z wDjwD4D@8888 8@" 999 9 d 9D" 1j1j1j 1j  1j d 1jj&$$** ..44  E  ,~ ,$ ,""D"t"sUYz"^adh^h"JtPSX\B-`! :!Dj! 4Z !:IXJg>O6P5QR)00A(B% d8[\(7==;\] pp%p :pDj pSpf XJgFW>X=YZ(/0A(B% d8WX - d@f gs0MRSX\]'--/JqQ $ %$ :$ :$   $  $ [$ N H:WEV=W #8*GL]D^C_`(/0A(B' %' a' T.DcNT1.2"c3h"D" +,-  )00A(B#%#  #LI -6|x||~!VV-0"E/.1#U0[-['M'T +,-   +,-  B  BJ !+M %L>d>:!%RPoRP #s(^RPjQQ AA sTz& g $ &&&c&(  2X2 2& +,-  BXB  B +,-  <  <  )>)> )>X     h      ?  ?   ?   ? +,- ,W\\ n(7<< _$w|| 5)00A(B;R R     R *+R 23R :&;R  R rM