Ë
    tPêjÆ; ã                   ó  — d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d d	lmZmZmZ d d
lmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:­ d dl;m<Z<m=Z= ddl>­ ddl?m@Z@mAZA ddlBmCZCmDZDmEZEmFZFmGZG ddlHmIZI e	rd dlJmKZK d dlLmMZN eNZMne
ZMdeMdz  fd„ZO G d„ d e«      ZPy)!é    N)ÚCallable)Ú	timedelta)ÚTYPE_CHECKINGÚAnyÚFinalÚLiteral)ÚAPIError)ÚTypeAdapter)Úverbose_loggerÚverbose_proxy_logger)Ú	DualCache)ÚHOURS_IN_A_DAYÚSLACK_DAILY_REPORT_LOCK_IDÚSLACK_MODEL_DEPRECATION_LOCK_ID)ÚCustomBatchLogger)Úget_budget_alert_type)ÚAlertingHangingRequestCheck)Úduration_in_seconds)Ú_add_key_name_and_team_to_alert)Úget_async_httpx_clientÚhttpxSpecialProvider)Ú	AlertTypeÚCallInfoÚInvitationModelÚInvitationNewÚLitellm_EntityTypeÚUserAPIKeyAuthÚVirtualKeyEventÚWebhookEvent)ÚInvitationLinkRepository)ÚTeamRepository)ÚUserRepository)Ú*)Ú*DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDSÚDEPRECATION_IDLE_POLL_SECONDSé   é   )Úsend_to_webhookÚsquash_payloads)ÚMS_TEAMS_ALERT_HEADERSÚMS_TEAMS_ALERTING_DESTINATIONÚMSTeamsAlertTextÚMSTeamsQueueItemÚget_ms_teams_webhook_url)Ú process_slack_alerting_variables)ÚPodLockManager)ÚRouterÚreturnc                  ó   — ddl m}  | S )Nr   ©Ú
llm_router)Úlitellm.proxy.proxy_serverr5   r4   s    úv/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/integrations/SlackAlerting/slack_alerting.pyÚ_proxy_llm_routerr8   N   s   € Ý5àÐó    c                   óê  ‡ — e Zd ZdZddg edi ddfdedz  dedz  dedz  dee   de	eee
   e
z  f   dz  de
dz  d	e	e
e	f   dz  fˆ fd
„Z	 	 	 	 	 	 	 dndedz  dedz  dee   dz  de	eee
   e
z  f   dz  de	dz  dedz  d	e	e
e	f   dz  fd„Zde	ez  ez  de	fd„Zde	dz  de	dz  fd„Zd„ Zd„ Zd„ Zd„ Zdod„Zd„ Zdedefd„Zdefd„Z	 dode	dz  fd„Zde
de
fd„Zd ed!   d"e fd#„Z!d"e d$ed%   dz  d&e
de"ed%   dz  e
f   fd'„Z#d"e defd(„Z$d"e de
fd)„Z%d*e
dz  d+e
dz  d,e
dz  d-edz  d.edz  f
d/„Z&d0ee   de
fd1„Z'd2ed3   d4ed5   d6e
d7e
d8e
dz  de(de
fd9„Z)d:e*d;e
ddfd<„Z+d:e*d;e
ddfd=„Z,d>e
d?e
d@e-fdA„Z.d>e
fdB„Z/defdC„Z0	 	 dpdedz  dDdEdefdF„Z1dDdEdefdG„Z2defdH„Z3dedz  dDdEdefdI„Z4e5dfdJe6g edz  f   dDdEddfdK„Z7dLe8defdM„Z9	 	 dpdNedOe
dz  dPe
dz  fdQ„Z:dRe
dz  dSe
de
fdT„Z;dLe8defdU„Z<dLe8d2e
defdV„Z=	 	 	 dqdWe
dXedY   d2edZe	d"e8dz  d[e
dz  d8e
dz  fd\„Z>d]e
d2eddfd^„Z?d_„ Z@d`„ ZAda„ ZBdb„ ZCdc„ ZD	 dodDdEdefdd„ZE	 	 dpde-dz  dDdEfde„ZF	 drdfe
fdg„ZGdh„ ZHdi„ ZIdjeJd2edke
fdl„ZKde	dz  defdm„ZLˆ xZMS )sÚSlackAlertingz(
    Class for sending Slack Alerts
    NÚinternal_usage_cacheÚalerting_thresholdÚalertingÚalert_typesÚalert_to_webhook_urlÚdefault_webhook_urlÚalert_type_configc	                 ól  •— |€d}|| _         || _        || _        |xs
 t        «       | _        t        t        j                  ¬«      | _        t        |¬«      | _
        d| _        t        di |¤Ž| _        || _        t        j                   «       | _        d| _        t'        | ¬«      | _        i | _        |rB|j-                  «       D ]/  \  }
}t/        |t0        «      rt3        di |¤Žn|| j*                  |
<   Œ1 i | _        t        j                   «       | _        t9        ‰| �t  di |	¤d| j"                  i¤Ž y )Ni,  )Úllm_provider©r@   F)Úslack_alerting_objectÚ
flush_lock© )r=   r>   r?   r   r<   r   r   ÚLoggingCallbackÚasync_http_handlerr/   r@   Ú
is_runningÚSlackAlertingArgsÚalerting_argsrA   ÚasyncioÚLockrG   Úperiodic_startedr   Úhanging_request_checkrB   ÚitemsÚ
isinstanceÚdictÚAlertTypeConfigÚdigest_bucketsÚdigest_lockÚsuperÚ__init__)Úselfr<   r=   r>   r?   r@   rM   rA   rB   ÚkwargsÚkeyÚvalÚ	__class__s               €r7   rY   zSlackAlerting.__init__Z   s  ø€ ð Ð%Ø!$ÐØ"4ˆÔØ ˆŒØ&ˆÔØ$8Ò$G¼I»KˆÔ!Ü"8ÔFZ×FjÑFjÔ"kˆÔÜ$DÐZnÔ$oˆÔ!ØˆŒÜ.Ñ?°Ñ?ˆÔØ#6ˆÔ Ü!Ÿ,™,›.ˆŒØ %ˆÔÜ%@Ø"&ô&
ˆÔ"ð >@ˆÔÙØ-×3Ñ3Ó5ò g‘��SÜHRÐSVÔX\ÔH]¬oÑ.DÀÒ.DÐcf�×&Ñ& sÒ+ðgà68ˆÔÜ"Ÿ<™<›>ˆÔÜ‰ÑÑ>˜6Ñ>¨d¯o©oÔ>r9   rM   r5   c                 óP  — |�1|| _         t        j                  | j                  «       «       d| _        |�|| _        |�|| _        |�Ft        di |¤Ž| _        | j                  s*t        j                  | j                  «       «       d| _        |�B|j                  «       D ]/  \  }}	t        |	t        «      rt        di |	¤Žn|	| j                  |<   Œ1 |�I| j                  €t        |¬«      | _        n+t        |¬«      xs i }
| j                  j!                  |
«       |�|| _        y y )NTrE   rH   )r>   rN   Úcreate_taskÚperiodic_flushrP   r=   r?   rL   rM   rR   rS   rT   rU   rB   r@   r/   Úupdater5   )rZ   r>   r=   r?   r@   rM   r5   rB   r\   r]   Ú_new_valuess              r7   Úupdate_valueszSlackAlerting.update_values   s-  € ð ÐØ$ˆDŒMÜ×Ñ × 3Ñ 3Ó 5Ô6Ø$(ˆDÔ!ØÐ)Ø&8ˆDÔ#ØÐ"Ø*ˆDÔØÐ$Ü!2Ñ!C°]Ñ!CˆDÔØ×(Ò(Ü×#Ñ# D×$7Ñ$7Ó$9Ô:Ø(,�Ô%ØÐ(Ø-×3Ñ3Ó5ò g‘��SÜHRÐSVÔX\ÔH]¬oÑ.DÀÒ.DÐcf�×&Ñ& sÒ+ðgð  Ð+à×(Ñ(Ð0Ü,LÐbvÔ,w�Õ)ä%EÐ[oÔ%pÒ%vÐtv�Ø×)Ñ)×0Ñ0°Ô=ØÐ!Ø(ˆD�Oð "r9   Úoutage_valuer2   c                 ól   — t        |«      }d|v r$t        |d   t        «      rt        |d   «      |d<   |S )zˆ
        Helper method to prepare outage value for Redis caching.
        Converts set objects to lists for JSON serialization.
        Údeployment_ids)rT   rS   ÚsetÚlist)rZ   re   Úcache_values      r7   Ú_prepare_outage_value_for_cachez-SlackAlerting._prepare_outage_value_for_cache¤   sB   € ô " ,Ó/ˆà˜{Ñ*¬z¸+ÐFVÑ:WÔY\Ô/]Ü,0°Ð=MÑ1NÓ,OˆKÐ(Ñ)ØÐr9   c                 ój   — |r0t        |j                  d«      t        «      rt        |d   «      |d<   |S )z”
        Helper method to restore outage value after retrieving from cache.
        Converts list objects back to sets for proper handling.
        rg   )rS   Úgetri   rh   )rZ   re   s     r7   Ú _restore_outage_value_from_cachez.SlackAlerting._restore_outage_value_from_cache°   s8   € ñ
 œJ |×'7Ñ'7Ð8HÓ'IÌ4ÔPÜ-0°Ð>NÑ1OÓ-PˆLÐ)Ñ*ØÐr9   c              ƒ   ó   K  — y ­w©NrH   ©rZ   s    r7   Údeployment_in_cooldownz$SlackAlerting.deployment_in_cooldown¹   ó	   è ø€ Øùó   ‚c              ƒ   ó   K  — y ­wrp   rH   rq   s    r7   Ú deployment_removed_from_cooldownz.SlackAlerting.deployment_removed_from_cooldown¼   rs   rt   c                 ó    — t        t        «      S rp   )ri   r   rq   s    r7   Ú_all_possible_alert_typesz'SlackAlerting._all_possible_alert_types¿   s   € ô ”I‹Ðr9   c                 ó>  — 	 ||z
  }|j                  «       }|j                  di «      }|j                  dd«      }t        j                  ||¬«      }|j                  dd «      }	|	€|j                  dd «      }	t	        |	«      d d }
||||
fS # t
        $ r}|‚d }~ww xY w)NÚlitellm_paramsÚmodelÚ ©r{   Úoptional_paramsÚmessagesÚinputéd   )Útotal_secondsrm   ÚlitellmÚget_api_baseÚstrÚ	Exception)rZ   r[   Ú
start_timeÚend_timeÚtime_differenceÚtime_difference_floatrz   r{   Úapi_baser   Ú	_messagesÚes               r7   Ú)_response_taking_too_long_callback_helperz7SlackAlerting._response_taking_too_long_callback_helperÅ   s°   € ð	Ø%-°
Ñ%:ˆOà+:×+HÑ+HÓ+JÐ!Ø$*§J¡JÐ/?ÀÓ$DˆNØ!Ÿ:™: g¨rÓ2ˆEÜ%×2Ñ2¸ÐP^Ô_ˆHØ—z‘z *¨dÓ3ˆHàÐØ!Ÿ:™: g¨tÓ4�ô  # 8›}¨T¨cÐ2ˆIà(¨%°¸9ÐDÐDøÜò 	ØˆGûð	ús   ‚B	B Â	BÂBÂBc           	      ó(  — |€y d|v r|	 d}|d   }t        |«      dk(  ry d }	 t        |j                  «       d„ ¬«      }t        |«      }|€y |j                  «       D ]  \  }}|d|› dt        |d«      › d	�z  }Œ d
|z   d
z   }|S y # t        $ r Y ŒGw xY w)NÚ_latency_per_deploymentr|   r   c                 ó   — | d   S )Nr'   rH   )Úxs    r7   ú<lambda>zBSlackAlerting._get_deployment_latencies_to_alert.<locals>.<lambda>ñ   s   € Ð\]Ð^_Ñ\`€ r9   ©r\   ú
ú: r&   Úsz```)ÚlenÚsortedrR   rT   r†   Úround)rZ   ÚmetadataÚ_message_to_sendÚ_deployment_latenciesÚ_deployment_latency_mapr‹   Úlatencys          r7   Ú"_get_deployment_latencies_to_alertz0SlackAlerting._get_deployment_latencies_to_alertÞ   sè   € ØÐØà$¨Ñ0ðð
  "ÐØ$,Ð-FÑ$GÐ!ÜÐ(Ó)¨QÒ.ØØ37Ð#ðä(.Ð/D×/JÑ/JÓ/LÑR`Ô(aÐ%Ü*.Ð/DÓ*EÐ'ð 'Ð.Øà%<×%BÑ%BÓ%Dò JÑ!�˜'Ø  b¨¨
°"´U¸7ÀAÓ5FÐ4GÀqÐ$IÑIÑ ðJà$Ð'7Ñ7¸%Ñ?ÐØ#Ð#ð5 1øô" ò Ùðús   ¢'B Â	BÂBc              ƒ   ó  K  — | j                   �| j                  €y | j                  |||¬«      \  }}}}t        j                  st        j
                  rd}d|› d|› d|› d�}	dt        |d«      › d	| j                  › d
�}
i }|| j                  kD  rx|�Ed|v rAd|d   v r:|d   d   }t        |	|¬«      }	| j                  |¬«      }|�|	d|› �z  }	d|v r|d   }| j                  |
|	z   dt        j                  |||¬«      ƒ d {  –—†  y y 7 Œ­w)N)r[   r‡   rˆ   z>Message not logged. litellm.redact_messages_in_exceptions=Truez
Request Model: `z`
API Base: `z`
Messages: `ú`z`Responses are slow - r&   z&s response time > Alerting threshold: zs`rz   r›   )Úrequest_infor›   )r›   z 
Available Deployment Latencies
Úalerting_metadataÚLow)ÚmessageÚlevelÚ
alert_typer¤   Úrequest_modelr‹   )r>   r?   rŽ   rƒ   Úturn_off_message_loggingÚredact_messages_in_exceptionsrš   r=   r   r    Ú
send_alertr   Úllm_too_slow)rZ   r[   Úcompletion_responser‡   rˆ   rŠ   r{   r‹   r   r£   Úslow_messager¤   Ú	_metadatarž   s                 r7   Ú!response_taking_too_long_callbackz/SlackAlerting.response_taking_too_long_callbackþ   s�  è ø€ ð �=‰=Ð  D×$4Ñ$4Ð$<Øð ×:Ñ:ØØ!Øð ;ó 
ñ	
Ø!ØØØô ×+Ò+¬w×/TÒ/TØWˆHØ+¨E¨7°.ÀÀ
È.ÐYaÐXbÐbcÐdˆØ/´Ð6KÈQÓ0OÐ/PÐPvÐw{÷  xOñ  xOð  wPð  PRð  SˆØ"$ÐØ  4×#:Ñ#:Ò:àÐ!Ð&6¸&Ñ&@ÀZÐSYÐZjÑSkÑEkØ)/Ð0@Ñ)AÀ*Ñ)M�	Ü>ÈLÐclÔm�à15×1XÑ1XÐbkÐ1XÓ1lÐ'Ø*Ð6Ø Ð&HÐI`ÐHaÐ$bÑb�Là&¨)Ñ3Ø(1Ð2EÑ(FÐ%Ø—/‘/Ø$ |Ñ3ØÜ$×1Ñ1Ø"3Ø#Ø!ð "ó ÷ ñ ð ;ðús   ‚DDÄD
ÄDÚdeployment_metricsc              ƒ   óÆ  K  — d}	 |j                   rR| j                  j                  |j                  › dt        j
                  j                  › �dd¬«      ƒ d{  –—†  |dz  }|j                  �\| j                  j                  |j                  › dt        j                  j                  › �|j                  d¬«      ƒ d{  –—†  |dz  }|S 7 Œs7 Œ# t        $ r Y yw xY w­w)ax  
        Store the perf by deployment in cache
        - Number of failed requests per deployment
        - Latency / output tokens per deployment

        'deployment_id:daily_metrics:failed_requests'
        'deployment_id:daily_metrics:latency_per_output_token'

        Returns
            int - count of metrics set (1 - if just latency, 2 - if failed + latency)
        r   ú:r'   N)r\   ÚvalueÚparent_otel_span)
Úfailed_requestr<   Úasync_increment_cacheÚidÚSlackAlertingCacheKeysÚfailed_requests_keyrµ   Úlatency_per_output_tokenÚlatency_keyr†   )rZ   r²   Ú
return_vals      r7   Úasync_update_daily_reportsz(SlackAlerting.async_update_daily_reports,  s  è ø€ ð ˆ
ð	à!×0Ò0Ø×/Ñ/×EÑEØ-×0Ñ0Ð1°Ô3I×3]Ñ3]×3cÑ3cÐ2dÐeØØ%)ð Fó ÷ ð ð ˜a‘�
ð "×:Ñ:ÐFØ×/Ñ/×EÑEØ-×0Ñ0Ð1°Ô3I×3UÑ3U×3[Ñ3[Ð2\Ð]Ø,×EÑEØ%)ð Fó ÷ ð ð ˜a‘�
àÐð%øðùô ò 	Ùð	üsN   ‚C!†AC ÁCÁA'C ÃCÃ
C ÃC!ÃC ÃC Ã	CÃC!ÃCÃC!c              ƒ   óš  ‡‡K  — |j                  «       }|D �cg c]!  }|› dt        j                  j                  › �‘Œ# }}|D �cg c]!  }|› dt        j                  j                  › �‘Œ# }}||z   }| j
                  j                  |¬«      ƒ d{  –—† }|€yd}|D ]  }	|	€Œ|	dkD  sŒd} n |ry|dt        |«       }
|t        |«      d }d}|
D �cg c]  }|�|n|‘Œ
 c}Št        t        t        ‰«      «      ˆfd„d¬«      dd	 }|D �cg c]  }‰|   dkD  sŒ|‘Œ }}d}|D �cg c]  }|�|n|‘Œ
 c}Št        t        t        ‰«      «      ˆfd
„d¬«      dd	 }|D �cg c]  }‰|   dkD  sŒ|‘Œ }}dt        j                  «       › d�}|dz  }|s|dz  }t        t        |«      «      D ]”  }|||      j                  d«      d   }|j                  |«      }t        |t        «      r|d   j                  dd«      }n yt!        j"                  ||�|d   ni ¬«      }|€d}‰||      }|d|dz   › d|› d|› d|› d�	z  }Œ– |dz  }|s|dz  }t        t        |«      «      D ]Œ  }|||      j                  d«      d   }|j                  |«      }|�|d   j                  dd«      }nd}t!        j"                  ||�|d   ni ¬«      }t%        ‰||      d«      }|d|dz   › d|› d|› d|› d�	z  }ŒŽ |D �cg c]  }|df‘Œ }}|D �cg c]  }|df‘Œ }}||z   }| j
                  j'                  |¬«      ƒ d{  –—†  |dt        j                  «       | j(                  j*                  z   › d �z  }| j-                  |d!t.        j0                  i ¬"«      ƒ d{  –—†  yc c}w c c}w 7 �Œc c}w c c}w c c}w c c}w c c}w c c}w 7 Œ�7 Œ2­w)#a�  
        Send a daily report on:
        - Top 5 deployments with most failed requests
        - Top 5 slowest deployments (normalized by latency/output tokens)

        Get the value from redis cache (if available) or in-memory and send it

        Cleanup:
        - reset values in cache -> prevent memory leak

        Returns:
            True -> if successfuly sent
            False -> if not sent
        r´   )ÚkeysNFTr   c                 ó   •— ‰|    S rp   rH   )ÚiÚreplaced_failed_valuess    €r7   r“   z2SlackAlerting.send_daily_reports.<locals>.<lambda>†  s   ø€ Ð0°Ñ3€ r9   )r\   Úreverseé   c                 ó   •— ‰|    S rp   rH   )rÃ   Úreplaced_slowest_valuess    €r7   r“   z2SlackAlerting.send_daily_reports.<locals>.<lambda>“  s   ø€ Ð1°!Ñ4€ r9   z	

Time: `u(   `s
Here are today's key metrics ðŸ“ˆ: 

u7   

*â�—ï¸� Top Deployments with Most Failed Requests:*

z	None
rz   r{   r|   r}   ú	r'   z. Deployment: `z`, Failed Requests: `z`,  API Base: `ú`
u#   

*ðŸ˜… Top Slowest Deployments:*

é   z`, Latency per output token: `zs/token`,  API Base: `ú`

)Ú
cache_listz

Next Run is at: `z`sr¥   ©r¦   r§   r¨   r¤   )Úget_model_idsrº   r»   rµ   r½   r<   Úasync_batch_get_cacher˜   r™   ÚrangeÚtimeÚsplitÚget_model_inforS   rT   rm   rƒ   r„   rš   Úasync_set_cache_pipelinerM   Údaily_report_frequencyr¬   r   Údaily_reports)rZ   ÚrouterÚidsr¹   Úfailed_request_keysÚlatency_keysÚcombined_metrics_keysÚcombined_metrics_valuesÚall_noner]   Úfailed_request_valuesÚlatency_valuesÚplaceholder_valuerµ   Útop_5_failedÚindexÚtop_5_slowestr¦   rÃ   r\   Ú_deploymentÚdeployment_namer‹   Úlatency_cache_keysÚfailed_request_cache_keysÚcombined_metrics_cache_keysrÄ   rÈ   s                             @@r7   Úsend_daily_reportsz SlackAlerting.send_daily_reportsS  sþ  ùè ø€ ð  ×)Ñ)Ó+ˆð loÖ%oÐeg¨¨¨AÔ.D×.XÑ.X×.^Ñ.^Ð-_Ò&`Ð%oÐÐ%oØ\_Ö`ÐVX "  QÔ'=×'IÑ'I×'OÑ'OÐ&PÒQÐ`ˆÐ`à':¸\Ñ'IÐà/3×/HÑ/H×/^Ñ/^Ø&ð 0_ó 0
÷ *
Ðð #Ð*ØàˆØ*ò 	ˆCØ‰ 3¨£7Ø �Ùð	ñ
 Øà'>Ð?YÄÐEXÓAYÐ'ZÐØ 7¼Ð<OÓ8PÐ8RÐ Sˆð ÐØavÖ!wÐX]¨5Ð+<¡%ÐBSÑ"SÒ!wÐô Ü”#Ð,Ó-Ó.Û3Øô
ð ˆ1ð	ˆð
 ,8Ö] %Ð;QÐRWÑ;XÐ[\Ó;\šÐ]ˆÐ]ð ÐØiwÖ)xÐ`e°5Ð3D©%ÐJ[Ñ*[Ò)xÐô Ü”#Ð-Ó.Ó/Û4Øô
ð ˆ1ð	ˆð
 -:Ö` 5Ð=TÐUZÑ=[Ð^_Ó=_šÐ`ˆÐ`ð  ¤§	¡	£˜}Ð,WÐXˆàÐPÑPˆÙØ�zÑ!ˆGÜ”s˜<Ó(Ó)ò 	ˆAØ% l°1¡oÑ6×<Ñ<¸SÓAÀ!ÑDˆCØ ×/Ñ/°Ó4ˆKÜ˜+¤tÔ,Ø"-Ð.>Ñ"?×"CÑ"CÀGÈRÓ"P‘áä×+Ñ+Ø%ØBMÐBY Ð-=Ò!>Ð_aôˆHð ÐØ�Ø*¨<¸©?Ñ;ˆEØØ�Q˜‘U�G˜?¨?Ð*;Ð;PÐQVÐPWÐWfÐgoÐfpÐpsÐtñ‰Gð	ð& 	Ð<Ñ<ˆÙØ�zÑ!ˆGÜ”s˜=Ó)Ó*ò 	NˆAØ˜}¨QÑ/Ñ0×6Ñ6°sÓ;¸AÑ>ˆCØ ×/Ñ/°Ó4ˆKØÐ&Ø"-Ð.>Ñ"?×"CÑ"CÀGÈRÓ"P‘à"$�Ü×+Ñ+Ø%ØBMÐBY Ð-=Ò!>Ð_aôˆHô Ð1°-ÀÑ2BÑCÀQÓGˆEØ˜˜A ™E˜7 /°/Ð1BÐB`ÐafÐ`gÐg}ð  Gð  ~Hð  HMð  Nñ  N‰Gð	Nð :FÖ$F°# c¨1¢XÐ$FÐÐ$FØ@SÖ+T¸¨S°!ªHÐ+TÐ!Ð+TØ-?ÐB[Ñ-[Ð#Ø×'Ñ'×@Ñ@ÐLgÐ@Óh×hÐhàÐ*¬4¯9©9«;¸×9KÑ9K×9bÑ9bÑ+bÐ*cÐceÐfÑfˆð �o‰oØØÜ ×.Ñ.Ø ð	 ó 
÷ 	
ð 	
ð ùòY &pùÚ`ð*
ûò, "xùò ^ùò
 *yùò aùòZ %GùÚ+Tàhøð
	
ús­   „O™&N¿OÁ&N!Á+&OÂN&ÂOÂ#OÂ)*OÃN)Ã -OÄN.ÄN.ÄOÄ'N3Ä4-OÅ!N8Å/N8Å3FOË7N=ÌOÌOÌ&OÌ9OÌ:AOÎO	ÎOÎ)OÏ	OÚrequest_datac              ƒ   óÄ   K  — | j                   �| j                  €y t        j                  | j                  vry | j                  j                  |¬«      ƒ d {  –—†  y 7 Œ­w)N)rë   )r>   r?   r   Úllm_requests_hangingrQ   Ú$add_request_to_hanging_request_check)rZ   rë   s     r7   Úresponse_taking_too_longz&SlackAlerting.response_taking_too_longÔ  sU   è ø€ ð �=‰=Ð  D×$4Ñ$4Ð$<Øä×)Ñ)°×1AÑ1AÑAØà×(Ñ(×MÑMÐ[gÐMÓh×hÒhús   ‚AA ÁAÁA Úerror_messageÚfailing_modelc              ƒ   ó„  K  — | j                   �| j                  €yd| j                  vry| j                  }d|z   }d|› �}|j                  |¬«      ƒ d{  –—† }|€\| j	                  |dt
        j                  i ¬«      ƒ d{  –—†  |j                  |d| j                  j                  ¬	«      ƒ d{  –—†  yy7 Œc7 Œ87 Œ
­w)
z¿
        Raise alert when tracking failed for specific model

        Args:
            error_message (str): Error message
            failing_model (str): Model that failed tracking
        NÚfailed_tracking_spendzFailed Tracking Cost for zbudget_alerts:failed_tracking:r”   ÚHighrÎ   ÚSENT©r\   rµ   Úttl)
r>   r?   r<   Úasync_get_cacher¬   r   ró   Úasync_set_cacherM   Úbudget_alert_ttl)rZ   rð   rñ   Ú_cacher¦   Ú
_cache_keyÚresults          r7   Úfailed_tracking_alertz#SlackAlerting.failed_tracking_alertà  så   è ø€ ð �=‰=Ð  D×$4Ñ$4Ð$<àØ"¨$×*:Ñ*:Ñ:Øà#'×#<Ñ#<ˆØ4°}ÑDˆØ<¸]¸OÐLˆ
Ø$×4Ñ4¸Ð4ÓD×DˆØˆ>Ø—/‘/ØØÜ$×:Ñ:Ø"$ð	 "ó ÷ ð ð ×(Ñ(ØØØ×&Ñ&×7Ñ7ð )ó ÷ ñ ð ð Eøðøðús6   ‚AC ÁB:Á,C ÂB<Â/C Â3B>Â4C Â<C Â>C Útype)	Útoken_budgetÚuser_budgetÚsoft_budgetÚmax_budget_alertÚteam_budgetÚorganization_budgetÚproxy_budgetÚprojected_limit_exceededÚproject_budgetÚ	user_infoc              ƒ   óÚ  K  — | j                   }| j                  �| j                  €yd| j                  vryt        |«      }|j	                  |«      }|j                  d¬«      }| j                  |«      }|j                  «       }|dk(  rdnd}	d}
|j                  €|j                  €y| j                  |	||¬«      \  }	}|	�¡|j                  �•d|	› d|› �}|j                  |¬	«      ƒ d{  –—† }|€pt        d|	|d
œ|¤Ž}
| j                  |dz   |z   dt        j                   |
i ¬«      ƒ d{  –—†  |j#                  |d| j$                  j&                  ¬«      ƒ d{  –—†  yy7 Œx7 Œ87 Œ
­w)z¶
        Send a budget alert on slack or webhook

        Args:
            type: The type of budget alert to send
            user_info: The user info to send the alert for
        NÚbudget_alertsT©Úexclude_noner  )Úeventr	  Úevent_messagezbudget_alerts:r´   r”   )r  r  ú

rô   )r¦   r§   r¨   r	  r¤   rõ   rö   rH   )r<   r>   r?   r   Úget_idÚ
model_dumpÚ_get_user_info_strÚget_event_messageÚ
max_budgetr  Ú_get_event_and_event_messageÚevent_grouprø   r   r¬   r   r  rù   rM   rú   )rZ   rÿ   r	  rû   Úbudget_alert_classÚ_idÚuser_info_jsonÚuser_info_strr  r  Úwebhook_eventrü   rý   s                r7   r  zSlackAlerting.budget_alertsÿ  sÒ  è ø€ ð4 $(×#<Ñ#<ˆà�=‰=Ð  D×$4Ñ$4Ð$<àØ $×"2Ñ"2Ñ2Øô %:¸$Ó$?ÐØ'×.Ñ.¨yÓ9ˆØ )× 4Ñ 4À$Ð 4Ó GˆØ#×6Ñ6°yÓAˆØ*×<Ñ<Ó>ˆð
 +/Ð2LÒ*LÑ&ÐRVð 	ð .2ˆð ×ÑÐ'¨I×,AÑ,AÐ,IØð  $×@Ñ@ØØØ'ð  Aó  
Ñˆˆ}ð Ð ×!6Ñ!6Ð!BØ"0°°°q¸¸Ð >ˆJØ"(×"8Ñ"8¸ZÐ"8Ó"H×HˆFØˆ~Ü ,ð !ØØ"/ñ!ð %ñ!�ð
 —o‘oØ)¨FÑ2°]ÑBØ Ü(×6Ñ6Ø+Ø&(ð &ó ÷ ð ð ×,Ñ,Ø"Ø Ø×*Ñ*×;Ñ;ð -ó ÷ ð ð Øð+ Iøðøðús7   ‚C*E+Ã,E%Ã-AE+Ä.E'Ä//E+ÅE)ÅE+Å'E+Å)E+r  )Úbudget_crossedÚthreshold_crossedÚsoft_budget_crossedr  r  c                 óf  — | j                  |¬«      }|j                  �.|j                  |j                  k\  rd}|d|j                  › d�z  }|j                  �V|j                  |j                  k\  rd}|d|j                  › d�z  }||fS |t        k  rd}|dz  }||fS |t
        k  rd}|d	z  }||fS )
zº
        Get the event and event message for a budget alert

        This will append any new information to the event_message

        Handles Max Budget and Soft Budget Alerts
        )r	  r  zTotal Soft Budget:`r¢   r  zBudget Crossed
 Total Budget:`r  z5% Threshold Crossed z15% Threshold Crossed)Ú_get_percent_of_max_budget_leftr  Úspendr  Ú"SLACK_ALERTING_THRESHOLD_5_PERCENTÚ#SLACK_ALERTING_THRESHOLD_15_PERCENT)rZ   r	  r  r  Úpercent_lefts        r7   r  z*SlackAlerting._get_event_and_event_messageT  sô   € ð  &*×%IÑ%IÐT]Ð%IÓ%^ˆð × Ñ Ð,Ø�‰ )×"7Ñ"7Ò7Ø-�ØÐ#6°y×7LÑ7LÐ6MÈQÐ!OÑO�ð ×ÑÐ+Ø�‰ )×"6Ñ"6Ò6Ø(�ØÐ#BÀ9×CWÑCWÐBXÐXYÐ!ZÑZ�ð �mÐ#Ð#ð Ô!CÒCØ+�ØÐ!8Ñ8�ð
 �mÐ#Ð#ð	 Ô!DÒDØ+�ØÐ!8Ñ8�à�mÐ#Ð#r9   c                 ó`   — d}|j                   }|j                  }|€|S |dk  r|S ||z
  |z  }|S )z@
        Get the percent of the max budget that is left
        g        r   )r"  r  )rZ   r	  r%  Úcurrent_spendr  s        r7   r!  z-SlackAlerting._get_percent_of_max_budget_left‚  sK   € ð "ˆØ&/§o¡oˆØ*3×*>Ñ*>ˆ
ØÐØÐØ˜Š?ØÐØ" ]Ñ2°jÑ@ˆØÐr9   c                 óÐ   — |j                  d¬«      }|j                  d«       d}|j                  «       D ]-  \  }}t        |t        «      r|j
                  }|d|› d|› d�z  }Œ/ |S )z>
        Create a standard message for a budget alert
        Tr  Útokenr|   r#   ú:* `rÊ   )r  ÚpoprR   rS   r   rµ   )rZ   r	  Ú_all_fields_as_dictÚmsgÚkÚvs         r7   r  z SlackAlerting._get_user_info_str�  s{   € ð &/×%9Ñ%9ÀtÐ%9Ó%LÐØ×Ñ Ô(ØˆØ'×-Ñ-Ó/ò 	%‰DˆAˆqÜ˜!Ô/Ô0Ø—G‘G�Ø�Q�q�c˜˜a˜S Ð$Ñ$‰Cð	%ð
 ˆ
r9   r)  Ú	key_aliasÚend_user_idÚresponse_costr  c              ƒ   óæ   K  — | j                   �_d| j                   v rP|�M|�J|�Gt        ||||d d d |d d dt        j                  d|› d|› �¬«      }| j	                  |¬«      ƒ d {  –—†  y y y y y y 7 Œ
­w)NÚwebhookÚspend_trackedz!Customer spend tracked. Customer=z, spend=)r"  r  r)  Úcustomer_idÚuser_idÚteam_idÚ
user_emailr0  Úprojected_exceeded_dateÚprojected_spendr  r  r  ©r  )r>   r   r   ÚEND_USERÚsend_webhook_alert)rZ   r)  r0  r1  r2  r  r  s          r7   Úcustomer_spend_alertz"SlackAlerting.customer_spend_alertž  s¯   è ø€ ð �M‰MÐ%Ø˜TŸ]™]Ñ*ØÐ'ØÐ!ØÐ)ô (Ø#Ø%ØØ'ØØØØ#Ø(,Ø $Ø%Ü.×7Ñ7Ø AÀ+ÀÈhÐWdÐVeÐfôˆEð  ×)Ñ)¸Ð)Ó>×>Ñ>ð' *ð "ð (ð +ð &ð. ?ús   ‚A"A1Á$A/Á%A1Úalertsc                 óÞ   — ddddœ}|D ]5  }|dk(  r|dxx   dz  cc<   Œ|dk\  r|dxx   dz  cc<   Œ)|dxx   dz  cc<   Œ7 d	}|j                  «       D ]  \  }}|dkD  sŒ|d
|› d|› d
�z  }Œ |S )zç
        Parameters:
        - alerts: List[int] -> list of error codes (either 408 or 500+)

        Returns:
        - str -> formatted string. This is an alert message, giving a human-friendly description of the errors.
        r   )úTimeout Errorsú
API ErrorsúUnknown Errorsé˜  rB  r'   éô  rC  rD  r|   r•   r–   )rR   )rZ   r@  Úerror_breakdownÚalertÚ	error_msgr\   rµ   s          r7   Ú_count_outage_alertsz"SlackAlerting._count_outage_alertsÀ  s«   € ð 56ÀQÐZ[Ñ!\ˆØò 	7ˆEØ˜Š|ØÐ 0Ó1°QÑ6Ô1Ø˜#’Ø Ó-°Ñ2Ô-àÐ 0Ó1°QÑ6Ô1ð	7ð ˆ	Ø)×/Ñ/Ó1ò 	3‰JˆC�Ø�q‹yØ˜r #  b¨¨¨rÐ2Ñ2‘	ð	3ð Ðr9   r¨   )ÚMajorÚMinorr\   )ÚModelÚRegionÚkey_valÚproviderr‹   c                 óö   — |› d�|d|i}|�||d<   d}|j                  «       D ]  \  }	}
|d|	› d|
› d�z  }Œ d|› d	|› d
| j                  |d   ¬«      › dt        t        j                  «       |d   z
  d«      › d�	S )z!Format an alert message for slackz NameÚProviderzAPI Baser•   r#   r*  rÊ   u   


*âš ï¸� z Service Outage*

z

*Errors:*
r@  )r@  z

*Last Check:* `Úlast_updated_até   z	s ago`


)rR   rJ  rš   rÒ   )rZ   r¨   r\   rO  rP  r‹   re   ÚheadersÚheaders_strr.  r/  s              r7   Ú_outage_alert_msg_factoryz'SlackAlerting._outage_alert_msg_factoryØ  sÆ   € ð !˜E ˜-¨°*¸hÐGˆØÐØ"*ˆG�JÑàˆØ—M‘M“Oò 	-‰DˆAˆqØ˜Q˜q˜c  a S¨Ð,Ñ,‰Kð	-ð	Ø	ˆð à€ð ð ×Ñ ,¨xÑ"8ÐÓ9Ð :ð ;ä”d—i‘i“k LÐ1BÑ$CÑCÀQÓGÐHð Ið	ð 		r9   Ú	exceptionÚdeployment_idc              ƒ   ó  K  — | j                   €y| j                   j                  |¬«      }|€y|j                  j                  }|j                  j                  }|€t        j                  |¬«      \  }}}}|j                  j                  }|€+t
        j                  j                  ||j                  ¬«      }|€y||z   }| j                  j                  |¬«      ƒ d{  –—† }	|	�| j                  |	«      }	t        |dd«      �*|j                  dk7  r|j                  dk  s| j                   €y|	€•t        «       }
|
j!                  |«       t#        ||j                  gd	d	t%        j$                  «       |
¬
«      }	| j'                  |	«      }| j                  j)                  ||| j*                  j,                  ¬«      ƒ d{  –—†  yt/        |	d   «      | j*                  j0                  k  r|	d   j3                  |j                  «       n	 |	d   }
|
j!                  |«       |
|	d<   t%        j$                  «       |	d<   |	d   d	u r{t/        |	d   «      | j*                  j4                  k\  rVt/        |
«      dkD  rH| j7                  dd|d|	|¬«      }| j9                  |dt:        j<                  i ¬«      ƒ d{  –—†  d|	d<   n�|	d   d	u rzt/        |	d   «      | j*                  j>                  k\  rUt/        |
«      dkD  rG| j7                  dd|d|	|¬«      }| j9                  |dt:        j<                  i ¬«      ƒ d{  –—†  d|	d<   | j'                  |	«      }| j                  j)                  ||¬«      ƒ d{  –—†  y7 �Œ˜7 �Œº7 ŒÈ7 ŒH7 Œ­w)zŠ
        Send slack alert if specific provider region is having an outage.

        Track for 408 (Timeout) and >=500 Error codes
        N©Úmodel_id©r{   )Úcustom_llm_providerrz   r”   Ústatus_coderE  rF  F)Úprovider_region_idr@  Úminor_alert_sentÚmajor_alert_sentrS  rg   rö   r@  rg   rS  ra  r'   rL  rN  ©r¨   r\   rO  r‹   re   rP  ÚMediumrÎ   Trb  rK  rô   ©r\   rµ   ) r5   Úget_deploymentrz   r{   r^  rƒ   Úget_llm_providerÚregion_nameÚutilsÚ_get_model_regionr<   rø   rn   Úgetattrr_  rh   ÚaddÚProviderRegionOutageModelrÒ   rk   rù   rM   Úregion_outage_alert_ttlr˜   Úmax_outage_alert_list_sizeÚappendÚminor_outage_alert_thresholdrW  r¬   r   Úoutage_alertsÚmajor_outage_alert_threshold)rZ   rX  rY  Ú
deploymentr{   rP  Ú_rh  Ú	cache_keyre   Ú_deployment_setrj   r-  s                r7   Úregion_outage_alertsz"SlackAlerting.region_outage_alertsô  sÉ  è ø€ ð �?‰?Ð"Øà ŸO™O×:Ñ:ÀMÐ:ÓRˆ
àÐØà×)Ñ)×/Ñ/ˆà×,Ñ,×@Ñ@ˆØÐÜ$+×$<Ñ$<À5Ô$IÑ!ˆE�8˜Q ð !×/Ñ/×;Ñ;ˆØÐÜ!Ÿ-™-×9Ñ9Ø$,¸Z×=VÑ=Vð :ó ˆKð ÐØð $ kÑ1ˆ	à?C×?XÑ?X×?hÑ?hÐmvÐ?hÓ?w×9wˆð Ð#Ø×@Ñ@ÀÓNˆLô �I˜}¨dÓ3Ð;Ø×%Ñ%¨Ò,°×1FÑ1FÈÒ1LØ�‰Ð&ààÐÜ!›eˆOØ×Ñ Ô.Ü4Ø#,Ø!×-Ñ-Ð.Ø!&Ø!&Ü $§	¡	£Ø.ôˆLð ×>Ñ>¸|ÓLˆKØ×+Ñ+×;Ñ;ØØ!Ø×&Ñ&×>Ñ>ð <ó ÷ ð ð
 äˆ|˜HÑ%Ó&¨×);Ñ);×)VÑ)VÒVØ˜Ñ"×)Ñ)¨)×*?Ñ*?Õ@àØ&Ð'7Ñ8ˆØ×Ñ˜MÔ*Ø)8ˆÐ%Ñ&Ü*.¯)©)«+ˆÐ&Ñ'ð Ð+Ñ,°Ñ5Ü�L Ñ*Ó+¨t×/AÑ/A×/^Ñ/^Ò^Ü�OÓ$ qÒ(à×0Ñ0Ø"ØØ#ØØ)Ø!ð 1ó ˆCð —/‘/ØØÜ$×2Ñ2Ø"$ð	 "ó ÷ ð ð 04ˆLÐ+Ò,ð Ð+Ñ,°Ñ5Ü�L Ñ*Ó+¨t×/AÑ/A×/^Ñ/^Ò^Ü�OÓ$ qÒ(à×0Ñ0Ø"ØØ#ØØ)Ø!ð 1ó ˆCð —/‘/ØØÜ$×2Ñ2Ø"$ð	 "ó ÷ ð ð 04ˆLÐ+Ñ,ð ×:Ñ:¸<ÓHˆØ×'Ñ'×7Ñ7¸IÈ[Ð7ÓY×YÑYðA :xùð8ùð>øð2øð 	Zús]   ‚C NÃ"M:Ã#C NÇM=ÇC3NÊ7N Ê8BNÌ9NÌ::NÍ4NÍ5NÍ=NÎ NÎNÎNc              ƒ   óú  K  — 	 | j                   j                  |¬«      ƒ d{  –—† }t        |dd«      �*|j                  dk7  r|j                  dk  s| j                  €y| j                  j                  |¬«      }|€y|j                  j                  }|j                  j                  }|€	 t        j                  |¬«      \  }}}}t        j                  ||j                  ¬	«      }|€ht        ||j                  gd
d
t        j                  «       ¬«      }| j                   j                  ||| j                   j"                  ¬«      ƒ d{  –—†  yt%        |d   «      | j                   j&                  k  r|d   j)                  |j                  «       n	 t        j                  «       |d<   |d   d
u rmt%        |d   «      | j                   j*                  k\  rH| j-                  dd||||¬«      }	| j/                  |	dt0        j2                  i ¬«      ƒ d{  –—†  d|d<   ns|d   d
u rlt%        |d   «      | j                   j4                  k\  rG| j-                  dd||||¬«      }	| j/                  |	dt0        j2                  i ¬«      ƒ d{  –—†  d|d<   | j7                  |«      }
| j                   j                  ||
¬«      ƒ d{  –—†  y7 �Œ«# t        $ r d}Y �Œw xY w7 �Œ•7 ŒÌ7 ŒZ7 Œ!# t        $ r Y yw xY w­w)a
  
        Send slack alert if model is badly configured / having an outage (408, 401, 429, >=500).

        key = model_id

        value = {
        - model_id
        - threshold
        - alerts []
        }

        ttl = 1hr
        max_alerts_size = 10
        r”   Nr_  rE  rF  r[  r]  r|   r}   F)r\  r@  ra  rb  rS  rö   r@  rS  ra  rL  rM  rc  rd  rÎ   Trb  rK  rô   re  )r<   rø   rk  r_  r5   rf  rz   r{   r^  rƒ   rg  r†   r„   ÚOutageModelrÒ   rù   rM   Úoutage_alert_ttlr˜   ro  rp  rq  rW  r¬   r   rr  rs  rk   )rZ   rX  rY  re   rt  r{   rP  ru  r‹   r-  rj   s              r7   rr  zSlackAlerting.outage_alerts|  s4  è ø€ ð&`	Ø59×5NÑ5N×5^Ñ5^ÐcpÐ5^Ó5q×/qˆLä˜	 =°$Ó7Ð?Ø×)Ñ)¨SÒ0°Y×5JÑ5JÈSÒ5PØ—?‘?Ð*àð !%§¡× >Ñ >ÈÐ >Ó VˆJØÐ!Øà×-Ñ-×3Ñ3ˆEØ!×0Ñ0×DÑDˆHØÐð"Ü,3×,DÑ,DÈ5Ô,QÑ)�E˜8 Q¨ô &×2Ñ2¸ÐPZ×PiÑPiÔjˆHàÐ#Ü*Ø*Ø%×1Ñ1Ð2Ø%*Ø%*Ü$(§I¡I£Kô �ð ×/Ñ/×?Ñ?Ø%Ø&Ø×*Ñ*×;Ñ;ð @ó ÷ ð ð
 ä�< Ñ)Ó*¨T×-?Ñ-?×-ZÑ-ZÒZØ˜XÑ&×-Ñ-¨i×.CÑ.CÕDàä.2¯i©i«kˆLÐ*Ñ+ð Ð/Ñ0°EÑ9Ü˜ XÑ.Ó/°4×3EÑ3E×3bÑ3bÒbà×4Ñ4Ø&ØØ!Ø%Ø!-Ø%ð 5ó �ð —o‘oØØ"Ü(×6Ñ6Ø&(ð	 &ó ÷ ð ð 48�Ð/Ò0àÐ/Ñ0°EÑ9Ü˜ XÑ.Ó/°4×3EÑ3E×3bÑ3bÒbà×4Ñ4Ø&ØØ!Ø%Ø!-Ø%ð 5ó �ð —o‘oØØ Ü(×6Ñ6Ø&(ð	 &ó ÷ ð ð 48�Ð/Ñ0ð "&×!EÑ!EÀlÓ!SˆKØ×+Ñ+×;Ñ;ÀÐU`Ð;Óa×aÑað{ 0rúô$ !ò "Ø!“Hð"úðùð8øð*øð bùÜò 	Ùð	üsÅ   ‚K;„K, £K¤;K, ÁK;Á K, Á>K;Á?.K, Â.K Ã	BK, ÅK#ÅK, ÅK;ÅCK, ÈK&ÈA3K, ÊK(Ê:K, ËK*Ë	K, ËK;ËK, ËK ËK, ËK Ë K, Ë&K, Ë(K, Ë*K, Ë,	K8Ë5K;Ë7K8Ë8K;Ú
model_nameÚlitellm_model_nameÚpassed_model_infoc              ƒ   ó  K  — t        |dd «      }i }d}|�'t        j                  j                  |i «      }d|› d�}n t        j                  j                  |i «      }d}|j	                  «       D ]  \  }}	|dk(  s|dk(  r|	d›}	||› d|	› d	�z  }Œ  d
|› d|› dt        j                  dd«      › d|› d|› d�}
| j                  |
dt        j                  i ¬«      }|�!t        j                  |«      r|ƒ d {  –—†  y y y 7 Œ­w)NÚ
base_modelr|   zBase Model: `rÊ   Úinput_cost_per_tokenÚoutput_cost_per_tokenz.8fr–   r•   u%   
*ðŸš… New Model Added*
Model Name: `zn

Usage OpenAI Python SDK:
```
import openai
client = openai.OpenAI(
    api_key="your_api_key",
    base_url=ÚPROXY_BASE_URLúhttp://0.0.0.0:4000z:
)

response = client.chat.completions.create(
    model="zÃ", # model to send to the proxy
    messages = [
        {
            "role": "user",
            "content": "this is a test request, write a short poem"
        }
    ]
)
```

Model Info: 
```
z
```
r¥   rÎ   )rk  rƒ   Ú
model_costrm   rR   ÚosÚgetenvr¬   r   Únew_model_addedrN   Úiscoroutine)rZ   r|  r}  r~  Úbase_model_from_userÚ
model_infor€  Úmodel_info_strr.  r/  r¦   Ú	alert_vals               r7   Úmodel_added_alertzSlackAlerting.model_added_alertñ  s\  è ø€ Ü&-Ð.?ÀÈtÓ&TÐØˆ
Øˆ
ØÐ+Ü ×+Ñ+×/Ñ/Ð0DÀbÓIˆJØ(Ð)=Ð(>¸cÐB‰Jä ×+Ñ+×/Ñ/Ð0BÀBÓGˆJØˆØ×$Ñ$Ó&ò 	,‰DˆAˆqØÐ*Ò*¨aÐ3JÒ.Jà˜�g�à   2 a S¨˜mÑ+‰Nð	,ðàˆ\ð Ø€ð ô �i‰iÐ(Ð*?Ó@ÐAð Bð ˆ<ð ð Ð ð ð5ˆð<  Ÿ?™?ØØÜ ×0Ñ0Ø ð	 +ó 
ˆ	ð Ð ¤W×%8Ñ%8¸Ô%CØ�O‰Oð &DÐ Øús   ‚C3C?Ã5C=Ã6C?c              ƒ   ó   K  — y ­wrp   rH   )rZ   r|  s     r7   Úmodel_removed_alertz!SlackAlerting.model_removed_alert*  rs   rt   c                 óZ   — | j                   d uxr t        j                  | j                  v S rp   )r>   r   Úmodel_deprecation_warningsr?   rq   s    r7   Ú_deprecation_alerts_enabledz)SlackAlerting._deprecation_alerts_enabled-  s(   € Ø�}‰} DÐ(Òe¬Y×-QÑ-QÐUY×UeÑUeÐ-eÐer9   Úpod_lock_managerzPodLockManager | Nonec           
   ƒ   ó>  K  — | j                  «       syddlm}m}  ||¬«      } ||«      }|€y| j	                  |«      ƒ d{  –—† sy|j
                  rdnd}| j                  ||t        j                  t        |j
                  «      t        |j                  «      t        |j                  «      dœ¬	«      ƒ d{  –—†  | j                  j                  t        j                  j                   t#        j"                  «       t$        ¬
«      ƒ d{  –—†  y7 ŒÒ7 Œ[7 Œ	­w)a+  Alert on the router's deprecated and imminent models, True when one was sent

        The daily lock is claimed only once there is something to say, so an empty pass never blocks a
        later real one, and a sent alert is stamped in the shared cache for a day so sibling pods stop asking
        Fr   )Úcollect_model_deprecationsÚ format_deprecation_alert_messager4   Nrô   rd  )Údeprecated_countÚimminent_countÚupcoming_countrÎ   rö   T)r“  Ú,litellm.proxy.common_utils.model_deprecationr–  r—  Ú!_claimed_deprecation_alert_windowÚ
deprecatedr¬   r   r’  r˜   ÚimminentÚupcomingr<   rù   rº   Údeprecation_alert_sent_keyrµ   rÒ   r$   )rZ   r5   r”  r–  r—  Úsnapshotr¦   r§   s           r7   Úsend_model_deprecation_alertz*SlackAlerting.send_model_deprecation_alert0  s  è ø€ ð ×/Ñ/Ô1Ø÷	
ñ
 5À
ÔKˆÙ9¸(ÓCˆØˆ?ØØ×;Ñ;Ð<LÓM×MÐMØàCK×CVÒCV¹Ð\dˆà�o‰oØØÜ ×;Ñ;ä$'¨×(;Ñ(;Ó$<Ü"% h×&7Ñ&7Ó"8Ü"% h×&7Ñ&7Ó"8ñð	 ó 	
÷ 		
ð 		
ð ×'Ñ'×7Ñ7Ü&×AÑA×GÑGÜ—)‘)“+Ü:ð 8ó 
÷ 	
ð 	
ð
 ð) Nøð
		
øð	
ús8   ‚ADÁDÁA8DÂ=DÂ>ADÄDÄDÄDÄDc              ƒ   ó`   K  — |€y|j                  t        t        d¬«      ƒ d{  –—† duS 7 Œ­w)zXWithout a redis backed lock there is no fleet to coordinate, so a lone pod always alertsNTF©Ú
cronjob_idr÷   Úallow_reentrant)Úacquire_lockr   r$   )rZ   r”  s     r7   rœ  z/SlackAlerting._claimed_deprecation_alert_window\  sD   è ø€ àÐ#Øà"×/Ñ/Ü:Ü>Ø %ð 0ó ÷ ð
 ðð 	ðús   ‚#.¥,¦.c              ƒ   óŠ   K  — | j                   j                  t        j                  j                  ¬«      ƒ d {  –—† d uS 7 Œ­w)Nr”   )r<   rø   rº   r   rµ   rq   s    r7   Ú$_deprecation_alert_sent_within_a_dayz2SlackAlerting._deprecation_alert_sent_within_a_dayh  s=   è ø€ à×+Ñ+×;Ñ;Ô@V×@qÑ@q×@wÑ@wÐ;Óx×xØðð 	Øxús   ‚8AºA»Ac              ƒ   ó    K  — |�| j                  «       sy| j                  «       ƒ d {  –—† ry| j                  ||¬«      ƒ d {  –—† S 7 Œ 7 Œ­w)NF©r5   r”  )r“  r©  r¢  )rZ   r5   r”  s      r7   Ú_run_deprecation_alert_passz)SlackAlerting._run_deprecation_alert_passm  sT   è ø€ ð Ð T×%EÑ%EÔ%GØØ×:Ñ:Ó<×<Ð<ØØ×6Ñ6À*Ð_oÐ6Óp×pÐpð =øàpús!   ‚'A©A
ªAÁAÁAÁAÚget_llm_routerc              ƒ   ó4  K  — 	 	 | j                   |«       |«      ƒ d{  –—†  t	        j
                  t        «      ƒ d{  –—†  ŒB7 Œ&# t        $ rB}t        j                  d|«       t	        j
                  t        «      ƒ d{  –—†7   Y d}~Œ‹d}~ww xY w7 ŒU­w)a¯  Poll every pass for a loaded router, the alert being on, and no alert in the last day, then alert

        A pass that could not alert (no router yet, alert type off, a sibling pod holds the daily lock, or a
        redis blip at claim time) is retried on the next poll instead of costing a day, while a pass that
        raised (a missing webhook, say) backs off a full day so a misconfiguration logs once, not every poll
        Nz)Error in model deprecation alert loop: %s)r¬  r†   r   rX  rN   Úsleepr$   r%   )rZ   r­  r”  r�   s       r7   Úrun_scheduled_deprecation_checkz-SlackAlerting.run_scheduled_deprecation_checkv  s„   è ø€ ð ðØ×6Ñ6±~Ó7GÐIYÓZ×ZÐZô
 —-‘-Ô =Ó>×>Ð>ð àZùÜò Ü$×.Ñ.Ð/ZÐ\]Ô^Ü—m‘mÔ$NÓO×OÑOÜûðúð ?úsU   ‚B…A ŸA A ¤BÁ BÁBÁA Á	BÁ2BÂBÂBÂ	BÂBÂBr  c              ƒ   ó  K  — t        j                  dd«      }|€t        d«      ‚|j                  «       }ddi}| j                  j                  |||¬«      ƒ d{  –—† }|j                  dk(  ryt        d	|j                  «       y
7 Œ+­w)zä
        Sends structured alert to webhook, if set.

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.

        Raises Exception
            - if WEBHOOK_URL is not set
        ÚWEBHOOK_URLNz$Missing webhook_url from environmentúContent-typeúapplication/json)ÚurlrU  ÚdataéÈ   Tz#Error sending webhook alert. Error=F)	r†  r‡  r†   Úmodel_dump_jsonrJ   Úpostr_  ÚprintÚtext)rZ   r  Úwebhook_urlÚpayloadrU  Úresponses         r7   r>  z SlackAlerting.send_webhook_alertŠ  s–   è ø€ ô  ŸY™Y }°dÓ;ˆØÐÜÐBÓCÐCà&×6Ñ6Ó8ˆØ(Ð*<Ð=ˆà $× 7Ñ 7× <Ñ <ØØØð !=ó !
÷ 
ˆð
 ×Ñ 3Ò&ØäÐ7¸¿¹ÔGàð
ús   ‚AB	ÁBÁ,B	Úpremium_userÚemail_logo_urlÚemail_support_contactc              ƒ   ór   K  — ddl m}m} |dur'|€|�"t        d|j                  j
                  › �«      ‚y y ­w)Nr   )ÚCommonProxyErrorsr¿  Tz$Trying to Customize Email Alerting
 )r6   rÃ  r¿  Ú
ValueErrorÚnot_premium_userrµ   )rZ   r¿  rÀ  rÁ  rÃ  s        r7   Ú%_check_if_using_premium_email_featurez3SlackAlerting._check_if_using_premium_email_feature©  sL   è ø€ ÷ 	Oà˜tÑ#ØÐ)Ð-BÐ-NÜ Ð#HÐIZ×IkÑIk×IqÑIqÐHrÐ!sÓtÐtð .Oð $ùs   ‚57Úrecipient_user_idÚbase_urlc              ƒ   ó,  K  — ddl m} ddlm} |�|€|S 	 t	        t
        t           «      j                  t        |«      j                  j                  d|iddi¬«      ƒ d {  –—† d¬	«      }|r|d   nCt	        t        «      j                   |t        |¬
«      t        |¬
«      ¬«      ƒ d {  –—† d¬	«      }|j#                  d«      › d|j$                  › �S 7 Œu7 Œ,# t        $ r,}t        j                  d|t!        |«      «       |cY d }~S d }~ww xY w­w)Nr   )Úcreate_invitation_for_user)Úprisma_clientr7  Ú
created_atÚdesc)ÚwhereÚorderT)Úfrom_attributes)r7  )r¶  Úuser_api_key_dictz1Error creating invitation link for user_id %s: %sú/z/ui/onboarding?invitation_id=)Ú0litellm.proxy.management_helpers.user_invitationrÊ  r6   rË  r
   ri   r   Úvalidate_pythonr    ÚtableÚ	find_manyr   r   r†   r   Úerrorr…   Úrstripr¹   )rZ   rÇ  rÈ  rÊ  rË  Úexisting_invitationsÚ
invitationr�   s           r7   Ú_construct_user_invitation_linkz-SlackAlerting._construct_user_invitation_linkµ  s1  è ø€ õ	
õ 	=àÐ$¨Ð(=ØˆOð	Ü*5´d¼?Ñ6KÓ*L×*\Ñ*\Ü.¨}Ó=×CÑC×MÑMØ$Ð&7Ð8Ø'¨Ð0ð Nó ÷ ð !%ð +]ó +Ð ñ (ð % QÒ'ä ¤Ó1×AÑAÙ4Ü*Ð3DÔEÜ*8ÐARÔ*Sô÷ ð %)ð Bó ð ð& —/‘/ #Ó&Ð'Ð'DÀZÇ]Á]ÀOÐTÐTð3øðùô ò 	Ü ×&Ñ&ØCØ!Ü�A“ôð
 �Oûð	üsZ   ‚D–AC Á"CÁ#A
C Â-CÂ.
C Â8 DÃC ÃC Ã	DÃ%!DÄDÄDÄDÄDc              ƒ   ó®  K  — 	 ddl m} | j                  �d| j                  vr!t        j                  d| j                  «       yddlm}m} t        j                  dt        j                  dd «      «      }t        j                  d	d «      }| j                  |||«      ƒ d {  –—†  |€t        }|€t        }|j                  }|j                  }|j                  }	|€A|	�?|�=t!        |«      j"                  j%                  d
|	i¬«      ƒ d {  –—† }
|
�|
j                  }|j&                  }|j(                  }t        j                  dd«      }d}|€%t        j                  d|j+                  «       ¬«       |j,                  dk(  rt.        j1                  ||||||¬«      }n½|j,                  dk(  r‰|j2                  }d}|�C|�At5        |«      j"                  j%                  d|i¬«      ƒ d {  –—† }|�|j6                  xs d}| j9                  |	|¬«      ƒ d {  –—† }t:        j1                  |||||¬«      }n%t        j                  d|j=                  «       ¬«       |j?                  «        |d|› �|dœ} ||d   |d   |d   ¬«      ƒ d {  –—†  y 7 �Œñ7 �Œ‹7 Œ­7 Œ‚7 Œ# t@        $ r)}t        j                  d!tC        |«      «       Y d }~yd }~ww xY w­w)"Nr   ©Ú
send_emailÚemailz;Error sending email alert - 'email' not in self.alerting %sF)r¿  rË  ÚSMTP_SENDER_LOGOÚEMAIL_LOGO_URLÚEMAIL_SUPPORT_CONTACTr7  )rÎ  rƒ  r„  úAlert from LiteLLM Serverú*Trying to send email alert to no recipient©ÚextraÚkey_created)rÀ  Úrecipient_emailÚ
key_budgetÚ	key_tokenrÈ  rÁ  Úinternal_user_createdzDefault Teamr8  ú-)rÇ  rÈ  )rÀ  rè  Ú	team_namerÈ  rÁ  z3Trying to send email alert on unknown webhook eventú	LiteLLM: ©ÚtoÚsubjectÚhtmlrð  rñ  rò  ©Úreceiver_emailrñ  rò  TzError sending email alert %s)"Úlitellm.proxy.utilsrÞ  r>   r   r×  r6   r¿  rË  r†  r‡  rÆ  ÚLITELLM_LOGO_URLÚLITELLM_SUPPORT_CONTACTr  r9  r7  r"   rÕ  Úfind_uniquer)  r  rT   r  ÚKEY_CREATED_EMAIL_TEMPLATEÚformatr8  r!   Ú
team_aliasrÛ  ÚUSER_INVITED_EMAIL_TEMPLATEr  r¸  r†   r…   )rZ   r  rÞ  r¿  rË  rÀ  rÁ  Ú
event_namerè  rÇ  Úuser_rowrê  ré  rÈ  Úemail_html_contentr8  rí  Úteam_rowÚinvitation_linkÚemail_eventr�   s                        r7   Ú&send_key_created_or_user_invited_emailz4SlackAlerting.send_key_created_or_user_invited_emailÛ  s  è ø€ ðZ	Ý6à�}‰}Ð$¨°t·}±}Ñ(Dä$×*Ñ*ØQØ—M‘Môð ßNäŸY™YÐ'9¼2¿9¹9ÐEUÐW[Ó;\Ó]ˆNÜ$&§I¡IÐ.EÀtÓ$LÐ!Ø×<Ñ<¸\È>Ð[pÓq×qÐqØÐ%Ü!1�Ø$Ð,Ü(?Ð%à -× ;Ñ ;ˆJØ+×6Ñ6ˆOØ'4×'<Ñ'<ÐØÐ&Ð+<Ð+HÈ]ÐMfÜ!/°Ó!>×!DÑ!D×!PÑ!PÐXaÐctÐWuÐ!PÓ!v×v�àÐ'Ø&.×&9Ñ&9�Oà,×2Ñ2ˆIØ -× 8Ñ 8ˆJÜ Ÿi™iÐ(8Ð:OÓPˆHà!<ÐØÐ&Ü$×*Ñ*Ø@Ø'×,Ñ,Ó.õð
 ×"Ñ" mÒ3Ü%?×%FÑ%FØ#1Ø$3Ø)Ø'Ø%Ø*?ð &Gó &Ñ"ð ×$Ñ$Ð(?Ò?à!.×!6Ñ!6�Ø*�	ØÐ&¨=Ð+DÜ,:¸=Ó,I×,OÑ,O×,[Ñ,[ÐclÐnuÐbvÐ,[Ó,w×&w�HØÐ+Ø$,×$7Ñ$7Ò$>¸3˜	Ø/3×/SÑ/SØ&7À(ð 0Tó 0÷ *�ô &A×%GÑ%GØ#1Ø$3Ø'Ø,Ø*?ð &Hó &Ñ"ô %×*Ñ*ØIØ'×2Ñ2Ó4õð
 ×)Ñ)Ô+à%Ø& z lÐ3Ø*ñ"ˆKñ Ø*¨4Ñ0Ø# IÑ.Ø  Ñ(ô÷ ð ð ðQ rùð wùð< 'xøð*øð.ùô ò 	Ü ×&Ñ&Ð'EÄsÈ1ÃvÔNÜûð	üs™   ‚K„A J  ÁKÁAJ  Â#JÂ$A(J  ÄJÄCJ  Ç,JÇ-,J  ÈJÈA4J  ÊJÊJ  ÊKÊJ  ÊJ  ÊJ  ÊJ  ÊJ  Ê 	KÊ)KËKËKËKc           	   ƒ   óÆ  K  — ddl m} ddlm} t	        j
                  dt	        j
                  dd«      «      }t	        j
                  dd«      }| j                  |||«      ƒ d{  –—†  |€t        }|€t        }|j                  }|j                  }|j                  }	|j                  }
d}|€%t        j                  d	|j                  «       ¬
«       |j                   dk(  rd|› d|	› d|
› d|› d�	}|j#                  «        |d|› �|dœ} ||d   |d   |d   ¬«      ƒ d{  –—†  |j$                  t&        j(                  k(  rddlm}  ||¬«      ƒ d{  –—†  y7 Œõ7 Œ;7 Œ	­w)z¦
        Sends structured Email alert to an SMTP server

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.
        r   )r¿  rÝ  rà  rá  Nrâ  rã  rä  rå  r  z
            <img src="zD" alt="LiteLLM Logo" width="150" height="50" />

            <p> Hi ze, <br/>

            Your LLM API usage this month has reached your account's <b> monthly budget of $a   </b> <br /> <br />

            API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month. <br /> <br />

            If you have any questions, please send an email to zY <br /> <br />

            Best, <br />
            The LiteLLM team <br />
            rî  rï  rð  rñ  rò  ró  )Úsend_team_budget_alertr<  F)r6   r¿  rõ  rÞ  r†  r‡  rÆ  rö  r÷  r  r9  r7  r  r   r×  rT   r  r¸  r  r   ÚTEAMÚ#litellm.integrations.email_alertingr  )rZ   r  r¨   r¿  rÞ  rÀ  rÁ  rý  rè  Ú	user_namer  rÿ  r  r  s                 r7   Úsend_email_alert_using_smtpz)SlackAlerting.send_email_alert_using_smtp8  s¡  è ø€ õ 	<Ý2äŸ™Ð#5´r·y±yÐAQÐSWÓ7XÓYˆÜ "§	¡	Ð*AÀ4Ó HÐØ×8Ñ8¸À~ÐWlÓm×mÐmàÐ!Ü-ˆNØ Ð(Ü$;Ð!à)×7Ñ7ˆ
Ø!.×!9Ñ!9ˆØ(×0Ñ0ˆ	Ø)×4Ñ4ˆ
Ø8ÐØÐ"Ü ×&Ñ&Ð'SÐ[h×[mÑ[mÓ[oÕpà×ÑÐ"2Ò2ð&Ø%Ð&ð 'à�;ð ]à]gÐ\hð i@ð AVÐ?Vð Wð"Ðð 	×%Ñ%Ô'à!Ø" : ,Ð/Ø&ñ
ˆñ Ø& tÑ,Ø 	Ñ*Ø˜VÑ$ô
÷ 	
ð 	
ð
 ×$Ñ$Ô(:×(?Ñ(?Ò?ÝRá(°}ÔE×EÐEàða 	nøðL	
øð Fús7   ‚A#E!Á%EÁ&B;E!Ä!EÄ"3E!ÅEÅE!ÅE!ÅE!r¦   r§   )r¥   rd  rô   r¤   r©   c              ‹   ó 	  K  — | j                   €y| j                  sN| j                   �Bt        | j                   «      dkD  r*t        j                  | j                  «       «       d| _        d| j                   v r!|dk(  r|�| j                  |¬«      ƒ d{  –—†  d| j                   v r"|dk(  r|�| j                  ||¬«      ƒ d{  –—†  d	| j                   v }	t        | j                   v }
|	s|
sy|| j                  vrydd
l
m
}  |j                  «       j                  d«      }t        j                  dd«      }t        |d|«      }d|› d�}|dk(  s|dk(  r||z   }n|› d|› d|› d|› �}|r$|j!                  «       D ]  \  }}|d|› d|› d�z  }Œ |r$|j!                  «       D ]  \  }}|d|› d|› d�z  }Œ |�	|d|› d�z  }|
r| j#                  ||¬«       |	s;t        | j$                  «      | j&                  k\  r| j)                  «       ƒ d{  –—†  yt        |dt+        |«      «      }| j,                  j/                  |«      }|�� |j0                  �r| j2                  �|| j2                  v r| j2                  |   }n/| j4                  �| j4                  }nt        j                  dd«      }|€t7        d«      ‚|› d|xs d› d|xs d› �}| j8                  4 ƒd{  –—†   |j                  «       }|| j:                  v r-| j:                  |   d xx   d!z  cc<   || j:                  |   d"<   n)t=        ||xs d|xs d||d!|||¬#«	      | j:                  |<   ddd«      ƒd{  –—†  y| j2                  �|| j2                  v r| j2                  |   }n/| j4                  �| j4                  }nt        j                  dd«      }|€t7        d«      ‚d$|i}d%d&i}t?        |t@        «      r(|D ]"  }| j$                  jC                  ||||d'œ«       Œ$ n | j$                  jC                  ||||d'œ«       t        | j$                  «      | j&                  k\  r| j)                  «       ƒ d{  –—†  yy7 �ŒÔ7 �Œ§7 �Œb7 �Œ–7 �Œ# 1 ƒd{  –—†7  sw Y   yxY w7 Œ*­w)(a„  
        Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298

        - Responses taking too long
        - Requests are hanging
        - Calls are failing
        - DB Read/Writes are failing
        - Proxy Close to max budget
        - Key Close to max budget

        Parameters:
            level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'.
            message: str - what is the alert about
            request_model: Optional[str] - model name for digest grouping
            api_base: Optional[str] - api base for digest grouping
        Nr   Tr4  r  r<  rß  )r  r¨   Úslack©Údatetimeú%H:%M:%Srƒ  ÚnameúAlert type: `r¢   r×   rˆ  z	
Level: `z`
Timestamp: `ú`

Message: r  ú: `rÌ   z

*Alerting Metadata*: 
ú

Proxy URL: `)Úformatted_messager¨   rµ   ÚSLACK_WEBHOOK_URLz*Missing SLACK_WEBHOOK_URL from environmentr´   r|   Úcountr'   Ú	last_time)	r¨   r©   r‹   Úfirst_messager§   r  r‡   r  r¼  r»  r³  r´  ©rµ  rU  r½  r¨   )"r>   rP   r˜   rN   r`   ra   r>  r	  r+   r?   r  ÚnowÚstrftimer†  r‡  rk  rR   Ú_enqueue_ms_teams_alertÚ	log_queueÚ
batch_sizeÚflush_queuer…   rB   rm   Údigestr@   rA   rÄ  rW   rV   ÚDigestEntryrS   ri   rp  )rZ   r¦   r§   r¨   r¤   r	  r©   r‹   r[   Úsend_to_slackÚsend_to_ms_teamsr  Úcurrent_timeÚ_proxy_base_urlÚalert_type_nameÚalert_type_formattedr  r\   rµ   Úalert_type_name_strÚ_atcÚ_digest_webhookÚ
digest_keyr  Úslack_webhook_urlr½  rU  rµ  s                               r7   r¬   zSlackAlerting.send_alertw  s  è ø€ ð6 �=‰=Ð Øð ×$Ò$¨¯©Ð)BÄsÈ4Ï=É=ÓGYÐ\]ÒG]Ü×Ñ × 3Ñ 3Ó 5Ô6Ø$(ˆDÔ!à˜Ÿ™Ñ%¨*¸Ò*GÈIÐLaØ×)Ñ)¸	Ð)ÓB×BÐBà�d—m‘mÑ#¨
°oÒ(EÈ)ÐJ_à×2Ñ2ÀÐWaÐ2Ób×bÐbà&¨$¯-©-Ð7ˆÜ"?À4Ç=Á=Ð"PÐÙÑ%5ØØ˜T×-Ñ-Ñ-Øå%à*˜hŸl™l›n×5Ñ5°jÓAˆÜ!#§¡Ð+;¸TÓ!BˆÜ!(¨°V¸ZÓ!HˆØ(5°oÐ5FÀaÐ&HÐØ˜Ò(¨JÐ:KÒ,KØ 4°wÑ >Ñð (Ð(¨
°5°'¸ÈÈÐVdÐelÐdmÐnð ñ Ø$Ÿl™l›nò A‘
��UØ! t¨C¨5°°E°7¸%Ð%@Ñ@Ñ!ðAáØ/×5Ñ5Ó7ò X‘
��UØ!Ð'BÀ3À%ÀsÈ5È'ÐQVÐ%WÑWÑ!ðXàÐ&ØÐ#3°OÐ3DÀAÐ!FÑFÐáØ×(Ñ(Ð;LÐYcÐ(ÔdáÜ�4—>‘>Ó" d§o¡oÒ5Ø×&Ñ&Ó(×(Ð(Øô &-¨Z¸Ä#ÀjÃ/Ó%RÐØ×,Ñ,×0Ñ0Ð1DÓEˆØÑ §£à×(Ñ(Ð4¸Àt×G`ÑG`Ñ9`Ø:>×:SÑ:SÐT^Ñ:_‘Ø×)Ñ)Ð5Ø"&×":Ñ":‘ä"$§)¡)Ð,?ÀÓ"F�ØÐ&Ü Ð!MÓNÐNà#6Ð"7°q¸Ò9LÈ"Ð8MÈQÈxÊ~Ð[]ÐN^Ð _ˆJà×'Ñ'÷ ñ Ø)˜XŸ\™\›^�Ø ×!4Ñ!4Ñ4Ø×'Ñ'¨
Ñ3°GÓ<ÀÑAÓ<ØCF�D×'Ñ'¨
Ñ3°KÒ@ä6AØ#6Ø&3Ò&9°rØ!)¢¨RØ&-Ø#ØØ#&Ø"%Ø$3ô
7�D×'Ñ'¨
Ñ3÷÷ ð" ð ×$Ñ$Ð0°ZÀ4×C\ÑC\Ñ5\Ø8<×8QÑ8QÐR\Ñ8]ÑØ×%Ñ%Ð1Ø $× 8Ñ 8Ñä "§	¡	Ð*=¸tÓ DÐàÐ$ÜÐIÓJÐJØ Ð"3Ð4ˆØ(Ð*<Ð=ˆäÐ'¬Ô.Ø(ò �Ø—‘×%Ñ%à"Ø#*Ø#*Ø&0ñ	õñð �N‰N×!Ñ!à,Ø&Ø&Ø",ñ	ôô ˆt�~‰~Ó $§/¡/Ò1Ø×"Ñ"Ó$×$Ñ$ð 2ð_ Cùð cùðH )ùð&ùð ù÷ ÷ ð ð" úðH %úsŽ   ‚BRÂQ(Â/RÃQ+ÃERÈQ.ÈCRËQ1ËRËA6Q7ÍRÍ Q4Í!D RÑ!RÑ"RÑ+RÑ.RÑ1RÑ4RÑ7R	Ñ=R Ñ>R	ÒRr  c                 ó¨   — t        «       }|€t        j                  d|«       y d|i}|t        ||t        dœ}| j
                  j                  |«       y )NzXMS Teams alerting is enabled but MS_TEAMS_WEBHOOK_URL is not set. Dropping alert type=%sr»  )rµ  rU  r½  r¨   rú  )r.   r   r×  r*   r+   r  rp  )rZ   r  r¨   Úms_teams_webhook_urlr½  Úitems         r7   r  z%SlackAlerting._enqueue_ms_teams_alert  s_   € Ü&>Ó&@ÐØÐ'Ü ×&Ñ&ØjØôð Ø,2Ð4EÐ+Fˆà'Ü-ØØ$Ü3ñ)
ˆð 	�‰×Ñ˜dÕ#r9   c           	   ƒ   ó$  K  — | j                   sy t        | j                   «      }|j                  «       D �cg c]  }t        | |d   |d   ¬«      ‘Œ }}t	        j
                  |Ž ƒ d {  –—†  | j                   j                  «        y c c}w 7 Œ$­w)Nr/  r  )ÚslackAlertingInstancer/  r  )r  r)   Úvaluesr(   rN   ÚgatherÚclear)rZ   Úsquashed_queuer/  Útaskss       r7   Úasync_send_batchzSlackAlerting.async_send_batch  sˆ   è ø€ Ø�~Š~Øä /°·±Ó ?ˆð '×-Ñ-Ó/ö
àô °$¸TÀ&¹\ÐQUÐV]ÑQ^Ö_ð
ˆð 
ô �n‰n˜eÐ$×$Ð$Ø�‰×ÑÕùò
ð 	%ús   ‚5B·B	ÁBÁ)BÁ*%Bc              ƒ   óÒ  K  — ddl m }  |j                  «       }g }| j                  4 ƒd{  –—†  | j                  j	                  «       D �]Y  \  }}|d   }| j
                  j                  |«      }|€Œ*||d   z
  j                  «       }||j                  k  rŒP|d   j                  d«      }	|d   j                  d«      }
|d   j                  d«      }|d   j                  d«      }d	|› d
|d   › d|› d|	› d|› d|
› d|d   › d|d   › �}t        j                  dd«      }|�	|d|› d�z  }d|i}ddi}|d   }t        |t        «      r(|D ]"  }| j                  j                  ||||dœ«       Œ$ n | j                  j                  ||||dœ«       |j                  |«       �Œ\ |D ]  }| j                  |= Œ ddd«      ƒd{  –—†  y7 �Œ¡7 Œ# 1 ƒd{  –—†7  sw Y   yxY w­w)zÓFlush any digest buckets whose interval has expired.

        For each expired bucket, formats a digest summary message and
        appends it to the log_queue for delivery via the normal batching path.
        r   r  Nr¨   r‡   r  r  ú%Y-%m-%dr  z` (Digest)
Level: `r§   z
`
Start: `ú z`
End: `z
`
Count: `r  r  r  rƒ  r  r¢   r»  r³  r´  r¼  r  )r  r  rW   rV   rR   rB   rm   r‚   Údigest_intervalr  r†  r‡  rS   ri   r  rp  )rZ   r  r  Úflushed_keysr\   Úentryr&  r)  ÚelapsedÚstart_tsÚend_tsÚ
start_dateÚend_dater  r%  r½  rU  r¼  rµ  s                      r7   Ú_flush_digest_bucketsz#SlackAlerting._flush_digest_buckets+  s`  è ø€ õ 	&à!�X—\‘\“^ˆØ)+ˆà×#Ñ#÷ 5	-ñ 5	-Ø"×1Ñ1×7Ñ7Ó9ó 1)‘
��UØ"'¨Ñ"5�Ø×-Ñ-×1Ñ1°/ÓB�Ø�<ØØ  |Ñ!4Ñ4×CÑCÓE�Ø˜T×1Ñ1Ò1Øð ! Ñ.×7Ñ7¸
ÓC�Ø˜{Ñ+×4Ñ4°ZÓ@�Ø" <Ñ0×9Ñ9¸*ÓE�
Ø  Ñ-×6Ñ6°zÓB�à# OÐ#4ð 5Ø$ W™~Ð.ð /Ø)˜l¨!¨H¨:ð 6Ø%˜J a¨ xð 0Ø$ W™~Ð.ð / Ø % oÑ 6Ð7ð9ð "ô #%§)¡)Ð,<¸dÓ"C�Ø"Ð.Ø%Ð+;¸OÐ;LÈAÐ)NÑNÐ%à!Ð#4Ð5�Ø)Ð+=Ð>�Ø# MÑ2�ä˜k¬4Ô0Ø*ò ˜ØŸ™×-Ñ-à'*Ø+2Ø+2Ø.=ñ	õñð —N‘N×)Ñ)à#.Ø'.Ø'.Ø*9ñ	ôð ×#Ñ# CÖ(ðc1)ðf $ò -�Ø×'Ñ'¨Ñ,ð-÷i5	-÷ 5	-ñ 5	-ùð 5	-ø÷ 5	-÷ 5	-ñ 5	-üsE   ‚*G'¬G­G'°FGÆ<G'ÇGÇG'ÇG'ÇG$ÇGÇG$Ç G'c              ƒ   ó"  K  — 	 t        j                  | j                  «      ƒ d{  –—†  	 | j                  «       ƒ d{  –—†  | j                  «       ƒ d{  –—†  ŒY7 Œ67 Œ# t        $ r }t        j                  d|«       Y d}~ŒAd}~ww xY w7 Œ5­w)z:Override base periodic_flush to also flush digest buckets.Nz!Error flushing digest buckets: %s)rN   r¯  Úflush_intervalrC  r†   r   Údebugr  )rZ   r�   s     r7   ra   zSlackAlerting.periodic_flushm  sƒ   è ø€ àÜ—-‘- × 3Ñ 3Ó4×4Ð4ðSØ×0Ñ0Ó2×2Ð2ð ×"Ñ"Ó$×$Ð$ð Ø4øà2ùÜò SÜ$×*Ñ*Ð+NÐPQ×RÑRûðSúà$úsU   ‚$B¦A§B¬A! ¿AÁ A! ÁBÁBÁBÁA! Á!	B
Á*BÂ BÂB
Â
Bc           	   ƒ   óâ  K  — 	 d| j                   v �r-|j                  di «      xs i }|j                  di «      xs i }|j                  dd«      xs d}||z
  }|}	t        |t        j                  «      rgt        |d«      r[|j                  �Ot        |j                  d«      r9|j                  j                  }
|
�!|
d	kD  rt        |j                  «       |
z  «      }	t        |	t        «      r|	j                  «       }	| j                  t        |d
|	t        j                  j                  «       ¬«      «      ƒ d{  –—†  yy7 Œ# t        $ r }t!        j"                  d|«       Y d}~yd}~ww xY w­w)zLog deployment latencyr×   rz   r‹  r¹   r|   ÚusageNÚcompletion_tokensr   F©r¹   r·   r¼   Ú
updated_atzT[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: %s)r?   rm   rS   rƒ   ÚModelResponseÚhasattrrH  rI  Úfloatr‚   r   r¿   ÚDeploymentMetricsri  Úget_utc_datetimer†   r   r×  )rZ   r[   Úresponse_objr‡   rˆ   rz   r‹  r\  Ú
response_sÚfinal_valuerI  r�   s               r7   Úasync_log_success_eventz%SlackAlerting.async_log_success_eventw  s_  è ø€ ð	Ø $×"2Ñ"2Ò2Ø(.¯
©
Ð3CÀRÓ(HÒ(NÈB�Ø$2×$6Ñ$6°|ÀRÓ$HÒ$NÈB�
Ø",§.¡.°°rÓ":Ò"@¸b�Ø/7¸*Ñ/D�
à(�ä˜l¬G×,AÑ,AÔBÜ˜L¨'Ô2Ø$×*Ñ*Ð6Ü × 2Ñ 2Ð4GÔHà/;×/AÑ/A×/SÑ/SÐ%Ø(Ð4Ð9JÈQÒ9NÜ&+¨J×,DÑ,DÓ,FÐIZÑ,ZÓ&[˜Ü˜k¬9Ô5Ø"-×";Ñ";Ó"=�Kà×5Ñ5Ü%Ø#Ø',Ø1<Ü#*§=¡=×#AÑ#AÓ#Cô	ó÷ ñ ð' 3ð&ùô ò 	Ü ×&Ñ&ØfÐhi÷ñ ûð	üsA   ‚E/„D6E Ä:EÄ;E Ä?E/ÅE Å	E,ÅE'Å"E/Å'E,Å,E/c           	   ƒ   óš  K  — |j                  di «      }|j                  di «      xs i }|j                  dd«      }	 d| j                  v rC	 | j                  t        |ddt        j
                  j                  «       ¬«      «      ƒ d{  –—†  t        |j                  d
d«      t        «      rZd| j                  v r| j                  |d
   |¬«      ƒ d{  –—†  d| j                  v r| j                  |d
   |¬«      ƒ d{  –—†  yyy7 Œ# t        $ r }t        j                  d	|«       Y d}~Œ¡d}~ww xY w7 Œa7 Œ7# t        $ r Y yw xY w­w)z Log failure + deployment latencyrz   r‹  r¹   r|   r×   TNrJ  zException raises -%srX  rr  )rX  rY  rx  )rm   r?   r¿   rO  rƒ   ri  rP  r†   r   rF  rS   r	   rr  rx  )	rZ   r[   rQ  r‡   rˆ   Ú_litellm_paramsÚ_model_infor\  r�   s	            r7   Úasync_log_failure_eventz%SlackAlerting.async_log_failure_eventš  sa  è ø€ à!'§¡Ð,<¸bÓ!AˆØ,×0Ñ0°¸rÓBÒHÀbˆØ%Ÿ/™/¨$°Ó3ˆð	Ø $×"2Ñ"2Ñ2ð
DØ×9Ñ9Ü)Ø'Ø+/Ø59Ü'.§}¡}×'EÑ'EÓ'Gô	ó÷ ð ô ˜&Ÿ*™* [°"Ó5´xÔ@Ø" d×&6Ñ&6Ñ6Ø×,Ñ,Ø"(¨Ñ"5Ø&.ð -ó ÷ ð ð
 *¨T×-=Ñ-=Ñ=Ø×3Ñ3¸fÀ[Ñ>QÐaiÐ3Ój×jÑjð >ð Aðùô !ò DÜ"×(Ñ(Ð)?À×CÑCûðDúð
øð kùÜò 	Ùð	üsƒ   ‚;E¾D< Á=D Â
D
ÂD ÂAD< ÃD8Ã+D< ÄD:ÄD< ÄEÄ
D Ä	D5ÄD0Ä+D< Ä0D5Ä5D< Ä:D< Ä<	EÅEÅEÅEc              ƒ   ó¤  K  — d}| j                   j                  t        j                  j                  d¬«      ƒ d{  –—† }t        j
                  «       }|€?| j                   j                  t        j                  j                  |¬«      ƒ d{  –—†  |S t        |t        «      rœ| j                  j                  }||z
  |k\  r~|�#|j                  t        |d¬«      ƒ d{  –—† du ry| j                  |¬«      ƒ d{  –—†  | j                   j                  t        j                  j                  |¬«      ƒ d{  –—†  d}|S 7 �Œ7 Œ·7 Œg7 ŒL7 Œ­w)z[
        Returns:
        - True -> report sent
        - False -> report not sent
        FN)r\   r¶   re  r¤  )rØ   T)r<   rø   rº   Úreport_sent_keyrµ   rÒ   rù   rS   rN  rM   rÖ   r§  r   rê   )rZ   r5   r”  Úreport_sent_boolÚreport_sentr$  Úinterval_secondss          r7   Ú_run_scheduler_helperz#SlackAlerting._run_scheduler_helper¹  sm  è ø€ ð !Ðà#'×#<Ñ#<×#LÑ#LÜ&×6Ñ6×<Ñ<Ø!ð $Mó $
÷ 
ˆô
 #Ÿi™i›kˆàÐØ×+Ñ+×;Ñ;Ü*×:Ñ:×@Ñ@Ø"ð <ó ÷ ð ð8  Ðô1 ˜¤UÔ+à&*×&8Ñ&8×&OÑ&OÐà˜kÑ)Ð-=Ò=à$Ð0à.×;Ñ;Ü'AÐGWÐinð <ó ÷ ð ñð !à×-Ñ-°ZÐ-Ó@×@Ð@à×/Ñ/×?Ñ?Ü.×>Ñ>×DÑDØ&ð @ó ÷ ð ð $(Ð àÐðI
ùðøðøð AøðúsY   ‚;E½E¾AEÂEÂAEÃ"E
Ã#EÃ?EÄ <EÄ<EÄ=	EÅEÅ
EÅEÅEc              ƒ   óJ  K  — |�| j                   €yd| j                   v r~	 | j                  ||¬«      ƒ d{  –—†  t        j                  | j                  j
                  dz
  | j                  j
                  dz   «      }t        j                  |«      ƒ d{  –—†  Œ}y7 Œg7 Œ­w)z¨
        If 'daily_reports' enabled

        Ping redis cache every 5 minutes to check if we should send the report

        If yes -> call send_daily_report()
        Nr×   r«  rË   )r?   r^  ÚrandomÚrandintrM   Úreport_check_intervalrN   r¯  )rZ   r5   r”  Úintervals       r7   Ú_run_scheduled_daily_reportz)SlackAlerting._run_scheduled_daily_reportë  s§   è ø€ ð Ð ×!1Ñ!1Ð!9Øà˜d×.Ñ.Ñ.ØØ×0Ñ0¸JÐYiÐ0Ój×jÐjÜ!Ÿ>™>Ø×&Ñ&×<Ñ<¸qÑ@Ø×&Ñ&×<Ñ<¸qÑ@ó�ô —m‘m HÓ-×-Ð-ð ð 	ð køð
 .ús"   ‚5B#·B¸A B#ÂB!ÂB#Â!B#Ú
time_rangec              ƒ   óâ  K  — | j                   �d| j                  vry	 ddlm} t	        |dd «      }|d   j                  «       dk7  rt        d«      ‚t        j                  j                  «       j                  «       }|t        j                  |¬«      z
  }d	|j                  d
«      › d|j                  d
«      › �}| j                  j                  |¬«      ƒ d{  –—† ry ||j                  d
«      |j                  d
«      ¬«      ƒ d{  –—† }|�|g g fk(  ry|\  }}	d|j                  d«      › d|j                  d«      › d|› d�}
|�3|
dz  }
|D ])  }t        t        |d   «      d«      }|
d|d   › d|› d�z  }
Œ+ |	�3|
dz  }
|	D ])  }t        t        |d   «      d«      }|
d|d   › d|› d�z  }
Œ+ | j!                  |
dt"        j$                  i ¬«      ƒ d{  –—†  | j                  j'                  |dt)        |«      ¬ «      ƒ d{  –—†  y7 �Œ47 �Œ7 Œ:7 Œ# t        $ r }t+        j,                  d!|«       Y d}~yd}~wt.        $ r }t+        j,                  d"|«       Y d}~yd}~ww xY w­w)#z¶
        Send a spend report for a configurable time range.

        Args:
            time_range: A string specifying the time range for the report, e.g., "1d", "7d", "30d"
        NÚspend_reportsr   ©Ú _get_spend_report_for_time_rangeéÿÿÿÿÚdz0Time range must be specified in days, e.g., '7d'©ÚdaysÚweekly_spend_report_sent_r9  ru  r”   ©rA  rB  u   *ðŸ’¸ Spend Report for `ú%m-%d-%Yú - z` (z days)*
ú
*Team Spend Report:*
Útotal_spendrT  úTeam: `rû  ú` | Spend: `$rÊ   ú
*Tag Spend Report:*
úTag: `Úindividual_request_tagr¥   rÎ   rõ   rö   zInvalid time range format: %szError sending spend report: %s)r>   r?   Ú7litellm.proxy.spend_tracking.spend_management_endpointsri  ÚintÚlowerrÄ  r  r  Údater   r  r<   rø   rš   rN  r¬   r   rg  rù   r   r   r×  r†   )rZ   re  ri  rm  Útodays_daterA  Ú_event_cache_keyÚ_respÚspend_per_teamÚspend_per_tagÚ_spend_messager"  Ú_team_spendÚ
_tag_spendÚver�   s                   r7   Úsend_weekly_spend_reportz&SlackAlerting.send_weekly_spend_report  sò  è ø€ ð �=‰=Ð  O¸4×;KÑ;KÑ$KØð:	Lõô
 ˜j¨¨"˜oÓ.ˆDØ˜"‰~×#Ñ#Ó%¨Ò,Ü Ð!SÓTÐTä!)×!2Ñ!2×!6Ñ!6Ó!8×!=Ñ!=Ó!?ˆKØ +¬h×.@Ñ.@ÀdÔ.KÑ KˆJð ,¨J×,?Ñ,?À
Ó,KÐ+LÈAÈk×NbÑNbÐcmÓNnÐMoÐpð ð ×.Ñ.×>Ñ>ÐCSÐ>ÓT×TÐTØá!AØ%×.Ñ.¨zÓ:Ø$×-Ñ-¨jÓ9ô"÷ ˆEð ˆ} ¨"¨b¨Ò 1Øà,1Ñ)ˆN˜Mà7¸
×8KÑ8KÈJÓ8WÐ7XÐX[Ð\g×\pÑ\pÐq{Ó\|Ð[}ð  ~Að  BFð  AGð  GPð  QˆNàÐ)ØÐ"<Ñ<�Ø+ò c�EÜ"'¬¨e°MÑ.BÓ(CÀQÓ"G�KØ"¨°°lÑ0CÐ/DÀMÐR]ÐQ^Ð^aÐ&bÑb‘Nðcð Ð(ØÐ";Ñ;�Ø*ò m�EÜ!&¤u¨U°=Ñ-AÓ'BÀAÓ!F�JØ"¨¨uÐ5MÑ/NÐ.OÈ}Ð]gÐ\hÐhkÐ&lÑl‘Nðmð —/‘/Ø&ØÜ$×2Ñ2Ø"$ð	 "ó ÷ ð ð ×+Ñ+×;Ñ;Ø$ØÜ'¨
Ó3ð <ó ÷ ñ ðC Uùðùð.øðùô ò 	LÜ ×&Ñ&Ð'FÈ×KÑKûÜò 	LÜ ×&Ñ&Ð'GÈ×KÑKûð	Lüs¢   ‚I/ŸB?H ÃHÃH Ã#I/Ã$+H ÄHÄH ÄI/ÄB?H ÇHÇ.H ÈHÈH ÈI/ÈH ÈH ÈH ÈH È	I,È%I È;I/É I,ÉI'É"I/É'I,É,I/c              ƒ   óˆ  K  — 	 ddl m} ddlm} t        j                  j                  «       j                  «       }|j                  d¬«      } ||j                  |j                  «      \  }}|t	        j                  |dz
  ¬«      z   }d|j                  d«      › d	|j                  d«      › �}| j                  j                  |¬
«      ƒ d{  –—† ry ||j                  d«      |j                  d«      ¬«      ƒ d{  –—† }|�|g g fk(  ry|\  }	}
d|j                  d«      › d|j                  d«      › d�}|	�7|dz  }|	D ]-  }|d   }t        |«      }t        |d«      }|d|d   › d|› d�z  }Œ/ |
�7|dz  }|
D ]-  }|d   }t        |«      }t        |d«      }|d|d   › d|› d�z  }Œ/ | j!                  |dt"        j$                  i ¬«      ƒ d{  –—†  | j                  j'                  |ddt(        z  dz  dz  ¬ «      ƒ d{  –—†  y7 �Œ=7 �Œ7 Œ>7 Œ# t*        $ r }t-        j.                  d!|«       Y d}~yd}~ww xY w­w)"r:  r   )Ú
monthrangerh  r'   )Údayrl  Úmonthly_spend_report_sent_r9  ru  r”   Nro  u    *ðŸ’¸ Monthly Spend Report for `rp  rq  z` *
rr  rs  rT  rt  rû  ru  rÊ   rv  rw  rx  r¥   rÎ   rõ   é   é<   rö   ú$Error sending weekly spend report %s)Úcalendarrˆ  ry  ri  r  r  r|  ÚreplaceÚyearÚmonthr   r  r<   rø   rN  rš   r¬   r   rg  rù   r   r†   r   rX  )rZ   rˆ  ri  r}  Úfirst_day_of_monthru  Úlast_day_of_monthr~  r  Úmonthly_spend_per_teamÚmonthly_spend_per_tagr‚  r"  rƒ  r„  r�   s                   r7   Úsend_monthly_spend_reportz'SlackAlerting.send_monthly_spend_reportM  sû  è ø€ ð<	VÝ+õô "*×!2Ñ!2×!6Ñ!6Ó!8×!=Ñ!=Ó!?ˆKØ(3×(;Ñ(;ÀÐ(;Ó(BÐÙ#-¨k×.>Ñ.>À×@QÑ@QÓ#RÑ ˆAÐ Ø 2´X×5GÑ5GÐM^ÐabÑMbÔ5cÑ cÐà!;Ð<N×<WÑ<WÐXbÓ<cÐ;dÐdeÐfw÷  gAñ  gAð  BLó  gMð  fNð   OÐØ×.Ñ.×>Ñ>ÐCSÐ>ÓT×TÐTØá!AØ-×6Ñ6°zÓBØ*×3Ñ3°JÓ?ô"÷ ˆEð
 ˆ} ¨"¨b¨Ò 1Øà<AÑ9Ð"Ð$9à?Ð@R×@[Ñ@[Ð\fÓ@gÐ?hÐhkÐl}÷  mGñ  mGð  HRó  mSð  lTð  TYð  ZˆNà%Ð1ØÐ"<Ñ<�Ø3ò c�EØ"'¨Ñ"6�KÜ"'¨Ó"4�Kä"'¨°QÓ"7�KØ"¨°°lÑ0CÐ/DÀMÐR]ÐQ^Ð^aÐ&bÑb‘Nðcð %Ð0ØÐ";Ñ;�Ø2ò m�EØ!& }Ñ!5�JÜ!& zÓ!2�Jä!& z°1Ó!5�JØ"¨¨uÐ5MÑ/NÐ.OÈ}Ð]gÐ\hÐhkÐ&lÑl‘Nðmð —/‘/Ø&ØÜ$×2Ñ2Ø"$ð	 "ó ÷ ð ð ×+Ñ+×;Ñ;Ø$ØØœ.Ñ(¨2Ñ-°Ñ2ð <ó ÷ ñ ðQ Uùðùð<øðùô ò 	VÜ ×*Ñ*Ð+QÐST×UÑUûð	Vüs�   ‚I„CH ÃHÃH ÃIÃ+H Ä HÄH ÄIÄCH ÇHÇ2H ÈHÈH ÈIÈH ÈH ÈH ÈH È	H?ÈH:È5IÈ:H?È?Ic              ƒ   óø   K  — 	 ddl m}  |«       ƒ d{  –—† }d|› �}| j                  |dt        j                  i ¬«      ƒ d{  –—†  y7 Œ57 Œ# t
        $ r }t        j                  d|«       Y d}~yd}~ww xY w­w)z¬
        Helper to send fallback statistics from prometheus server -> to slack

        This runs once per day and sends an overview of all the fallback statistics
        r   )Ú#get_fallback_metric_from_prometheusNz*Fallback Statistics:*
r¥   rÎ   r�  )Ú6litellm.integrations.prometheus_helpers.prometheus_apir˜  r¬   r   Úfallback_reportsr†   r   r×  )rZ   r˜  Ú!falllback_success_info_prometheusÚfallback_messager�   s        r7   Ú#send_fallback_stats_from_prometheusz1SlackAlerting.send_fallback_stats_from_prometheus�  s�   è ø€ ð	Rõñ
 >aÓ=b×7bÐ-à(@ÐAbÐ@cÐ&dÐà—/‘/Ø(ØÜ$×5Ñ5Ø"$ð	 "ó ÷ ñ ð	 8cøðùô ò 	RÜ ×&Ñ&Ð'MÈq×QÑQûð	RüsP   ‚A:„A ”A
•/A ÁAÁA Á	A:Á
A ÁA Á	A7ÁA2Á-A:Á2A7Á7A:Ú	key_eventrý  c              ƒ   óž  K  — 	 d|› d�}|j                  «       }|dz  }|j                  «       D ]  \  }}d|v sŒ||› d|› d�z  }Œ |dz  }|j                  }|j                  «       D ]  \  }}|dk(  rŒ||› d|› d�z  }Œ | j                  |d|i ¬	«      ƒ d
{  –—†  y
7 Œ# t        $ r }	t        j                  d|	«       Y d
}	~	y
d
}	~	ww xY w­w)z¹
        Handles sending Virtual Key related alerts

        Example:
        - New Virtual Key Created
        - Internal User Updated
        - Team Created, Updated, Deleted
        r¢   rÊ   z*Action Done by:*
Ú
created_byr  z
*Arguments passed:*
rÑ  rô   rÎ   Nz-Error sending send_virtual_key_event_slack %s)r  rR   Úrequest_kwargsr¬   r†   r   r×  )
rZ   rž  r¨   rý  r¦   Úkey_event_dictr\   rµ   r¡  r�   s
             r7   Úsend_virtual_key_event_slackz*SlackAlerting.send_virtual_key_event_slack§  s  è ø€ ð	[Ø˜*˜ SÐ)ˆGà$-×$8Ñ$8Ó$:ˆNð Ð,Ñ,ˆGØ,×2Ñ2Ó4ò 5‘
��UØ 3Ò&Ø #  c¨%¨°Ð4Ñ4‘Gð5ð
 Ð0Ñ0ˆGØ$-×$<Ñ$<ˆNØ,×2Ñ2Ó4ò 1‘
��UØÐ-Ò-ØØ˜c˜U # e W¨CÐ0Ñ0‘ð1ð
 —/‘/ØØØ%Ø"$ð	 "ó ÷ ò ùô ò 	[Ü ×&Ñ&Ð'VÐXY×ZÑZûð	[üsF   ‚C„5B! ºAB! ÂBÂB! ÂCÂB! Â!	C
Â*CÃ CÃC
Ã
Cc              ƒ   óø   K  — |€y|j                  dd«      dk7  r\|j                  dd«      dk7  rG|j                  dd«      }| j                  j                  d|› �d	¬
«      ƒ d{  –—† }|�|dk(  s|dk(  ry	y7 Œ­w)z[
        Returns True if the request is completed - either as a success or failure
        NFÚlitellm_statusr|   ÚsuccessÚfailÚlitellm_call_idzrequest_status:T)r\   Ú
local_only)rm   r<   rø   )rZ   rë   r¨  Ústatuss       r7   Ú_request_is_completedz#SlackAlerting._request_is_completedÒ  s    è ø€ ð ÐØà×ÑÐ,¨bÓ1°YÒ>À<×CSÑCSÐTdÐfhÓCiÐmsÒCsà%1×%5Ñ%5Ð6GÈÓ%LˆOØ.2×.GÑ.G×.WÑ.WØ% oÐ%6Ð7ÀDð /Xó /÷ )ˆFð Ð! v°Ò':¸fÈÒ>NØØð)ús   ‚A#A:Á%A8Á&A:)NNNNNNNrp   )NN)NNN)Ú7d)NÚ__name__Ú
__module__Ú__qualname__Ú__doc__ÚDEFAULT_ALERT_TYPESr   rN  ri   r   rT   r…   rY   r1   rd   rm  rz  rk   rn   rr   rv   rx   rŽ   r    r±   rO  rz  r¿   Úboolrê   rï   rþ   r   r   r  Útupler  r!  r  r?  rJ  ÚBaseOutageModelrW  r	   rx  rr  r   rŽ  r�  r“  r¢  rœ  r©  r¬  r8   r   r°  r   r>  rÆ  rÛ  r  r	  r¬   r  r7  rC  ra   rT  rX  r^  rd  r†  r–  r�  r   r£  r«  Ú__classcell__)r^   s   @r7   r;   r;   T   sk  ø„ ñð 26Ø+/Ø "Ø':àØØ*.Ø48ñ#?à'¨$Ñ.ð#?ð " D™Lð#?ð ˜‘+ð	#?ð
 ˜)‘_ð#?ð # 9¨d°3©i¸#©oÐ#=Ñ>Ø
ñð#?ð ! 4™Zð#?ð    T 	™?¨TÑ1õ#?ðN !%Ø+/Ø.2ØHLØ%)Ø$(Ø48ñ#)à˜‘+ð#)ð " D™Lð#)ð ˜)‘_ tÑ+ð	#)ð
 # 9¨d°3©i¸#©oÐ#=Ñ>ÀÑEð#)ð ˜d‘{ð#)ð ˜T‘Mð#)ð    T 	™?¨TÑ1ó#)ðJ
¸DÐC\Ñ<\Ð_jÑ<jð 
Ðosó 
ð¸TÀD¹[ð ÈTÐTXÉ[ó òòòòó2$ò@,ð\%ÐCTð %ÐY\ó %ðN°$ó ðF %)ñ
ià˜T‘kó
ið¸ð ÈSó ð>Sàðñ

ðSð óSðj,$àð,$ð ÐoÑpÐswÑwð,$ð ð	,$ð
 
ØÐhÑiÐlpÑpØð	ñ
ó,$ð\¸ð Àeó ð¨Hð ¸ó ð ?à�T‰zð ?ð ˜‘:ð ?ð ˜4‘Zð	 ?ð
 ˜t‘|ð ?ð ˜D‘Ló ?ðD¨4°©9ð ¸ó ð0àÐ,Ñ-ðð Ð&Ñ'ðð ð	ð
 ðð ˜‘*ðð &ðð 
óð8FZàðFZð ðFZð 
ó	FZðPsàðsð ðsð 
ó	sðj7°#ð 7È3ð 7Ðcfó 7ðr°Có ðf¨Tó fð
 %)Ø48ñ*à˜T‘Mð*ð 2ð*ð 
ó	*ðX
ÐH_ð 
Ðdhó 
ð¸Dó ð
qØ  4™-ðqØ;Rðqà	óqð 7HØ48ñ?à   V¨d¡]Ð!2Ñ3ð?ð 2ð?ð 
ó	?ð(°lð Àtó ðD &*Ø,0ñ	
uàð
uð ˜d™
ð
uð  # T™zó	
uð$UÀsÈTÁzð $UÐ]`ð $UÐehó $UðL[È,ð [Ð[_ó [ðz=¸|ð =ÐY\ð =Ðaeó =ðJ *.Ø$(Ø#ñT%àðT%ð Ð.Ñ/ðT%ð ð	T%ð
  ðT%ð   $Ñ&ðT%ð ˜T‘zðT%ð ˜‘*óT%ðl$¸ð $È)ð $ÐX\ó $ò$
ò@-òD%ò!òFðD 59ñ0 ð 2ð0 ð 
ó	0 ðh "&Ø48ñà˜$‘Jðð 2óð6 ñGLàóGLòR>Vò@Rð4)[à"ð)[ð ð)[ð ó	)[ðV¸¸t¹ð È÷ r9   r;   )QrN   r  r†  r`  rÒ   Úcollections.abcr   r   Útypingr   r   r   r   Úopenair	   Úpydanticr
   rƒ   Úlitellm.litellm_core_utilsÚ*litellm.litellm_core_utils.litellm_loggingÚlitellm.typesÚlitellm._loggingr   r   Úlitellm.caching.cachingr   Úlitellm.constantsr   r   r   Ú(litellm.integrations.custom_batch_loggerr   Ú5litellm.integrations.SlackAlerting.budget_alert_typesr   Ú8litellm.integrations.SlackAlerting.hanging_request_checkr   Ú*litellm.litellm_core_utils.duration_parserr   Ú2litellm.litellm_core_utils.exception_mapping_utilsr   Ú&litellm.llms.custom_httpx.http_handlerr   r   Úlitellm.proxy._typesr   r   r   r   r   r   r   r   Ú'litellm.repositories.table_repositoriesr    Ú$litellm.repositories.team_repositoryr!   Ú$litellm.repositories.user_repositoryr"   Ú)litellm.types.integrations.slack_alertingÚ%litellm.types.proxy.model_deprecationr$   r%   Úemail_templates.templatesÚbatching_handlerr(   r)   Úms_teamsr*   r+   r,   r-   r.   ri  r/   Ú6litellm.proxy.db.db_transaction_queue.pod_lock_managerr0   Úlitellm.routerr1   Ú_Routerr8   r;   rH   r9   r7   ú<module>rÒ     sÎ   ðó Û Û 	Û Û Ý $Ý ß 5Ó 5å Ý  ã Û !Û 1Û ß AÝ -÷ñ õ
 GÝ Wõõ Kõ÷÷	÷ 	ó 	õ MÝ ?Ý ?Ü 7÷ô
 *ß >÷õ õ 4áÝUÝ0à�Fà€Fð˜6 D™=ó ôMÐ%õ Mr9   