Ë
    uPêjÈ+  ã                   ó  — d Z ddlZddlmZmZ ddlmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZmZmZ ddl m!Z! erddl"Z" G d„ de«      Z# G d„ de#«      Z$ G d„ de#«      Z% G d„ de«      Z&y)z\
Translates from OpenAI's `/v1/chat/completions` endpoint to Triton's `/generate` endpoint.
é    N)ÚAsyncIteratorÚIterator)ÚTYPE_CHECKINGÚAnyÚFinalÚLiteral)ÚHeadersÚResponse)ÚDEFAULT_MAX_TOKENS_FOR_TRITON)Úprompt_factory)ÚBaseModelResponseIterator)Ú
BaseConfigÚBaseLLMExceptionÚLiteLLMLoggingObj)ÚAllMessageValues)ÚChatCompletionToolCallChunkÚChatCompletionUsageBlockÚChoicesÚGenericStreamingChunkÚMessageÚModelResponseé   ©ÚTritonErrorc                   óš  — e Zd ZdZdededeez  defd„Z		 	 d#deded	e
e   d
edededz  dedz  defd„Zdede
fd„Zded
edededef
d„Z	 d$dedz  dedz  ded
edededz  defd„Z	 	 d#dededededed	e
e   d
ededddedz  dedz  defd„Zded	e
e   d
edededefd„Zdeded   fd„Z	 d%d ee   ee   z  ez  d!ededz  defd"„Zy)&ÚTritonConfigzx
    Base class for Triton configurations.

    Handles routing between /infer and /generate triton completion llms
    Úerror_messageÚstatus_codeÚheadersÚreturnc                 ó   — t        |||¬«      S )N)r   Úmessager   r   )Úselfr   r   r   s       úr/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/llms/triton/completion/transformation.pyÚget_error_classzTritonConfig.get_error_class*   s   € Ü {¸MÐSZÔ[Ð[ó    NÚmodelÚmessagesÚoptional_paramsÚlitellm_paramsÚapi_keyÚapi_basec                 ó
   — ddiS )NzContent-Typezapplication/json© )r#   r   r'   r(   r)   r*   r+   r,   s           r$   Úvalidate_environmentz!TritonConfig.validate_environment-   s   € ð Ð 2Ð3Ð3r&   c                 ó
   — ddgS ©NÚ
max_tokensÚmax_completion_tokensr.   )r#   r'   s     r$   Úget_supported_openai_paramsz(TritonConfig.get_supported_openai_params9   s   € ØÐ5Ð6Ð6r&   Únon_default_paramsÚdrop_paramsc                 óV   — |j                  «       D ]  \  }}|dk(  s|dk(  sŒ|||<   Œ |S r1   )Úitems)r#   r5   r)   r'   r6   ÚparamÚvalues          r$   Úmap_openai_paramszTritonConfig.map_openai_params<   sB   € ð /×4Ñ4Ó6ò 	/‰LˆE�5Ø˜Ò$¨Ð1HÓ(HØ).� Ò&ð	/ð Ðr&   Ústreamc                 óZ   — |€t        d«      ‚| j                  |«      }|dk(  r|r|dz   S |S )Nzapi_base is requiredÚgenerateÚ_stream)Ú
ValueErrorÚ_get_triton_llm_type)r#   r,   r+   r'   r)   r*   r<   Úllm_types           r$   Úget_complete_urlzTritonConfig.get_complete_urlH   s@   € ð ÐÜÐ3Ó4Ð4Ø×3Ñ3°HÓ=ˆØ�zÒ!¡fØ˜iÑ'Ð'Øˆr&   Úraw_responseÚmodel_responseÚlogging_objÚrequest_dataÚencodingútiktoken.Encoding | NoneÚ	json_modec                 óð   — |j                  dd«      }| j                  |«      }|dk(  r$t        «       j                  |||||||||	|
|¬«      S |dk(  r$t	        «       j                  |||||||||	|
|¬«      S |S )Nr,   Ú r>   )r'   rD   rE   rF   rG   r(   r)   r*   rH   r+   rJ   Úinfer)ÚgetrA   ÚTritonGenerateConfigÚtransform_responseÚTritonInferConfig)r#   r'   rD   rE   rF   rG   r(   r)   r*   rH   r+   rJ   r,   rB   s                 r$   rP   zTritonConfig.transform_responseX   s¸   € ð )×,Ñ,¨Z¸Ó<ˆØ×3Ñ3°HÓ=ˆØ�zÒ!Ü'Ó)×<Ñ<ØØ)Ø-Ø'Ø)Ø!Ø /Ø-Ø!ØØ#ð =ó ð ð ˜Ò Ü$Ó&×9Ñ9ØØ)Ø-Ø'Ø)Ø!Ø /Ø-Ø!ØØ#ð :ó ð ð Ðr&   c                 óØ   — |j                  dd«      }| j                  |«      }|dk(  rt        «       j                  |||||¬«      S |dk(  rt	        «       j                  |||||¬«      S i S )Nr,   rL   r>   )r'   r(   r)   r*   r   rM   )rN   rA   rO   Útransform_requestrQ   )r#   r'   r(   r)   r*   r   r,   rB   s           r$   rS   zTritonConfig.transform_request†   s“   € ð )×,Ñ,¨Z¸Ó<ˆØ×3Ñ3°HÓ=ˆØ�zÒ!Ü'Ó)×;Ñ;ØØ!Ø /Ø-Øð <ó ð ð ˜Ò Ü$Ó&×8Ñ8ØØ!Ø /Ø-Øð 9ó ð ð ˆ	r&   )r>   rM   c                 óf   — |j                  d«      ry|j                  d«      ryt        d|› �«      ‚)Nz	/generater>   z/inferrM   zInvalid Triton API base: )Úendswithr@   )r#   r,   s     r$   rA   z!TritonConfig._get_triton_llm_type¢   s7   € Ø×Ñ˜[Ô)ØØ×Ñ˜xÔ(ØäÐ8¸¸
ÐCÓDÐDr&   Ústreaming_responseÚsync_streamc                 ó   — t        |||¬«      S )N)rV   rW   rJ   )ÚTritonResponseIterator)r#   rV   rW   rJ   s       r$   Úget_model_response_iteratorz(TritonConfig.get_model_response_iteratorª   s   € ô &Ø1Ø#Øô
ð 	
r&   ©NN)N)F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚintÚdictr	   r   r%   Úlistr   r/   r4   Úboolr;   rC   r
   r   r   rP   rS   r   rA   r   r   r   rZ   r.   r&   r$   r   r   #   sx  „ ñð\¨Sð \¸sð \ÈTÐT[É^ð \Ð`pó \ð #Ø#ñ
4àð
4ð ð
4ð Ð'Ñ(ð	
4ð
 ð
4ð ð
4ð �t‘ð
4ð ˜‘*ð
4ð 
ó
4ð7°ð 7¸ó 7ð
à ð
ð ð
ð ð	
ð
 ð
ð 
ó
ð& #ñà˜‘*ðð �t‘ðð ð	ð
 ðð ðð �t‘ðð 
óð6 #Ø!%ñ,àð,ð ð,ð &ð	,ð
 'ð,ð ð,ð Ð'Ñ(ð,ð ð,ð ð,ð -ð,ð �t‘ð,ð ˜$‘;ð,ð 
ó,ð\àðð Ð'Ñ(ðð ð	ð
 ðð ðð 
óð8E¨Sð E°WÐ=PÑ5Qó Eð "'ñ	

à$ S™M¨M¸#Ñ,>Ñ>ÀÑNð

ð ð

ð ˜$‘;ð	

ð
 
ô

r&   r   c                   ó†   — e Zd ZdZdedee   dedededefd„Z	 	 dded
e	de
dededee   dededdded	z  ded	z  de
fd„Zy	)rO   zP
    Transformations for triton /generate endpoint (This is a trtllm model)
    r'   r(   r)   r*   r   r    c                 óä   — |j                  «       }|j                  dd«      }t        ||¬«      dt        |j	                  dt
        «      «      it        |«      dœ}|d   j                  |«       |S )Nr<   F)r'   r(   r2   )Ú
text_inputÚ
parametersr<   rh   )ÚcopyÚpopr   ra   rN   r   rd   Úupdate)	r#   r'   r(   r)   r*   r   Úinference_paramsr<   Údata_for_tritons	            r$   rS   z&TritonGenerateConfig.transform_request¼   sx   € ð #2×"6Ñ"6Ó"8ÐØ(×,Ñ,¨X°uÓ=ˆä(¨u¸xÔHàœc /×"5Ñ"5°lÔDaÓ"bÓcðô ˜6“lñ2
ˆð 	˜Ñ%×,Ñ,Ð-=Ô>ØÐr&   NrD   rE   rF   rG   rH   rI   r+   rJ   c                 óÄ   — 	 |j                  «       }t        dt        |d   ¬«      ¬«      g|_        |S # t        $ r" t        |j                  |j                  ¬«      ‚w xY w)N©r"   r   r   Útext_output©Úcontent©Úindexr"   )ÚjsonÚ	Exceptionr   Útextr   r   r   Úchoices)r#   r'   rD   rE   rF   rG   r(   r)   r*   rH   r+   rJ   Úraw_response_jsons                r$   rP   z'TritonGenerateConfig.transform_responseÐ   si   € ð	_Ø'3×'8Ñ'8Ó':Ðô #*°¼7ÐK\Ð]jÑKkÔ;lÔ"mÐ!nˆÔàÐøô	 ò 	_Ü l×&7Ñ&7À\×E]ÑE]Ô^Ð^ð	_ús	   ‚4 ´+Ar[   ©r\   r]   r^   r_   r`   rc   r   rb   rS   r
   r   r   rd   rP   r.   r&   r$   rO   rO   ·   sÝ   „ ñðàðð Ð'Ñ(ðð ð	ð
 ðð ðð 
óð> #Ø!%ñàðð ðð &ð	ð
 'ðð ðð Ð'Ñ(ðð ðð ðð -ðð �t‘ðð ˜$‘;ðð 
ôr&   rO   c                   ó†   — e Zd ZdZdedee   dedededefd„Z	 	 dded
e	de
dededee   dededdded	z  ded	z  de
fd„Zy	)rQ   zj
    Transformations for triton /infer endpoint (his is an infer model with a custom model on triton)
    r'   r(   r)   r*   r   r    c                 óT  — |d   j                  dd«      }dddgd|gdœgi}|j                  «       D ]T  \  }}	|d	k(  rŒ|d
k(  rŒt        |	t        «      rdnd}
t        |	t        «      rdn|
}
|d   j                  |dg|
|	gdœ«       ŒV d|vr|d   j                  ddgddgdœ«       |S )Nr   rr   rL   Úinputsrg   é   ÚBYTES)ÚnameÚshapeÚdatatypeÚdatar<   Úmax_retriesÚINT32ÚFP32r2   é   )rN   r8   Ú
isinstancera   ÚfloatÚappend)r#   r'   r(   r)   r*   r   rg   rm   ÚkÚvr‚   s              r$   rS   z#TritonInferConfig.transform_requestì   sç   € ð % Q™KŸO™O¨I°rÓ:ˆ
àà(Ø˜SØ 'Ø'˜Lñ	ðð	"
ˆð $×)Ñ)Ó+ò 	o‰DˆAˆqØ˜“M Q¨-Ó%7Ü&0°´CÔ&8™7¸g�Ü%/°´5Ô%9™6¸x�Ø Ñ)×0Ñ0¸!ÀqÀcÐW_ÐjkÐilÑ1mÕnð		oð ˜Ñ.Ø˜HÑ%×,Ñ,à(Ø˜SØ 'Ø˜Dñ	ôð Ðr&   NrD   rE   rF   rG   rH   rI   r+   rJ   c                 ó   — 	 |j                  «       }|d   d   d   }d }t        |t        «      rdj                  |«      }n|}t        dt        |¬«      ¬«      g|_
        |S # t        $ r" t        |j                  |j                  ¬«      ‚w xY w)Nro   Úoutputsr   rƒ   rL   rq   rs   )ru   rv   r   rw   r   rˆ   rc   Újoinr   r   rx   )r#   r'   rD   rE   rF   rG   r(   r)   r*   rH   r+   rJ   ry   Ú_triton_response_dataÚtriton_response_datas                  r$   rP   z$TritonInferConfig.transform_response  s©   € ð	_Ø'3×'8Ñ'8Ó':Ðð (9¸Ñ'CÀAÑ'FÀvÑ'NÐØ+/ÐÜÐ+¬TÔ2Ø#%§7¡7Ð+@Ó#AÑ à#8Ð ô ØÜÐ(<Ô=ôð"
ˆÔð Ðøô# ò 	_Ü l×&7Ñ&7À\×E]ÑE]Ô^Ð^ð	_ús   ‚A" Á"+Br[   rz   r.   r&   r$   rQ   rQ   ç   sÞ   „ ñð#àð#ð Ð'Ñ(ð#ð ð	#ð
 ð#ð ð#ð 
ó#ð` #Ø!%ñ!àð!ð ð!ð &ð	!ð
 'ð!ð ð!ð Ð'Ñ(ð!ð ð!ð ð!ð -ð!ð �t‘ð!ð ˜$‘;ð!ð 
ô!r&   rQ   c                   ó   — e Zd Zdedefd„Zy)rY   Úchunkr    c           	      ó,  — 	 d}d }d}d}d }d }t        |j                  dd«      «      }|j                  dd«      }|j                  dd«      }|j                  dd«      }t        |||||||¬«      S # t        j                  $ r t        d	|› �«      ‚w xY w)
NrL   Frt   r   rp   Ústop_reasonÚis_finished)rw   Útool_user–   Úfinish_reasonÚusagert   Úprovider_specific_fieldsz"Failed to decode JSON from chunk: )ra   rN   r   ru   ÚJSONDecodeErrorr@   )	r#   r“   rw   r—   r–   r˜   r™   rš   rt   s	            r$   Úchunk_parserz#TritonResponseIterator.chunk_parser6  s¼   € ð	KØˆDØBFˆHØˆKØˆMØ<@ˆEØ.2Ð$Ü˜uŸy™y¨°!Ó4Ó5ˆEð —9‘9˜]¨BÓ/ˆDØ!ŸI™I m°RÓ8ˆMØŸ)™) M°5Ó9ˆKä(ØØ!Ø'Ø+ØØØ)Aôð øô ×#Ñ#ò 	KÜÐAÀ%ÀÐIÓJÐJð	Kús   ‚A.A1 Á1"BN)r\   r]   r^   rb   r   rœ   r.   r&   r$   rY   rY   5  s   „ ðK $ð KÐ+@ô Kr&   rY   )'r_   ru   Úcollections.abcr   r   Útypingr   r   r   r   Úhttpxr	   r
   Úlitellm.constantsr   Ú3litellm.litellm_core_utils.prompt_templates.factoryr   Ú)litellm.llms.base_llm.base_model_iteratorr   Ú)litellm.llms.base_llm.chat.transformationr   r   r   Úlitellm.types.llms.openair   Úlitellm.types.utilsr   r   r   r   r   r   Úcommon_utilsr   Útiktokenr   rO   rQ   rY   r.   r&   r$   ú<module>r¨      s‚   ðñó ß 3ß 5Ó 5ç #å ;Ý NÝ O÷ñ õ
 7÷÷ õ 'áÛôQ
�:ô Q
ôh-˜<ô -ô`K˜ô Kô\KÐ6õ Kr&   