Ë
    cQêjR1  ã                   óÜ   — d dl Z d dlZd dlmZ d dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZmZ d dlmZmZ d dlmZmZ d d	lmZ  e j,                  e«      Zd
efd„Zg d¢Zg d¢Z G d„ d«      Zy)é    N)ÚPath)Úfloat_to_float16_max_diff)Ú	OnnxModel)Úoptimize_model)Ú	T5DecoderÚT5DecoderHelper)ÚT5EncoderDecoderInitÚT5EncoderDecoderInitHelper)ÚMT5ForConditionalGenerationÚT5ForConditionalGeneration)ÚInferenceSessionÚpathc                 óª   — 	 t        j                  | fddi|¤ŽS # t        $ r/ t        j	                  d| «       t        j                  | fi |¤ŽcY S w xY w)NÚweights_onlyTz€Current PyTorch version does not support torch.load(..., weights_only=True); falling back to default torch.load behavior for %s.)ÚtorchÚloadÚ	TypeErrorÚloggerÚwarning)r   Úkwargss     úq/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/models/t5/t5_helper.pyÚ_torch_load_weights_onlyr      sZ   € ð*Ü�z‰z˜$Ñ<¨TÐ<°VÑ<Ð<øÜò *Ü�‰ðBàô	
ô
 �z‰z˜$Ñ) &Ñ)Ò)ð*ús   ‚ š5AÁA)zt5-smallzt5-basezt5-largezt5-3bzt5-11b)zgoogle/mt5-smallzgoogle/mt5-basezgoogle/mt5-largezgoogle/mt5-xlzgoogle/mt5-xxlc                   ó–  — e Zd Ze	 	 d$dededededef
d„«       Ze	 	 	 d%dededej                  d	ed
edede	ee
ez  f   fd„«       Ze	 	 	 	 d&dee
z  dej                  dededededefd„«       Ze	 	 	 d'dedee   dz  dedefd„«       Ze	 	 	 	 d(dedededededededed efd!„«       Zedee
z  d"edej                  defd#„«       Zy))ÚT5HelperÚ
output_dirÚmodel_name_or_pathÚsuffixÚ
new_folderÚreturnc                 ó6  — |}t         j                  j                  |«      rt        |«      j                  d   }n|j                  d«      d    ||z  }|r t         j                  j                  | |«      n| }t         j                  j                  ||dz   «      S )aÊ  Build onnx path

        Args:
            output_dir (str): output directory
            model_name_or_path (str): pretrained model name, or path to the model checkpoint
            suffix (str, optional): suffix like "_encoder" or "_decoder_fp16" will be appended to file name. Defaults to None.
            new_folder (bool, optional): create a new directory for the model. Defaults to False.

        Returns:
            str: path of onnx model
        éÿÿÿÿú/z.onnx)Úosr   Úisdirr   ÚpartsÚsplitÚjoin)r   r   r   r   Ú
model_nameÚ	directorys         r   Úget_onnx_pathzT5Helper.get_onnx_path.   s   € ð$ (ˆ
Ü�7‰7�=‰=Ð+Ô,ÜÐ0Ó1×7Ñ7¸Ñ;‰Jà×Ñ˜SÓ! "Ò%à�fÑˆ
á<F”B—G‘G—L‘L ¨ZÔ8ÈJˆ	Ü�w‰w�|‰|˜I z°GÑ';Ó<Ð<ó    Ú	cache_dirÚdeviceÚ
model_typeÚstate_dict_pathÚencoder_decoder_initc                 óä  — |dk(  rt        j                  | |¬«      }n(|dk(  rt        j                  | |¬«      }nt        d«      ‚|r|j	                  t        |«      «       t        |j                  |j                  |j                  «      }|j                  «       j                  |«       t        |j                  |j                  |j                  |j                  d| ¬«      }|rdnd}	|	|d	|iS )
a{  Load model given a pretrained name or path, then build models for ONNX conversion.

        Args:
            model_name_or_path (str): pretrained model name or path
            cache_dir (str): cache directory
            device (torch.device): device to run the model
            model_type (str, optional): model type "t5" or "mt5"
            state_dict_path(str, optional): state dictionary path
            encoder_decoder_init (bool, optional): combine encoder and decoder kv cache initialization into one model.
        Returns:
            Dict[str, torch.nn.Module]: mapping from name to modules for ONNX conversion.
        Út5)r,   Úmt5z only support mode_type=t5 or mt5N)Údecoder_start_token_idÚoutput_cross_onlyr0   ÚencoderÚdecoder)r   Úfrom_pretrainedr   Ú
ValueErrorÚload_state_dictr   r   r7   Úlm_headÚconfigÚevalÚtor	   r6   )
r   r,   r-   r.   r/   r0   Úmodelr7   r6   Úencoder_names
             r   Ú
load_modelzT5Helper.load_modelK   sØ   € ð* ˜ÒÜ.×>Ñ>Ð?QÐ]fÔg‰EØ˜5Ò Ü/×?Ñ?Ð@RÐ^gÔh‰EäÐ?Ó@Ð@áØ×!Ñ!Ô":¸?Ó"KÔLä˜EŸM™M¨5¯=©=¸%¿,¹,ÓGˆØ�‰‹×Ñ˜&Ô!ä&Ø�M‰MØ�M‰MØ�M‰MØ�L‰LØ#'Ø"6Ð6ô
ˆñ 2FÑ-È9ˆØ˜g y°'Ð:Ð:r+   r?   Úonnx_model_pathÚverboseÚuse_external_data_formatÚuse_decoder_input_idsÚuse_int32_inputsc           	      ó�   — t        | t        «      rt        j                  | ||||||«       y t	        j                  | |||||«       y )N)Ú
isinstancer	   r
   Úexport_onnxr   )r?   r-   rB   rC   rD   rE   rF   s          r   rI   zT5Helper.export_onnxy   sT   € ô �eÔ1Ô2Ü&×2Ñ2ØØØØ%ØØ(Ø õô ×'Ñ'ØØØØØ(Ø õr+   NÚ
onnx_modelÚop_block_listÚforce_fp16_logitsÚuse_symbolic_shape_inferc                 ó*  — |€g d¢}| j                  «       D �ch c]  }|j                  ’Œ }}t        |«      }|j                  |«      }t        j                  d|› d|› �«       | j                  «       j                  d   j                  }d}	| j                  «       }
||
v sJ ‚|
|   }d}|j                  dk(  r‚|}t        j                  d|j                  › �«       d}|j                  D ]  }| j                  |«      }|€Œ n t        |«      }t        j                  d	|j                  › d
|› �«       |dk  }	n/t        j                  d|j                  › d|j                  › �«       g }g }|	s|�|s|g}|j                  g}d|v�r| j                  «       }d}d}| j!                  d«      }|rÀd}|D ]¶  }|j                  |vsŒ| j#                  ||
«      }| j%                  ||«      }|D �cg c]  }|j                  |v s||v sŒ|‘Œ }}|D �cg c]  }|j                  |v s||v sŒ|‘Œ }}t'        |«      t'        |«      z   dkD  sŒ•|j)                  |j                  «       |dz  }d}Œ¸ |rŒÀt'        |«      |z
  }t        j                  d|› d|› �«       t        j                  d|› �«       ||||	dœ}t        j                  d|› �«       |r | j*                  dddi|¤Ž |S ddlm}  || j.                  fddi|¤Ž |S c c}w c c}w c c}w )a¢  Convert model to mixed precision.
           It detects whether original model has fp16 precision weights, and set parameters for float16 conversion automatically.
        Args:
            onnx_model (OnnxModel): optimized ONNX model
            op_block_list (List[str], optional): operators need to run in fp32.
            force_fp16_logits (bool, optional): force logits and last MatMul node to be in float16. Defaults to False.
            use_symbolic_shape_infer (bool, optional): use symbolic shape inference to convert float to float16. Defaults to True.
        Returns:
            parameters(dict): a dictionary of parameters used in float16 conversion
        N)ÚSimplifiedLayerNormalizationÚ SkipSimplifiedLayerNormalizationÚReluÚAddz	fp32 op: z
 fp16 op: r   FÚMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node z: g�íµ ÷Æ°>z-Failed to find MatMul node for logits. Found z	 of node rR   Té   z#node counter of Add operator: fp32=z fp16=znode_block_list: )Úkeep_io_typesrK   Únode_block_listÚforce_fp16_initializersz!auto_mixed_precision parameters: rM   )Úconvert_float_to_float16Údisable_shape_infer© )ÚnodesÚop_typeÚsetÚ
differencer   ÚinfoÚgraphÚoutputÚnameÚoutput_name_to_nodeÚinputÚget_initializerr   Údebugr   Úinput_name_to_nodesÚget_nodes_by_op_typeÚget_parentsÚget_childrenÚlenÚappendrX   Úfloat16r?   )rJ   rK   rL   rM   ÚnodeÚop_full_setÚfp32_op_setÚfp16_op_setÚlogits_output_nameÚis_weight_fp16_precisionrc   Úlast_matmul_nodeÚinitializerrd   Úmax_diffrU   rV   rg   Úfp32_addÚchangedÚ	add_nodesÚparentsÚchildrenÚchildÚblocked_childrenÚparentÚblocked_parentsÚfp16_addÚ
parametersrX   s                                 r   Úauto_mixed_precisionzT5Helper.auto_mixed_precision—   sŒ  € ð" Ð òˆMð 1;×0@Ñ0@Ó0BÖC¨�t—|“|ÐCˆÐCÜ˜-Ó(ˆØ!×,Ñ,¨[Ó9ˆÜ�‰�i ˜}¨J°{°mÐDÔEð (×-Ñ-Ó/×6Ñ6°qÑ9×>Ñ>Ðð $)Ð Ø(×<Ñ<Ó>ÐØ!Ð%8Ñ8Ð8Ð8Ø"Ð#5Ñ6ˆØÐØ�<‰<˜8Ò#Ø#ÐÜ�K‰KÐ=¸d¿i¹i¸[ÐIÔJØˆKØŸ™ò �Ø(×8Ñ8¸Ó?�ØÑ*Ùðô 1°Ó=ˆHÜ�L‰LÐNÈtÏyÉyÈkÐY[Ð\dÐ[eÐfÔgØ'/°$¡Ñ$ä�N‰NÐJÈ4Ï<É<È.ÐXaÐbf×bkÑbkÐalÐmÔnàˆØˆÙ(Ð/?Ð/KÑUfà/Ð0ˆMØ/×4Ñ4Ð5ˆOà˜Ò%Ø",×"@Ñ"@Ó"BÐØˆHØˆGØ"×7Ñ7¸Ó>ˆIÙØ�Ø%ò +�DØ—y‘y¨Ò7Ø",×"8Ñ"8¸Ð?RÓ"S˜Ø#-×#:Ñ#:¸4ÐATÓ#U˜à/7ö,Ø&+¸5¿=¹=ÈMÑ;YÐ]bÐfuÒ]ušEð,Ð(ð ,ð 29ö+Ø'-¸F¿N¹NÈmÑ<[Ð_eÐixÒ_xšFð+˜ð +ô  Ð 0Ó1´C¸Ó4HÑHÈAÓMØ+×2Ñ2°4·9±9Ô=Ø$¨™M˜HØ&*™Gð+ò ô" ˜9“~¨Ñ0ˆHÜ�K‰KÐ=¸h¸ZÀvÈhÈZÐXÔYä�‰Ð'¨Ð'8Ð9Ô:ð +Ø*Ø.Ø'?ñ	
ˆ
ô 	�‰Ð7¸
°|ÐDÔEÙ#Ø/ˆJ×/Ñ/Ñ\ÈÐ\ÐQ[Ò\ð Ðõ 9á$Ø× Ñ ñà$(ðð òð Ðùòm Dùòb,ùò+s   ™LÇ/LÈLÈLÈ)LÚoptimized_model_pathÚ
is_float16Únum_attention_headsÚhidden_sizer‚   Úuse_gpuÚforce_fp16_ioc	           	      óà   — ddl m}	 d}
|r |	d«      }
| |
_        t        | d||d|
|¬«      }|r,|rt        j                  ||¬«       n|j                  |¬«       |j                  ||d¬	«       y)
zHOptimize ONNX model with an option to convert it to use mixed precision.r   )ÚFusionOptionsNr2   )r.   Ú	num_headsr†   Ú	opt_levelÚoptimization_optionsr‡   )rL   )Úcast_input_outputT)Úall_tensors_to_one_file)Úfusion_optionsrŠ   Úenable_skip_layer_normr   r   r‚   Ú convert_model_float32_to_float16Úsave_model_to_file)rB   rƒ   r„   r…   r†   rD   r‚   r‡   rˆ   rŠ   r�   Úms               r   Úoptimize_onnxzT5Helper.optimize_onnx  sŠ   € õ 	1à#ÐÙÙ#0°Ó#6Ð à>RÐ:RÐ Ô7äØØØ)Ø#ØØ!5Øô
ˆñ Ù#Ü×-Ñ-¨aÀ=Ð-ÕQà×2Ñ2À]Ð2ÔSà	×ÑÐ1Ð3KÐeiÐÕjr+   Úort_sessionc                 ó‚   — t        | t        «      rt        j                  | |||«      S t	        j                  | |||«      S )zQCompare the result from PyTorch and OnnxRuntime to verify the ONNX model is good.)rH   r	   r
   Úverify_onnxr   )r?   r–   r-   rF   s       r   r˜   zT5Helper.verify_onnx0  s?   € ô �eÔ1Ô2Ü-×9Ñ9¸%ÀÈfÐVfÓgÐgä×*Ñ*¨5°+¸vÐGWÓXÐXr+   )Ú F)r2   r™   F)TFTF)NFT)FTFF)Ú__name__Ú
__module__Ú__qualname__ÚstaticmethodÚstrÚboolr*   r   r-   Údictr	   r   rA   rI   r   Úlistr‚   Úintr•   r   r˜   rZ   r+   r   r   r   -   sM  „ Øð Ø ñ	=Øð=àð=ð ð=ð ð	=ð
 
ò=ó ð=ð8 ð
 Ø!Ø%*ñ+;Øð+;àð+;ð —‘ð+;ð ð	+;ð
 ð+;ð #ð+;ð 
ˆcÐ'¨)Ñ3Ð3Ñ	4ò+;ó ð+;ðZ ð
 Ø).Ø&*Ø!&ñØÐ/Ñ/ðà—‘ðð ðð ð	ð
 #'ðð  $ðð òó ðð: ð +/Ø"'Ø)-ñ	nØðnà˜C‘y 4Ñ'ðnð  ðnð #'ò	nó ðnð` ð */Ø%)ØØ#ñ%kØð%kà!ð%kð ð%kð !ð	%kð
 ð%kð #'ð%kð #ð%kð ð%kð ò%kó ð%kðN ð
YØÐ/Ñ/ð
Yà%ð
Yð —‘ð
Yð ò	
Yó ñ
Yr+   r   )Úloggingr#   Úpathlibr   r   rm   r   rJ   r   Ú	optimizerr   Ú
t5_decoderr   r   Út5_encoder_decoder_initr	   r
   Útransformersr   r   Úonnxruntimer   Ú	getLoggerrš   r   rž   r   ÚPRETRAINED_T5_MODELSÚPRETRAINED_MT5_MODELSr   rZ   r+   r   ú<module>r­      se   ðó Û 	Ý ã Ý -Ý  Ý $ß 1ß Tß På (à	ˆ×	Ñ	˜8Ó	$€ð	* 3ó 	*ò NÐ òÐ ÷NYò NYr+   