cQj&ddlZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl Z ddl Z ddl m Z ddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZmZmZ dd lmZej@e!Z"gd Z#e jHd e jJde jLdiZ'GddeZ(e GddeZ)ndZ)GddeZ*GddeZ+GddeZ,e+ddfe,ddfe*ddfdZ-Gdd Z.Gd!d"Z/y#e$rdZYwxYw)#N)Path) Precision)float_to_float16_max_diff) FusionOptions)IOBindingHelper) OnnxModel)optimize_model)torch_onnx_export) GPT2ConfigGPT2LMHeadModel GPT2Model) TFGPT2Model) distilgpt2gpt2z gpt2-mediumz gpt2-largezgpt2-xlMb@?g?g@c,eZdZdZfdZfdZxZS)GPT2ModelNoPastState2Here we wrap a class to disable past state output.c$t||yNsuper__init__selfconfig __class__s u/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/models/gpt2/gpt2_helper.pyrzGPT2ModelNoPastState.__init__/  c(t||ddS)NF) use_cache return_dict)rforwardr input_idsrs rr$zGPT2ModelNoPastState.forward2swyEuMMr __name__ __module__ __qualname____doc__rr$ __classcell__rs@rrr,s<!NNr rc,eZdZdZfdZfdZxZS)TFGPT2ModelNoPastStaterc2d|_t| |y)NF)r"rrrs rrzTFGPT2ModelNoPastState.__init__;s$F  G V $r c&t||dS)NF)r")rcallr%s rr$zTFGPT2ModelNoPastState.forward?s7< U<; ;r r'r-s@rr/r/8s@ % < Y.3vay|3D3I IIG9% II1a2215vay|A7P7PQR7ST 1IuW~. . r ct|||||d}tj||jj SNF) position_idsattention_maskpast_key_valuesr#rr$r4rGrn_layerrr&rJrKpastrCrs rr$zMyGPT2Model.forward_sD %)  ! '' 0C0CDDr ) r(r)r*r+r staticmethodrGr$r,r-s@rr4r4Fs+W!$EEr r4c,eZdZdZfdZfdZxZS)MyGPT2LMHeadModelzSHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state.c$t||yrrrs rrzMyGPT2LMHeadModel.__init__mrr ct|||||d}tj||jj SrIrMrOs rr$zMyGPT2LMHeadModel.forwardpsD %)  ! '' 0C0CDDr r'r-s@rrSrSjs]! E Er rSc,eZdZdZfdZfdZxZS)MyGPT2LMHeadModel_NoPaddingaHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and no padding. When you always use batch_size=1 in inference, there is no padding in inputs. In such case, position_ids and attention_mask need no be in inputs. c$t||yrrrs rrz$MyGPT2LMHeadModel_NoPadding.__init__rr c|t|||d}tj||jj S)NF)rLr#rM)rr&rPrCrs rr$z#MyGPT2LMHeadModel_NoPadding.forwards4DeT'' 0C0CDDr r'r-s@rrWrW|s !EEr rWlogitsTF last_state)r GPT2LMHeadModel_NoPaddingr c0eZdZdZdefdZdefdZdZy) Gpt2Inputsc<||_||_||_||_yr)r&rJrKrP)rr&rJrKrPs rrzGpt2Inputs.__init__s +4.:WeFJ r returnc|j|j|jfDcgc]}|| }}|jr|j |j|Scc}wr)r&rJrKrPextend)rv input_lists rto_listzGpt2Inputs.to_listsW"&..$2C2CTEXEX!YkA]^]jak k 99   dii ( ls AAc~td|j|j|j|jfDS)Nc3&K|] }|| ywr).0rcs r z&Gpt2Inputs.to_tuple..su1ghgtQus)r;r&rJrKrP)rs rto_tuplezGpt2Inputs.to_tuples3u1B1BDDWDWY]YbYb cuuur cd}|j]|jjtjk(r*|jj tj n |j}|j Dcgc]"}|j tj $}}t|j|j||Scc}w)N)dtype) rKrmr@float16tofloat32rPr^r&rJ)rrKprPs rto_fp32zGpt2Inputs.to_fp32s    *''-->##&&U]]&;((  4899=a5==)==$..$*;*;^TRR>s:'CN) r(r)r*rr<rer;rkrrrhr rr^r^s(K v%v Sr r^c"peZdZdZedddej ej ej dfdededededed ed ed ejd e d e de dejdejdejde de f dZ e dAdededede dedeeeeff dZedZedBdZedBdZedCdZedDdZeddddej ej ej fdede d e d e de dejdejdejfd!Ze dEd#Zegd$fd%ed&eefd'ZedFd(e d)efd*ZedFd(e d)efd+Zed,ZedGd-Ze dHd(e d.eeej>fd/eeeefd)ed0e d1e f d2Z ed3Z!ed4Z"edd5d5d6d7ddddej ej ej d"ddfd8Z#edd9ddddej ej ej d:d7d;f d<Z$edId=Z%edddgd>fdefd?Z&y@)J Gpt2HelperzEA helper class for Gpt2 model conversion, inference and verification.FT batch_sizepast_sequence_lengthsequence_lengthnum_attention_heads hidden_sizerD vocab_sizedevicernhas_position_idshas_attention_maskinput_ids_dtypeposition_ids_dtypeattention_mask_dtypeleft_side_paddingr`c|rtjntj}d|||t||z g}t |Dcgc] }tj |||dzdz "}}tj d|dz ||f| |}d}| re||z}tj||g| |}|dk\rAt |D]3}tj d|dz }|r d||d|f<(d||||z df<5d}| rQ|jjd dz }|j|dkd|dd|dfj| }t||||Scc}w) zCreate random inputs for GPT2 model. Returns torch tensors of input_ids, position_ids, attention_mask and a list of past state tensors. r8rmr{g@?rr7)lowhighsizermr{N)r@rnrpintr>randrandintonesrandomlongcumsum masked_fill_ror^)rurvrwrxryrDrzr{rnr|r}r~rrr float_type past_shape_rPr&rKtotal_sequence_lengthrFpadding_lengthrJs rget_dummy_inputszGpt2Helper.get_dummy_inputss*'.U]]5==    11 2  `een_opZ[JjH3NQTTppMMao.!   $8?$J !"ZZ23*N %)z*XA%+^^A7Lq7P%QN(=>q/>/'9:VWq*?.*P*R'RS X )..077;a?L  % %lQ&6 :'+?+@(@ADDEWXL)\>4HHCqs%Er r model_classc |j}|j}|j}|j}t|d} ||| dk(r|n|g} d||||zt ||z g} | | i} t |D]} | | dt| z<| S)zAReturns a dictionary with output name as key, and shape as value.r7rZr8present_)rxrynum_hidden_layersrz MODEL_CLASSESrr>str)rurvrwrrrxryrDrz output_namelast_state_shapepresent_state_shape output_shapesrFs rget_output_shapeszGpt2Helper.get_output_shapess%88(( ,, && #K03   %1J{     ? 2  11 2  %&67 y! EA1DM*s1v- . Er c|D]|}||vsJ||}tj|||jkDs8tjtj|||j |j ||<~y)Nr)numpyprodnelementr@emptyrmr{)output_buffersrkeybuffers rauto_increase_buffer_sizez$Gpt2Helper.auto_increase_buffer_sizesx  C.( ((#C(Fzz-,-0AA&+kkJJ}S12 ,,!=='s#  r c|rtjntj}i}|jD]3\}}tjt j |||||<5|S)zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.r)r@rnrpitemsrrr)rr{ is_float16 data_typernameshapes rget_output_bufferszGpt2Helper.get_output_buffers%s_&0EMMU]] (..0 bKD%#(;;uzz%/@ Z`#aN4  br c|djj}tj||dz }|r.tj|tj|dzz Stj|S)zGReturns the maximum difference between PyTorch and OnnxRuntime outputs.rư>)cpurabsamax) torch_outputs ort_outputsrelativeexpected_outputsdiffs r diff_outputszGpt2Helper.diff_outputs/sn)+//1779yy)KN:; ::deii0@&AD&HIJ J::d# #r c tj|d|djj||}tj d||}t |dz }t |D]g}tj|d|z|d|jj||}tj d|d|d||xr|}i|s/tj||} tjd| d |S) zReturns True if torch and ORT outputs are close for given thresholds, and False otherwise. Note: need kwargs since Gpt2BeamSearchHelper.compare_outputs has an extra parameter model_class r)rtolatolz9PyTorch and OnnxRuntime output 0 (last_state) are close: r7zPyTorch and OnnxRuntime layer z state (present_z ) are close:z@PyTorch and OnnxRuntime results are not all close: max_abs_diff=.5f) rallcloserloggerdebugr=r>rtrinfo) rrrrkwargsis_close is_all_close num_layerslayer max_abs_diffs rcompare_outputszGpt2Helper.compare_outputs9s >>+a.-2B2F2F2H2N2N2PW[bfg PQYPZ[\ %) :& 5E~~AI&a '++-335 H LL9%@PQVPWWcdlcmn o'4HL 5%22=+NL KKZ[ghkZlm nr cd}d}g}g}tt|D]}||}|dk(r|dn |d|dz jj} tj|| |d} |j tj tj| |z |xr| }tj| jrtjd|dtj| jrtjd|dtj|jrtjd |dtj|jrtjd |dtj|| z } tj| j| j } |j d | | d d | d || d dt#| | d |dk(stjtj|d|j } tjtj| d| j }tj$| |}|j't)|}|t)||||fS)aCompare outputs from PyTorch and OnnxRuntime Args: torch_outputs (Tuple[Torch.Tensor]): PyTorch model output ort_outputs (List[numpy.ndarray]): OnnxRuntime output atol (float, optional): Absolute tollerance. Defaults to 1e-06. Returns: is_all_close(bool): whether all elements are close. max_abs_diff(float): maximum absolute difference. messages(str): a list of debug message for each output TFrr7)rrzPyTorch output z has nanz has infz ORT output zdiff=z.9fz index=z ort=z torch=N)axis)r>r=rrrr?rrisnananyrrisinffabs unravel_indexargmaxrfloat array_equalindexmax)rrrris_top1_matched max_diffsmessagesrF ort_output torch_outputrridx ort_max_indextorch_max_indexmax_diff_output_indexs rcompare_outputs_v2zGpt2Helper.compare_outputs_v2Tsg  s;'( TA$QJ01QM!,M!rr?rbr=rrr&rPrparentmkdirtempfileTemporaryDirectoryospathjoinr r;onnx load_modelrsave)modelr{rrrr|r}r~rrrrD dummy_inputsrdoutputsrF past_names present_namesrrrr tmp_dir_nametemp_onnx_model_paths r export_onnxzGpt2Helper.export_onnxs#\\NN !22!" & : :**((-1+1!53 "))+ ]]_ )Z(G ),1+;f\EVEVWXEYE_E_D``hipqrisiyiyhz{DELMNEOPQEREXEXDY Z  _$$**4$*G #,,. ,')ww||L+'N$)*1177t7T!z**"& +!-!-"$(,-1# (L$ L14 L6%BL;$L.;Mrc td} t|d||d| d} |r5|rtj| nd|vrd|d<| jdddi|| j ||| S) zHOptimize ONNX model with an option to convert it to use mixed precision.rrF) model_type num_headsry opt_leveloptimization_optionsuse_gpu keep_io_typesuse_symbolic_shape_inferTrh)rr rtauto_mixed_precisionconvert_float_to_float16save_model_to_file) roptimized_model_pathrrxryrrstagerrms r optimize_onnxzGpt2Helper.optimize_onnxs -V4  )#!5  #//2"&0.3F?+***SDSFS 13KLr )AddLayerNormalizationSkipLayerNormalizationFastGeluEmbedLayerNormalization onnx_model op_block_listcn|jDchc]}|j}}t|}|j|}tj d|d||j jdj}d}|j}||vsJ||}d} |jdk(r|} tj d|jd} |jD]} |j| } | nt| } tjd|jd | | d k}n/tjd |jd |jg} g}|s| |g} | jg}| |||d }tj d||jdddi||Scc}w)a?Convert GPT-2 model to mixed precision. It detects whether original model has fp16 weights, and set parameters for float16 conversion automatically. Args: onnx_model (OnnxModel): optimized ONNX model op_block_list (List[str], optional): operators to compute in fp32. Defaults to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu", "EmbedLayerNormalization"] Returns: parameters(dict): a dictionary of parameters used in float16 conversion z fp32 op: z fp16 op: rFNMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node : rz-Failed to find MatMul node for logits. Found z of node )rr"node_block_listforce_fp16_initializersz!auto_mixed_precision parameters: rTrh)nodesop_typeset differencerrgraphoutputroutput_name_to_nodeinputget_initializerrrwarningr)r!r"node op_full_set fp32_op_set fp16_op_setlogits_output_nameis_weight_fp16_precisionr.last_matmul_node initializerr/max_diffrr& parameterss rrzGpt2Helper.auto_mixed_precision&s(1;0@0@0BCt||C C-( !,,[9  i }J{mDE(--/66q9>>$) (<<>!%8888"#56 <<8 ##  KK=dii[I JK (88? * 1=H LLNtyykY[\d[ef g'/$ $ NNJ4<<.Xabfbkbkalm n (/?/K/0M/445O+*.'?    7 |DE+ ++XTXZXaDsF2inputs total_runsc`tjd|jj}t j 5||}ddd|dk(rSg}t j 5t |D]A}tj}||}|jtj|z C dddt|dzt|z }tjdjt|d|fS#1swYxYw#1swYaxYw)zfRun inference of PyTorch model, and returns average latency in ms when total_runs > 0 besides outputs.zstart pytorch_inferenceNrzPyTorch inference time = {} ms.2f) rrrrrer@rr>timer?sumr=format) rr<r=rdrlatencyrstartaverage_latencys rpytorch_inferencezGpt2Helper.pytorch_inferencels  ./^^%--/ ]]_ )Z(G ) ?N ]]_ 4:& 4 ,tyy{U23 4 4 g,-G < 5<|j>tj|jj j|d<|j>tj|jj j|d<|jd|}|dk(r|Sg}t|D]N}tj} |jd|}|jtj| z Pt|dzt|z } tjd j!t!| d || fS) zcRun inference of ONNX model, and returns average latency in ms when total_runs > 0 besides outputs.zstart onnxruntime_inferencer&NrrKrJrr?z"OnnxRuntime Inference time = {} msr@)rrrascontiguousarrayr&rrP enumeraterKrJrunr>rAr?rBr=rC) ort_sessionr<r= ort_inputsrFpast_irrDrrErFs ronnxruntime_inferencez Gpt2Helper.onnxruntime_inferences  23!5#:#:6;K;K;O;O;Q;W;W;Y#Z[ ;; "&v{{3 X 6*/*A*A&**,BTBTBV*W U1#;' X  ,+0+B+B6CXCXC\C\C^CdCdCf+gJ' (    *).)@)@ATATAXAXAZA`A`Ab)cJ~ &!oodJ7 ? z" 0AIIKE%//$ ;K NN499;. / 0 g,-G < 9@@Y^A_`aO++r c 8tj|||||||S)z)Returnas IO binding object for a session.)rprepare_io_binding)rLr&rJrKrPrrs rrQzGpt2Helper.prepare_io_bindings,11         r c2tj||||S)z3Copy results to cpu. Returns a list of numpy array.)r"get_outputs_from_io_binding_buffer)rLrr return_numpys rrSz-Gpt2Helper.get_outputs_from_io_binding_buffers AA    r rrrTinclude_copy_output_latencyc htjdtj||j|j |j |j||}|j|tj||||}|dk(r|Sg} t|D]g} tj} |j||rtj||||} | jtj| z it| dzt| z } tjd| || fS)zUInference with IO binding. Returns outputs, and optional latency when total_runs > 0.z*start onnxruntime_inference_with_binded_iorr?z4OnnxRuntime with IO binding inference time = %.2f ms)rrrtrQr&rJrKrPrun_with_iobindingrSr>rAr?rBr=) rLr<rrr=rTrU io_bindingrrDrrErFs r$onnxruntime_inference_with_binded_ioz/Gpt2Helper.onnxruntime_inference_with_binded_ios"  AB 22         ! ! KK    &&z2!CC    ? z" 0AIIKE  * *: 6*AA  NN499;. / 0g,-G < K_]O++r cTtd|dd5}tj||dddtj d|dtd|dd5}tj||dddtj d|dy#1swYkxYw#1swY/xYw)N ort_outputs_.picklewbz$ORT output are saved to ort_outputs_torch_outputs_z(Torch output are saved to torch_outputs_openpickledumprr)rFrrrs r save_outputszGpt2Helper.save_outputss L7+T 2 (a KK Q ' ( :1#WEF N1#W-t 4 * KK q ) * >qcIJ  ( ( * *sBBBB'ctd|dd5}tj||dddtj d|dy#1swY#xYw)N dummy_inputs_r\r]z!inputs are saved to dummy_inputs_r_)rFrrrrs r save_inputszGpt2Helper.save_inputssP M!G,d 3 )q KK a ( ) 7s'BC ) )s A  Ari'r7c|j}tjd|d|d|d|d| d|d d}d }d }d }|r0tj ||||| }tj |||}d }d }g}d g|z}||z}t |D]E}t||z }tjd |}|d k(rd ntjd |} tjd |}!tjd|!d| dtj|!| ||j|j|j|j||| | | | |d}"tj!||"}#|rtj#||"}$n1tj |!| ||| }%tj%||"||%}$tj'|#|$|\}&}'}(})}*t)j*|'s|j-|'|&r|d z }|*r|d z }||xxd z cc<|rr|&sptjd|d|!d| d|d|' t/|)D]>\}}+tjd|d|j1|j2d|+@|st)j*|'s |'d|zkDstj5||"tj7||$|#H|r*dD,cic]},d|,t)j8||,d }-},ndD,cic]},d|,d }-},|d z|z |-d!<|D.cgc] }.|.d z|z  c}.|-d"<|d z|z |-d#<|t;|z d z|z |-d$<tjd%|d&|d'|t;|z d(||d)|zkDr)tjd*t|dz|z d+d,|-Scc},wcc},wcc}.w)-zKGenerate random inputs and compare the results of PyTorch and Onnx Runtime.zRunning parity test (atol=z , test_cases=z, runs=z, use_io_binding=z, model_class=z , is_float16=z) ...r8Nrr7z#Running parity test for batch_size=z past_sequence_length=z...T)r~rrr)rz test_case=z batch_size=z sequence_length=z MaxDiff= z: Name=z, d)2Z_cmax_diff_percentile_rnanrtop1_match_ratetop1_match_rate_per_rundiff_pass_ratenan_ratezParity Test Cases=z ; Passed=z; Nan=z; Top1_Matched=gffffff?zParity is good: passed rate=z.0f%)rrrrtrrr>rrrrrrxryrNrzrGrOrYrrrr?rJ get_outputsrrfrc percentiler=)/rLrr{rrrtest_cases_per_runr=use_io_bindingrr|r}r~rrrrenable_pickle_outputrmax_batch_sizemax_past_seq_len max_seq_lenrmax_output_shapespassed_test_casestop1_matched_casesmax_abs_diff_listtop1_matched_cases_per_runtotal_test_casesrFrun_idrwrvrurrrrrrrrrmessagerqrCxs/ r test_parityzGpt2Helper.test_paritys.#\\ (m:J LL5j\AWXlWmmpq &66$**""!! " /#5%9"&7L"!225,GG(>>{LY * < <(# ! )MM~} --g{-N  %;;|,!((6!Q&!"a'"*62a72|  < |;QRfQggxzIyJJST`Sab#,H"5_JAwKK"QCw{/F/F/H/K/P/P.QQST[S\ ]^_$\)BlUX[_U_F_&&q,7'';@GC AJ eu`a&qc*u/?/?@QST/UVY.Z[FBRRA,QC0%7RFR$6$"Orkrh c|j}d}|r0tj| || ||}tj|||}tj | || |j |j |j|j||||| | | }|rtj|||\}}|Stj||||\}}|S)zCGenerate random inputs and measure average latency of Onnx Runtime.N)r~rr) rrtrrrrxryrNrzrOrY)rLrr{rr=rzrr|r}r~rrrurwrvrrrrrrDs rtest_performancezGpt2Helper.test_performances(#\\ &880/6;M(::=&R\]N!22    & &    NN       +1!53 " #99+|U_`JAw  $HH\>=*JAwr c tjddd|j|j|j|j |d|| j }tjj||S)zJIT trace for TorchScript.r7F) rurvrwrxryrDrzr{rnr|r}) rtrrxryrNrzrer@jittrace)rrr{r|r}rds r torchscriptzGpt2Helper.torchscriptss 00!" & : :**nn((-11   ') yyuj11r rawfp32fp16int8c |}tjj|rt|jd}n|j dd|dk7r|d|zz }|r|dz }|rdddd d }d D]}tjj ||||z} tjj| sI||vr/ tj| tjd | |tjd|d| tjj tjj |||dztjj tjj ||dz|dztjj tjj ||dz|dztjj tjj ||d z|dzd Stjj ||dztjj ||dztjj ||dztjj ||dzd S#t$r0} tjd | d | jYd} ~ dd} ~ wwxYw)z=Build a path name for given model based on given attributes.r/r r_past_fp32_fp16_int8rzRemoved the existed directory: zFailed to remove the directory r%NzDirectory for z existed: z.onnxz _fp32.onnxz _fp16.onnxz _int8.onnx)rrisdirrpartssplitrexistsshutilrmtreerrOSErrorstrerror) output_dirmodel_name_or_pathrhas_past new_folderremove_existing model_namesuffixrnew_dires rget_onnx_pathszGpt2Helper.get_onnx_pathssg( 77==+ ,0177;J   S !" % + + # + +J  ' !J '7SF= V '',,z:z@R3RS77>>'*!_4c"MM'2"KK*I'(ST nZL 7)$TU Vww||BGGLLZ$H*W^J^_ GGLLZ'-AB- GGLLZ'-AB- GGLLZ'-AB- "77<< J,@AGGLLZ,-FGGGLLZ,-FGGGLLZ,-FG   - 'c"KK*I'RTUVU_U_T`(abbcs=-J K(%KKN)r )F)MbP?r)r)FFr)r)T)rTF)TT)'r(r)r*r+rQr@int32rr{boolrmr^rr rdictr<rrrrrrr rrrrGrOrQrSTensorrYrcrfrrrrrhr rrtrtsO!%#'',{{*/++,1KK"&>I>I!>I>I! >I  >I  >I>I >I>I>I!>I>I"KK>I$kk>I >I !>I>I@ -  !      c49n   D  $$43 3 j ).!%#'',{{*/++,1KKuu u #' u  u!uu"KKu$kkuun"'"!!F$ CCCyCCJ((((2,:,3,,>  (   !,10,0,S%,,./0,CcN+ 0,  0,  0,&*0,0,dKKDD     %  ;;"[["%EEN %  ;;"[[44l22"-7 : : : r rt)0loggingrrarrrrApathlibrrrr@benchmark_helperrrnrfusion_optionsrio_binding_helperrr!r optimizerr torch_onnx_export_helperr transformersr r r r ImportError getLoggerr(rPRETRAINED_GPT2_MODELSFLOAT32FLOAT16INT8DEFAULT_TOLERANCErr/r4rSrWrr^rtrhr rrs9   &-(- $6??(   8 $Wv s NNCN9N<<"!E)!EHEE$ E/ E"*8T:"=x!O|T2 SS>_  _  iKs$DDD