cQjSddlZddlZddlZddlmZddlmZddlZddl Z ddl Z ddl m Z ddl mZddl mZmZddlmZddlmZdd lmZdd lmZmZmZmZdd lmZej<eZ Gd d e jBjDZ#y)N)chain)Path)convert_float_to_float16)RepeatedCompositeFieldContainer) ModelProtoValueInfoProto) OnnxModel)PastKeyValuesHelper) WhisperConfig)convert_inputs_for_ortget_model_dynamic_axesget_sample_decoder_inputsgroup_past_key_values)InferenceSessioncbeZdZdZd%dedej jdede ffd Z d&dejd ejdzd e e ejdzfd Z d&dejd ejdzd e e ejdzfd Z d&dejd ejdzd e e ejdzfd ZdZdZdZd%de de de fdZd'dede de fdZd%dede fdZdefdZdede fdZ d(dedede d e de de d!e d"e fd#Zdedede de fd$ZxZS))WhisperDecoderz/A Whisper decoder with optional past key valuesconfigmodel model_implno_beam_search_opct|||_|j|_||_||_|dk(rdn|j j|_|dk(rdn |j|_|dk(r|nd|_|jj|_ |jj|_ |jj|jz|_ yNopenai)super__init__rdevicerrrdecoderproj_outmax_source_positionsdecoder_attention_heads num_headsd_model head_size)selfrrrr __class__s |/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_decoder.pyrzWhisperDecoder.__init__%s  ll $!2)X5t5;;;N;N *h 6ENN (H4U$ $(KK$D$D!<<,,>Ndecoder_input_idsencoder_hidden_statespast_key_valuesc|j|||d}|j|j}|j}|||fSt j |\}}||fS)NT)r) input_idsr* use_cache)rrlast_hidden_stater*r group_by_self_and_cross) r$r(r)r*outputslogitspresent_key_values present_self present_crosss r& hf_forwardzWhisperDecoder.hf_forward4s| ,,"7'+   w889$44  "-- - ':&Q&QRd&e# m|##r'c i}|_t|\}}|Dcgc]}|jdd}}|Dcgc]%}|jg|jddd'}}|Dcgc]}|jdd}}|Dcgc]%}|jg|jddd'}}t |j j jD]\}} |d|z|| jj<|d|zdz|| jj<|d|z|| jj<|d|zdz|| jj<|j j\} } |j j |||} ||j j jD]} tj|| jj| | jjgdj| | jj<tj|| jj| | jjgdj| | jj<gg}} |j j jD]} | j!| | jj| j!| | jj|V|j!| | jj|j!| | jj| Dcgc]A}|jg|jddd|j"jddC} }|Dcgc]A}|jg|jddd|j"jddC}}| D]}|j%|)t'j(| |zt+| dz}| |fS| | fScc}wcc}wcc}wcc}wcc}wcc}w)N)xxakv_cache)dim)r transposereshapeshape enumeraterrblocksattnkeyvalue cross_attninstall_kv_cache_hookstorchcatdetachappendr#remover group_by_layerlen)r$r(r)r* past_kv_cacheself_attn_kv_cachescross_attn_kv_cachespast_kvidxblockr<hooksr1r3r4 present_kvhookr2s r& oai_forwardzWhisperDecoder.oai_forwardPsI   &8Mo8^ 5 !5J]"^w7#4#4Q#:"^ "^\o"pQX7??3KW]]2A5F3K3K#L"p "pK_#`G$5$5a$;#` #`]q#rRYGOO4LgmmBQ6G4L4L$M#r #r' (:(:(A(AB Z U0CAG0L ejjnn-2Ea#gPQk2R ejj../6J1s76S e..2238LQQTWWX[8Y e..445  Z**;;=%##&7>2HUZZ^^4LMST,&((.3YY"5::#3#34huzz?O?O6PQWX.&())*  ')"m ZZ''.. GE    8 9   )9)9 : ;&$$Xe.>.>.B.B%CD$$Xe.>.>.D.D%EF G+    J!1!1"1!5JrJ4>>J K U UVWYZ [  ,    J!1!1"1!5JrJ4>>J K U UVWYZ [   D KKM   "!4!C!C},c,.?1.D" -- -|##]#_"p#`#rl  s$Q *Q'Q*Q/AQ;AQ"cl|jdk(r|j|||S|j|||Sr)rrXr5)r$r(r)r*s r&forwardzWhisperDecoder.forwards< ??h &##$57Lo^ ^02GYYr'c |jrddg}|Sddgttjdt |j j D}|S)Nr,r)c3@K|]}d|d|d|d|fyw)past_key_self_past_value_self_past_key_cross_past_value_cross_N.0is r& z-WhisperDecoder.input_names..sC(*!-1A!/EYZX[G\`qrsqt^uv( first_passlistr from_iterablerangerdecoder_layers)r$ input_namess r&rmzWhisperDecoder.input_namessl ??&(?@K' ''(!&t{{'A'A!B( Kr'c 6|jrGdgttjdt |j j D}|Sdgttjdt |j j D}|S)Nr1c3@K|]}d|d|d|d|fyw)present_key_self_present_value_self_present_key_cross_present_value_cross_Nrarbs r&rez.WhisperDecoder.output_names..sD( 0s31!50421#6 (rfc30K|]}d|d|fyw)rprqNrarbs r&rez.WhisperDecoder.output_names..s*(QR,QC04Gs2KL(srg)r$ output_namess r&ruzWhisperDecoder.output_namess ?? ''("'t{{'A'A!B(  L.''(V[\`\g\g\v\vVw(Lr'c`t|j||}d|vr|js|dd=|S)Nr,r7)r rr)r$rmru dynamic_axess r&rwzWhisperDecoder.dynamic_axess7-dkk; U , &t/E/E[)!,r'use_fp16_inputsuse_int32_inputs return_dictc t|j|jd|jrdnd|jrdnd||}|r|jr|d=|S|jr |d|dfS|d|d|dfS) Nr8rr7) batch_sizepast_sequence_lengthsequence_lengthuse_fp16 use_int32r*r(r))rrrrh)r$rxryrzinputss r&rzWhisperDecoder.inputss* KK KK'+!A"&//Qq$&  ,-M ??*+./  & ' * + $ %  r'iois_cross is_outputcJ|jjjjd}d|jvr!|j |j |_|jjjjd}d|jvr/|j |r|j|_n |rdnd|_|jjjjd}d|jvr!|j |j|_|S)Nr7_dim_r8total_sequence_lengthr~) type tensor_typer@r= dim_paramClearr! dim_valuerr#)r$rrrr!rr#s r&fix_key_value_cache_dimsz'WhisperDecoder.fix_key_value_cache_dimssGG''--11!4 i)) ) OO "&..I ''--3377: o// /  ! ! #,0,E,E)GP,CVl)GG''--11!4 i)) ) OO "&..I  r'io_listcg}g}g}|D]}d|jvr d|jvr|j|1d|jvrD|j|d|}|jr|j|q|j||j|d|}|jr|j||j||js|||zz }|S)Npastpresentr$F)rrT)namerKrr)r$rr reordered_iorPrQrnew_ios r&fix_iozWhisperDecoder.fix_ios  ! 8BRWW$"'')A##B'277"66rEU^6_)) ''/'..v666rDT]6^)) ''/(//7! 8$%% /2FF FLr'c|j|jjd}t|jjdkDrG|jjj t|jjdkDrG|jjj ||j|jj d}t|jj dkDrG|jj j t|jj dkDrG|jj j ||S)NF)rrT)rgraphinputrNpopextendoutput)r$rreordered_inputsreordered_outputss r&fix_inputs_and_outputsz%WhisperDecoder.fix_inputs_and_outputs0s ;;u{{'8'8E;J%++##$q( KK   ! ! #%++##$q(   !12 KK (:(:dKK%++$$%) KK   " " $%++$$%) !!"34 r'c>|jdk(r |r t|}|Sr)rr)r$rrxs r&fix_layernorm_weightsz$WhisperDecoder.fix_layernorm_weights>s! ??h &?-U3E r'onnx_model_pathproviderverboseuse_external_data_formatuse_encoder_hidden_statesuse_kv_cache_inputsc T|xr| |_| xr||_|js|jsJd|j||} |j} |j } |j | | } t |jjddtj5} tjj| d}t |jjdd|r|n|}tjj!|| |d| | | ddd| tj"|| }|j%|}|j'||}t)j*|||d d d d |j-||||y #1swYxYw) alExport decoder to ONNX Args: onnx_model_path (str): path to save ONNX model provider (str): provider to use for verifying parity on ONNX model verbose (bool, optional): print verbose information. Defaults to True. use_external_data_format (bool, optional): use external data format or not. Defaults to False. use_fp16_inputs (bool, optional): use float16 inputs for the KV caches. Defaults to False. use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids. Defaults to True. use_encoder_hidden_states (bool, optional): use encoder_hidden_states as model input for decoder-init/decoder-without-past models. Defaults to False. use_kv_cache_inputs (bool, optional): use KV caches as model inputs for decoder-with-past models. Defaults to True. zVOnly one of `use_encoder_hidden_states` and `use_kv_cache_inputs` can be true at once.)rxryT)parentsexist_okz decoder.onnxF) argsf export_paramsrmrurw opset_versiondo_constant_foldingdynamor)load_external_data)save_as_external_dataall_tensors_to_one_fileN)rh later_passrrmrurwrparentmkdirtempfileTemporaryDirectoryospathjoinrHonnxexport load_modelrrr save verify_onnx)r$rrrrrxryrrrrmrurw tmp_dir_nametemp_onnx_model_pathout_pathrs r& export_onnxzWhisperDecoder.export_onnxJsH4O(,  - : (OEUV;  s 7C FF'c|j||d}g}|jr|jd i|}|j|dj j j |dD]D}|D]=} |j| j j j ?Fn|jd i|}|j|dj j j |dD]=} |j| j j j ?t||g} | jdt|| } t|jD]g\} }tj|| | | z }tjd|dtjd tj |iy#YyxYw) awVerify ONNX model outputs and PyTorch model outputs match Args: onnx_model_path (str): path to save ONNX model provider (str): execution provider for ONNX model use_fp16_inputs (bool, optional): use float16 inputs for the KV caches use_int32_inputs (bool, optional): use int32 inputs for the decoder_input_ids T)rxryrzrr7) providersNz Comparing z...z Max diff: ra)rrhrZrKrJcpunumpyrrunr rArunpabsloggerwarningmax)r$rrrxryr pt_outputsoutpresent_key_value_layerpresent_key_valuepresent_self_key_valuesess ort_outputsrd output_namediffs r&rzWhisperDecoder.verify_onnxs8_O_mqr ??$,,((C   c!fmmo11399; <+.q6 P')@P%%%&7&>&>&@&D&D&F&L&L&NOP P$,,((C   c!fmmo11399; <*-a& Q&!!"8"?"?"A"E"E"G"M"M"OP Q H:Fhht%;FD%IJ  "+D,=,=,?"@ <;vvjmk!n<=K=<=BFF4L>:; <  s BHH)F)NN)FF)TFFTFT)__name__ __module__ __qualname____doc__r rHnnModulestrboolrTensorritupler5rXrZrmrurwrrrrrrrrrr __classcell__)r%s@r&rr"s29 ?} ?UXX__ ?RU ?jn ?$6:<@ $ <<$ %||d2$eELL12T9 $>6:<@ X$ <<X$ %||d2X$eELL12T9 X$z6:<@ Z <<Z %||d2ZeELL12T9 Z 6 d d QU 4>T^b(=$8 J  :   ). %!%*/$(PWPWPW PW #' PW  PWPW$(PW"PWd555 5  5r'r)$loggingrr itertoolsrpathlibrrrrrHfloat16r#google.protobuf.internal.containersrrr onnx_modelr past_helperr transformersr whisper_inputsr r rr onnxruntimer getLoggerrrrrrrar'r&rsh  ,O+ +&)   8 $oUXX__or'