Ë
    cQêjä3  ã                   óh   — d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	  ee
«      Zd„ Z G d„ de«      Zy)	é    )Ú	getLoggerN)ÚFusionGptAttentionPastBase)Úhelper)Ú	OnnxModelc                 ó$   — t        | |z
  «      dk  S )Ng�íµ ÷Æ°>)Úabs)ÚvalueÚexpected_values     ú{/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_gpt_attention_megatron.pyÚis_closer      s   € Üˆu�~Ñ%Ó&¨$Ñ.Ð.ó    c                   ó>   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zd„ Zd„ Z	ˆ xZ
S )ÚFusionGptAttentionMegatronz^
    Fuse GPT-2 Attention with past state subgraph from Megatron into one Attention node.
    ÚmodelÚ	num_headsc                 ó&   •— t         ‰| �  ||«       y )N)ÚsuperÚ__init__)Úselfr   r   Ú	__class__s      €r   r   z#FusionGptAttentionMegatron.__init__   s   ø€ Ü‰Ñ˜ 	Õ*r   c                 óR  — | j                   j                  d«      }| j                  |«      }	|j                  d   }
|j                  d   |j                  d   k(  rdnd}t        j                  d||j                  d   |j                  |   |	|g|
|g|¬«      }d|_        |j                  j                  t        j                  d| j                  «      t        j                  dd«      g«       | j                  �C|j                  j                  t        j                  d	t        | j                  «      «      g«       |g}| j                  j                  |«       |D ]%  }| j                  | j                   |j"                  <   Œ' | j$                  j'                  |«       d
| _        y )NÚGptAttentionr   é   Ú	Attention)ÚinputsÚoutputsÚnamezcom.microsoftr   ÚunidirectionalÚmask_filter_valueT)r   Úcreate_node_nameÚcast_attention_maskÚoutputÚinputr   Ú	make_nodeÚdomainÚ	attributeÚextendÚmake_attributer   r   ÚfloatÚnodes_to_addÚthis_graph_nameÚnode_name_to_graph_namer   Únodes_to_removeÚappendÚprune_graph)r   Úmatmul_before_splitÚadd_before_splitÚpastÚpresentr#   Úreshape_qkvÚmaskÚattention_node_nameÚ
int32_maskr"   ÚiÚattention_noder*   Únodes                  r   Úfuse_attention_nodez.FusionGptAttentionMegatron.fuse_attention_node   s˜  € ð #Ÿj™j×9Ñ9¸.ÓIÐØ×-Ñ-¨dÓ3ˆ
Ø×#Ñ# AÑ&ˆØ"×(Ñ(¨Ñ+Ð/B×/IÑ/IÈ!Ñ/LÒL‰AÐSTˆÜ×)Ñ)ØàØ#×)Ñ)¨!Ñ,Ø ×&Ñ& qÑ)ØØðð ˜WÐ%Ø$ô
ˆð !0ˆÔØ× Ñ ×'Ñ'ä×%Ñ% k°4·>±>ÓBÜ×%Ñ%Ð&6¸Ó:ðô	
ð ×!Ñ!Ð-Ø×$Ñ$×+Ñ+¬V×-BÑ-BÐCVÔX]Ð^b×^tÑ^tÓXuÓ-vÐ,wÔxà&Ð'ˆØ×Ñ× Ñ  Ô.à ò 	KˆDØ6:×6JÑ6JˆD×(Ñ(¨¯©Ò3ð	Kð 	×Ñ×#Ñ# KÔ0ð  ˆÕr   c                 óD  — | j                   j                  |g d¢g d¢«      }|€t        j                  d«       y |\  }}}}	t	        |«      dkD  r@|d   j
                  dk(  r.| j                   j                  |d   «      \  }
}|dk7  r| | _        |j                  d   |j                  d   k7  rt        j                  d«       y | j                  j                  |dd	«      st        j                  d
«       y | j                  j                  |dd«      st        j                  d«       y | j                   j                  |	j                  d   «      st        j                  d«       y | j                  j                  |ddg«      st        j                  d«       y | j                  j                  |ddg«      st        j                  d«       y| j                  j                  |ddg«      st        j                  d«       y| j                  j                  |	ddg«      st        j                  d«       y | j                  j                  |	ddg«      st        j                  d«       y | j                   j                  |g d¢g d¢«      }|�|d   |k7  rt        j                  d«       y | j                   j                  |	g d¢g d¢«      }|�|d   |k7  rt        j                  d«       y | j                   j                  |	g d¢g d¢«      }|�|d   |k7  rt        j                  d«       y | j                   j                  |	g d¢g d ¢«      }|€!| j                   j                  |	g d!¢g d ¢«      }|�|d   |k7  rt        j                  d"«       y |	j                  d   S )#N)ÚMulÚSubÚSlicer?   )r   r   r   r   z8fuse_attention: failed to match unidirectional mask pathr   r   r=   i'  zCfuse_attention failed: mul_qk.input[1] != last_slice_mask.output[0]g     ˆÃ@z?fuse_attention failed: mul_mask input 1 is not constant 10000.0g      ð?z;fuse_attention failed: sub_mask input 0 is not constant 1.0z+expect slick_mask input 0 to be graph inputzKfuse_attention failed: last_slice_mask input 1 (starts) is not constant [0]é   zIfuse_attention failed: last_slice_mask input 3 (axes) is not constant [3]Fé   zJfuse_attention failed: last_slice_mask input 4 (steps) is not constant [1]é   zDfuse_attention failed: slice_mask input 3 (axes) is not constant [2]zEfuse_attention failed: slice_mask input 4 (steps) is not constant [1])Ú	UnsqueezeÚGatherÚShapeÚMatMul)rB   r   r   r   éÿÿÿÿz/fuse_attention: failed to match last slice pathz0fuse_attention: failed to match first slice path)rC   r>   rD   rE   rF   )r   r   r   r   r   z3fuse_attention: failed to match last slice sub path)rC   r>   rD   rE   ÚLayerNormalization)r   r   r   r   r   )rC   r>   rD   rE   ÚSkipLayerNormalizationz5fuse_attention: failed to match last slice sub path 1)r   Úmatch_parent_pathÚloggerÚdebugÚlenÚop_typeÚget_constant_inputr   r#   r"   ÚutilsÚcheck_node_input_valueÚfind_graph_inputÚinfo)r   Úsub_qkÚmul_qkÚ	matmul_qkÚlayernorm_before_attentionÚ
mask_nodesÚmul_maskÚsub_maskÚlast_slice_maskÚ
slice_maskÚ_Úmul_valÚlast_slice_pathÚfirst_slice_pathÚfirst_slice_subÚfirst_slice_sub_1s                   r   Ú
match_maskz%FusionGptAttentionMegatron.match_maskJ   s5  € Ø—Z‘Z×1Ñ1°&Ò:ZÒ\hÓiˆ
ØÐÜ�L‰LÐSÔTØØ<FÑ9ˆ�8˜_¨jäˆz‹?˜QÒ :¨a¡=×#8Ñ#8¸EÒ#AØŸ™×6Ñ6°zÀ!±}ÓE‰JˆAˆwØ˜%ÒØ*1¨�Ô&à�<‰<˜‰?˜o×4Ñ4°QÑ7Ò7Ü�L‰LÐ^Ô_Øà�z‰z×0Ñ0°¸1¸gÔFÜ�L‰LÐZÔ[Øà�z‰z×0Ñ0°¸1¸cÔBÜ�L‰LÐVÔWØà�z‰z×*Ñ*¨:×+;Ñ+;¸AÑ+>Ô?Ü�K‰KÐEÔFØà�z‰z×0Ñ0°À!ÀaÀSÔIÜ�L‰LÐfÔgØà�z‰z×0Ñ0°À!ÀaÀSÔIÜ�L‰LÐdÔeØà�z‰z×0Ñ0°À!ÀaÀSÔIÜ�L‰LÐeÔfØà�z‰z×0Ñ0°¸QÀÀÔDÜ�L‰LÐ_Ô`Øà�z‰z×0Ñ0°¸QÀÀÔDÜ�L‰LÐ`ÔaØàŸ*™*×6Ñ6ØÒGÊó
ˆð Ð" o°bÑ&9¸YÒ&FÜ�L‰LÐJÔKØàŸ:™:×7Ñ7ØÒBÂLó
Ðð Ð#Ð'7¸Ñ';¸yÒ'HÜ�L‰LÐKÔLØàŸ*™*×6Ñ6ØÚ=Úó
ˆð
 Ð" o°bÑ&9¸YÒ&FÜ�L‰LÐNÔOØà ŸJ™J×8Ñ8ØÚIÚó
Ðð Ð$Ø $§
¡
× <Ñ <ØÚQÚó!Ðð Ð$Ð(9¸"Ñ(=ÐA[Ò([Ü�L‰LÐPÔQØà×Ñ Ñ"Ð"r   c           	      ó
  — d }d }|j                   dk(  }d }|s$| j                  j                  |g d¢g d¢|¬«      }n#| j                  j                  |g d¢g d¢|¬«      }|€y d }|s|\  }	}
}}}}|	j                  d   }n|\  }
}}}}|j                  d   }| j                  j                  |g d¢g d	¢«      }|€!| j                  j                  |g d
¢g d	¢«      }|€t        j                  d«       y |\  }}}}}}}|j                   dk(  r(||j                  d   k7  rt        j                  d«       y |j                   dk(  r(||j                  d   k7  rt        j                  d«       y | j                  j                  |g d¢g d¢«      }|€t        j                  d«       y |\  }}}}| j                  j                  |d«      dk7  rt        j                  d«       y | j                  ||||«      }| j                  j                  |g d¢g d¢«      }|€t        j                  d«       y |\  }}} }!||!k7  rt        j                  d«       y | j                  j                  |g d¢g d¢«      }"|"€t        j                  d«       y |"\  }#}$}%}&}'}(||(k7  rt        j                  d«       y | j                  j                  |'«      \  })}*t        |*t        j                  «      r9t        |*j                  «      dgk(  r |*d   dk(  r|*d   dk(  r|*d   dkD  r|*d   dkD  st        j                  d«       y |*d   }+|+| j                  k7  r,t        j!                  d|+› d | j                  › �«       |+| _        |*d   },| j                  j                  |#«      \  })}*t#        t        j$                  t        j$                  |,«      «      «      }-t'        |*|-«      st        j                  d!|*› d"|-› �«       y | j                  j                  |«      \  })}*t'        |*|-«      st        j                  d#|*› d"|-› �«       y | j)                  |%||«      }|€t        j                  d$«       y | j                  j+                  |«      st        j                  d%«       | j-                  ||«      }|€t        j                  d&«       y | j                  j/                  |«      st        j!                  d'«       y | j1                  |||||j                  d   ||«       y )(NrI   )ÚAddre   rF   ÚReshapeÚ	TransposerF   )r   r   Nr   r   r   )Úoutput_name_to_node)re   rF   rf   rg   rF   )r   Nr   r   r   r   )ÚConcatrg   rf   ÚSplitre   rF   rH   )r   r   r   r   r   Nr   )ri   rg   rf   rj   re   rF   rI   z&fuse_attention: failed to match v pathrH   zAfuse_attention: skip_input != layernorm_before_attention.input[0]r@   )ÚSoftmaxr>   r=   rF   )r   r   r   r   z'fuse_attention: failed to match qk pathÚaxisz+fuse_attention failed: softmax_qk axis != 3)ÚDivrg   rf   rj   z&fuse_attention: failed to match q pathz-fuse_attention: skip since split_v != split_q)rm   rg   ri   rg   rf   rj   )r   r   r   r   r   r   z&fuse_attention: failed to match k pathz-fuse_attention: skip since split_v != split_krA   r   rB   z:fuse_attention: reshape constant input is not [0, 0, N, H]zDetected num_heads=z. Ignore user specified value zfuse_attention: div_k value=z
 expected=zfuse_attention: div_q value=z!fuse_attention: match past failedz(fuse_attention: past is not graph input.z$fuse_attention: match present failedz1fuse_attention: expect present to be graph output)rN   r   rJ   r#   rK   rL   r"   Úget_node_attributerc   rO   Ú
isinstanceÚnpÚndarrayÚlistÚshaper   rS   r)   Úsqrtr   Úmatch_past_pattern_2rR   Úmatch_presentÚfind_graph_outputr;   ).r   Únormalize_nodeÚinput_name_to_nodesrh   r2   r3   Úis_normalize_node_skiplayernormÚ	qkv_nodesÚ
skip_inputÚadd_skipÚadd_after_attentionÚmatmul_after_attentionr4   Útranspose_qkvÚ
matmul_qkvÚv_nodesÚconcat_vÚtranspose_vÚ	reshape_vÚsplit_vr1   r0   rW   Úqk_nodesÚ
softmax_qkrT   rU   rV   Úattention_maskÚq_nodesÚdiv_qÚtranspose_qÚ	reshape_qÚsplit_qÚk_nodesÚdiv_kr]   Úconcat_kÚtranspose_kÚ	reshape_kÚsplit_kr8   r	   r   Úhidden_size_per_headr
   s.                                                 r   ÚfusezFusionGptAttentionMegatron.fuse¤   s  € ØˆØˆà*8×*@Ñ*@ÐD\Ñ*\Ð'Øˆ	á.ØŸ
™
×4Ñ4ØÚJÚ%Ø$7ð	 5ó ‰Ið Ÿ
™
×4Ñ4ØÚCÚ"Ø$7ð	 5ó ˆIð ÐØàˆ
Ù.ð ñØØ#Ø&ØØØð "Ÿ™¨Ñ*‰Jð ñØ#Ø&ØØØð (×-Ñ-¨aÑ0ˆJà—*‘*×.Ñ.Øòò %ó
ˆð ˆ?Ø—j‘j×2Ñ2Øòò )óˆGð ˆ?Ü�L‰LÐAÔBØð ñ	
ØØØØØØØ&ð '×.Ñ.Ð2FÒFØÐ8×>Ñ>¸qÑAÒAä�L‰LÐ\Ô]Øð '×.Ñ.Ð2JÒJØÐ8×?Ñ?ÀÑBÒBä�L‰LÐ\Ô]Øà—:‘:×/Ñ/°
Ò<_ÒamÓnˆØÐÜ�L‰LÐBÔCØØ2:Ñ/ˆ�V˜V YØ�:‰:×(Ñ(¨°VÓ<ÀÒAÜ�L‰LÐFÔGØàŸ™¨°¸ÐD^Ó_ˆà—*‘*×.Ñ.¨yÒ:bÒdpÓqˆØˆ?Ü�L‰LÐAÔBØØ3:Ñ0ˆ�˜Y¨Ø�gÒÜ�L‰LÐHÔIØà—*‘*×.Ñ.ØÚKÚó
ˆð
 ˆ?Ü�L‰LÐAÔBØØ@GÑ=ˆ��8˜[¨)°WØ�gÒÜ�L‰LÐHÔIØà—:‘:×0Ñ0°Ó;‰ˆˆ5ä�uœbŸj™jÔ)Ü�U—[‘[Ó! a SÒ(Ø�a‘˜A’Ø�a‘˜A’Ø�a‘˜1’Ø�a‘˜1’ä�L‰LÐUÔVØà˜!‘Hˆ	Ø˜Ÿ™Ò&Ü�K‰KÐ-¨i¨[Ð8VÐW[×WeÑWeÐVfÐgÔhØ&ˆDŒNà$ Q™xÐØ—:‘:×0Ñ0°Ó7‰ˆˆ5ÜœrŸw™w¤r§w¡wÐ/CÓ'DÓEÓFˆÜ˜˜~Ô.Ü�L‰LÐ7¸°w¸jÈÐHXÐYÔZØà—:‘:×0Ñ0°Ó7‰ˆˆ5Ü˜˜~Ô.Ü�L‰LÐ7¸°w¸jÈÐHXÐYÔZØð ×(Ñ(¨°8Ð=PÓQˆØˆ<Ü�L‰LÐ<Ô=ØØ�z‰z×*Ñ*¨4Ô0Ü�L‰LÐCÔDð ×$Ñ$ XÐ/BÓCˆØˆ?Ü�L‰LÐ?Ô@ØØ�z‰z×+Ñ+¨GÔ4Ü�K‰KÐKÔLØà× Ñ ØØØØØ&×-Ñ-¨aÑ0ØØõ	
r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   r;   rc   r–   Ú__classcell__)r   s   @r   r   r      s.   ø„ ñð+˜ið +°Cõ +ò- ò^X#öt
r   r   )Úloggingr   Únumpyrp   Úfusion_gpt_attentionr   Úonnxr   Ú
onnx_modelr   r—   rK   r   r   © r   r   ú<module>r£      s4   ðõ
 ã Ý ;Ý Ý  á	�8Ó	€ò/ôP
Ð!;õ P
r   