Ë
    cQêjhH  ã                   óf   — d dl Z d dlZd dlmZ d dlmZ  e j
                  e«      Z G d„ de«      Zy)é    N)Únumpy_helper)ÚBertOnnxModelTFc                   ó`   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zd„ Zˆ xZS )ÚBertOnnxModelKerasc                 ó(   •— t         ‰| �  |||«       y ©N)ÚsuperÚ__init__)ÚselfÚmodelÚ	num_headsÚhidden_sizeÚ	__class__s       €ús/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/onnx_model_bert_keras.pyr
   zBertOnnxModelKeras.__init__   s   ø€ Ü‰Ñ˜ 	¨;Õ7ó    c                 ó    — | j                  |g d¢g d¢«      }|�|S | j                  |g d¢g d¢«      }|�|S | j                  |g d¢g d¢«      }|S )N)ÚMulÚSubÚReshapeÚCast)é   Nr   r   )r   r   r   ÚSliceÚ	Unsqueeze)r   r   r   r   r   )r   r   r   r   r   )r   Nr   r   r   )Úmatch_parent_path)r   Úadd_or_sub_before_softmaxÚ
mask_nodess      r   Úmatch_mask_pathz"BertOnnxModelKeras.match_mask_path   sq   € Ø×+Ñ+Ø%Ú-Úó
ˆ
ð
 Ð!ØÐà×+Ñ+Ø%Ú8Úó
ˆ
ð
 Ð!ØÐà×+Ñ+Ø%Ú<Úó
ˆ
ð
 Ðr   c                 ó0  — g }|||fD ]‰  }|j                   d   }||   }	|	|k(  rŒ|	j                  dk(  r1|	j                   d   |j                  d   k(  r|j                  |	«       Œ]t        j                  d|› d|j                  d   › �«       dg fc S  d|fS )Nr   r   zCheck attention input failed:z, FT)ÚinputÚop_typeÚoutputÚappendÚloggerÚdebug)
r   Úmatmul_qÚmatmul_kÚmatmul_vÚparentÚoutput_name_to_nodeÚreshape_nodesÚxÚ
root_inputÚ	root_nodes
             r   Úcheck_attention_inputz(BertOnnxModelKeras.check_attention_input+   s´   € Øˆà˜H hÐ/ò 		ˆAØŸ™ ™ˆJØ+¨JÑ7ˆIØ˜FÒ"ØØ× Ñ  IÒ-°)·/±/À!Ñ2DÈÏÉÐVWÑHXÒ2XØ×$Ñ$ YÔ/ØÜ�L‰LÐ8¸¸ÀBÀvÇ}Á}ÐUVÑGWÐFXÐYÔZØ˜"�9Òð		ð �]Ð"Ð"r   c                 ó*	  — | j                  «        | j                  «       }g }d}| j                  d«      }|D �]  }| j                  |d«      }|�|j                  dvr|j                  dk(  rI| j                  |d«      }|�|j                  dvrOt
        j                  d|�|j                  nd › �«       Œ~t
        j                  d|�|j                  nd › �«       Œ¥	 | j                  |g d¢g d¢«      }|€t
        j                  d	«       ŒÕ|\  }}	}
}}}t
        j                  d
«       | j                  |g d¢g d¢«      }|€t
        j                  d«       �Œ#|\  }}}}}| j                  |g d¢g d¢«      }|�)|\  }}}| j                  |g d¢g d¢«      }|�{|\  }}}}}}nq| j                  |g d¢g d¢«      }|€0| j                  |g d¢g d¢«      }|€t
        j                  d«       �Œ¶|\  }}} }| j                  |g d¢g d¢«      }|�|\  }}}}}|€t
        j                  d«       �Œ÷| j                  |g d¢g d¢«      }!|!€t
        j                  d«       �Œ'|!\  }"}#}$}%}&| j                  |d   «      }'|'€t
        j                  d«       �Œ\| j                  |'d   d«      st
        j                  d«       �Œˆ| j                  |&|||«      \  }(})|(�rh| j                  j                  |'d   j                  d   «      }*t
        j                  d«       | j                  j                  |*||&||$|| j                   | j"                  |j$                  d   |j$                  d   ¬«      }+|+€�Œ;| j'                  |+«       |dz  }|j)                  |||g«       |j)                  |«       |j)                  |«       |j)                  |!«       |j)                  |«       |j)                  |'«       |j)                  |)«       |j+                  |	«       | j-                  ||	j$                  d   |
j$                  d   «       �Œt
        j                  d«       �Œ" | j/                  |«       | j1                  «        t
        j3                  d|› �«       y ) Nr   ÚSkipLayerNormalization)r0   ÚEmbedLayerNormalizationÚAddr   zFirst input for skiplayernorm: )r2   r   ÚMatMulr   Ú	Transposer3   )Nr   r   r   r   r   zFailed to match qkv nodeszMatched qkv nodes)r4   r   r2   r   r3   )r   r   r   r   r   zFailed to match v path)ÚSoftmaxr   r3   ©r   r   r   )r   r4   r   r2   r   r3   )r   Nr   r   r   r   )r5   r2   r   r3   )r   r   r   N)r5   r2   ÚDivr3   zFailed to match qk path)r   r   r   r   r   zFailed to match q pathzFailed to match k pathzFailed to match mask pathz;Sub node expected to have an input with constant value 1.0.éÿÿÿÿzCreate an Attention node.)Ú
mask_indexÚq_matmulÚk_matmulÚv_matmulÚq_addÚk_addÚv_addr   r   Úfirst_inputr!   zRoot node not matched.zFused Attention count:)Úinput_name_to_nodesr)   Úget_nodes_by_op_typeÚ
get_parentr    r#   r$   r   r   Úhas_constant_inputr.   Úattention_maskÚprocess_maskr   Úattention_fusionÚcreate_attention_noder   r   r!   Úadd_nodeÚextendr"   Úreplace_node_inputÚremove_nodesÚupdate_graphÚinfo),r   r)   Únodes_to_removeÚattention_countÚskip_layer_norm_nodesÚnormalize_noder(   Ú	qkv_nodesÚaddÚextra_reshape_0ÚmatmulÚreshape_qkvÚtranspose_qkvÚ
matmul_qkvÚv_nodesÚtranspose_vÚ	reshape_vÚadd_vÚextra_reshape_1r'   Úqk_nodesÚ
softmax_qkÚsub_qkÚ	matmul_qkÚq_nodesÚmul_qÚtranspose_qÚ	reshape_qÚadd_qÚextra_reshape_2r%   Úadd_qkÚmul_qkÚk_nodesÚtranspose_kÚ	reshape_kÚadd_kÚextra_reshape_3r&   r   Úis_same_rootr*   r9   Úattention_nodes,                                               r   Úfuse_attentionz!BertOnnxModelKeras.fuse_attention;   sÕ  € Ø× Ñ Ô"Ø"×6Ñ6Ó8ÐàˆØˆà $× 9Ñ 9Ð:RÓ SÐØ3ó O	ˆNà—_‘_ ^°QÓ7ˆFØˆ~ §¡ð 8ñ "ð —>‘> UÒ*Ø!Ÿ_™_¨^¸QÓ?�FØ�~¨¯©ð @ñ *ô Ÿ™Ð'FÐY_ÐYkÀvÇ~Â~ÐquÐFvÐ%wÔxØ ä—L‘LÐ#BÐU[ÐUgÀ6Ç>Â>ÐmqÐBrÐ!sÔtØð à×.Ñ.ØÚNÚ%óˆIð
 Ð Ü—‘Ð8Ô9Øð ñØØØØØØä�L‰LÐ,Ô-à×,Ñ,ØÚDÚóˆGð
 ˆÜ—‘Ð5Ô6ÙØIPÑFˆ[˜) U¨O¸Xà×-Ñ-¨jÒ:VÒXaÓbˆHØÐ#Ø2:Ñ/�˜V YØ×0Ñ0ØÚOÚ)ó�ð
 Ð&ð  ñØØ#Ø!ØØ'Ù ð  ×1Ñ1°*Ò>aÒcrÓs�ØÐ#Ø#×5Ñ5°jÒBeÒgvÓw�HØÐ'ÜŸ™Ð%>Ô?Ù Ø:BÑ7�˜V V¨Yà×0Ñ0ØÚHÚ#ó�ð
 Ð&ØQXÑN�[ )¨U°OÀXàˆÜ—‘Ð5Ô6Ùà×,Ñ,ØÚDÚóˆGð
 ˆÜ—‘Ð5Ô6ÙØIPÑFˆ[˜) U¨O¸Xà×-Ñ-¨h°q©kÓ:ˆJØÐ!Ü—‘Ð8Ô9ÙØ×*Ñ*¨:°a©=¸!Ô<Ü—‘ÐZÔ[Ùà*.×*DÑ*DØ˜( H¨fÐ6Ió+Ñ'ˆL˜-ò Ø!×0Ñ0×=Ñ=¸jÈ¹n×>RÑ>RÐSTÑ>UÓV�
Ü—‘Ð8Ô9Ø!%×!6Ñ!6×!LÑ!LØ)Ø%Ø%Ø%ØØØØ"Ÿn™nØ $× 0Ñ 0Ø &§¡¨aÑ 0Ø&×-Ñ-¨aÑ0ð "Mó "�ð "Ð)Ùà—‘˜nÔ-Ø 1Ñ$�à×&Ñ&¨°]ÀJÐ'OÔPØ×&Ñ& xÔ0Ø×&Ñ& wÔ/Ø×&Ñ& wÔ/Ø×&Ñ& wÔ/Ø×&Ñ& zÔ2Ø×&Ñ& }Ô5Ø×&Ñ& Ô7Ø×'Ñ'¨¨_×-CÑ-CÀAÑ-FÈÏÉÐVWÑHXÖYä—‘Ð5Ô6Ùð_O	ð` 	×Ñ˜/Ô*Ø×ÑÔÜ�‰Ð,¨_Ð,=Ð>Õ?r   c                 ód   — | j                  «        | j                  «        | j                  «        y r   )Úprocess_embeddingÚ	fuse_maskÚskip_reshape)r   s    r   Ú
preprocesszBertOnnxModelKeras.preprocess×   s$   € Ø×ÑÔ Ø�‰ÔØ×ÑÕr   c                 ó@  — | j                  «        | j                  «        d}| j                  d«      }|D ]H  }| j                  |d«      }|€Œ|j                  dk(  sŒ(|j
                  d   |j
                  d<   |dz  }ŒJ |dkD  rt        j                  d|› �«       y y )Nr   r   r   zSkip consequent Reshape count: )rA   r)   rB   rC   r    r   r#   rN   )r   Úcountr*   Úreshape_noder(   s        r   rv   zBertOnnxModelKeras.skip_reshapeÜ   s¢   € Ø× Ñ Ô"Ø× Ñ Ô"àˆØ×1Ñ1°)Ó<ˆØ)ò 	ˆLØ—_‘_ \°1Ó5ˆFØÑ! f§n¡n¸	Ó&AØ(.¯©°Q©�×"Ñ" 1Ñ%Ø˜‘
‘ð		ð �1Š9Ü�K‰KÐ9¸%¸ÐAÕBð r   c                 ó	  — |j                   dk(  sJ ‚t        j                  d|j                  d   › d�«       | j	                  |g d¢g d¢|«      }|€t        j                  d«       y|\  }}}| j                  |j                  d   «      }|€t        j                  d	«       yt        j                  |«      }t        |j                  «      d
k(  r<t        j                  d|j                  › d|j                  › �«       |j                  }	n0t        j                  d|j                  › d|j                  › �«       y| j                  |j                  d   «      }
|
�÷t        j                  |
«      }t        |j                  «      dk(  rš|j                  d   dk(  rˆt        j                  |j                  |j                  d   |j                  d
   f«      d«      }| j                  |«       t        j                  d|
j                  › d|j                  dd  › �«       d}�n4t        j                  d|
j                  › d|j                  › �«       y| j	                  |ddgddg|«      }|€t        j                  d«       y|\  }}| j                  |j                  d   «      }
|
€t        j                  d«       yt        j                  |
«      }t        |j                  «      d
k(  r<t        j                  d|
j                  › d|j                  › �«       |
j                  }n0t        j                  d|
j                  › d|j                  › �«       y| j!                  |d|«      }|�|j                   dk7  rt        j                  d«       y| j                  |j                  d   «      }|€t        j                  d«       yt        j                  |«      }t        |j                  «      d
k(  r<t        j                  d|j                  › d|j                  › �«       |j                  }n0t        j                  d|j                  › d|j                  › �«       yt        j                  d«       | j#                  ||	||«       y)NÚLayerNormalizationz-start fusing embedding from node with output=r   z...)r2   r2   ÚGatherr6   zfailed to match word_embed_pathFzfailed to get word initializeré   zFound word embedding. name:z, shape:z$Failed to find word embedding. name:r   é   Úposition_embeddingzFound position embedding. name:z(Failed to find position embedding. name:r}   r   zfailed to match pos_embed_pathzfailed to get pos initializerzfailed to get gatherz!failed to get segment initializerzFound segment embedding. name:z'Failed to find segment embedding. name:zCreate Embedding nodeT)r    r#   r$   r!   r   Úget_initializerr   r   Úto_arrayÚlenÚshaperN   ÚnameÚ
from_arrayÚreshapeÚadd_initializerrC   Úcreate_embedding_subgraph)r   Únoder)   Úword_embed_pathÚ	skip_noderI   Úgather_nodeÚword_initializerÚtempÚword_embeddingÚpos_initializerÚtensorr€   Úpos_embed_pathÚ
pos_gatherÚ	pos_sliceÚgatherÚsegment_initializerÚsegment_embeddings                      r   Úfuse_embeddingz!BertOnnxModelKeras.fuse_embeddingë   s  € Ø�|‰|Ð3Ò3Ð3Ð3Ü�‰ÐDÀTÇ[Á[ÐQRÁ^ÐDTÐTWÐXÔYØ×0Ñ0°Ò7OÒQZÐ\oÓpˆØÐ"Ü�L‰LÐ:Ô;Øà+:Ñ(ˆ	�8˜[à×/Ñ/°×0AÑ0AÀ!Ñ0DÓEÐØÐ#Ü�L‰LÐ9Ô:Øä×$Ñ$Ð%5Ó6ˆÜˆt�z‰z‹?˜aÒÜ�K‰KÐ5Ð6F×6KÑ6KÐ5LÈHÐUY×U_ÑU_ÐT`ÐaÔbØ-×2Ñ2‰Nä�K‰KÐ>Ð?O×?TÑ?TÐ>UÐU]Ð^b×^hÑ^hÐ]iÐjÔkØà×.Ñ.¨x¯~©~¸aÑ/@ÓAˆØÐ&Ü×(Ñ(¨Ó9ˆDÜ�4—:‘:‹ !Ò#¨¯
©
°1©¸Ò(:Ü%×0Ñ0°·±¸t¿z¹zÈ!¹}ÈdÏjÉjÐYZÉmÐ>\Ó1]Ð_sÓt�Ø×$Ñ$ VÔ,Ü—‘Ð=¸o×>RÑ>RÐ=SÐS[Ð\`×\fÑ\fÐghÐgiÐ\jÐ[kÐlÔmØ%9Ò"ä—‘ÐFÀ×G[ÑG[ÐF\Ð\dÐei×eoÑeoÐdpÐqÔrØà!×3Ñ3°H¸xÈÐ>QÐTUÐWXÐSYÐ[nÓoˆNØÐ%Ü—‘Ð=Ô>Øà$2Ñ!ˆJ˜	Ø"×2Ñ2°:×3CÑ3CÀAÑ3FÓGˆOØÐ&Ü—‘Ð<Ô=Øä×(Ñ(¨Ó9ˆDÜ�4—:‘:‹ !Ò#Ü—‘Ð9¸/×:NÑ:NÐ9OÈxÐX\×XbÑXbÐWcÐdÔeØ%4×%9Ñ%9Ñ"ä—‘ÐFÀ×G[ÑG[ÐF\Ð\dÐei×eoÑeoÐdpÐqÔrØà—‘ ¨AÐ/BÓCˆØˆ>˜VŸ^™^¨xÒ7Ü�L‰LÐ/Ô0Øà"×2Ñ2°6·<±<À±?ÓCÐØÐ&Ü�L‰LÐ<Ô=Øä×$Ñ$Ð%8Ó9ˆÜˆt�z‰z‹?˜aÒÜ�K‰KÐ8Ð9L×9QÑ9QÐ8RÐRZÐ[_×[eÑ[eÐZfÐgÔhØ 3× 8Ñ 8Ñä�K‰KÐAÐBU×BZÑBZÐA[Ð[cÐdh×dnÑdnÐcoÐpÔqØä�‰Ð+Ô,Ø×&Ñ& t¨^Ð=NÐPbÔcØr   c                 óÂ   — t         j                  d«       | j                  «       }| j                  «       D ]'  }|j                  dk(  sŒ| j                  ||«      r y y y)zM
        Automatically detect word, segment and position embeddings.
        z#start processing embedding layer...r|   N)r#   rN   r)   Únodesr    r™   )r   r)   rŠ   s      r   rt   z$BertOnnxModelKeras.process_embedding7  sY   € ô 	�‰Ð9Ô:Ø"×6Ñ6Ó8ÐØ—J‘J“Lò 	ˆDØ�|‰|Ð3Ó3Ø×&Ñ& tÐ-@ÔAÙÙñ		r   c                 ó,  — g }| j                  «       D �]¥  }|j                  dk(  sŒ| j                  |d«      sŒ'| j                  |g d¢g d¢«      }|€ŒA|\  }}}}| j                  j                  «       }|j                  d   |k7  rt        d|j                  d   › d|› �«       Œ“t        j                  j                  d|gd	gd
dg¬«      }	t        j                  j                  dd	gdgddg¬«      }
t        j                  j                  ddgdg¬«      }|j                  j                  t        j                  j                  dd«      g«       | j                  ||j                  d   d«       |j                  |||g«       | j                  |	«       | j                  |
«       | j                  |«       �Œ¨ | j!                  |«       t#        |«      dkD  r| j%                  «        t&        j)                  t#        |«      dkD  rd«       y d«       y )Nr   iðØÿÿ)r   r   r   r   )r   r   r   r   r   zCast input z is not mask input r   Úmask_fuse_unsqueeze1_outputÚMask_UnSqueeze_1r   )ÚinputsÚoutputsr…   ÚaxesÚmask_fuse_unsqueeze2_outputÚMask_UnSqueeze_2r~   r   Úmask_fuse_cast_output)rŸ   r    Útoz
Fused maskzFailed to fuse mask)r›   r    rD   r   rE   Úget_first_maskr   ÚprintÚonnxÚhelperÚ	make_nodeÚ	attributerJ   Úmake_attributerK   rI   rL   rƒ   Úprune_graphr#   rN   )r   rO   rŠ   Ú	mask_pathÚsub_nodeÚ	cast_nodeÚ
slice_nodeÚunsqueeze_nodeÚmask_input_nameÚunsqueeze_added_1Úunsqueeze_added_2Úcast_node_2s               r   ru   zBertOnnxModelKeras.fuse_maskC  s  € ØˆØ—J‘J“Ló (	+ˆDØ�|‰|˜uÓ$¨×)@Ñ)@ÀÀvÕ)NØ ×2Ñ2°4Ò9^Ò`lÓm�	ØÐ$ØØBKÑ?�˜) Z°à"&×"5Ñ"5×"DÑ"DÓ"F�Ø!×'Ñ'¨Ñ*¨oÒ=Ü˜K¨×(<Ñ(<¸QÑ(?Ð'@Ð@SÐTcÐSdÐeÔfØä$(§K¡K×$9Ñ$9ØØ+Ð,Ø:Ð;Ø+Ø˜ð %:ó %Ð!ô %)§K¡K×$9Ñ$9ØØ9Ð:Ø:Ð;Ø+Ø˜ð %:ó %Ð!ô #Ÿk™k×3Ñ3ØØ9Ð:Ø4Ð5ð 4ó �ð
 ×%Ñ%×,Ñ,¬d¯k©k×.HÑ.HÈÈqÓ.QÐ-RÔSØ×'Ñ'¨°(·.±.ÀÑ2CÐE\Ô]à×&Ñ&¨
°NÀIÐ'NÔOØ—‘Ð/Ô0Ø—‘Ð/Ô0Ø—‘˜kÖ*ðQ(	+ðT 	×Ñ˜/Ô*ô ˆÓ !Ò#Ø×ÑÔä�‰¤C¨Ó$8¸1Ò$<�LÕXÐBWÕXr   c           
      óž  — | j                  d«      }d}|D ]´  }| j                  |g d¢g d¢«      }|€Œ|\	  }}}}}	}
}}}|j                  d   |
j                  d<   | j	                  |«       |	j                  d   |j                  d<   | j	                  |«       |j                  d   |j                  d<   | j	                  |«       |dz  }Œ¶ |S )Nr0   r   )	r2   r   r3   r   ÚGelur2   r   r3   r0   )	r   r   r   r   r   r   r   r   r   r   ©rB   r   r!   r   Úremove_node)r   Úskiplayernorm_nodesÚreshape_removedÚskiplayernorm_nodeÚpathÚadd_1Ú	reshape_1Úmatmul_1Ú	reshape_2ÚgeluÚadd_2Ú	reshape_3Úmatmul_2Úskiplayernorms                 r   Úremove_extra_reshapez'BertOnnxModelKeras.remove_extra_reshapew  sõ   € Ø"×7Ñ7Ð8PÓQÐØˆØ"5ò $	!ÐØ×)Ñ)Ø"ò
ò ,óˆDð ˆ|Øð ñ
ØØØØØØØØØà%Ÿ_™_¨QÑ/ˆE�K‰K˜‰NØ×Ñ˜YÔ'Ø $§¡¨A¡ˆH�N‰N˜1ÑØ×Ñ˜YÔ'Ø%Ÿ_™_¨QÑ/ˆE�K‰K˜‰NØ×Ñ˜YÔ'Ø˜qÑ ‰OðI$	!ðL Ðr   c                 óú  — | j                  d«      }d}|D ]â  }| j                  |g d¢g d¢«      }|€Œ|\
  }}}}}	}
}}}}|j                  d   |j                  d<   | j	                  |«       |j                  d   |
j                  d<   | j	                  |«       |	j                  d   |j                  d<   | j	                  |«       |j                  d   |j                  d<   | j	                  |«       |dz  }Œä |S )Nr0   r   )
r2   r   r3   r   r¸   r2   r   r3   r   r0   )
Nr   r   r   r   r   r   r   r   r   é   r¹   )r   r»   r¼   r½   r¾   r¿   rÀ   rÁ   rÂ   rÃ   rÄ   rÅ   rÆ   Ú	reshape_4rÇ   s                  r   Úremove_extra_reshape_2z)BertOnnxModelKeras.remove_extra_reshape_2¢  s!  € Ø"×7Ñ7Ð8PÓQÐØˆØ"5ò -	!ÐØ×)Ñ)Ø"òò 2óˆDð  ˆ|Øð ñØØØØØØØØØØð !.× 4Ñ 4°QÑ 7ˆH�N‰N˜1ÑØ×Ñ˜YÔ'à%Ÿ_™_¨QÑ/ˆE�K‰K˜‰NØ×Ñ˜YÔ'à $§¡¨A¡ˆH�N‰N˜1ÑØ×Ñ˜YÔ'à%Ÿ_™_¨QÑ/ˆE�K‰K˜‰NØ×Ñ˜YÔ'à˜qÑ ‰Oð[-	!ð^ Ðr   c                 ó˜   — | j                  «       | j                  «       z   }t        j                  d|› d�«       | j	                  «        y )NzRemove z Reshape nodes.)rÈ   rÌ   r#   rN   r­   )r   r¼   s     r   ÚpostprocesszBertOnnxModelKeras.postprocessÖ  s@   € Ø×3Ñ3Ó5¸×8SÑ8SÓ8UÑUˆÜ�‰�g˜oÐ.¨oÐ>Ô?à×ÑÕr   )Ú__name__Ú
__module__Ú__qualname__r
   r   r.   rr   rw   rv   r™   rt   ru   rÈ   rÌ   rÎ   Ú__classcell__)r   s   @r   r   r      sI   ø„ ô8òò0#ò Z@òxò
CòJòX
ò2Yòh)òV2öhr   r   )	Úloggingr¨   r   Úonnx_model_bert_tfr   Ú	getLoggerrÏ   r#   r   © r   r   ú<module>r×      s3   ðó ã Ý Ý .à	ˆ×	Ñ	˜8Ó	$€ôK˜õ Kr   