Ë
    cQêjSa  ã                   óv   — d dl Z d dlZd dlZd dlmZmZmZ d dlmZ  e j                  e
«      Z G d„ de«      Zy)é    N)ÚTensorProtoÚhelperÚnumpy_helper)ÚBertOnnxModelc                   ór   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚBertOnnxModelTFc                 ó(   •— t         ‰| �  |||«       y ©N)ÚsuperÚ__init__)ÚselfÚmodelÚ	num_headsÚhidden_sizeÚ	__class__s       €úp/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/onnx_model_bert_tf.pyr   zBertOnnxModelTF.__init__   s   ø€ Ü‰Ñ˜ 	¨;Õ7ó    c                 ón  — g }| j                  «       D ]n  }|j                  dk(  sŒ| j                  |j                  d   «      rŒ2| j	                  |j                  d   |j
                  d   «       |j                  |«       Œp | j                  |«       t        j                  dt        |«      › �«       y )NÚIdentityr   zRemoved Identity count: )ÚnodesÚop_typeÚfind_graph_outputÚoutputÚreplace_input_of_all_nodesÚinputÚappendÚremove_nodesÚloggerÚinfoÚlen)r   Únodes_to_removeÚnodes      r   Úremove_identityzBertOnnxModelTF.remove_identity   s–   € ØˆØ—J‘J“Lò 	1ˆDØ�|‰|˜zÓ)Ø×-Ñ-¨d¯k©k¸!©nÕ=Ø×3Ñ3°D·K±KÀ±NÀDÇJÁJÈqÁMÔRØ#×*Ñ*¨4Õ0ð		1ð
 	×Ñ˜/Ô*Ü�‰Ð.¬s°?Ó/CÐ.DÐEÕFr   c                 ó    — | j                  |g d¢g d¢«      }|�|S | j                  |g d¢g d¢«      }|�|S | j                  |g d¢g d¢«      }|S )N)ÚMulÚSubÚReshapeÚCast)é   Nr)   r   )r%   r&   r(   ÚSliceÚ	Unsqueeze)r)   r   r)   r   r   )r%   r&   r(   r+   r+   )r)   Nr)   r   r   )Úmatch_parent_path)r   Úadd_or_sub_before_softmaxÚ
mask_nodess      r   Úmatch_mask_pathzBertOnnxModelTF.match_mask_path   sq   € Ø×+Ñ+Ø%Ú-Úó
ˆ
ð
 Ð!ØÐà×+Ñ+Ø%Ú8Úó
ˆ
ð
 Ð!ØÐà×+Ñ+Ø%Ú<Úó
ˆ
ð Ðr   c                 ó  — | j                  |g «      }i }|D ]n  }|j                  D ]]  }| j                  |«      }|sŒt        j                  |«      }t        |j                  «      dk(  sŒE|j                  ||j                  <   Œ_ Œp |S )zi
        Find initializers that is 2D. Returns a dictionary with name as key and shape as value.
        é   )Úget_parent_subgraph_nodesr   Úget_initializerr   Úto_arrayr    ÚshapeÚname)r   Úcurrent_nodeÚparent_nodesÚinitializersr"   r   ÚinitializerÚtemps           r   Ú)get_2d_initializers_from_parent_subgraphsz9BertOnnxModelTF.get_2d_initializers_from_parent_subgraphs7   s�   € ð ×5Ñ5°lÀBÓGˆØˆØ ò 	DˆDØŸ™ò D�Ø"×2Ñ2°5Ó9�ÚÜ'×0Ñ0°Ó=�DÜ˜4Ÿ:™:“¨!Ó+Ø9=¿¹˜ [×%5Ñ%5Ò6ñDð	Dð Ðr   c                 óò  — | j                  «       }||vry ||   }t        |«      dk7  ry | j                  |d   d¬«      }t        |«      dkD  rt        d|«       y t        |«      dk(  r|d   |k7  r|d   S |d   j                  d   }| j                  |d   g d¢g d¢fg d¢g d	¢fgd «      \  }}}|rÅ|rÃ||d
   j                  d   k(  r®t        j                  d«       |d   }	| j                  |	«      j                  }
| j                  t        j                  d|gdg¬«      |
«       t        j                  |	j                  d   «      }| j                  t        j                  ddgdg|¬«      |
«       d}|S )Nr)   r   T©Ú	recursivez(Found multiple candidates of segment_ids)ÚConstantOfShaper(   ÚConcatr*   r(   ÚShape)r)   r   r   r   r   r   ©r@   r(   rA   r+   ÚSqueezer*   r(   rB   )r)   r   r   r   r   r   r   r   éÿÿÿÿúSimplify semgent id path...rB   Úinput_shape©ÚinputsÚoutputsr@   Úzeros_for_input_shape©rI   rJ   Úvalue)Úinput_name_to_nodesr    Úget_graph_inputsÚprintr   Úmatch_parent_pathsr   ÚdebugÚget_graph_by_noder6   Úadd_noder   Ú	make_nodeÚget_attribute_valueÚ	attribute)r   Úsegment_embeddingÚ	input_idsrN   r   Úgraph_inputsÚsegment_idsÚ_Úsegment_id_pathÚconstantofshape_nodeÚ
graph_nameÚconstantofshape_values               r   Úfind_segment_idsz BertOnnxModelTF.find_segment_idsG   sµ  € Ø"×6Ñ6Ó8ÐØÐ$7Ñ7Øà#Ð$5Ñ6ˆÜˆu‹:˜Š?Øà×,Ñ,¨U°1©XÀÐ,ÓFˆÜˆ|Ó˜qÒ ÜÐ<¸lÔKØäˆ|Ó Ò! l°1¡o¸Ò&BØ ‘?Ð"ð ˜A‘h—n‘n QÑ'ˆØ $× 7Ñ 7Ø�!‰Hò TÚ&ðò
	ò -ððð& ó+!
Ñˆˆ?˜Añ0 ™y¨Y¸/È"Ñ:M×:SÑ:SÐTUÑ:VÒ-VÜ�L‰LÐ6Ô7Ø#2°1Ñ#5Ð Ø×/Ñ/Ð0DÓE×JÑJˆJØ�M‰MÜ× Ñ  °)°À}ÀoÔVØôô %+×$>Ñ$>Ð?S×?]Ñ?]Ð^_Ñ?`Ó$aÐ!Ø�M‰MÜ× Ñ Ø%Ø)˜?Ø4Ð5Ø/ô	ð ôð 2ˆKØÐr   c                 óÀ   — | j                  «       }||vry ||   }t        |«      dk7  ry | j                  |d   d¬«      }t        |«      dk(  r|d   S t        d|«       y )Nr)   r   Tr>   z&Found multiple candidates of input_ids)rN   r    rO   rP   )r   Úword_embeddingrN   r   rZ   s        r   Úfind_input_idszBertOnnxModelTF.find_input_ids‡   ss   € Ø"×6Ñ6Ó8ÐØÐ!4Ñ4Øà# NÑ3ˆÜˆu‹:˜Š?Øà×,Ñ,¨U°1©XÀÐ,ÓFˆÜˆ|Ó Ò!Ø ‘?Ð"äÐ6¸ÔEØr   c           	      óî  — | j                  «       D �]×  }|j                  dk(  sŒ| j                  |g d¢g d¢«      }|€Œ.|\  }}}}}}	| j                  |d«      sŒJ| j                  |d«      sŒ]| j	                  |d¬«      }
|
D �cg c]	  }||vsŒ|‘Œ }}t        |«      dkD  rt        d|«        y t        |«      dk(  r|d	   c S | j                  |d
   g d¢g d¢«      }|
D �cg c]	  }||v sŒ|‘Œ }}|rä|rât        |«      dk(  rÔ|d	   |d
   j                  d	   k(  r¼t        j                  d«       |d	   }t        j                  |j                  d	   «      }| j                  |«      j                  }| j                  t        j                   d|d	   gdg¬«      |«       | j                  t        j                   ddg|	j                  d	   g|¬«      |«       |	j                  d	   c S  y c c}w c c}w )NÚSoftmax)ÚAddr%   r&   r(   r*   r+   )r   r)   Nr)   r   r   iðØÿÿr)   Tr>   z'Found multiple candidates of mask inputr   rE   rC   )r   r   r   r   r   r   r   r   rF   rB   Úinput_shape_for_maskrH   r@   rL   )r   r   r,   Úhas_constant_inputrO   r    rP   r   r   rR   r   rV   rW   rS   r6   rT   rU   )r   Úexcluded_graph_inputsr"   Ú	mask_pathrT   Úmul_nodeÚsub_nodeÚ	cast_nodeÚ
slice_nodeÚunsqueeze_noderZ   r   rI   Úpath_to_be_simplifiedÚduplicated_inputsr^   r`   r_   s                     r   Úfind_mask_inputzBertOnnxModelTF.find_mask_input—   s@  € Ø—J‘J“Ló E	3ˆDØ�|‰|˜yÓ(Ø ×2Ñ2ØÚGÚ)ó�	ð
 Ð$Øð ñØØØØØØ"à×*Ñ*¨8°VÕ<À×AXÑAXÐYaÐcdÕAeØ#'×#8Ñ#8¸ÈTÐ#8Ó#R�LØ1=Öd¨ÀÐNcÒAcšeÐd�FÐdÜ˜6“{ Q’ÜÐGÈÔPÙ#Ü˜6“{ aÒ'Ø% a™yÒ(à,0×,BÑ,BØ! "™ò	ò 1ó-Ð)ð =IÖ(k°5ÈEÐUjÒLjªÐ(kÐ%Ð(kñ .Ù-ÜÐ 1Ó2°aÒ7Ø-¨aÑ0Ð4IÈ"Ñ4M×4SÑ4SÐTUÑ4VÒVäŸ™Ð%BÔCØ/DÀQÑ/GÐ,Ü06×0JÑ0JÐK_×KiÑKiÐjkÑKlÓ0mÐ-Ø%)×%;Ñ%;Ð<PÓ%Q×%VÑ%V˜
ØŸ™Ü"×,Ñ,Ø 'Ø(9¸!Ñ(<Ð'=Ø)?Ð(@ôð
 'ôð Ÿ™Ü"×,Ñ,Ø 1Ø(>Ð'?Ø)7×)=Ñ)=¸aÑ)@Ð(AØ&;ô	ð 'ôð *×/Ñ/°Ñ2Ò2ðKE	3ðL ùòg eùò* )ls   Â	G-ÂG-Ã$	G2Ã.G2c                 ó´  — | j                  |«      }|€t        j                  d«       y| j                  ||«      }|€t        j                  d«       y| j	                  ||g«      }|€t        j                  d«       y|||g| _        | j                  d«      }| j                  j                  ||«       | j                  |«      j                  j                  j                  t        j                  k7  r| j                  j!                  |«      \  }	}| j                  |«      r| j                  j!                  |«      \  }	}n| j                  j#                  |«      \  }}
| j                  |«      r| j                  j!                  |«      \  }	}n| j                  j#                  |«      \  }}| j                  d«      }t$        j&                  j)                  d||||||j*                  d   |j*                  d	   |g||gd
¬«      }d|_        | j/                  |j0                  d   |«       | j3                  || j5                  |«      j6                  «       y )Nz6Failed to find input_ids. Cannot fuse embedding layer.Fz8Failed to find segment_ids. Cannot fuse embedding layer.z7Failed to find input_mask. Cannot fuse embedding layer.Ú
mask_indexÚembed_outputÚEmbedLayerNormalizationr)   r1   Ú
EmbedLayer©rI   rJ   r6   zcom.microsoftr   )rd   r   r   ra   rs   Úbert_inputsÚcreate_node_nameÚattention_maskÚset_mask_indiceÚfind_graph_inputÚtypeÚtensor_typeÚ	elem_typer   ÚINT32ÚutilsÚcast_graph_input_to_int32Úcast_input_to_int32Úonnxr   rU   r   Údomainr   r   rT   rS   r6   )r   Únormalize_noderc   rX   Úposition_embeddingrY   r[   Ú
mask_inputru   ÚcastedÚsegment_id_cast_nodeÚmask_input_cast_noderv   Ú
embed_nodes                 r   Úcreate_embedding_subgraphz)BertOnnxModelTF.create_embedding_subgraphà   s'  € Ø×'Ñ'¨Ó7ˆ	ØÐÜ�K‰KÐPÔQØà×+Ñ+Ð,=¸yÓIˆØÐÜ�K‰KÐRÔSØà×)Ñ)¨;¸	Ð*BÓCˆ
ØÐÜ�K‰KÐQÔRØà% {°JÐ?ˆÔà×*Ñ*¨<Ó8ˆ
Ø×Ñ×+Ñ+¨J¸
ÔCà× Ñ  Ó+×0Ñ0×<Ñ<×FÑFÌ+×J[ÑJ[Ò[Ø $§
¡
× DÑ DÀYÓ OÑˆF�Ià× Ñ  Ô-Ø"&§*¡*×"FÑ"FÀ{Ó"SÑˆF‘Kà04·
±
×0NÑ0NÈ{Ó0[Ñ-ˆKÐ-à× Ñ  Ô,Ø!%§¡×!EÑ!EÀjÓ!QÑˆF‘Jà/3¯z©z×/MÑ/MÈjÓ/YÑ,ˆJÐ,à×,Ñ,¨^Ó<ˆÜ—[‘[×*Ñ*Ø%àØØØ"Ø!Ø×$Ñ$ QÑ'Ø×$Ñ$ QÑ'Øð	ð " :Ð.Øð +ó 
ˆ
ð ,ˆ
ÔØ×'Ñ'¨×(=Ñ(=¸aÑ(@À,ÔOØ�‰�j $×"8Ñ"8¸Ó"H×"MÑ"MÕNr   c                 ó`  — t         j                  d«       | j                  «       }| j                  d«      }|D �]ò  }| j	                  |g d¢g d¢|«      }|€Œ|\  }}}| j                  |j                  d   «      }|€ŒFt        j                  |«      }	t        |	j                  «      dk(  r<t         j                  d|j                  › d	|	j                  › �«       |j                  }
n1t         j                  d
|j                  › d	|	j                  › �«        y| j                  |d|«      }|€Œö|j                  dk(  s�Œ| j                  |«      }t        |«      dk7  rt         j                  d|› �«        yd}d}|j!                  «       D ]H  \  }}|d   dk(  r|}t         j                  d|› d	|› �«       Œ,|}t         j                  d|› d	|› �«       ŒJ |�|€t         j                  d«        yt         j                  d«       | j#                  ||||
«       | j%                  «         y y)zM
        Automatically detect word, segment and position embeddings.
        z#start processing embedding layer...ÚLayerNormalization)rg   r'   r*   )r   r)   r   Nr   r1   zFound position embedding. name:z, shape:z(Failed to find position embedding. name:rg   zFFailed to find two embeddings (word and segment) from Add node. Found zFound segment embedding. name:zFound words embedding. name:z.Failed to find both word and segment embeddingzCreate Embedding node)r   r   Úoutput_name_to_nodeÚget_nodes_by_op_typer,   r3   r   r   r4   r    r5   r6   Ú
get_parentr   r<   ÚwarningÚitemsr�   Úprune_graph)r   r’   Úlayer_norm_nodesÚlayer_norm_nodeÚpos_embed_pathrT   Úreshape_nodero   r:   r;   r‰   Úfirst_parentÚ
embeddingsrc   rX   r6   r5   s                    r   Úprocess_embeddingz!BertOnnxModelTF.process_embedding  sG  € ô 	�‰Ð9Ô:Ø"×6Ñ6Ó8Ðà×4Ñ4Ð5IÓJÐØ/ó 7	ˆOØ!×3Ñ3ØÚ+ÚØ#ó	ˆNð Ð%Øà1?Ñ.ˆH�l JØ×.Ñ.¨z×/?Ñ/?ÀÑ/BÓCˆKØÐ"Øä×(Ñ(¨Ó5ˆDÜ�4—:‘:‹ !Ò#Ü—‘Ð=¸k×>NÑ>NÐ=OÈxÐX\×XbÑXbÐWcÐdÔeØ%0×%5Ñ%5Ñ"ä—‘ÐFÀ{×GWÑGWÐFXÐX`Ðae×akÑakÐ`lÐmÔnÙàŸ?™?¨8°QÐ8KÓLˆLØÑ'¨L×,@Ñ,@ÀEÔ,IØ!×KÑKÈLÓY�
Ü�z“? aÒ'Ü—N‘NØ`ÐakÐ`lÐmôñ à!%�Ø$(Ð!Ø#-×#3Ñ#3Ó#5ò Z‘K�D˜%Ø˜Q‘x 1’}Ø,0Ð)ÜŸ™Ð&DÀTÀFÈ(ÐSXÐRYÐ$ZÕ[à)-˜ÜŸ™Ð&BÀ4À&ÈÐQVÐPWÐ$XÕYðZð "Ð)Ð->Ð-FÜ—K‘KÐ PÔQÙä—‘Ð3Ô4Ø×.Ñ.Ø#Ø"Ø%Ø&ô	ð × Ñ Ô"Ùño7	r   c                 óž   — |||fD ]E  }|j                   d   }||   }||k(  rŒt        j                  d|› d|j                  d   › �«        y y)Nr   zCheck attention input failed:z, FT)r   r   rR   r   )	r   Úmatmul_qÚmatmul_kÚmatmul_vÚparentr’   ÚxÚ
root_inputÚ	root_nodes	            r   Úcheck_attention_inputz%BertOnnxModelTF.check_attention_inputW  sg   € Ø˜H hÐ/ò 	ˆAØŸ™ ™ˆJØ+¨JÑ7ˆIØ˜FÒ"ØÜ�L‰LÐ8¸¸ÀBÀvÇ}Á}ÐUVÑGWÐFXÐYÔZÙð	ð r   c                 óœ  — | j                  «       }g }d}g }| j                  d«      }| j                  d«      }|j                  |«       |j                  |«       |D �]³  }| j                  |«      j                  }|j
                  dk(  r| j                  |dd«      }	|	�|	}nŒG| j                  |d«      }
|
�|
j
                  dvr8| j                  |d«      }
|
�|
j
                  dvrt        j                  d«       Œ¡| j                  |g d¢g d	¢«      }|€I| j                  |g d
¢g d¢«      }|€0| j                  |g d¢g d¢«      }|€t        j                  d«       �Œ|d   }| j                  |g d¢g d¢«      }|€0| j                  |ddgddg«      }|€t        j                  d«       �ŒQ|d   }|d   }| j                  |g d¢g d¢«      }|€0| j                  |g d¢g d¢«      }|€t        j                  d«       �Œ¤|d   }| j                  |g d¢g d¢«      }|€0| j                  |ddgddg«      }|€t        j                  d«       �Œò|d   }|d   }| j                  |g d¢g d¢«      }|€0| j                  |g d¢g d¢«      }|€t        j                  d«       �ŒE|d   }|d   }| j                  |d   «      }|€t        j                  d«       �Œ|| j                  |d   d«      st        j                  d«       �Œ¨| j                  |d   dgdg«      xs | j                  |d   dgdg«      }d }|d!z   }|€€t        |«      d"k(  rr| j                  |d   j                  d   «      €Q|d   j                  d   }| j!                  t#        j$                  d|g|g|dg¬#«      |«       ||d   j                  d<   | j'                  ||||
|«      }|�r$| j(                  j+                  |d   j                  d   «      }t        j                  d$«       | j,                  j/                  |||||||| j0                  | j2                  |
j4                  d   |d%   j4                  d   ¬&«      }|€�Œ|d   j
                  dk(  �r(t#        j6                  |d   j                  d'z   t8        j:                  d(gt=        j>                  dd| j0                  tA        | j2                  | j0                  z  «      gg«      jC                  «       d)¬*«      } | jE                  | |«       t#        j$                  d+|j4                  d   |d   j                  d'z   g|d   j                  d,z   g|d   j                  d-z   ¬.«      }!|d   j                  d,z   |d   j                  d<   | j!                  |!|«       |
j
                  d+k(  r½tG        jH                  | jK                  |
j                  d   «      «      d   }"t#        j6                  |
j                  d/z   t8        j:                  d0gt=        j>                  dd|"gg«      jC                  «       d)¬*«      } | jE                  | |«       |
j                  d/z   |
j                  d<   | j!                  ||«       |dz  }|j                  |d%d  «       |j                  |«       |j                  |«       |j                  |«       |j                  |«       |j                  |«       �ŒŸt        j                  d1«       �Œ¶ | jM                  |«       | jO                  «        t        jQ                  d2|› �«       y )3Nr   ÚSkipLayerNormalizationr‘   rg   r)   )r©   r‘   r'   z(Failed to match parent of normalize_node)rg   ÚMatMulr'   Ú	Transposerª   )r   r   r   r   r   )rª   r'   r«   rª   )r)   r   r   r   )rg   ÚEinsumr¬   )r   r   r   zFailed to match qkv nodesrE   )r«   r'   rg   rª   r¬   zFailed to match v pathéþÿÿÿ)rf   rg   r%   rª   )r   r   r   r   )rf   rg   r¬   zFailed to match qk_pathszFailed to match q path)r%   rg   r¬   )r)   r   r   zFailed to match k pathzCannot find mask_nodes.z;Sub node expected to have an input with constant value 1.0.rD   ÚExpandÚSqueeze_3d_to_2d_maskÚ_outputé   )ÚaxeszCreate an Attention node.r1   )ru   Úq_matmulÚk_matmulÚv_matmulÚq_addÚk_addÚv_addr   r   Úfirst_inputr   Ú	_newshapeé   T)r6   Ú	data_typeÚdimsÚvalsÚrawr'   Ú_reshape_outputÚ_reshapery   Ú	_modifiedé   zRoot node not matched.zFused Attention count:))r’   r“   ÚextendrS   r6   r   Úmatch_parentr”   r   rR   r,   r/   ri   r    r~   r   rT   r   rU   r§   r|   Úprocess_maskÚattention_fusionÚcreate_attention_noder   r   r   Úmake_tensorr   ÚINT64ÚnpÚint64ÚintÚtobytesÚadd_initializerr   r4   r3   r   Úupdate_graphr   )#r   r’   r!   Úattention_countÚstart_nodesÚskip_layer_norm_nodesr˜   rˆ   r_   Úadd_before_layernormr£   Ú	qkv_nodesÚ
matmul_qkvÚv_nodesÚadd_vr¢   Úqk_nodesÚ	matmul_qkÚq_nodesÚadd_qr    Úk_nodesÚadd_kr¡   r.   Úsqueeze_nodeÚsqueeze_node_nameÚsqueeze_output_namerŠ   Úis_same_rootru   Úattention_nodeÚtensorÚreshape_r   s#                                      r   Úfuse_attentionzBertOnnxModelTF.fuse_attentionb  s‚  € Ø"×6Ñ6Ó8ÐàˆØˆàˆØ $× 9Ñ 9Ð:RÓ SÐØ×4Ñ4Ð5IÓJÐð 	×ÑÐ0Ô1Ø×ÑÐ+Ô,à)ó x	ˆNØ×/Ñ/°Ó?×DÑDˆJà×%Ñ%Ð)=Ò=Ø'+×'8Ñ'8¸ÈÐPQÓ'RÐ$Ø'Ð3Ø%9‘NàØ—_‘_ ^°QÓ7ˆFØˆ~ §¡ð 8ñ "ð
 Ÿ™¨¸Ó;�Ø�> V§^¡^ð <ñ &ô
 —L‘LÐ!KÔLØà×.Ñ.ØÚCÚóˆIð
 Ð Ø ×2Ñ2Ø"Ú@Ú ó�	ð
 Ð$Ø $× 6Ñ 6°~ÒGbÒdmÓ n�IØ Ð(ÜŸ™Ð%@ÔAÙ à" 2™ˆJØ×,Ñ,¨ZÒ9bÒdpÓqˆGØˆØ×0Ñ0°¸eÀXÐ=NÐQRÐTUÐPVÓW�Ø�?Ü—L‘LÐ!9Ô:Ùà˜B‘KˆEØ˜r‘{ˆHØ×-Ñ-¨jÒ:]Ò_kÓlˆHØÐØ×1Ñ1°*Ò>ZÒ\eÓf�ØÐ#Ü—L‘LÐ!;Ô<ÙØ  ™ˆIà×,Ñ,¨YÒ8aÒcoÓpˆGØˆØ×0Ñ0°¸UÀHÐ<MÐPQÐSTÈvÓV�Ø�?Ü—L‘LÐ!9Ô:Ùà˜B‘KˆEØ˜r‘{ˆHà×,Ñ,¨YÒ8aÒcoÓpˆGØˆØ×0Ñ0°Ò<TÒV_Ó`�Ø�?Ü—L‘LÐ!9Ô:ÙØ˜B‘KˆEØ˜r‘{ˆHà×-Ñ-¨h°q©kÓ:ˆJàÐ!Ü—‘Ð6Ô7Ùà×*Ñ*¨:°a©=¸!Ô<Ü—‘ÐZÔ[Ùð  ×1Ñ1°*¸R±.À9À+ÐPQÈsÓSò ÐW[×WmÑWmØ˜2‘  
¨Q¨CóXˆLð !8ÐØ"3°iÑ"?ÐØÐ#¬¨J«¸1Ò(<À×AVÑAVÐWaÐbdÑWe×WkÑWkÐlmÑWnÓAoÐAwØ'¨™^×1Ñ1°!Ñ4�
Ø—‘Ü×$Ñ$Ø!Ø#˜Ø,Ð-Ø)Ø˜Sôð ô	ð +>�
˜2‘×$Ñ$ QÑ'à×5Ñ5°hÀÈ(ÐTZÐ\oÓpˆLÚØ!×0Ñ0×=Ñ=¸jÈ¹n×>RÑ>RÐSTÑ>UÓV�
Ü—‘Ð8Ô9ð "&×!6Ñ!6×!LÑ!LØ)Ø%Ø%Ø%ØØØØ"Ÿn™nØ $× 0Ñ 0Ø &§¡¨aÑ 0Ø$ Q™<×.Ñ.¨qÑ1ð "Mó "�ð "Ð)Ùà˜Q‘<×'Ñ'¨8Ó3ä#×/Ñ/Ø& q™\×.Ñ.°Ñ<Ü"-×"3Ñ"3Ø˜SÜŸX™Xð %&Ø$%Ø$(§N¡NÜ$'¨×(8Ñ(8¸4¿>¹>Ñ(IÓ$Jð	!"ðó	÷ "™'›)Ø ô�Fð  ×(Ñ(¨°Ô<Ü%×/Ñ/Ø!à*×1Ñ1°!Ñ4Ø% a™L×-Ñ-°Ñ;ð ð "+¨1¡×!2Ñ!2Ð5FÑ!FÐ GØ& q™\×.Ñ.°Ñ;ô �Hð -6°a©L×,=Ñ,=Ð@QÑ,Q�I˜a‘L×&Ñ& qÑ)Ø—M‘M (¨JÔ7Ø—>‘> YÒ.ä".×"7Ñ"7¸×8LÑ8LÈVÏ\É\ÐZ[É_Ó8]Ó"^Ð_`Ñ"a�KÜ#×/Ñ/Ø#Ÿ[™[¨;Ñ6Ü"-×"3Ñ"3Ø˜SÜŸX™X¨¨2¨{Ð';Ð&<Ó=×EÑEÓGØ ô�Fð ×(Ñ(¨°Ô<Ø&,§k¡k°KÑ&?�F—L‘L ‘Oà—‘˜n¨jÔ9Ø 1Ñ$�à×&Ñ& y°° }Ô5Ø×&Ñ& xÔ0Ø×&Ñ& wÔ/Ø×&Ñ& wÔ/Ø×&Ñ& wÔ/Ø×&Ñ& zÖ2ä—‘Ð5Ô6Ùðqx	ðr 	×Ñ˜/Ô*Ø×ÑÔÜ�‰Ð,¨_Ð,=Ð>Õ?r   c                 ód   — | j                  «        | j                  «        | j                  «        y r
   )r#   rž   Úskip_reshape©r   s    r   Ú
preprocesszBertOnnxModelTF.preprocess-  s&   € Ø×ÑÔØ×ÑÔ Ø×ÑÕr   c                 ó   — d}| j                  d«      }|D ]H  }| j                  |d«      }|€Œ|j                  dk(  sŒ(|j                  d   |j                  d<   |dz  }ŒJ |dkD  rt        j                  d|› �«       y y )Nr   r'   r)   zSkip consequent Reshape count: )r“   r”   r   r   r   r   )r   ÚcountÚreshape_nodesr›   r£   s        r   rè   zBertOnnxModelTF.skip_reshape2  sŠ   € ØˆØ×1Ñ1°)Ó<ˆØ)ò 	ˆLØ—_‘_ \°1Ó5ˆFØÑ! f§n¡n¸	Ó&AØ(.¯©°Q©�×"Ñ" 1Ñ%Ø˜‘
‘ð		ð �1Š9Ü�K‰KÐ9¸%¸ÐAÕBð r   c                 ó  — | j                  d«      }|D ]t  }| j                  |ddgddg«      }|€Œt        j                  d«       |\  }}| j	                  |j
                  d   |j                  d   «       | j                  |«        y  y )NÚ	Attentionr'   rw   r   z+Remove Reshape before first Attention node.)r“   r,   r   r   r   r   r   Úremove_node)r   Úattention_nodesrã   ÚpathÚreshaper\   s         r   Ú%remove_reshape_before_first_attentionz5BertOnnxModelTF.remove_reshape_before_first_attention>  s“   € Ø×3Ñ3°KÓ@ˆØ-ò 	ˆNØ×)Ñ)¨.¸9ÐF_Ð:`ÐcdÐfgÐbhÓiˆDØˆ|ØÜ�K‰KÐEÔFØ‰JˆG�QØ×+Ñ+¨G¯N©N¸1Ñ,=¸w¿}¹}ÈQÑ?OÔPØ×Ñ˜WÔ%Ùñ	r   c                 óD   — | j                  «        | j                  «        y r
   )rô   r—   ré   s    r   ÚpostprocesszBertOnnxModelTF.postprocessJ  s   € Ø×2Ñ2Ô4Ø×ÑÕr   )Ú__name__Ú
__module__Ú__qualname__r   r#   r/   r<   ra   rd   rs   r�   rž   r§   ræ   rê   rè   rô   rö   Ú__classcell__)r   s   @r   r   r      sY   ø„ ô8òGòò2ò >ò@ò GòR4Oòl?òB	òI@òVò

Cò
ör   r   )ÚloggingÚnumpyrË   r†   r   r   r   Úonnx_model_bertr   Ú	getLoggerr÷   r   r   © r   r   ú<module>r      s9   ðó ã Û ß 2Ñ 2Ý )à	ˆ×	Ñ	˜8Ó	$€ô|�mõ |r   