Ë
    cQêjú.  ã                   óÀ  — d dl Z d dlZd dlZd dlZ	 d dlmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ  ej6                  e«       ej6                  d«      k  r ede› d�«      ‚g d¢Zdefd„Z	 ddededz  defd„Z de!e"eef      defd„Z#de!e"eef      dedefd„Z$d„ Z%dd„Z&e'dk(  r e%«        yy# e$ r d dlmZmZ Y ŒÂw xY w)é    N)ÚPackageNotFoundErrorÚversion)ÚPath)ÚAny)Úload_dataset)Ú	evaluator)ÚORTModelForQuestionAnswering)Ú__version__)r   )ÚAutoTokenizerÚpipelinez1.13.1z1Please install optimum>=1.13.1. Current version: ú.)z5bert-large-uncased-whole-word-masking-finetuned-squadzdeepset/roberta-base-squad2z%distilbert-base-cased-distilled-squadÚpackage_namec                 ó8   — 	 t        | «      S # t        $ r Y y w xY w©N)r   r   )r   s    út/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/models/bert/eval_squad.pyÚget_package_versionr   7   s$   € ðÜ�|Ó$Ð$øÜò Ùðús   ‚
 �	˜Úmodel_idÚ	onnx_pathÚuse_io_bindingc                 ó”  — |€zt        j                  | d||¬«      }t        j                  j	                  d| «      }|j                  |«       t        j                  j	                  |d«      }t        d|«       ||fS t        j                  t        j                  j                  |«      t        |«      j                  ||¬«      }||fS )a¨  Load onnx model given pretrained model name and optional ONNX model path. If onnx_path is None,
    the default onnx model from optimum will be used.

    Args:
        model_id (str): pretrained model name or checkpoint path
        onnx_path (Optional[str], optional): path of onnx model to evaluate. Defaults to None.

    Returns:
        model: ORTModel for the onnx model
        onnx_path: the path of onnx model
    T)ÚexportÚproviderr   r   z
model.onnxzModel is exported to onnx file:)Ú	file_namer   r   )
r	   Úfrom_pretrainedÚosÚpathÚjoinÚsave_pretrainedÚprintÚdirnamer   Úname)r   r   r   r   ÚmodelÚsave_onnx_dirs         r   Úload_onnx_modelr$   >   sµ   € ð Ðä,×<Ñ<Ø˜T¨HÀ^ô
ˆô Ÿ™Ÿ™ S¨(Ó3ˆØ×Ñ˜mÔ,Ü—G‘G—L‘L °Ó=ˆ	ÜÐ/°Ô;ð �)ÐÐô -×<Ñ<Ü�G‰G�O‰O˜IÓ&Ü˜9“o×*Ñ*ØØ)ô	
ˆð �)ÐÐó    ÚresultsÚcsv_filenamec                 ó  — t        |ddd¬«      5 }g d¢}t        j                  ||¬«      }|j                  «        | D ]  }|j	                  |«       Œ |j                  «        ddd«       t        d|› �«       y# 1 sw Y   ŒxY w)	zµOutput a CSV file with detail of each test results.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
    ÚaÚ Úascii©ÚmodeÚnewlineÚencoding)Úpretrained_model_namer   r   Údisable_fused_attentionÚ
batch_sizeÚsequence_lengthr   ÚexactÚf1ÚtotalÚHasAns_exactÚ	HasAns_f1ÚHasAns_totalÚ
best_exactÚbest_exact_threshÚbest_f1Úbest_f1_threshÚtotal_time_in_secondsÚsamples_per_secondÚlatency_in_seconds©Ú
fieldnamesNz&Detail results are saved to csv file: )ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowÚflushr   )r&   r'   Úcsv_fileÚcolumn_namesÚ
csv_writerÚresults         r   Úoutput_detailsrM   b   s‰   € ô 
ˆl ¨b¸7Ô	Cð Àxò
ˆô. —^‘^ H¸ÔFˆ
Ø×ÑÔ Øò 	(ˆFØ×Ñ Õ'ð	(ð 	�‰Ô÷;ô> 
Ð2°<°.Ð
AÕB÷?ð ús   �AA;Á;BÚmetric_namec           	      ó
  — t        |ddd¬«      5 }g d¢}t        | D �ch c]  }|d   ’Œ	 c}«      }|j                  «        t        | D �ch c]  }|d   ’Œ	 c}«      }|j                  «        t        | D �ch c]  }|d   ’Œ	 c}«      }|j                  «        g }	|D ]   }
|D ]  }|	j                  d	|› d
|
› �«       Œ Œ" t	        j
                  |||	z   ¬«      }|j                  «        |D ]Ë  }i }i }|j                  t        j                  |	d«      «       | D ]k  }|d   |k(  sŒ||   sŒ|j                  «       D ��ci c]  \  }}||v sŒ||“Œ }}}|s|j                  |«       |d   }|d   }
d	|› d
|
› �}||	v sŒd||   ||<   Œm |sŒŸ|	D ]  }|j                  |d«      ||<   Œ |j                  |«       ŒÍ |j                  «        ddd«       t        d|› d|› �«       yc c}w c c}w c c}w c c}}w # 1 sw Y   Œ0xY w)a  Output a CSV file with summary of a metric on combinations of batch_size and sequence_length.

    Args:
        results (List[Dict[str, Any]]): list of JSON results.
        csv_filename (str): path of output CSV file
        metric_name (str): the metric to summarize
    r)   r*   r+   r,   )r0   r   r   r1   r   r   r2   r3   ÚbÚ_srA   NzSummary results for z are saved to csv file: )rC   ÚlistÚsortÚappendrD   rE   rF   ÚupdateÚdictÚfromkeysÚitemsÚgetrG   rH   r   )r&   r'   rN   rI   Úheader_namesrL   Ú
model_listÚbatch_sizesÚsequence_lengthsÚ	key_namesr3   r2   rK   r"   ÚrowÚvaluesÚkÚvÚheadersÚkeys                       r   Úoutput_summaryre   ‹   sK  € ô 
ˆl ¨b¸7Ô	Cð 4Àxò
ˆô ¸WÖE°6˜6 +Ó.ÒEÓFˆ
Ø�‰Ôä¸wÖG°V˜F <Ó0ÒGÓHˆØ×ÑÔäÈÖ Q¸v Ð(9Ó!:Ò QÓRÐØ×ÑÔàˆ	Ø/ò 	FˆOØ)ò F�
Ø× Ñ  1 Z L°°?Ð2CÐ!DÕEñFð	Fô —^‘^ H¸È	Ñ9QÔRˆ
Ø×ÑÔ àò 	)ˆEØˆCð ˆFØ�M‰Mœ$Ÿ-™-¨	°2Ó6Ô7à!ò :�Ø˜+Ñ&¨%Ó/°F¸;Ó4GØ06·±³×T©¨¨1À!À|ÒBS˜q !™tÐT�GÑTÙØŸ
™
 7Ô+à!'¨Ñ!5�JØ&,Ð->Ñ&?�OØ˜j˜\¨¨OÐ+<Ð=�Cà˜iÒ'Ø&,¨[Ñ&9˜˜sšð:ò Ø$ò 3�CØ%Ÿz™z¨#¨rÓ2�C˜’Hð3à×#Ñ# CÕ(ð1	)ð4 	�‰Ô÷i4ôl 
Ð   Ð-EÀlÀ^Ð
TÕUùò[ Fùò Hùò !Rùó* U÷I4ð 4ús^   �G9žG$
ª G9Á
G)
Á G9Á6G.
ÂB#G9Ä&G9Ä,G9Ä?G3ÅG3Å,G9Å>G9Æ?G9Ç$G9Ç9Hc                  óP  — t        «       } t        | «       dD ]  }t        |«      }|sŒt        |› d�|«       Œ! | j                  }| j                  rAt
        j                  j                  | j                  «      st        d| j                  › �«      ‚t
        j                  j                  dd«      dk(  }g }t        j                  |«      }| j                  D �]'  }||_        t        |dz  d«      |_        | j                  €t        d	«       t#        j"                  «       }t%        || j                  | j&                  | j(                  «      \  }	}
t#        j"                  «       |z
  }t        d
|d›d�«       t        |	j*                  «       ||	j*                  j,                  kD  rt        d«      ‚t/        d|	|d| j0                  ¬«      }t3        d«      }t        d«       t#        j"                  «       }t5        d| j6                  dkD  rd| j6                  › d�nd¬«      }t#        j"                  «       |z
  }t        d|d›d�«       t        d«       t#        j"                  «       }|j9                  ||dd¬«      }t#        j"                  «       |z
  }t        d|d›d�«       | j&                  |d<   ||d<   ||d<   |
|d <   | j0                  |d!<   ||d"<   | j(                  |d#<   t        |«       |j;                  |«       �Œ* t=        |d$«       d%D ]  }t?        ||› d&�|«       Œ y )'N)zonnxruntime-gpuÚonnxruntimeÚonnxÚtorchÚtransformersÚoptimumÚdatasetsÚevaluatez versionz Onnx model path does not exist: ÚORT_DISABLE_FUSED_ATTENTIONÚ0Ú1é   é€   z4Exporting onnx model. It might take a few minutes...z!Onnx model exported or loaded in z.1fz secondszTsequence length should not be larger than {ort_model.config.max_position_embeddings}zquestion-answeringT)r"   Ú	tokenizerÚquestion_firstr2   zLoading dataset...Úsquadr   zvalidation[:ú]Ú
validation)ÚsplitzDataset loaded in z<Evaluating squad_v2 with ORT. It might take a few minutes...Úsquad_v2)Úmodel_or_pipelineÚdataÚmetricÚsquad_v2_formatzEvaluation done in r   r1   r0   r   r2   r3   r   z
detail.csv)r5   r4   r?   z.csv) Úparse_argumentsr   r   Ú
model_namerh   r   r   ÚexistsÚRuntimeErrorÚenvironrY   r   r   r]   Úmodel_max_lengthÚminÚ
doc_strideÚtimer$   r   r   ÚconfigÚmax_position_embeddingsr   r2   r   r   r6   ÚcomputerT   rM   re   )Úargsr!   Úpackage_versionr0   r1   Úall_resultsrs   r3   Ú
start_timeÚ	ort_modelr   ÚlatencyÚqa_pipelineÚtask_evaluatorÚsquad_datasetrL   rN   s                    r   Úmainr“   Ì   sý  € ÜÓ€DÜ	ˆ$„Kàvò 6ˆÜ-¨dÓ3ˆÚÜ�T�F˜(Ð# _Õ5ð6ð
 !ŸO™OÐØ‡y‚yœŸ™Ÿ™¨¯	©	Ô2ÜÐ=¸d¿i¹i¸[ÐIÓJÐJä Ÿj™jŸn™nÐ-JÈCÓPÐTWÑWÐà€KÜ×-Ñ-Ð.CÓD€IØ×0Ñ0ó -#ˆØ%4ˆ	Ô"Ü" ?°aÑ#7¸Ó=ˆ	ÔØ�9‰9ÐÜÐHÔIÜ—Y‘Y“[ˆ
Ü.Ð/DÀdÇiÁiÐQU×Q^ÑQ^Ð`d×`sÑ`sÓtÑˆ	�9Ü—)‘)“+ 
Ñ*ˆÜÐ1°'¸#°¸hÐGÔHäˆi×ÑÔØ˜Y×-Ñ-×EÑEÒEÜÐuÓvÐväØ ¨	¸YÐW[Ðhl×hwÑhwô
ˆô #Ð#7Ó8ˆÜÐ"Ô#Ü—Y‘Y“[ˆ
Ü$ WÐTX×T^ÑT^ÐabÒTb°lÀ4Ç:Á:À,ÈaÑ4PÐhtÔuˆÜ—)‘)“+ 
Ñ*ˆÜÐ" 7¨3 -¨xÐ8Ô9äÐLÔMÜ—Y‘Y“[ˆ
Ø×'Ñ'Ø)ØØØ ð	 (ó 
ˆô —)‘)“+ 
Ñ*ˆÜÐ# G¨C =°Ð9Ô:à!Ÿ]™]ˆˆzÑØ,CˆÐ(Ñ)Ø*?ˆÐ&Ñ'Ø'ˆˆ{ÑØ#Ÿ™ˆˆ|ÑØ$3ˆÐ Ñ!Ø#'×#6Ñ#6ˆÐÑ ÜˆfŒà×Ñ˜6Ö"ð[-#ô^ �; Ô-à<ò GˆÜ�{ { m°4Ð$8¸+ÕFñGr%   c           	      óî  — t        j                  «       }|j                  dddt        t        d   dt        › �¬«       |j                  ddd	t
        d
gd¬«       |j                  ddt
        dd¬«       |j                  ddt
        dd¬«       |j                  ddt        d d¬«       |j                  dddd¬«       |j                  dddd¬«       |j                  d¬«       |j                  | «      }|S ) Nz-mz--model_nameFr   z=Checkpoint directory or pre-trained model names in the list: )ÚrequiredÚtypeÚdefaultÚhelpz-sz--sequence_lengthsú+i€  zFSequence lengths for onnx model inputs. It could have multiple values.)Únargsr–   r—   r˜   z-bz--batch_sizeé   zbatch size for inference.)r–   r—   r˜   z-tz--totalz+Total samples to test. 0 means all samples.z--onnxzbOptional onnx model path. If not specified, optimum will be used to export onnx model for testing.z
--providerÚCUDAExecutionProviderzRSelect which Execution Provider to use for runs. Default is CUDAExecutionProvider.)r•   r—   r˜   z--use_io_bindingÚ
store_truezUse IO Binding for GPU.)r•   Úactionr˜   )r   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚPRETRAINED_SQUAD_MODELSÚintÚset_defaultsÚ
parse_args)ÚargvÚparserrŠ   s      r   r~   r~     s0  € Ü×$Ñ$Ó&€Fà
×ÑØØØÜÜ'¨Ñ*ØLÔMdÐLeÐfð ô ð ×ÑØØØÜØ�ØUð ô ð ×ÑØØÜØØ(ð ô ð ×Ñ˜˜i¬c¸1ÐCpÐÔqà
×ÑØØÜØØqð ô ð ×ÑØØØ'Øað	 ô ð ×ÑÐ*°UÀ<ÐVoÐÔpØ
×Ñ uÐÔ-à×Ñ˜TÓ"€Dà€Kr%   Ú__main__)Nrœ   Fr   )(rŸ   rD   r   r†   Úimportlib.metadatar   r   ÚImportErrorÚimportlib_metadataÚpathlibr   Útypingr   rl   r   rm   r   Úoptimum.onnxruntimer	   Úoptimum.versionr
   Úoptimum_versionÚ	packagingÚversion_checkrj   r   r   Úparser£   r¢   r   Úboolr$   rR   rV   rM   re   r“   r~   Ú__name__© r%   r   ú<module>r¸      s:  ðó2 Û 
Û 	Û ðAß@õ Ý å !Ý Ý <Ý :Ý .ß 0à€=×Ñ�Ó'Ð*=¨-×*=Ñ*=¸hÓ*GÒGÙ
ÐIÈ/ÐIZÐZ[Ð\Ó
]Ð]òÐ ð có ð kpñ!Øð!Ø! D™jð!Øcgó!ðH&C˜D  c¨3 h¡Ñ0ð &CÀó &CðR>V˜D  c¨3 h¡Ñ0ð >VÀð >VÐRUó >VòBCGóL3ðl ˆzÒÙ…Fð øðQ	 ò Aß@Ñ@ðAús   ’C ÃCÃC