Ë
    cQêjÉ;  ã                   ó°   — d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
  ej                  e«      Zd„ Zd„ Zd„ Zd„ Zd„ Zed	k(  r e«        yy)
é    N)Úsetup_logger)ÚBenchmarkRecordc                  óÐ  — t        j                  «       } | j                  ddt        d¬«       | j                  ddt        d¬«       | j                  dd	t        d
¬«       | j                  ddt        d¬«       | j                  dddd¬«       | j                  dddd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  d t        d!d"¬#«       | j                  d$t        d!g d%¢d&¬'«       | j                  d(t        d!d)d*gd+¬'«       | j                  d,t        d-d.¬«       | j                  d/ddd0¬«       | j                  d1t        d2d3¬«       | j                  d4t        d d5¬«       | j                  «       }t        |d6|j                  j                  d7«      d8   j                  d9d:«      «       d;|j                  › d<|j                  › �}|j                  s||_        t        j                  |j                  d!¬=«       |xj                  d>z  c_        |S )?Nz-bú--batch-sizesz1 2)ÚtypeÚdefaultz-sú--sequence-lengthsz8 16 32 64 128 256 512z-wú--warmup-runsé   z-nú
--num-runséè  z--hf-pt-eagerFÚ
store_truez,Benchmark in PyTorch without `torch.compile`)r   ÚactionÚhelpz--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`ú--hf-ort-dir-pathÚ zDPath to folder containing ONNX models for Optimum + ORT benchmarking)r   r   r   z--ort-msft-model-pathzAPath to ONNX model from https://github.com/microsoft/Llama-2-Onnxz --ort-convert-to-onnx-model-pathz'Path to ONNX model from convert_to_onnxú--cache-dirz./model_cachez-Cache dir where Hugging Face files are storedú--model-nameTzModel name in Hugging Face)r   Úrequiredr   ú--precision)Úint4Úint8Úfp16Úfp32zPrecision to run model)r   r   Úchoicesr   ú--deviceÚcpuÚcudazDevice to benchmark modelsz--device-idr   zGPU device IDz	--verbosezPrint detailed logsz	--timeouté
   z8Number of mins to attempt the benchmark before moving onú--log-folderz'Path to folder to save logs and resultsÚ
model_sizeú/éÿÿÿÿú.ú-z./Ú_)Úexist_oké<   )ÚargparseÚArgumentParserÚadd_argumentÚstrÚintÚ
parse_argsÚsetattrÚ
model_nameÚsplitÚreplacer!   Ú	precisionÚ
log_folderÚosÚmakedirsÚtimeout)ÚparserÚargsÚlog_folder_names      úx/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/benchmark_all.pyÚget_argsr<      s×  € Ü×$Ñ$Ó&€Fà
×ÑØØÜØð	 ô ð ×ÑØØÜØ(ð	 ô ð ×ÑØØÜØð	 ô ð ×ÑØØÜØð	 ô ð ×ÑØØØØ;ð	 ô ð ×ÑØØØØ8ð	 ô ð ×ÑØÜØØSð	 ô ð ×ÑØÜØØPð	 ô ð ×ÑØ*ÜØØ6ð	 ô ð ×ÑØÜØØ<ð	 ô ð ×ÑØÜØØ)ð	 ô ð ×ÑØÜØÚ0Ø%ð ô ð ×ÑØÜØØ˜�Ø)ð ô ð ×ÑØÜØØð	 ô ð ×ÑØØØØ"ð	 ô ð ×ÑØÜØØGð	 ô ð ×ÑØÜØØ6ð	 ô ð ×ÑÓ€DäˆD�, §¡× 5Ñ 5°cÓ :¸2Ñ >× FÑ FÀsÈCÓ PÔQØ˜4Ÿ?™?Ð+¨1¨T¯^©^Ð,<Ð=€OØ�?Š?Ø)ˆŒÜ‡K�K�—‘¨$Õ/ð 	‡L‚L�BÑ…Là€Kó    c           
      ó`  — g }d\  }}}d\  }}}	}
d}d}d}d}d}d}d	}t        |«      5 }|D �]h  }|j                  d
d«      }||v rt        |t        |«      d  «      }Œ2||v rt        |t        |«      d  «      }ŒN||v rd}ŒU||v rd}Œ\||v r,t	        |t        |«      |j                  d«       «      }|dz  }ŒŒ||v r't	        |t        |«      |j                  d«       «      }	Œ·||v sŒ¼d|v r3t	        ||j                  d«      dz   |j                  d«       «      dz  }
nY||j                  |«      t        |«      z   d  j                  dd«      }t        j                  |«      |    d   }t	        |«      dz  }
|||||||	|
gz   }|j                  |«       �Œk 	 d d d «       |S # 1 sw Y   |S xY w)N)NNN)NNNNzBatch Size: zSequence Length: zto get past_key_valueszwith past_key_valuesz	Latency: zThroughput: zpeak=ú
r   Úpromptz	per-tokenú r   ÚCPUú=é   z MBú'ú"Úmax_used_MB)
Úopenr2   r-   ÚlenÚfloatÚrfindÚfindÚjsonÚloadsÚappend)Ú	device_idÚlog_fileÚbase_resultsÚentriesÚ
batch_sizeÚsequence_lengthÚstepÚ	latency_sÚ
latency_msÚ
throughputÚmemoryÚbatch_patternÚsequence_patternÚprompt_step_patternÚper_token_step_patternÚlatency_patternÚthroughput_patternÚmemory_patternÚfÚ
input_lineÚlineÚpeakÚusageÚentrys                           r;   Úprocess_log_filerh   ž   s  € Ø€GØ(8Ñ%€J� Ø0FÑ-€Iˆz˜: và"€MØ*ÐØ2ÐØ3ÐØ!€OØ'ÐØ€Nä	ˆh‹ð '&˜1Øó &	&ˆJØ×%Ñ% d¨BÓ/ˆDà Ñ$Ü  ¤c¨-Ó&8Ð&:Ð!;Ó<‘
Ø! TÑ)Ü"% d¬3Ð/?Ó+@Ð+BÐ&CÓ"D‘Ø$¨Ñ,Ø‘Ø'¨4Ñ/Ø"‘Ø  DÑ(Ü! $¤s¨?Ó';¸d¿j¹jÈ»oÐ"NÓO�	Ø&¨Ñ-‘
Ø# tÑ+Ü" 4¬Ð,>Ó(?À$Ç*Á*ÈSÃ/Ð#RÓS‘
Ø 4Ò'Ø˜D‘=ô # 4¨¯
©
°3«¸!Ñ(;¸d¿j¹jÈÓ>OÐ#PÓQÐTXÑX‘Fð   §	¡	¨.Ó 9¼CÀÓ<OÑ OÐ QÐR×ZÑZÐ[^Ð`cÓd�DÜ ŸJ™J tÓ,¨YÑ7¸ÑF�EÜ" 5›\¨DÑ0�Fð %ØØ#ØØØØØð(ñ �ð —‘˜uÖ%ñM&	&÷'&ðR €N÷S'&ðR €Nús   ©B>F#Ã(B0F#Æ#F-c           	      ó‚  — dd l }|j                  | g d¢¬«      }|d   j                  d«      |d<   |d   j                  d«      |d<   |d   j                  d«      |d<   |d   j                  d«      |d<   |d	   j                  d
«      |d	<   |d   j                  d
«      |d<   |d   j                  d
«      |d<   |d   j                  d
«      |d<   dd l}|j                  }t        |D �cg c],  }|j                  dv sŒ|j                  › d|j                  › �‘Œ. c}«      }d}d}	|r.|d   j                  d«      d   }|d   j                  d«      d   }	g }
|j                  «       D �]y  \  }}|d   dv rt        |d   |d   d|d   ||	«      }nY|d   dv r6t        |d   |d   d|d   t        j                  t        j                  «      }nt        |d   |d   |d   |d   dd«      }|d   |j                  _        |d   |j                  _        |d   |j                  _        |d   |j                  _        |d   |j                  j&                  d<   |d   |j                  j&                  d<   |d	   |j(                  j&                  d<   |d   |j(                  _        |d   |j(                  j&                  d<   |d   |j(                  _        |
j/                  |«       �Œ| t        j0                  ||
«       t        j2                  |j5                  dd «      |
«       t6        j9                  d!|› d"�«       y c c}w )#Nr   )úWarmup RunsúMeasured Runsú
Model NameÚEngineÚ	PrecisionÚDeviceú
Batch SizeúSequence LengthÚStepúLatency (s)úLatency (ms)úThroughput (tps)úMemory (GB))Úcolumnsrj   r-   rk   rp   rq   rs   rJ   rt   ru   rv   )Úonnxruntimezonnxruntime-gpuz==r   rD   rm   )úoptimum-ortrx   rl   rn   rx   ro   )úpytorch-eagerúpytorch-compileÚpytorchrr   Úmeasure_stepÚengineÚlatency_s_meanÚthroughput_tpsú.csvz.jsonzResults saved in ú!)ÚpandasÚ	DataFrameÚastypeÚpkg_resourcesÚworking_setÚsortedÚkeyÚversionr1   Úiterrowsr   ÚtorchÚ__name__Ú__version__ÚconfigÚwarmup_runsÚmeasured_runsrT   Ú
seq_lengthÚ
customizedÚmetricsÚlatency_ms_meanÚmax_memory_usage_GBrO   Úsave_as_csvÚsave_as_jsonr2   ÚloggerÚinfo)ÚresultsÚfilenameÚpdÚdfr†   Úinstalled_packagesÚiÚinstalled_packages_listÚort_pkg_nameÚort_pkg_versionÚrecordsr&   ÚrowÚrecords                 r;   Úsave_resultsr§   ×   sf  € Ûà	�‰Øò
ð 
ó 
€Bð( ˜=Ñ)×0Ñ0°Ó7€B€}ÑØ˜_Ñ-×4Ñ4°UÓ;€B€ÑØ˜,Ñ'×.Ñ.¨uÓ5€B€|ÑØÐ0Ñ1×8Ñ8¸Ó?€BÐÑØ˜=Ñ)×0Ñ0°Ó9€B€}ÑØ˜NÑ+×2Ñ2°7Ó;€B€~ÑØÐ 2Ñ3×:Ñ:¸7ÓC€BÐÑØ˜=Ñ)×0Ñ0°Ó9€B€}Ñó à&×2Ñ2ÐÜ$Ø*<Öl QÀÇÁÐIkÒ@kˆA�E‰Eˆ7�"�Q—Y‘Y�KÒ	 ÒlóÐð €LØ€OÙØ.¨qÑ1×7Ñ7¸Ó=¸aÑ@ˆØ1°!Ñ4×:Ñ:¸4Ó@ÀÑCˆð €GØ—+‘+“-ó ‰ˆˆ3Øˆx‰=Ð:Ñ:Ü$Ø�LÑ! 3 {Ñ#3°]ÀCÈÁMÐS_Ðapó‰Fð �‰]ÐBÑBÜ$Ø�LÑ! 3 {Ñ#3°YÀÀHÁÌuÏ~É~Ô_d×_pÑ_pó‰Fô % S¨Ñ%6¸¸KÑ8HÈ#ÈhÉ-ÐY\Ð]eÑYfÐhjÐlnÓoˆFØ$'¨Ñ$6ˆ�‰Ô!Ø&)¨/Ñ&:ˆ�‰Ô#Ø#& |Ñ#4ˆ�‰Ô Ø#&Ð'8Ñ#9ˆ�‰Ô Ø36°v±;ˆ�‰× Ñ  Ñ0Ø-0°©]ˆ�‰× Ñ  Ñ*Ø69¸-Ñ6Hˆ�‰×!Ñ!Ð"2Ñ3Ø),¨^Ñ)<ˆ�‰Ô&Ø69Ð:LÑ6Mˆ�‰×!Ñ!Ð"2Ñ3Ø-0°Ñ-?ˆ�‰Ô*à�‰�vÖð-ô0 ×Ñ ¨'Ô2Ü× Ñ  ×!1Ñ!1°&¸'Ó!BÀGÔLÜ
‡K�KÐ# H :¨QÐ/Õ0ùòK 	ms   Ã+L<Ã?L<c                 ól  — |› dt         j                   j                  «       d›d�}t        j                  j	                  | j
                  |«      }t        |d«      5 }t        j                  |||¬«      }	 |j                  | j                  «       d d d «       t        j                  d«       | j                  | j                   | j"                  || j$                  | j&                  g}t)        | j*                  ||«      }|S # t        j                  $ r |j                  «        Y Œ•w xY w# 1 sw Y   ŒšxY w)Nr&   ú%Y-%m-%d_%H:%M:%Sz.logÚw)ÚstdoutÚstderrz Gathering data from log files...)ÚdatetimeÚnowr5   ÚpathÚjoinr4   rH   Ú
subprocessÚPopenÚwaitr7   ÚTimeoutExpiredÚkillr™   rš   r�   Únum_runsr0   r3   Údevicerh   rP   )	r9   Úbenchmark_cmdr~   Úlog_filenameÚlog_pathrQ   ÚprocessrR   r›   s	            r;   Ú	benchmarkr¼   $  sú   € Ø�X˜Qœx×0Ñ0×4Ñ4Ó6Ð7HÐIÈÐN€LÜ�w‰w�|‰|˜DŸO™O¨\Ó:€HÜ	ˆh˜Ó	ð  Ü×"Ñ" =¸È(ÔSˆð	Ø�L‰L˜Ÿ™Ô&÷ô ‡K�KÐ2Ô3Ø×$Ñ$ d§m¡m°T·_±_ÀfÈdÏnÉnÐ^b×^iÑ^iÐj€LÜ˜tŸ~™~¨x¸ÓF€Gà€Nøô ×(Ñ(ò 	Ø�L‰LŽNð	ú÷	ð ús*   ÁD*Á6DÄ#D'Ä$D*Ä&D'Ä'D*Ä*D3c                  óä	  — t        «       } t        | j                  «       t        j	                  | j
                  «       dt        j                  j                  _	        g }t        | j                  «      t        j                  d<   | j                  r¹dddddd| j                  d	| j                   d
| j"                  d| j$                  d| j&                  dt        | j(                  «      dt        | j*                  «      d| j,                  d| j.                  dg}t        j	                  d«       t        | |d«      }|j1                  |«       | j2                  r¹dddddd| j                  d	| j                   d
| j"                  d| j$                  d| j&                  dt        | j(                  «      dt        | j*                  «      d| j,                  d| j.                  dg}t        j	                  d«       t        | |d«      }|j1                  |«       | j4                  rÅdddddd| j4                  d| j                  d	| j                   d
| j"                  d| j$                  d| j&                  dt        | j(                  «      dt        | j*                  «      d| j,                  d| j.                  dg}t        j	                  d«       t        | |d«      }|j1                  |«       | j6                  rÄdddddd| j6                  d| j                  d	| j                   d
| j"                  d| j$                  d| j&                  dt        | j(                  «      dt        | j*                  «      d| j,                  d| j.                  g}t        j	                  d«       t        | |d«      }|j1                  |«       | j8                  rÄdddddd| j8                  d| j                  d	| j                   d
| j"                  d| j$                  d| j&                  dt        | j(                  «      dt        | j*                  «      d| j,                  d| j.                  g}t        j	                  d«       t        | |d «      }|j1                  |«       | j:                  › d!| j                   › d!t<        j<                  j?                  «       d"›d#�}tA        |t        jB                  jE                  | j,                  |«      «       y )$NTÚCUDA_VISIBLE_DEVICESÚpythonz-mzmodels.llama.benchmarkz--benchmark-typezhf-pt-eagerr   r   r   r	   r   r
   r   r    r   z--authz'Benchmark PyTorch without torch.compilerz   zhf-pt-compilez$Benchmark PyTorch with torch.compiler{   zhf-ortr   z Benchmark Optimum + ONNX Runtimery   zort-msftz--ort-model-pathz)Benchmark Microsoft model in ONNX Runtimezort-convert-to-onnxz/Benchmark convert_to_onnx model in ONNX Runtimerx   r&   r©   r�   )#r<   r   Úverboser™   rš   Ú__dict__rŒ   ÚbackendsÚcudnnr¼   r,   rP   r5   ÚenvironÚhf_pt_eagerr0   r3   Úbatch_sizesÚsequence_lengthsr·   r�   r¶   r4   Ú	cache_dirÚextendÚhf_pt_compileÚhf_ort_dir_pathÚort_msft_model_pathÚort_convert_to_onnx_model_pathr!   r­   r®   r§   r¯   r°   )r9   Úall_resultsr¸   r›   Úcsv_files        r;   ÚmainrÐ   6  sc  € Ü‹:€DÜ�—‘ÔÜ
‡K�K�—‘ÔØ%)„E‡N�N×ÑÔ"à€KÜ),¨T¯^©^Ó)<„B‡J�JÐ%Ñ&ð ×ÒàØØ$ØØØØ�O‰OØØ�N‰NØØ×ÑØ Ø×!Ñ!ØØ�K‰KØÜ�× Ñ Ó!ØÜ�—‘ÓØØ�O‰OØØ�N‰NØð1
ˆô4 	�‰Ð=Ô>Ü˜D -°ÓAˆØ×Ñ˜7Ô#ð ×ÒàØØ$ØØØØ�O‰OØØ�N‰NØØ×ÑØ Ø×!Ñ!ØØ�K‰KØÜ�× Ñ Ó!ØÜ�—‘ÓØØ�O‰OØØ�N‰NØð1
ˆô4 	�‰Ð:Ô;Ü˜D -Ð1BÓCˆØ×Ñ˜7Ô#ð ×ÒàØØ$ØØØØ× Ñ ØØ�O‰OØØ�N‰NØØ×ÑØ Ø×!Ñ!ØØ�K‰KØÜ�× Ñ Ó!ØÜ�—‘ÓØØ�O‰OØØ�N‰NØð5
ˆô8 	�‰Ð6Ô7Ü˜D -°Ó?ˆØ×Ñ˜7Ô#ð ×ÒàØØ$ØØØØ×$Ñ$ØØ�O‰OØØ�N‰NØØ×ÑØ Ø×!Ñ!ØØ�K‰KØÜ�× Ñ Ó!ØÜ�—‘ÓØØ�O‰OØØ�N‰Nð3
ˆô6 	�‰Ð?Ô@Ü˜D -°Ó<ˆØ×Ñ˜7Ô#ð ×*Ò*àØØ$ØØ!ØØ×/Ñ/ØØ�O‰OØØ�N‰NØØ×ÑØ Ø×!Ñ!ØØ�K‰KØÜ�× Ñ Ó!ØÜ�—‘ÓØØ�O‰OØØ�N‰Nð3
ˆô6 	�‰ÐEÔFÜ˜D -°Ó?ˆØ×Ñ˜7Ô#à—/‘/Ð" ! D§N¡NÐ#3°1´X×5FÑ5F×5JÑ5JÓ5LÐM^Ð4_Ð_cÐd€HÜ�œbŸg™gŸl™l¨4¯?©?¸HÓEÕFr=   Ú__main__)r)   r­   rM   Úloggingr5   r±   rŒ   Úbenchmark_helperr   r”   r   Ú	getLoggerr�   r™   r<   rh   r§   r¼   rÐ   © r=   r;   ú<module>rÖ      si   ðó Û Û Û Û 	Û ã Ý )Ý #à	ˆ×	Ñ	˜8Ó	$€òGòT6òrJ1òZò$nGðb ˆzÒÙ…Fð r=   