uPjUdZddlZddlZddlmZddlmZddlmZddl Z ddl Z ddl m Z ddl mZddlmZdd lmZdd lmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#dd l$m%Z%dd l&m'Z'm(Z(eZ)d Z*ee+d<e( d7dede,ede-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdefdZ2e( d7dede,ede-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzfdZ3e( d8de4dzde-dzde-dzde-dzde-dzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdefd Z5e( d8de4dzde-dzde-dzde-dzde-dzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzfd!Z6e( d9d"e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdef d#Z7e( d9d"e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzf d$Z8e( d7d"e-de-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdefd%Z9e( d7d"e-de-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzfd&Z:e( d9d"e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdef d'Z;e( d9d"e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzf d(Ze( d7d"e-d+e.e-e/fde-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde!fd,Z?e( d7d"e-d+e.e-e/fde-dzde.e-e/fdzde.e-e/fdzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde!ee/e/e!fzfd-Z@e( d:d"e-de4dzde-dzde-dzde-dzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde fd.ZAe( d:d"e-de4dzde-dzde-dzde-dzde.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde ee/e/e fzfd/ZBe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde!fd1ZCe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde!ee/e/e!fzfd2ZDe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdefd3ZEe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzdeee/e/efzfd4ZFe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde"fd5ZGe( d9d"e-d0e-de.e-e/fdzde.e-e/fdzde0e jbzdzde-dzde"ee/e/e"fzfd6ZHy);zw Main entry point for Evals API operations Provides create, list, get, update, delete, and cancel operations for evals N) Coroutine)partial)Final)request_timeout)Logging)BaseEvalsAPIConfig)BaseLLMHTTPHandler)CancelEvalResponseCancelRunResponseCreateEvalRequestCreateRunRequestDataSourceConfigDeleteEvalResponseEval GraderConfigListEvalsParamsListEvalsResponseListRunsParamsListRunsResponseRunRunDeleteResponseUpdateEvalRequest)GenericLiteLLMParams)ProviderConfigManagerclientzhttps://api.openai.comDEFAULT_OPENAI_API_BASEdata_source_configtesting_criterianamemetadata extra_headers extra_query extra_bodytimeoutcustom_llm_providerreturnc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)ae Async: Create a new evaluation Args: data_source_config: Configuration for the data source testing_criteria: List of graders for all eval runs name: Optional name for the evaluation metadata: Optional additional metadata (max 16 key-value pairs) extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T acreate_eval) rrrr r!r"r#r$r%Nmodelr%original_exceptioncompletion_kwargs extra_kwargs) localsasyncioget_event_loopr create_eval contextvars copy_contextrunrun_in_executor iscoroutine Exceptionlitellmexception_type)rrrr r!r"r#r$r%kwargs local_varsloopfuncctxfunc_with_context init_responseresponsees W/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/evals/main.pyr(r(,s>J" ,,.!%~   1-'#! 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    Y CA2B.B*B.B, B.%C&B.)C*B.,B.. C7CCCc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |||d }|||d <|r|j||xsi}|j|| }|j||| }|jxst}|j|d}| j| d|d| i|t j#|||||| ||xst$| | jd| jd }|S#t&$r}t j(d||| | d}~wwxYw)a^ Create a new evaluation Args: data_source_config: Configuration for the data source testing_criteria: List of graders for all eval runs name: Optional name for the evaluation metadata: Optional additional metadata (max 16 key-value pairs) extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object litellm_logging_objlitellm_call_idNr(FTopenaiproviderz!CREATE eval is not supported for )rrrheaderslitellm_params)create_requestrMrLevals)api_baseendpointr:r*optional_paramsrMr%rshared_session url request_bodyevals_api_provider_configr%rM logging_objr!r$ _is_asyncrrTr))r.getpoprrget_provider_evals_api_configr8 LlmProviders ValueErrorupdatevalidate_environmenttransform_create_eval_requestrPrget_complete_urlupdate_from_kwargsbase_llm_http_handlercreate_eval_handlerrr7r9)rrrr r!r"r#r$r%r:r;rFrGrZrMrXrNrLrWrPrVrArBs rCr1r1qs>JQ 8> CX8Y-3ZZ8I4-P!::ne<D !5 >v >  &"* @U?r?r))*=>@ ! % ,@AT@UVW W#5 04   %)N6 "   ! !* - %2+@@ao@p8UU))V )11L5L.??\c?d ..(!?!4 / 0CC%&? 3)+!.::h'!::&67D    $$ 3 (    sEE F'FFlimitafterbeforeorderorder_byc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)a Async: List all evaluations Args: limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns evals after this ID before: Cursor for pagination - returns evals before this ID order: Sort order ('asc' or 'desc', default 'desc') order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListEvalsResponse object T alist_evals) rhrirjrkrlr!r"r$r%Nr)) r.r/r0r list_evalsr2r3r4r5r6r7r8r9)rhrirjrkrlr!r"r$r%r:r;r<r=r>r?r@rArBs rCrnrns>J" ,,. $}   '# 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    rDc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |i}|||d <|||d <|||d <|||d <|||d<|r|j||xsi}|j||}|j|||\}}| j| d|d| i|tj|||||| ||xst| | jd| jd }|S#t $r}t j"d||| | d}~wwxYw)a List all evaluations Args: limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns evals after this ID before: Cursor for pagination - returns evals before this ID order: Sort order ('asc' or 'desc', default 'desc') order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListEvalsResponse object rFrGNrnFTrHrIz LIST evals is not supported for rhrirjrkrlrK) list_paramsrMrLrRrrT rV query_paramsrXr%rMrYr!r$rZrrTr)r[)r.r\r]rrr^r8r_r`rarbtransform_list_evals_requestrerflist_evals_handlerrr7r9)rhrirjrkrlr!r"r$r%r:r;rFrGrZrMrXrqrLrVrsrArBs rCroro*s>JR 8> CX8Y-3ZZ8I4-P!::mU;tC !5 >v >  &"* @U?r?r))*=>@ ! % ,?@S?TUV V/1  #(K  #(K  $*K !  #(K  &.K #    { + %2+@@ao@p6RR#)S \ ..(!?!4 / 0BB%&? 3)+!.::h'!::&67C    $$ 3 (    sEE E;E66E;eval_idc Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)ax Async: Get an evaluation by ID Args: eval_id: The ID of the evaluation to fetch extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T aget_evalrvr!r"r$r%Nr)) r.r/r0rget_evalr2r3r4r5r6r7r8r9rvr!r"r$r%r:r;r<r=r>r?r@rArBs rCrxrxs.J ,,."{  '# 3   !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    Y CA.B*<B&=B*B(B*!C"B*%C&B*(B** C3C  CCc t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} |j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)aq Get an evaluation by ID Args: eval_id: The ID of the evaluation to fetch extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object rFrGNrxFTrHrIzGET eval is not supported for rKrvrPrMrLrvrRrrT rVrXr%rMrYr!r$rZrrTr)r[)r.r\r]rrr^r8r_r`rbrPrtransform_get_eval_requestrerfget_eval_handlerrr7r9rvr!r"r$r%r:r;rFrGrZrMrXrLrPrVrArBs rCrzrzs.JB 8> CX8Y-3ZZ8I4-P!::k59TA !5 >v >  &"* @U?r?r))*=>@ ! % ,=>Q=RST T %2+@@ao@p)11L5L0KK) L W ..&0!?!4 / 0@@&? 3)+!.::h'!::&67A    $$ 3 (    D$D11 E:EEc Kt} tj} d|d<ttf||||||||d|} t j } t| j| } | jd| d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| |d}~wwxYww)a Async: Update an evaluation Args: eval_id: The ID of the evaluation to update name: Updated name metadata: Updated metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T aupdate_eval)rvrr r!r"r#r$r%Nr)) r.r/r0r update_evalr2r3r4r5r6r7r8r9)rvrr r!r"r#r$r%r:r;r<r=r>r?r@rArBs rCrr5s:J! ,,.!%~   '#! 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    sY CA1B-?B)B-B+B-$C%B-(C)B-+B-- C6CCCc ht} |jd} |jdd} |jdddu} tdi|} |d}t j t j|}|td |i}|||d <|3hd }|jDcic] \}}||vs ||}}}|r||d <|r|j||xsi}|j|| }| jxst}|j|||| |\}}}| j|d|d| i|t j#||||| | ||xst$| |jd|jd }|Scc}}w#t&$r}t j(d||| |d}~wwxYw)a Update an evaluation Args: eval_id: The ID of the evaluation to update name: Updated name metadata: Updated metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object rFrGNrFTrHrIz!UPDATE eval is not supported for r>rLrQ user_agent user_api_keyuser_api_key_authuser_api_key_hashrequester_metadatauser_api_key_aliasuser_api_key_spendlitellm_api_versionuser_api_key_org_idrequester_ip_addressuser_api_key_team_iduser_api_key_user_iduser_api_key_metadatauser_api_key_max_budgetuser_api_key_team_aliasuser_api_key_team_spenduser_api_key_user_emailuser_api_key_user_spendlitellm_parent_otel_spanuser_api_key_end_user_iduser_api_key_auth_metadatauser_api_key_request_routeglobal_max_parallel_requestsuser_api_end_user_max_budgetuser_api_key_budget_reset_atuser_api_key_team_max_budgetuser_api_key_user_max_budgetuser_api_key_model_max_budgetr rK)rvupdate_requestrPrMrLrRrrTrUr)r[)r.r\r]rrr^r8r_r`itemsrarbrPrtransform_update_eval_requestrerfupdate_eval_handlerrr7r9)rvrr r!r"r#r$r%r:r;rFrGrZrMrXr internal_keyskvfiltered_metadatarLrPrVrWrArBs rCrrwsD:Jy 8> CX8Y-3ZZ8I4-P!::ne<D !5 >v >  &"* @U?r?r))*=>@ ! % ,@AT@UVW W46  %)N6 "  $MB:B9I'dAQVcMc1'd 'd ->z*   ! !* - %2+@@ao@p)11L5L & C C)) D     ..(!?!4 / 0CC%&? 3)+!.::h'!::&67D  i(ej  $$ 3 (    s1B!F - F:F?CF F F1F,,F1c Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)a Async: Delete an evaluation Args: eval_id: The ID of the evaluation to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: DeleteEvalResponse object T adelete_evalryNr)) r.r/r0r delete_evalr2r3r4r5r6r7r8r9r{s rCrr.J ,,.!%~  '# 3   !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    r|c t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} |j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a} Delete an evaluation Args: eval_id: The ID of the evaluation to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: DeleteEvalResponse object rFrGNrFTrHrIz!DELETE eval is not supported for rKr~rvrRrrTrr)r[)r.r\r]rrr^r8r_r`rbrPrtransform_delete_eval_requestrerfdelete_eval_handlerrr7r9rs rCrrJs.JB 8> CX8Y-3ZZ8I4-P!::ne<D !5 >v >  &"* @U?r?r))*=>@ ! % ,@AT@UVW W %2+@@ao@p)11L5L0NN) O W ..&0!?!4 / 0CC&? 3)+!.::h'!::&67D    $$ 3 (    rc Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)a Async: Cancel a running evaluation Args: eval_id: The ID of the evaluation to cancel extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: CancelEvalResponse object T acancel_evalryNr)) r.r/r0r cancel_evalr2r3r4r5r6r7r8r9r{s rCrrrr|c t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} }|j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Cancel a running evaluation Args: eval_id: The ID of the evaluation to cancel extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: CancelEvalResponse object rFrGNrFTrHrIz!CANCEL eval is not supported for rKr~rvrRrrTrr)r[)r.r\r]rrr^r8r_r`rbrPrtransform_cancel_eval_requestrerfcancel_eval_handlerrr7r9)rvr!r"r$r%r:r;rFrGrZrMrXrLrPrVrWrArBs rCrrs.JF 8> CX8Y-3ZZ8I4-P!::ne<D !5 >v >  &"* @U?r?r))*=>@ ! % ,@AT@UVW W %2+@@ao@p)11L5L & C C) D     ..&0!?!4 / 0CC&? 3)+!.::h'!::&67D    $$ 3 (    sD%D22 E;EE data_sourcec Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)ap Async: Create a new run for an evaluation Args: eval_id: The ID of the evaluation to run data_source: Data source configuration for the run (can be jsonl, completions, or responses type) name: Optional name for the run metadata: Optional additional metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object T acreate_run) rvrrr r!r"r#r$r%Nr)) r.r/r0r create_runr2r3r4r5r6r7r8r9)rvrrr r!r"r#r$r%r:r;r<r=r>r?r@rArBs rCrrFs>J" ,,. $}   #'#! 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    rDc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |d |i}|||d <|r|j||xsi}|j|| }|jxst}|j|||| \}}| j| d|d| i|tj!|||||| ||xst#j$dd| | jd| jd }|S#t&$r}t j(d||| | d}~wwxYw)a Create a new run for an evaluation Args: eval_id: The ID of the evaluation to run data_source: Data source configuration for the run (can be jsonl, completions, or responses type) name: Optional name for the run metadata: Optional additional metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout (default 600s for long-running operations) custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object rFrGNrFTrHrIz CREATE run is not supported for rrrK)rvrNrMrLrRg@g@)r$connectrrTrUr)r[)r.r\r]rrr^r8r_r`rarbrPrtransform_create_run_requestrerfcreate_run_handlerhttpxTimeoutr7r9)rvrrr r!r"r#r$r%r:r;rFrGrZrMrXrNrLrPrVrWrArBs rCrrs>JP 8> CX8Y-3ZZ8I4-P!::mU;tC !5 >v >  &"* @U?r?r))*=>@ ! % ,?@S?TUV V ;3   %)N6 "   ! !* - %2+@@ao@p)11L5L5RR)) S \ ..(!?!4 / 0BB%&? 3)+!Hu}}UCH::h'!::&67C    $$ 3 (    sEE F(FFc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)a Async: List all runs for an evaluation Args: eval_id: The ID of the evaluation limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns runs after this ID before: Cursor for pagination - returns runs before this ID order: Sort order ('asc' or 'desc', default 'desc') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListRunsResponse object T alist_runs) rvrhrirjrkr!r"r$r%Nr)) r.r/r0r list_runsr2r3r4r5r6r7r8r9)rvrhrirjrkr!r"r$r%r:r;r<r=r>r?r@rArBs rCrrs>J" ,,.#|   '# 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    rDc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |i}|||d <|||d <|||d <|||d <|r|j||xsi}|j||}|j||||\}}| j| dd|i|d| i|tj|||||| ||xst| | jd| jd }|S#t $r}t j"d||| | d}~wwxYw)a List all runs for an evaluation Args: eval_id: The ID of the evaluation limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns runs after this ID before: Cursor for pagination - returns runs before this ID order: Sort order ('asc' or 'desc', default 'desc') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListRunsResponse object rFrGNrFTrHrIzLIST runs is not supported for rhrirjrkrK)rvrqrMrLrvrRrrTrrr)r[)r.r\r]rrr^r8r_r`rarbtransform_list_runs_requestrerflist_runs_handlerrr7r9)rvrhrirjrkr!r"r$r%r:r;rFrGrZrMrXrqrLrVrsrArBs rCrrCs >JQ 8> CX8Y-3ZZ8I4-P!::lE:dB !5 >v >  &"* @U?r?r))*=>@ ! % ,>?R>STU U.0  #(K  #(K  $*K !  #(K     { + %2+@@ao@p6QQ#) R \ ..&@<@!?!4 / 0AA%&? 3)+!.::h'!::&67B    $$ 3 (    sEE E9E44E9run_idc Kt} tj}d|d<ttf||||||d|} t j } t| j| } |jd| d{} tj| r | d{} | S| } | S7)7 #t$r}tjd||||d}~wwxYww)a Async: Get a specific run Args: eval_id: The ID of the evaluation run_id: The ID of the run to retrieve extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object Taget_runrvrr!r"r$r%Nr)) r.r/r0rget_runr2r3r4r5r6r7r8r9rvrr!r"r$r%r:r;r<r=r>r?r@rArBs rCrrs2J ,,.!z   '# 3    !--/#*377D#9%)%9%9$@Q%RR   } -**H%H S+  $$ 3 (    Y CA/B+=B'>B+B)B+"C#B+&C'B+)B++ C4CCCc t} |jd}|jdd} |jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst}| j|||| | \}} |j|d||d d| i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Get a specific run Args: eval_id: The ID of the evaluation run_id: The ID of the run to retrieve extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object rFrGNrFTrHrIzGET run is not supported for rKrvrrPrMrLrvrrRrrTrr)r[)r.r\r]rrr^r8r_r`rbrPrtransform_get_run_requestrerfget_run_handlerrr7r9)rvrr!r"r$r%r:r;rFrGrZrMrXrLrPrVrArBs rCrrs2JC 8> CX8Y-3ZZ8I4-P!::j%8D@ !5 >v >  &"* @U?r?r))*=>@ ! % ,<=P CX8Y-3ZZ8I4-P!::mU;tC !5 >v >  &"* @U?r?r))*=>@ ! % ,?@S?TUV V %2+@@ao@p)11L5L & B B) C     ..(/6B!?!4 / 0BB&? 3)+!.::h'!::&67C    $$ 3 (    D'D44 E=EEc Kt} tj}d|d<ttf||||||d|} t j } t| j| } |jd| d{} tj| r | d{} | S| } | S7)7 #t$r}tjd||||d}~wwxYww)a Async: Delete a run Args: eval_id: The ID of the evaluation run_id: The ID of the run to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: RunDeleteResponse object T adelete_runrNr)) r.r/r0r delete_runr2r3r4r5r6r7r8r9rs rCrrrrc t} |jd}|jdd} |jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst}| j|||| | \}} }|j|d||d d| i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Delete a run Args: eval_id: The ID of the evaluation run_id: The ID of the run to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: RunDeleteResponse object rFrGNrFTrHrIz DELETE run is not supported for rKrrrRrrTrr)r[)r.r\r]rrr^r8r_r`rbrPrtransform_delete_run_requestrerfdelete_run_handlerrr7r9rs rCrr4rr)NNNNNNN) NNNNNNNNN)NNNN)NNNNNNNN)I__doc__r/r2collections.abcr functoolsrtypingrrr8litellm.constantsr*litellm.litellm_core_utils.litellm_loggingrLiteLLMLoggingObj*litellm.llms.base_llm.evals.transformationr*litellm.llms.custom_httpx.llm_http_handlerr litellm.types.llms.openai_evalsr r r r rrrrrrrrrrrlitellm.types.routerr litellm.utilsrrrfr__annotations__liststrdictobjectfloatrr(r1intrnrorxrzrrrrrrrrrrrrrrrrr[rCrs % -SII"67+,!99)-.2,0+/,0&*A (A <(A  *A 3;$& A V $t+ A c6k"T) A S&[!D(A U]] "T )A tA  A A H)-.2,0+/,0&*p (p <(p  *p 3;$& p V $t+ p c6k"T) p S&[!D(p U]] "T )p tp  Iffd* ++p p f.2,0,0&*A :A :A  $JA  : A Dj A V $t+ A c6k"T)A U]] "T )A tA A A H.2,0,0&*q :q :q  $Jq  : q Dj q V $t+ q c6k"T)q U]] "T )q tq 663D#DEEq q h/3,0,0&* 5 5 V $t+5 c6k"T)5 U]] "T ) 5 t 5  5 5 p/3,0,0&* Y Y V $t+Y c6k"T)Y U]] "T ) Y t Y  Iffd* ++Y Y x)-.2,0+/,0&*> > *> 3;$&> V $t+ > c6k"T) > S&[!D( > U]] "T )> t>  > > B)-.2,0+/,0&*V V *V 3;$&V V $t+ V c6k"T) V S&[!D( V U]] "T )V tV  Iffd* ++V V r/3,0,0&* 5 5 V $t+5 c6k"T)5 U]] "T ) 5 t 5 5 5 p/3,0,0&* Y Y V $t+Y c6k"T)Y U]] "T ) Y t Y )FF4F$FGGY Y x/3,0,0&* 5 5 V $t+5 c6k"T)5 U]] "T ) 5 t 5 5 5 p/3,0,0&* ] ] V $t+] c6k"T)] U]] "T ) ] t ] )FF4F$FGG] ] J)-.2,0+/,0&*A A c6k"A  *A 3;$& A V $t+ A c6k"T) A S&[!D(A U]] "T )A tA  A A H)-.2,0+/,0&*o o c6k"o  *o 3;$& o V $t+ o c6k"T) o S&[!D(o U]] "T )o to  9VVS( ))o o d.2,0,0&*A A :A  :A  $J A : A V $t+ A c6k"T)A U]] "T )A tA A A H.2,0,0&*p p :p  :p  $J p : p V $t+ p c6k"T)p U]] "T )p tp  &&2B"BCCp p f/3,0,0&* 8 8 8 V $t+8 c6k"T) 8 U]] "T ) 8 t 8  8 8 v/3,0,0&* \ \ \ V $t+\ c6k"T) \ U]] "T ) \ t \  9VVS( ))\ \ ~/3,0,0&* 8 8 8 V $t+8 c6k"T) 8 U]] "T ) 8 t 8 8 8 v/3,0,0&* ` ` ` V $t+` c6k"T) ` U]] "T ) ` t ` 663D#DEE` ` P/3,0,0&* 8 8 8 V $t+8 c6k"T) 8 U]] "T ) 8 t 8 8 8 v/3,0,0&* ` ` ` V $t+` c6k"T) ` U]] "T ) ` t ` 663D#DEE` ` r