tPj UdZddlZddlZddlZddlZddlmZmZmZm Z ddl m Z ddl m Z m Z ddlmZddlmZddlmZdd lmZmZmZdd lmZmZmZmZmZmZdd lm Z dd l!m"Z"dd l#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7m8Z8erddl9m:Z:ddl;mZ>ddl6m?Z?dZ@eeAd<dZBeeAd<dZCeeAd<d ZDeeAd!<d"ZEeeAd#<d$ZFeeAd%<eiZGeeeHeIfeAd&<eeeHeIfZJeeAd'<eeKeeHeIfd(fZLeeAd)<eeKeId(fZMeeAd*<d+eeHeIfd,eKeeHeIfd(ffd-ZNd+eeHeIfd,eeHeIffd.ZOd+eeHeIfd/eeHeIfd,eeHeIffd0ZPeQd1ZReeAd2<d+eeHeIfd3eeHeIfd,eeHeIffd4ZSd+eeHeIfd3eeHeIfd,eeHeIffd5ZTd6eId,eHfd7ZUd6eId,eHfd8ZVGd9d:ZWeWePeUd;<ZXeeAd=<eWeSeUd><ZYeeAd?<eWeTeVd><ZZeeAd@<eQdAZ[eeAdB<dCeeHeIfd,e\fdDZ]eQdEZ^eeAdF<dGeId,e\fdHZ_dIeWd+eeHeIfd/eeHeIfd6eId,eKeKeeHeIfd(feeHeIfeHfdzf dJZ`eeXeXeYeZeZdKZaeeeHeWfeAdL<dMZbeeAdN<GdOdPeZce3ecZdeeAdQ<dReHdSeHdTeed,e\fdUZfdVegd,eHfdWZhdXeId,eefdYZidXeId,eefdZZjd[eHd\e\d,eHfd]Zkd^eHd_eHd`eHd,eHfdaZldbeeHeIfd,eefdcZmdSeHd,eHfddZndeeHdfeedgeed,eefdhZodeeHdieed,dfdjZpdSeHdkdld,dfdmZqdneeHeIfd,e\fdoZrdneeHeIfd,eHdzfdpZsdneeHeIfd,eeHeIffdqZtdneeHeIfd,eHdzfdrZudneeHeIfd,eefdsZvdCeeHeIfdteHd,e\fduZwe d>d>vGdwdxZxe d>d>vGdydzZye d>d>vGd{d|ZzGd}d~eZ{deIde|deed,e{dzfdZ}e"de@Z~eeAd<dZeeAd<Gdde&ZeiZeeeHeKe{d(ffeAd<ddZy)agShadow Eval Logger: samples a shadowed key's successful LLM requests (chat completions, Anthropic Messages, and Responses API surfaces, each normalized to chat shape), duplicates each against the job's other arm in a detached task (the auto-router for a forward job, the fixed baseline model for a reverse one), blind-judges real vs shadow, and appends one ``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write. Counts, status, and spend derive from those rows at read time, so nothing can disagree across pods or stop races; the hook reads active jobs through a short-TTL cache.N) AwaitableCallableMappingSequence) dataclass)datetimetimezone)groupby) itemgetter)MappingProxyType) TYPE_CHECKINGFinalLiteral) BaseModel ConfigDict TypeAdapterValidationErrorfield_validatormodel_validator)verbose_logger) InMemoryCache)!INTERNAL_CALL_ORIGIN_METADATA_KEY) CustomLogger) get_litellm_metadata_from_kwargs)#sanitized_forwardable_call_metadata)default_router_providerextract_text_from_contentjudge_acompletionparse_json_verdict)should_redact_message_logging)type_to_response_format_param)ShadowEvalDirection)SHADOW_EVAL_JUDGE_CALL_ORIGINSHADOW_EVAL_ROUTER_CALL_ORIGIN)ShadowEvalFunnelStage) PrismaClient)Router)StandardLoggingPayload _JOBS_CACHE_TTL_SECONDS_MAX_CONCURRENT_SHADOW_TASKSi@_MAX_JUDGE_RESPONSE_CHARSi]_MAX_JUDGE_PROMPT_CHARSiJUDGE_MAX_OUTPUT_TOKENSi_MAX_ERROR_CHARS_EMPTY_METADATA_CHAT_REQUEST_ADAPTER._CHAT_MESSAGES_ADAPTER_MESSAGE_ITEMS_ADAPTERkwargsreturncl|jd}t|trtd|DSdS)Nmessagesc3BK|]}t|ts|ywN) isinstancer.0ms l/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/integrations/shadow_eval_logger.py z!_chat_messages..Ms:q:a#9:s)getr;rtuple)r5raws r?_chat_messagesrEKs0J'C>Hh>W5:C: :_]__c|jd}t|tr|jdnd}t|tr|jdnd}t|tr|StS)Nlitellm_paramsproxy_server_requestbodyrBr;rr1)r5rHrequestrJs r?_proxy_wire_bodyrMPs\"JJ'78NCMn^eCf^''(>?lpG)3GW)E'++f%4DdG,4A/ArFmodel_parametersc6ti|dt|iS)zQChat requests are already chat-shaped: the logged model_parameters forward as-is.r8)r rE)r5rNs r?_chat_request_from_chatrPWs" T/T^F=ST UUrF) max_tokens temperaturetop_ptop_kreasoning_effort_ANTHROPIC_SAMPLING_PARAM_KEYS_model_parametersc  ddlm}|}t|}|jdxs|jd}t |j t zt dz ttg fd|jDdt|jdxsdfdtj|jdxsd f|d|ffnd }|j|\}}|S) a/v1/messages logs surface-native block messages with ``system`` top-level: the native provider path carries it in kwargs, the openai-compatible bridge path only in the proxy's snapshot of the client's wire body. Params come from the wire body alone, because the logged optional_params switch dialect per provider path (the bridge's inner completion rewrites them to chat shape mid-flight); the adapter translates them alongside the messages, and sampling params copy through untranslated.r)LiteLLMAnthropicMessagesAdaptersystem)r8rZc36K|]\}}|vs ||fywr:rA)r=kv param_keyss r?r@z8_chat_request_from_anthropic_messages..ysJTQ!z/1a&Js  modelr8rA)Hlitellm.llms.anthropic.experimental_pass_through.adapters.transformationrYrMrB frozensettranslatable_anthropic_paramsrVr dictitemsstrr3validate_pythontranslate_anthropic_to_openai) r5rWrYadapter wire_bodyrZrL translated_r^s @r?%_chat_request_from_anthropic_messagesrmcs 56G'/IJJx(CIMM(,CF'779:=[[()*J&  JY__%6J #fjj17R89 3CCFJJzDZD`^`ab ,2+=Hf%'2    G99'BMJ rFc ddlm}ddlmt |}|j dxs|j d}t tgfd|jD|d|ffnd}tj|jt|j dxsd|j d | S) a/v1/responses logs the raw ``input`` under ``kwargs["messages"]``, an alias function_setup creates for responses call types: a bare string, chat-shaped dicts, or item dicts; ``instructions`` is the system prompt. Params come from the wire body for the same reason as the messages surface; the transformer translates them with the input (max_output_tokens to max_tokens, Responses tools to chat tools, reasoning to reasoning_effort) and never reads surface-only keys like previous_response_id.r) LiteLLMCompletionResponsesConfig)!ResponsesAPIOptionalRequestParams instructionsc3JK|]\}}|jvs||fywr:)__annotations__)r=r\r]rps r?r@z/_chat_request_from_responses..s'qTQ!?`?p?p:p1a&qs# #rAr_r`r8)r_inputresponses_api_request) Blitellm.responses.litellm_completion_transformation.transformationrolitellm.types.llms.openairprMrBr rdrer2rg:transform_responses_api_request_to_chat_completion_requestrf)r5rWrorjrqresponses_requestrps @r?_chat_request_from_responsesrzsL'/I **^4U n8UL/  qY__%6q 7C7ONL13UW    ! 0 0(ccfjj)/R0**Z("3 d  rF response_objc2 t|tr |dddn|jdjttr jnfd}|ds|dryt|dS#tt t tf$rYywxYw) z~The assistant's text, or empty when the turn carries tool calls: only text-final turns produce a judgeable A/B comparison.choicesrmessager`ct|dSr:)getattr)keyr~s r?z"_chat_final_text..swW^`ceiOjrF tool_calls function_callcontent) r;rr}r~AttributeErrorKeyError IndexError TypeErrorrBr)r{readr~s @r?_chat_final_textrs,0  #A &y 1%%a(00 ",GW!='++CjD LT/2 $T)_ 55 Hj) <s4A::BBc.ddlm} t|tr|j |n|}t |dd}t|tsytd|D}td|Drytt |ddxsdS#t $rYywxYw) a=The turn's aggregated output text, or empty when the turn carries tool calls. A dict-shaped payload is validated into the owner type first, because ``output_text`` is a derived property rather than a serialized field, so it never exists on a dict; a dict the owner type rejects is unjudgeable and skipped.r)ResponsesAPIResponser`outputNc3`K|]&}t|tr|jn|(ywr:r;r model_dumpr=items r?r@z(_responses_final_text..s&gX\jy.I*tSgs,.c3hK|]*}t|t xs|jddv,yw)type)rcustom_tool_callNr;rrBrs r?r@z(_responses_final_text..s6 gkJtW %%b&)9=b)bb s02 output_text) rwrr;rmodel_validaterrrrCanyrf)r{rresponserres r?_responses_final_textrs ?AKLZaAb / / =ht  Hh5F fh 'g`fggE  ot  wx39r :: s#B BBcdeZdZdZdZdeeeefeeefgeeeffdeegefde ddfdZ y) _SurfaceOpsaPOne row per sampled call_type: how its logged request becomes a chat-shaped request (messages plus translated generation params) and how its response yields the judgeable final text. Membership in this table IS the sampling allowlist; unknown call types fail closed. ``wire_params`` marks the surfaces whose params come from the proxy's wire-body snapshot, which is taken before the guardrail pre-call hook: those rows must not sample a request a pre-call guardrail rewrote, or the shadow call would replay content (tools, unmasked entities) the guardrail removed. chat_request final_text wire_paramsrrrr6Nc.||_||_||_yr:r)selfrrrs r?__init__z_SurfaceOps.__init__s )$&rF) __name__ __module__ __qualname____doc__ __slots__rrrfobjectboolrrArFr?rrsq>I'V 4gc6k6JKWUXZ`U`Maab'fXs]+' '  'rFrF)r _CHAT_OPST_ANTHROPIC_OPS_RESPONSES_OPS) during_call post_call logging_onlyduring_mcp_call post_mcp_callrealtime_input_transcription_NON_MUTATING_GUARDRAIL_MODESrequest_metadatac|jd}t|tr|nd}td|D}t d|DS)aWhether a guardrail that can rewrite the outbound request ran on this one, read from the same guardrail-information entries spend logging uses. str-enum modes compare equal to their plain-string values, and an entry whose mode is missing or unrecognized counts as mutating.&standard_logging_guardrail_informationrAc3`K|]&}t|ts|jd(yw)guardrail_modeNr)r=entrys r?r@z2_request_mutating_guardrail_ran..s&"r5WabgipWq599-=#>"rs..c 3vK|]1}tdt|ttzr|n|fD 3yw)c3,K|] }|tvywr:)r)r=modes r?r@z<_request_mutating_guardrail_ran...s 6:D1 1 sN)allr;listrC)r=modess r?r@z2_request_mutating_guardrail_ran..sH   HRSXZ^afZfHgunsmu   s79)rBr;rrCr)rrDentriesmodes_per_entrys r?_request_mutating_guardrail_ranrsQ "%%&NOC&sH5S2G""rG"rrO %  rF)r_r8streamstream_optionsmetadata_UNFORWARDED_REQUEST_KEYSvaluecN|duxs t|txrt|dk(S)Nr)r;rlen)rs r?_forwards_nothingrs% D= IZt4HUqIrFopsc  |j||}tj|jd}tjt d|D}|j|}|r|syt|jD cic]\} } | tvst| r| | c} } } || |fS#t $r }tjd|Yd}~yd}~wwxYwcc} } w)zThe normalized chat conversation, the forwardable generation params, and the judgeable final text; None when this request's shapes cannot be sampled (tool-final turn, empty text, or a shape the owner transformations reject).r8c3dK|](}t|tr|jdn|*yw)T) exclude_noneNrr<s r?r@z$_judgeable_sample..s+dYZZ95M!,,D,1STTds.0z7shadow_eval: request normalization failed, skipping: %sN)rr4rgrBr3rC Exceptionrdebugrr rerr) rr5rNr{rLrer8e real_textr\r]paramss r?_judgeable_sampler s))&2BC-==gkk*>UV0@@ d^cd d  ~~l3I 9$!--/m$!QQ6O-OXijkXlAmF VY && VXYZ ns*AB: C&  C& (C& : C#CC#) completion acompletionanthropic_messages aresponses responses _SURFACE_OPSarYou are an impartial quality judge comparing two responses to the same conversation. The responses are labeled A and B in random order. You do not know which system produced which. Criteria: correctness, completeness, clarity, conciseness. Return ONLY valid JSON in this exact format, no other text: { "preference": "A" | "B" | "tie", "confidence": <0.0 to 1.0> }PAIRWISE_JUDGE_SYSTEM_PROMPTc,eZdZUdZeded<eed<y)PairwiseVerdictaDThe judge's blind A/B verdict: the response_format schema sent with the judge call and the validation contract on its reply. Both fields are required and preference is closed over the prompt's labels, so a malformed or truncated reply is an unparseable-verdict error row, never a defaulted or fabricated verdict.)ABtie preference confidenceN)rrrrrrsfloatrArFr?rr<sO ((rFrPAIRWISE_JUDGE_RESPONSE_FORMAT request_idjob_id percentagectj|d|jj}tj |dddt dz }|dz|kS)zDeterministically decide whether a request falls in the shadowed slice: hash-based rather than random so retries sample the same way and pods agree without coordination.:NbiglgY@)hashlibsha256encodedigestint from_bytesr)rrrrbuckets r? _sample_hitsrIs]NNfXQzl#;#B#B#DELLNFNN6"1:u5e DF E>J &&rFrctj|j}|r5d|djj dddd|dj nd}t |j|d|S)zException class, message, and the raising frame, so an attempt's error row names the faulty code path without needing debug logs on the pod.z at /rr`: ) traceback extract_tb __traceback__filenamersplitlinenorr)rframeslocations r?_failure_detailrQsw((9F]cVBZ0077Q?CDAfRjFWFWEXYikH1g zA3 //rFrctt|dddd}t|r|dnd}t|ttfr t |St |S)aPrice one eval-arm call with the figure the spend pipeline bills: the router client stamps _hidden_params.response_cost from the deployment's own pricing, which the public price map lookup below cannot see (it reads 0 for deployment-priced models)._hidden_paramsNrB response_cost)rcallabler;rr_price_map_cost)rgetterstampeds r? _call_costr YsRGH.>EudSF080@VO,dG'C<(W~ 8 $$rFcVddl} |j|xsdS#t$rYywxYw)zPPublic price map fallback, treating an unmapped model as free rather than fatal.rN)completion_response)litellmcompletion_costr)rrs r?r r ds5&&8&DKK s  ((raw_preference real_is_acl|jj}|dk(r|rdSdS|dk(r|rdSdSy)z>Map the judge's blind A/B/tie verdict back to real/shadow/tie.arealshadowbr)striplower)rr normalizeds r?_unmask_preferencernsF&,,.446JS"v00S$x0&0 rF conversation response_a response_bc|dt}|dt}tt|z t|z }d|| dd|d|dS)zThe judge prompt under one total character budget: each response is capped, and the conversation tail gets whatever budget the responses left over.NzConversation: z Response A: z Response B: z Which response is better?)r-r.r)rrrrrconversation_budgets r?_judge_user_promptr"xso445A445A!83q6!ACF!J ,(;';'<=>?ss$ $rFsumsc0tfddDS)Nc3K|]8}tj|xttfr tnd:yw)rN)r;rBrr)r=columnrDr#s r?r@z"_leg_eval_spend..s= !(8!83,Gc SPs>A judge_cost shadow_costshadow_classifier_cost)sum)r#rDs`@r?_leg_eval_spendr,s M rFc d|S)Nzspend:shadow_eval:rA)rs r?_job_spend_counter_keyr.s x ((rF counter_keyfallback_spend max_budgetc@Kddlm}||||d{S7w)zThe leg's spend through the cross-pod counter the key budget gates read. The owner degrades internally to the fill-time DB floor and raises only under fail-closed enforcement, which the caller honors by skipping the sample.r)get_current_spend)r/r0r1N)litellm.proxy.proxy_serverr3)r/r0r1r3s r?_job_spend_from_counterr5s!="{>fpq qq qs costcK ddlm}|||d{y7#t$r!}tjd||Yd}~yd}~wwxYww)aFAdvance the counter the moment a cost is known, so even a lost row closes the gate. Known failure mode: a Redis outage freezes the counter (the owner invalidates it), the gate degrades to the fill floor, and overshoot grows to in-flight plus one TTL of samples, the same degradation the key budget counters accept.r)increment_spend_counter)r/ incrementNz6shadow_eval: spend counter increment failed for %s: %s)r4r8rrwarning)r/r6r8rs r?_add_job_spend_to_counterr;sH iF%+NNN iWYdfghhis5A A  A AA A  A stager%c~ ddlm}|||y#t$r!}tjd||Yd}~yd}~wwxYw)Nr)record_shadow_eval_funnel_eventz/shadow_eval: funnel increment failed for %s: %s)#litellm.proxy.db.shadow_eval_funnelr>rrr)rr<r>rs r?_record_funnel_eventr@s?[W'6 [NPVXYZZ[s <7<rcK ddlm}ddlm}ddlm}m}m}ddlm }m }m }|jd} t| |sy || |d {| jr1|| j||d d {} || | | d {y#t$rYywxYw7Q7)7#|$rYy t $r } t#j$d | Yd } ~ yd } ~ wwxYww)aWhether the shadowed key or its team is over budget, decided by the same owners the request path uses, so counter keys and thresholds can never drift from auth's. Advisory and fail-open: real traffic on an over-budget key is already rejected at auth (so nothing reaches the success hook), and this gate only closes the race where the key crosses its budget while a request is in flight. r)BudgetExceededError)UserAPIKeyAuth)_team_max_budget_check_virtual_key_max_budget_checkget_team_object) prisma_clientproxy_logging_objuser_api_key_cacheFuser_api_key_auth) valid_tokenrHNT)team_idrGrIcheck_cache_only) team_objectrKrHz#shadow_eval: budget read failed: %s)litellm.exceptionsrBlitellm.proxy._typesrClitellm.proxy.auth.auth_checksrDrErFr4rGrHrI ImportErrorrBr;rLrrr) rrBrCrDrErFrGrHrIauthteamrs r?_key_or_team_is_over_budgetrUs :7  dc,,23D dN + G+Pabbb << / +#5!% !D )Tt_pq q q )  c r  GBAFF GsC BC B(B")B(:B$;B( B&B(C BCBC"B($B(&B((C-C/C7C CCCcP|jd}t|tr|r|SdS)zThe shadowed key's team, the identity the judge call already carries in its metadata and the router already selects deployments with. Read here too so the arm choice, which happens before the router sees the call, is made under the same team.user_api_key_team_idN)rBr;rf)rrLs r?_forwarded_team_idrXs)\\"89G #.77DDrFcT|jd}t|tr|StS)zThe routing decision a pre-routing strategy wrote to a call's metadata, empty when a plain model served it. Read off the sampled request for the control arm, and off the shadow call's own write-back for the shadow arm.routing_decisionrK)rdecisions r?_routing_decisionr\s'll#56H!(G48I/IrFc~t|}|jdxs|jd}| t|SdS)N tier_labeltier)r\rBrf)rr[rDs r? _routed_tierr`s;'1Hl+Cx||F/CC3s80D0rFc|t|jd}t|ttfr t |SdS)zWhat the arm's own routing decision says its classifier call billed: the money a completion cost alone omits, and 0 for a plain model that never classifies.classifier_costr)r\rBr;rr)rrDs r?_decision_classifier_costrcs6#8,001BCC#C#u65:?C?rF router_namec<t|jd|k(S)aNWhether the router under evaluation served this request, which is what decides the direction it belongs to. A forward job skips its own router's traffic, since duplicating it would compare the router to itself: guaranteed ties, judge spend for zero information. A reverse job samples exactly that traffic and nothing else.router_model_name)r\rB)rrds r?_request_was_routed_byrgs! - . 2 23F G; VVrF)frozenslotsc8eZdZUdZeed<dZeed<dZeed<y) _CallFailurezA shadow or judge call that produced no usable response. cost carries any spend the failed call still billed, so job-level spend figures never undercount.errorrr6rbN) rrrrrfrsr6rrbrArFr?rkrks#R JD% OU rFrkcJeZdZUdZeed<eed<edzed<eed<eed<y)_ShadowResponsez>h!G> HH LaPs, AAA)max_size_in_memory default_ttl _jobs_cachezshadow_eval:active_jobs_JOBS_CACHE_KEYc!eZdZdZ d8degdfdzdegdfdzdedzdeeeegeefdzd eeegedfdzd eed gdfdzd dfd Z d e ee e dfffdZ dee de eefded e e dffdZde eefdededed df dZd9dZde dedee eefdedededed ed!edzd"e eefd#e eefd dfd$Zdd%dd&d&d&dd'd(dde ded!edzd)ededed ed*edzded+edzd,ed-ed.ed/edzd df d0Zd1edee eefd"e eefd#e eefd d2f d3Zd4edee eefded5ed#e eefd d6f d7Zy):ShadowEvalLoggerzPFires blind pairwise shadow evaluations for keys with an active shadow-eval job.Nrouter_providerz Router | Noneprisma_providerPrismaClient | None jobs_cachejob_spend_readerjob_spend_writerfunnel_recorderr%r6c|xst|_|xst|_|xst|_|xst |_|xst|_|xst|_ d|_ i|_ y)zProviders are callables so the proxy's lazily-initialized globals are resolved at call time, not at logger construction. The spend reader and writer wrap the proxy's cross-pod spend counter; tests inject a plain in-memory pair.rN) r_router_provider_default_prisma_provider_prisma_providerrr5_read_job_spendr;_write_job_spendr@_record_funnel_inflight_shadow_tasks _job_starts)rrrrrrrs r?rzShadowEvalLogger.__init__Usi!0 J3J / K3K%4/J3J 0 M4M-E1E+,#,.rF.c b K|jjtd{}||S|j}|tS |j j jddtjtjidd{}|r[|j jjdgddddddd|Dcgc]}t|jc}ii d{nd }|xsgDcic]6}t|dt!|d d t#|d xst$f8c} t't) fd|xsgDt+d}t-t/|t+dD cic]\}} |t'd| Dc} }} |jj1t| d{i|_| S77Ncc}w7cc}wcc} }w7%#t4$r&} t7j8d| tcYd} ~ Sd} ~ wwxYww)zActive jobs by api_key_id, cache-first. A key holds at most one job per direction, so the value is a collection. A DB fault returns empty without caching, so sampling pauses for that request and the next one retries.Ngt) stopped_atr})whererTr'in)bycountr+rrA_count_all_sumc 3K|]L}t|gjt|jdxt|jfNyw))rrN)rrBrfrv api_key_id)r=r attempt_statsrs r?r@z0ShadowEvalLogger._active_jobs..sW"#1&#g=;L;LSQWQZQZ^]e;f#ggCtV../5sAAr)rc3&K|] \}}| ywr:rA)r=rlrs r?r@z0ShadowEvalLogger._active_jobs..s4FAsC4sz'shadow_eval: active-job read failed: %s)rasync_get_cacherr _EMPTY_JOBSdblitellm_shadowevaljob find_manyrnowr rlitellm_shadowevalattemptgroup_byrfrvrr,r1rCsortedr r r async_set_cacherrrr)rcachedprismarecordsrgroupedrowby_keyrgroupjobsrrrs @@r? _active_jobszShadowEvalLogger._active_jobsls1#..>>OO  M--/ > + #)99#B#B#L#L"& $hll8<<&@A$M$Gii99BB z'+D\`a#d',RS^,R%ST C  #=b $  CM"H f-.#CK$B?C%$M"&-m #1  F+GNv[efg[hGijee4e444jD""22?DI I I!D K]P-S $$k I   !JA N  s#H/G# H/ AG=G&3G= G)( G=5G.6G=;G0?AG= G5 *+G=G; G="H/&G=)G=0 G== H,H'!H,"H/'H,,H/ active_jobsrrcDg}tjtj}|D]}||jk\s|j |j j|jdz|jk\sK|j|j|jk\s&t||j|jdk(k7rt||j|j s|j#|jd|j%|t'|S)acThe jobs that sample this request. A key can hold one job per direction, and a request routed by one job's router while bypassing the other's qualifies for both; each is separately budgeted, so both fire. An admitting job that loses the sampling dice is counted, so results can weigh judged rows against the traffic they stand for.rr not_sampled)rrr rr}r~rrBrvr|r1rrgrdrxrrzrappendrC)rrrreligiblerrs r? _sampled_jobszShadowEvalLogger._sampled_jobss/1\\(,,/ !Cs{{"<<$"2"2"6"6svvq"AAS]]RNN.3993N)*:COOLQTQ^Q^bkQkl CFFC4I4IJ##CFFM: OOC  !XrFr5r{ start_timeend_timec,K |jd}|ytt|}t|tr|nt }|jt rytt|ry|jdxst }|jd} | sy|jdxsd} | sytjt|jdxsd} | y| jr t|ry|j|jd{jt| d|| } | syt| |tt|jdxsi|} | $| D]}|j!|j"d  y| \}}}t%|}t'|jd xsd }|jd d u}t)|}| D]}|j*t,k\r|j!|j"d3|j.j|j"ddz|j.|j"<|xj*dz c_t1j2|j5|| |||jdxsd|||||tt| j7|j8y7#t:$r }t=j>d|Yd}~yd}~wwxYww)Nstandard_logging_objectruser_api_key_hashrvr` call_typerArN unjudgeablerr cache_hitTshedrrr_) rrr8r real_model real_costreal_classifier_costreal_cache_hit control_tier shadow_paramsparent_metadataz(shadow_eval: failed to schedule task: %s) rBrrdr;rr1rr rrfrrrrrr rrvr`rrcrr,rasyncio create_task_run_shadow_evaladd_done_callback_release_shadow_slotrrr)rr5r{rrpayloadraw_metarr api_key_hashrrrsamplerr8rrrrrrrs r?async_log_success_eventz(ShadowEvalLogger.async_log_success_eventsB PtF|LH2?17 .HmY"./?"@L$W[[%A%HSII$+KK $<$DN*CDT*U  ?..2NN''7+/+;+;+?+?+JQ+N  (++q0+##))#-!)"+#*;;w#7#=2"+-A'5%1&3(8>N9O(P* $#D$=$=>) ?'+P P  !KQ O O PsLK(LAK(LK(.L/,K(LK(3L43K('L(K(?LK(K%%K(LAK(LEK($L%K(( L1L L LLc.|xjdzc_y)Nr)r)r_tasks r?rz%ShadowEvalLogger._release_shadow_slot s ##q(#rFrr8rrrrrrrrc K|j} | |j|jdyt| d{r|j|jdy|jo |j t |j|j|jd{} | |jk\r|j|jdy|j|j|| | d{}t|t rC|j| ||| d|j"|j$|j&||| d{y |j)|j*|||j,| d{}t|t rO|j| ||| d|j"||j$|j$|j&||| d{y|j| ||| |j.|||j0|j$|j$|j&||| d{y77#t$rG}tjd|j||j|jdYd}~yd}~wwxYw7#t$rG}tjd|||j| ||| dd|||| d{7Yd}~yd}~wwxYw77p77#t$r^}tjd|||j| ||| dd|||j$|j&||| d{7Yd}~yd}~wwxYww) aJBudget gate -> shadow call -> blind judge -> one attempt row, and every exit in exactly one coverage bucket: the gates that decline to spend on an admitted sample (no DB to record into, an over-budget key, an unverifiable or exhausted eval budget) count it withheld, so eligible traffic still reconciles as not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits above the dispatch so no provider spend happens without a place to record the outcome, and the budget read lives here rather than in the success hook.Nwithheldz;shadow_eval: budget unverifiable for %s, sample skipped: %sz'shadow_eval: pipeline failed for %s: %srlzpipeline error: )outcomerlrrr)rrlr)r*rrr)r{r8r shadow_textr) rrlrr(r)r*rrr) rrrrr(r)r*rrr)rrlrr)r*rrr)rrrvrUr1rr.rrrr:_call_router_shadowrr_record_attemptr;rkrlr6rb _call_judger{rorr)rrrr8rrrrrrrrrrrrverdicts r?rz!ShadowEvalLogger._run_shadow_evals*--/ ~##CFFJ70AAA##CFFJ7~~))-)=)=>TUXU[U[>\^a^g^giliwiw)x#xE CNN*'' ;"&":":3;L;LhXegv"wwF fl +&&ll"KK'-'='=#%9-'    8 #'#3#3OO!#"KK / $4$G'<0** #!--!&|| & +1+A+A')=#1+&&**%"--"<<"KK'-'='=#%9-'  KB $y "**+hjmjpjprst'' ;x   !JJXY Z&&(,#%9-'             !JJXY Z&&(,"KK'-'='=#%9-'    sDM'JM'JI J"M'# J0=I -I.I 2+JM'!J?JJA M'K2M'-K=K5AK=#K8$K=(M')AK==K;>K=M'JI J=JJM'JJ K/(7K*K" K*%M'*K//M'5K=8K=;K== M$AMMMM'M$$M'r`r)rrrr(r)r*rlrrrrr(r)r*rlc K| | z|z}|dkDr-|jt|j|d{|y |jjj |j|||j dk(r|n| r | jnd| xsd| r | jnd| | | |||||r |dtnddd{y77#t$r!}tjd||Yd}~yd}~wwxYww)Nrr)rrrr_r shadow_modelrr(r)r*rrrrl)dataz,shadow_eval: attempt write failed for %s: %s) rr.rvrrcreaterxr_r_r0rrr)rrrrrrrrrrrrr(r)r*rl eval_spendrs r?rz ShadowEvalLogger._record_attempts&'47MM >''(>svv(F S S S >  `))55<<!ff",&,/MMY,FL\bV[[hl","44:FLL",",#..D!*,@&49>U#4$45D=   T $ `  !OQ[]^ _ _ `sL6C6CC6A>C ?CC C6C C3C.)C6.C33C6 target_modelz_ShadowResponse | _CallFailurec K|j}| tdSt|t} |jd||Dcgc] }t |c}|dgd|d{}t|} | s tdt|t| St| tt|d dxst!|j#d xsd t%|t|t| Scc}w7#t $rB} tjd| tdt| t|cYd} ~ Sd} ~ wwxYww)a7Send the prompt through the arm nobody was served: the auto-router under evaluation, or a reverse job's fixed baseline model. The metadata carries the shadowed key's identity (spend attribution) and receives a routing decision write-back, which a plain baseline model simply never makes.Nz no router configured on this podr)r_r8r num_retries fallbacksz#shadow_eval: router call failed: %szshadow router call failed: )rbz(shadow router returned an empty response)r6rbr_ routed_modelr`)ror_r_r6rbrA)rrkrr$rrdrrrrrcrr rnrfrr\rBr`) rrr8rrroutershadow_metadatar>rrros r?rz$ShadowEvalLogger._call_router_shadows_--/ > BC C /A_ `  $6F$6$6 %"%- !DG) %  % H 'x0:) 9/ J  gh6v:KO:\:`:`ao:pvtvwo.H%5oF   +    !F J-oa.@-AB 9/ J  sS.E C;C4 C;#C9$C;(B E 4C;; E7E;E<E EE r{rz_JudgeVerdict | _CallFailurec >Ktjdk}|r|n|}|r|n|}djd|D} t|t} dtddt | ||dg} t |j|| t|dtt| d {} | d dd dxsd}tj!t#|}t't)|j*|t-dt/d|j0t%| S7~#t$r.} tjd | td | cYd } ~ Sd } ~ wwxYw#t$r9} tjd| td| t%| cYd } ~ Sd } ~ wwxYww)zHBlind pairwise judge with A/B labels randomized to cancel position bias.g? c3K|][}|jdHt|jddjdt|jd]yw)rNroleuserr)rBrfupperrr<s r?r@z/ShadowEvalLogger._call_judge..s]( uuY+155()//12"5NquuU^O_5`4a b( sA!A#rZ)rrr r)rLrRrQresponse_formatrNz"shadow_eval: judge call failed: %szjudge call failed: r}r~rr`z*shadow_eval: unparseable judge verdict: %szunparseable judge verdict: )r6rg?)rrr6)randomjoinrr#rr"rrrXr/rrrrrkrrrr rrrrmaxminr)rr{r8rrrrrrrjudge_metadatajudge_messagesrrrDrs r?rzShadowEvalLogger._call_judges"==?S0 )2I +4K) "ii( (  !DOUr s*F G-lJ S !  ;$5%%'*?;2 >' % H ^!),Q/ :9EKC,;;r)sT BB!'"00jj+9?;Ta UJX]I0%: "$#')e($)5(!''"&%%/?/CwsF{+,C +73;+?@u@ +E'#v+2F2K,L MM +E&#+,> ??`73;/`E'#v+:NPS:S4T` BWS&[1Bgc6k6JBV CK V4;CK4HV S&[V)22u(vv CK 5 ??@ AXrF