tPj UdZddlmZmZddlmZmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZddlmZdd lmZmZmZmZdd lZdd lmZdd lmZdd lmZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&ddl'm(Z(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/m0Z0ddl1m2Z2ddl3m4Z4ddl5m6Z6ddl7m8Z8m9Z9m:Z:ddl;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLerddlMmNZNmOZOmPZPmQZQmRZRddlSmTZTddlUmVZVn ddlMmNZNmOZOmPZPmQZQmRZReNZXeeYd<GddeZZGddeZ[Gd d!eZ\Gd"d#eZ]Gd$d%eZ^Gd&d'eZ_Gd(d)eZ`Gd*d+eZad,d-d.eZfd/Zbd,d-d.e\fd0Zcd,d-d.e^fd1Zdd,d-d.e`fd2Zed,d-d.eafd3Zfd,d-d4egd5ehd.eeegehffd6Zid7e&d8egd zd.d fd9Zjd:eFd.ekegd;ffd<Zld:eFd7e&d=d>d.d fd?ZmeXjd@dAgeOe+geEeRjBdCeDd7ee&eOe+fd.eEfdDZpeXjdEdAgeOe+geCeRjBdCeBd7ee&eOe+fd.eCfdFZqGdGdHeZreeserZteeYdI<dJegd.efdKZudLevdMevd.evfdNZwdOexdPexd.e@fdQZydRerd.e?fdSZzdRerd.e=fdTZ{dUeerd.erfdVZ|dWehd.ekegegfd zfdXZ}d=dYdZe~ekegegfd.eke=d;ffd[ZeXjd\d]eOe+fe>^ dd7ee&eOe+fd_eegd zeQd`afdbeegd zeQdcafd.e>fddZd7e&deegd.d fdfZd7e&deegd.d fdgZd=d>dhegd.efdiZdjegd.efdkZd=dYdjegdlegdmeegd zd.d f dnZdmeegd zd.egfdoZe~dpdqhZee~e8eYdr<d=dYdhegd.ekekegegfd;ffdsZd=dYdCeLd8egd zd.ekekegegfd;ffdtZd=dYdCeLdmeegd zd.d fduZdved.efdwZGdxdyeZeeseZeeYdz<d{ZeeYd|<d}ezZeeYd~<dezZeeYd<dezZeeYd<dZeeYd<dZeeYd<dZeeYd<dZeeYd<dZeeYd<GddeZGddeZeeseZeeYd<dZeeYd<dZeeYd<GddeZeeseZeeYd<dLexdMexd.evfdZdUeed.ekeKd;ffdZGddeZeeseZeeYd<d,d-deed.eegeffdZdegdeedeegefd.eIfdZdZeekegd zegd zfeYd<d,d-deeId.ekeId;ffdZd,d-deed.eJd zfdZeXjdd]eOe+feIeRjTBdCeLd7ee&eOe+fd.eIfdZeXjdd]eOe+feseI^ dd7ee&eOe+fdeegd zeQdafdeexeQdddfd.ekeId;ffdZeXjdd]eOe+feI^degd7ee&eOe+fd.eIfdZeXjdd]eOe+feI^degd7ee&eOe+fd.eIfdZy #eW$rYwxYw)z AUTO ROUTER MANAGEMENT ENDPOINTS POST /auto_router/test_routing - Route one request through an unsaved complexity-router config POST /auto_router/validate_complexity_router_config - Dry-run the complexity-router write gate without saving )MappingSequence)datetime timedeltatimezone)groupby) attrgetter)MappingProxyType) TYPE_CHECKING AnnotatedFinalProtocol)uuid4) BaseModel ConfigDict TypeAdapterfield_validatorN)verbose_proxy_logger)BudgetExceededError judge_target)CommonProxyErrorsLiteLLM_TeamTableLitellmUserRolesProxyErrorTypesProxyExceptionUserAPIKeyAuth)_virtual_key_max_budget_checkcan_key_call_resolved_model)user_api_key_auth)AUTOROUTER_BENCHMARKS_SQL)LiteLLMProxyRequestSetup*refresh_proxy_server_request_body_snapshot)SupportsModelDump)TeamRepository)ComplexityRouter)StrategyRouterDependencyRoleclassify_strategy_router_modelstrategy_router_dependencies)SHADOW_EVAL_TURN_VALVEAutoRouterBenchmarkGroupAutoRouterBenchmarksResponseAutoRouterBenchmarkTotalsAutoRouterCacheBucketAutoRouterCacheStatsAutoRouterRoutingTestRequestAutoRouterRoutingTestResponse'ComplexityRouterConfigValidationRequest(ComplexityRouterConfigValidationResponseRequestComplexityRouterConfigShadowEvalDirectionShadowEvalJobKeyResponseShadowEvalJobResponseShadowEvalResultShadowEvalSliceStartShadowEvalRequest) APIRouterDepends HTTPExceptionQuerystatus) PrismaClient)Routerrouterc,eZdZdeeefdedzfdZy) _TeamTablewherereturnNc KywNselfrEs }/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/proxy/management_endpoints/auto_router_endpoints.py find_uniquez_TeamTable.find_uniqueT)__name__ __module__ __qualname__rstrobjectr$rMrIrLrDrDSs`'#v+*>`CTW[C[`rUrDcveZdZedefdZededzfdZededzfdZededzfdZy)_VerificationTokenRowrFcyrHrIrKs rLtokenz_VerificationTokenRow.tokenXsrUNcyrHrIrYs rL key_aliasz_VerificationTokenRow.key_alias[s'*rUcyrHrIrYs rLkey_namez_VerificationTokenRow.key_name^s&)rUcyrHrIrYs rLteam_idz_VerificationTokenRow.team_idas%(rU) rPrQrRpropertyrSrZr\r^r`rIrUrLrWrWWsZ s *3:** )#*)) (t((rUrWcLeZdZdeeefdedzfdZdeeefdeefdZ y)_VerificationTokenTablerErFNc KywrHrIrJs rLrMz#_VerificationTokenTable.find_uniquefrNrOc KywrHrIrJs rL find_manyz!_VerificationTokenTable.find_manyhrNrO) rPrQrRrrSrTrWrMrrfrIrUrLrcrces;d'#v+*>dCX[_C_deV (<eJ_A`erUrcc"eZdZedefdZy)_ShadowEvalJobRowrFcyrHrIrYs rLidz_ShadowEvalJobRow.idlsrUN)rPrQrRrarSrjrIrUrLrhrhks CrUrhcLeZdZdeeefdeefdZdeeeefde fdZ y)_ShadowEvalJobTablerErFc KywrHrIrJs rLrfz_ShadowEvalJobTable.find_manyqrNrOdatac KywrHrI)rKrns rL create_manyz_ShadowEvalJobTable.create_manysrNrON) rPrQrRrrSrTrrhrfintrprIrUrLrlrlps9aV (<aJ[A\aQhwsF{/C&DQQrUrlc(eZdZededzfdZy)_ShadowEvalAttemptRowrFNcyrHrIrYs rLerrorz_ShadowEvalAttemptRow.errorws#&rU)rPrQrRrarSrurIrUrLrsrsvs &sTz&&rUrsc0eZdZdeeeefdedefdZ y)_ShadowEvalFunnelTablernskip_duplicatesrFc KywrHrI)rKrnrxs rLrpz"_ShadowEvalFunnelTable.create_many|rNrON) rPrQrRrrrSrTboolrqrprIrUrLrwrw{s$hhwsF{/C&DhW[h`chrUrwc:eZdZdeeefdeeefdedzfdZy)_ShadowEvalAttemptTablerEorderrFNc KywrHrI)rKrEr}s rL find_firstz"_ShadowEvalAttemptTable.find_firsts '*rO)rPrQrRrrSrTrsrrIrUrLr|r|s6+V ,+5z-_models_this_test_can_call..s      sN)tupleuses_llm_classifierclassifier_llm_configrsemantic_keyword_matchingembedding_model)rs rL_models_this_test_can_callrs]  ))f.J.J.V  ( ( . .&,&F&FF " "D     rU llm_routerrAcFKt|}|syddlm}|D]#}t||j||d{% t ||d{y77#t $r;}t|jtjdtj|d}~wwxYww)a'Hold a classifier or embedding call to the caller's model access and key budget. Those calls go through the router rather than through /v1/chat/completions, so the model checks a real request gets in user_api_key_auth would otherwise be skipped, letting a caller spend on a model their key cannot call, and this route is not an LLM API route, so the key's own budget is not checked either. Test Connection gets both for free by routing its calls through the proxy. Team and member budgets are already enforced on every route. Nr)proxy_logging_obj)rllm_model_list valid_tokenr)rr)messagetypeparamcode) rrrr model_listrrrrrbudget_exceededr?HTTP_400_BAD_REQUEST)rrrmodelsrres rL$_authorize_models_this_test_can_callrs/v6F < )%00)!      +)/      II 00,,    sE6B!AB!AAAB!A B#6BBB!z./auto_router/validate_complexity_router_configzmodel management)tags dependenciesresponse_modelrrncKt||jd{ddlm}||j}t |du|S7+w)a Validate a complexity-router config without saving it. Runs the same check every write path runs (the router's own pydantic model), so a form can show the backend's exact verdict while the operator is still editing rather than after a rejected save. Gated exactly like the save it rehearses: a proxy admin, or a team admin naming their own team. Nothing is created, routed, or billed. rr`Nr)'validate_complexity_router_config_write)validru)rr`-litellm.router_utils.auto_router_model_namingrcomplexity_router_configr3)rnrrrus rL!validate_complexity_router_configr sN& $6GQUQ]Q] ^^^;4;X;XYE 3%4-u UU_sA A ,A z/auto_router/test_routingc Kddlm}m}m}m}m}m}ddlm}t||jd{|'tddtjjit|j || d{t#|j$||j j'd |j(d } t+j,i|j/iddid|d} t1|  | j3|j$| | dd{} | | j:tdddi||||||||j|d{} t=| j>| j>tA| v| j:S7f7 7#t4$r.} t7j8d| tddd| i| d} ~ wwxYw7}w)a4 Route a single request through a complexity-router config and report where it landed. Answers "which model would this request get?" for a config that only exists in a form, so an auto router can be checked before it is created. The request is classified by the same pre-routing hook a live request runs, over the same messages, system prompt and tool definitions, then dropped: nothing is sent to the model it routed to, and no auto router is created. A heuristic config therefore spends nothing, while an `llm` classifier or semantic keyword matching bills its classifier/embedding call to the calling key, like Test Connection does. Send `messages` to classify a real turn, with `system` and `tools` beside it when the surface carries them top level, as Anthropic /v1/messages does. `prompt` is the single-ask shorthand and routes as one user turn with nothing around it. **Example Request:** ```json { "messages": [ {"role": "system", "content": "You are a database migration assistant"}, {"role": "user", "content": "the index is not unique"}, {"role": "assistant", "content": "Then two workers can both insert. Add a unique index"}, {"role": "user", "content": "ok do it"} ], "tools": [{"type": "function", "function": {"name": "Bash", "description": "Run a command"}}], "complexity_router_config": { "tiers": {"SIMPLE": ["gpt-4o-mini"], "REASONING": ["o3"]}, "classifier_type": "heuristic" } } ``` r)general_settingsrrruser_api_key_cache user_model)get_available_models_for_userrNrrur)rrrT) exclude_noneF) model_namelitellm_router_instancer default_modelderive_savings_baselinebody)metadataproxy_server_requestr)rnr_metadata_variable_namemessages)rrequest_kwargsrz2Auto router routing test failed. Due to error - %srzCould not route this prompt: zVThe router made no decision for this prompt. Check that at least one tier has a model.)rrrrrrr`r) routed_modelrouted_model_configuredrouting_decision)!rrrrrrrlitellm.proxy.utilsrrr`r=r no_llm_routerrrrr& router_namerrr")add_user_api_key_auth_to_request_metadata wire_bodyr#async_pre_routing_hook Exceptionr exceptionrr1r frozenset)rnrrrrrrrrcomplexity_routerr hook_responseravailable_modelss rLpreview_auto_router_routingr's,VB #6GQUQ]Q] ^^^*88>>   /,,+  0## *!%!>!>!I!IW[!I!\(( %  5^^ nn %+TN , *N/~> %6%M%M"")#J/&N&   > > Fq  %B+)#+ - %  )"(( - 3 3yAQ7R R&77 I_4 &&'[]^_8<   " se1GFAG<F=A>> && m6KE7Rj4kllms 4AA# numerator denominatorc2|dkrytd|z|z dS)NrgY@)roundrrs rL_pctrs$a "[0! 44rUrhitsc2t||t||S)N)rr  hit_rate_pct)r.r)rr s rL _cache_bucketr#s u4d4QVFW XXrUrowc|j|jz }|j|jz}|j}t ||j |r|j |z nd|r|j|z nd|r|j|z nd|j|j|t|j||r|j|z ndtt|j|j t|j|jt|j|jt|j |j"t|j|j|j$|j&|j(t+||j&z |j(z d|j,|j.  S)Nrr) coverage_pctr" same_model first_visitreturn_to_tierrreturn_misses_expiredreturn_misses_within_ttlreturn_misses_unknownrr) rravg_turns_per_sessionavg_session_secondsavg_tokens_per_sessionrr baseline_spend saved_pctsaved_per_sessioncache)rrrr rr-rr rrr/rrr#rrrrrrrmaxrr)r$ return_missesr0rs rL_benchmark_totalsr6sj++coo=MII7NllH $ii6>cii(2C>FC//(:C>Fs//(:CiiOO%s78@#//H4c"c//;cnnc.?.?@$S%9%93;N;NO%c&;&;S=Q=QR()9)93??K//"%";";%(%A%A"%mc6O6O&ORURnRn&npq"r))))   rUcbt|}t|j|j|j|j |j |j|j|j|j|j|j|j|j|jS)N)rrrrrr-r.r/rr r0r1r2r3)r6r+rrrrrr-r.r/rr r0r1r2r3)r$totalss rL_benchmark_groupr9s%c*F #OOOO>>ll$::"66%<<ll&&,,"" 22ll rUrowsc  td+idddddtidtd|Ddtd|Dd td |Dd td |Dd td|Ddtd|Ddtd|Ddtd|Ddtd|Ddtd|Ddtd|Ddtd|Ddtd|Ddtd |Dd!td"|Dd#td$|Dd%td&|Dd'td(|Dd)td*|DS),Nrrrrc34K|]}|jywrH)rrr$s rLrz"_summed_agg_row.. s2cS\\2rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..,#)),r?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..@CC//@r?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..s<#++s rLrz"_summed_agg_row..s6#s~~6r?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..BcS11Br?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rCr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..sDc33Dr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rGr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..:c)):r?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..8C8r?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..s!L#";";!Lr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..s$RcS%A%A$Rr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rLr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rLr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rLr?rc34K|]}|jywrH)rr>s rLrz"_summed_agg_row..rAr?r c34K|]}|jywrH)r r>s rLrz"_summed_agg_row..rNr?r c34K|]}|jywrH)r r>s rLrz"_summed_agg_row..rCr?rI)rr sumr:s rL_summed_agg_rowrYs    $B' 2T22   ,t,,   @4@@  .Gs%  ?s'1 c32K|]}t|ywrH)ra)rrZs rLrz&_idle_router_groups..Isc(4cc 3rK|].\}}tjt||d0yw)rrupdateN)r9 model_copyr )rnamer`zeros rLrz&_idle_router_groups..Ls: D$ 0@QUfjAk0lmns47)rYrrrsorted)rrbidleros ` @rL_idle_router_groupsrr9s`!"%DczG\G\Gb`bcD  , rUz/auto_router/benchmarks)z auto router)rrr start_datez?YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date)) descriptionend_datez-YYYY-MM-DD UTC, inclusive (defaults to today)c Kddlm}m}t|d|%t dt j j|r t|n6tjtjjddddd}|r t|n|td z }||kr t d d t|t |j#|td zj#d{}t$j'|xsd }gd|Dt)|t+d|D} t-|j/d|j/dt1| t3t5|| S7w)a[ Benchmarks for the auto-router dashboard: session shape, savings against the configured baseline, and prompt-caching behaviour bucketed by what the router did. Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time, so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over that bucket's turns. The rollup supplies the measures, never the list. Which routers appear comes from the model registry, so one shows up as soon as it is configured and reads zero until it serves traffic, and `routers_in_scope` counts those too rather than only the routers the window recorded. rrrz1view auto-router benchmarks across the deploymentNrr)hourminutesecond microsecondrdaysrz,end_date must not be earlier than start_daterrIc32K|]}t|ywrH)r9r>s rLrz-get_auto_router_benchmarks..s 0C 3  0rhc3LK|]}|j|jfywrHrjr>s rLrz-get_auto_router_benchmarks..s2fZ]COOS__3U2fs"$r)rsrurouters_in_scoper8groups)rrr_require_admin_viewerr=rrrrrnowrrrrrr! isoformatr validate_pythonrrrr,strftimelenr6rY) rrsrurrend_day start_dayraw_rowsr:rs rLget_auto_router_benchmarksrRsk8E+-`a4E4\4\4b4bcc  X& \\(,, ' / /QqXYbf / g .s  xs)any auto_routerscomplexity_routersadaptive_routersquality_routers)rrs `rL#_is_configured_pre_routing_strategyrsE   # #  ) )  ' '  & &   rUrc tj|\}}}}|dk7s tjstjryddlm}ddlm|js|jrytfddD S) N)r anthropicFr)AnthropicModelInfo)!secret_manager_would_be_consultedc3.K|] }|ywrHrI)r secret_namers rLrz>_sdk_model_is_missing_anthropic_credentials..s;F)+6rf)ANTHROPIC_API_KEYANTHROPIC_AUTH_TOKEN) litellmget_llm_provider anthropic_keyapi_key#litellm.llms.anthropic.common_utilsrlitellm.secret_managers.mainr get_api_keyget_auth_tokenr)r_providerrrs @rL+_sdk_model_is_missing_anthropic_credentialsrsu00u=AxA;'"7"77??FN%%'+=+L+L+NJw rU field_nameteam_idscXtrtd|ddtfd|D}td|D}|rtd|ddt|ztd |D}|syt sytd|dd t|z) aVReject a model the dispatch path cannot resolve, at start rather than as a silently growing error count once the job is already sampling and billing. Both the judge and a reverse job's baseline must be plain models: an auto-router in either slot would re-route per turn, so the comparison would have no fixed arm to attribute results to. Resolvability is asked once per team the job samples for, because that is the identity the call carries: a name only one team can reach fails every turn for the other keys, which is the growing error count this check exists to prevent.Nrz 'z-' is an auto-router; it must be a plain modelrc3<K|]}|t|fywrHr)rteamrrs rLrz(_validate_plain_model..s^TD,z5$"GH^sc3FK|]\}}|jdk(s|yw)nothingNviarrtargets rLrz(_validate_plain_model..s [ f6::QZCZt[!!z{' is neither a model configured on this proxy nor a provider-qualified public model name (e.g. 'anthropic/claude-sonnet-5')c3FK|]\}}|jdk(s|yw)sdkNrrs rLrz(_validate_plain_model..sUldFuATTUrzm' uses the LiteLLM SDK but required credentials are not configured: ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN)rr=r _for_teamsr)rrrrtargets unreachable sdk_teamss`` rL_validate_plain_modelrs"EjRW"X \E7*WX  ^U]^^G[[[K,b(ZZ\fgr\st  UgUUI  6u = l"UG$8 8:DY:O P rUchttd|D}|rddj|SdS)zMName the teams a fault applies to, when it does not apply to every key alike.c3&K|] }|| ywrHrI)rrs rLrz_for_teams..sNT=MNsz for team , r<)rrpjoin)rnameds rLrrs5NNNOE.3Z %() *;;rUtierdefault _JUDGED_ROLESc||j|nd}ttjd|xsdDS)a``(role, model_name)`` for every model the router under evaluation can answer with. Drawn from ``strategy_router_dependencies``, the single answer to "what does this router call", so this cannot disagree with the health check's reading of the same deployment. Only the roles that SERVE are arms: the classifier and embedding models pick the tier, they never produce a response anyone judges, so a judge sharing them carries no self-preference. A semantic auto-router keeps its routes in an opaque config blob or a file, so only its default model is enumerable and the guard below is incomplete for it. That direction is deliberate: it can miss a collision, never invent one. Which tiers a router declares is a property of its config and not of who is calling, so this lookup is unscoped; what each tier NAME resolves to is the team-dependent half, and it belongs to the caller that compares them. N)rc3K|]A}t|dD].}|jtvr|j|jf0Cyw)r\N)r)rolerr)rrZ dependencys rLrz%_router_arm_models..sS ::FV;WX -/__j33 4 4 sAA rI)get_model_listrdictfromkeys)rr deploymentss rL_router_arm_modelsrsL"OYNd22k2JjnK   )/R   rUct|jjtfdgt |j |j d|j ffndDS)aD``(role, model_name)`` for each arm the judge would also be, as one team's keys see it. Both sides resolve under the SAME team, since two names are the same model only for a caller who can reach both; resolving the judge for one team against an arm for another invents a collision no request could produce. c3`K|]%\}}t|jzr||f'ywrH)rr)rrrjudgerr`s rLrz-_judge_collisions_for_team..s: D% < E7;BB B u +.baselinerI)r judge_modelrrrrbaseline_model)rrnr`rs` `@rL_judge_collisions_for_teamr st D,<,.5s5 !?YZdfjls?t 2;I  s!Nrz judge_model 'z'' is also an arm this job would judge: rc32K|]\}}|d|dyw)z model ''NrI)rrrs rLrz5_validate_judge_is_not_a_candidate..?s!PkdE4&q1Prhz. A judge scores its own answers higher than a rival's, so the win rates would measure the judge; pick a judge that serves neither armr)rrrr=rr)rrnr collisionss`` rL"_validate_judge_is_not_a_candidater!sx&  %-  J  D,,--T UiiPZPP QF F rUruc ddlm}t ||S#t$r-dt|j vxs dt|vcYSwxYw)agWhether a Prisma create failed on a unique index. One active job per key and direction lives in a partial unique index (raw SQL in the migration; schema.prisma cannot express partial indexes), so the read-then-create check above it is advisory: two concurrent starts pass the read, and the loser must surface as the same 409 rather than a 500.r)UniqueViolationErrorzunique constraintP2002) prisma.errorsr ImportErrorrSlowerr^)rurs rL_is_unique_violationrFsT R6 e1 22 R"c%j&6&6&88QGs5z= j.max_turns OR ( j.max_budget IS NOT NULL AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget ) ) _SWEEP_FINISHED_JOBS_SQLa SELECT COUNT(*) FILTER (WHERE outcome != 'error')::int AS judged_count, COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count, COALESCE(SUM(judge_cost), 0)::float AS judge_spend FROM "LiteLLM_ShadowEvalAttempt" WHERE job_id = ANY($1::text[]) _ATTEMPT_TOTALS_SQLaO SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0)::float AS spend FROM "LiteLLM_ShadowEvalAttempt" a JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at) GROUP BY a.job_id _ATTEMPT_COUNTS_SQLa/ SELECT COUNT(*)::int AS legs_with_rows, COALESCE(SUM(not_sampled), 0)::int AS not_sampled, COALESCE(SUM(unjudgeable), 0)::int AS unjudgeable, COALESCE(SUM(shed), 0)::int AS shed, COALESCE(SUM(withheld), 0)::int AS withheld FROM "LiteLLM_ShadowEvalFunnel" WHERE job_id = ANY($1::text[]) _FUNNEL_TOTALS_SQLah UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp) WHERE group_id = $1 AND stopped_by IS NULL AND ends_at > (NOW() AT TIME ZONE 'utc') AND EXISTS ( SELECT 1 FROM "LiteLLM_ShadowEvalJob" k WHERE k.group_id = $1 AND k.stopped_at IS NULL AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns AND ( k.max_budget IS NULL OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget ) ) _STOP_JOB_SQLc@eZdZUeed<eed<eed<eed<eed<y)_FunnelTotalsRowlegs_with_rows not_sampled unjudgeableshedwithheldN)rPrQrRrqr rIrUrLrrs IMrUrc,eZdZUeed<eed<eed<y)_AttemptCountRowjob_id attempt_countrNrrIrUrLrrs K LrUr_ATTEMPT_COUNT_ROWSz SELECT * FROM "LiteLLM_ShadowEvalJob" WHERE group_id IN ( SELECT group_id FROM "LiteLLM_ShadowEvalJob" GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int ) _LIST_LEGS_SQLz SELECT * FROM "LiteLLM_ShadowEvalJob" WHERE group_id IN ( SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int ) _LIST_LEGS_BY_KEY_SQLc,eZdZUeed<eed<eed<y)_AttemptTotalsRow judged_count error_count judge_spendN)rPrQrRrqr r rIrUrLrrsrUr_ATTEMPT_TOTALS_ROWSct||SrH)rrs rL_pct_ofrs  ; ''rUc@tdt|ddDS)Nc 3K|]}t|j|jt|j|jt|j |jt|j |jt|jxsdd|j|j|j yw)r) grouprreal_win_rate_pctshadow_win_rate_pct tie_rate_pctavg_judge_confidencerrrN) r9rrrrrrrrrrrr>s rLrz_slices..s   ''~~%cmmS^^D ' H 3>>:!&s'9'9'@S!!D~~))//  sCCc|jSrHr)rs rLz_slices..s allrUTrereverse)rrprXs rL_slicesr#s(  $$:DI   rUceZdZUdZedZeed<eed<eed<eed<eed<d Z ed zed <eed <e ed <e ed <d Z e d zed<e ed<e ed<d Ze d zed<d Zed zed<edddede d zde d zfdZy )_LegRowa:One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is one key's leg of a job; the legs of a job share group_id and identical config, written together by one create_many. The API's job id is the group id, so leg ids never leave the server (attempts reference them internally).T)from_attributesrjgroup_id api_key_idr directionNrrshadow_percentage max_turns max_budget created_atends_at stopped_at stopped_byrrFcb| |j|S|jtjS)zThe columns store naive UTC wall time (prisma's convention); prisma reads hand back aware datetimes while raw SQL reads hand back naive ones, so this boundary makes every read aware UTC before anything compares or serializes them.r)rrrr)clsrs rL _as_aware_utcz_LegRow._as_aware_utcs, =ELL4L}}HLL}11rU)rPrQrR__doc__r model_configrSr r5rr rqr,rr/r0r classmethodr3rIrUrLr%r%s8 d3L GMO""!%NC$J%N#J # "&J4&!Jd !\9l;2(T/2ho2<2rUr% _LEG_ROWSlegsc K|s tiStjt|t|Dcgc]}|j c}d{xsd}t|Dcic]}|j |c}Scc}w75cc}ww)aEach leg's attempt count and recorded spend by leg id, judged and errored alike, in one grouped read. They are the same figures the sampler budgets against max_turns and max_budget, so the derived status flips to completed exactly when sampling actually ends. A stamped leg's figures freeze at its stopped_at: in-flight attempts that land after the stamp are excluded, so they can never reclassify a leg that was stopped under budget as budget-spent.NrI)r r rrrrjr)rr8legr:r$s rL_leg_attempt_countsr;s| ##%55(;PT=Ucff=UVV  D =SZZ_= >>>VV>s.,BA< B B B!B5 BBr'attempt_countsc&|d}t|tfdt|dD|j|j|j |j |j|j|jtd|Dd S)aThe one constructor of a job response: the caller names the group and passes that group's legs. Config is read off the first leg because every leg carries the same copy, written by one create_many. No caller may serialize a raw row (that would leak a leg id as the job id).rc 3K|]~}t|j|j|j|jj |j xr jndrtjdndyw)rr)r(r+r,r/r rN) r6r(r+r,r/r_rjr rr)rr:r<statss rLrz"_group_response../s{   %>>-->>>>?M?Q?QRURXRX?Y6Ye6Ye11`a/4eEKK+#     sBBc|jSrH)r()r:s rLr z!_group_response..8s CNNrUrec3NK|]}|j|jywrH)r0rr:s rLrz"_group_response..AsVC3>>;UVs%%N) rkeysrr)rrr*r-r.r0) r7rrprr)rrr*r-r.next)r'r8r<firstr@s ` @rL_group_responserH%s7E    d(BC   %%//++%%11## V4VX\]) rU)NN_NO_KEY_LABELS responsesc6K|syttd|D}t|jdd|iid{}|xsdDcic]%}|j|j |j f'c}tfd|DS7Mcc}ww)zResolve every scoped key's hash to its alias and masked name in one batched read, so the UI can say whose traffic a job shadows. Deleted keys resolve to None.rIc3VK|]!}|jD]}|j#ywrH)rEr()rresponseres rLrz#_with_key_labels..Os($gYaYfYf$gRUS^^$g^$gs')rZinrNc3~K|]4}|jdtfd|jDi6yw)rEc3K|][}|jj|jtdj|jtdd]yw)rr)r\r^rkN)rmr_r(rI)rrelabelss rLrz-_with_key_labels...Ys^ NN)/CNNN)STU)V(. 3>>>(RST(U #sA!A$rkN)rmrrE)rrMrQs rLrz#_with_key_labels..VsM   (}}   s:=)rprrrfrZr\r^r)rrJtokenskey_rowsr$rQs @rL_with_key_labelsrTHs 9$gI$gghF0?IIv'JH=ENB58 CMM3<<00BF "  Bs$ABB B*B;BBc b K|Dcgc]}|j}}tjt|t|d{xsd}|sytjt|t |d{xsd}t |Dcic]}|j|jc} tjt|t|d{xsd}t fd|D}td|D}t|t|d{} | rtj| dnd} | | jt|k(r| nd} t!t#|t#|t#|t%td|D|t%td|D|td|Dtd |D| | j&nd| | j(nd| | j*nd| | j, Sd Scc}w77cc}w7X7w) aAll three stratifications of one job's verdicts. Tier answers "where does the router do well"; the model stratification groups by whichever model served the real arm, so it answers "which of the models these keys use today would the router beat" forward, and "for the turns the router sent to X, did X beat the baseline" in reverse; key answers "which key's traffic does the router suit". Reads are bounded by the job's own attempts (<= the sum of its keys' max_turns) via the job_id index.NrIc3`K|]%}|jd|ji'yw)rrkN)rmr)rr$ key_by_legs rLrz'_shadow_eval_results..|s2  uj&9:;rc34K|]}|jywrHrrrs rLrz'_shadow_eval_results..s;aQ\\;r?rc34K|]}|jywrH)rrYs rLrz'_shadow_eval_results..s/O! /Or?c34K|]}|jywrH)rrYs rLrz'_shadow_eval_results..s(AA(Ar?c34K|]}|jywrH)rrYs rLrz'_shadow_eval_results..s=q||=r?c34K|]}|jywrH)rrYs rLrz'_shadow_eval_results..s AA Ar?) by_tierby_current_modelby_keyoverall_shadow_win_rate_pctoverall_tie_rate_pctsampled_real_spendsampled_shadow_spendnot_sampled_countunjudgeable_count shed_countwithheld_count)rjrrrrrr r(rrrWrrrrrr8r#rrrrr) rr8r:leg_idsr^by_modelby_legr` total_turns funnel_rowscountedfunnelrWs @rL_shadow_eval_resultsrphs)--cff-G-&66(@'JJPbG '77(A7KKQrH)D)QS#&&#..*@)QRJ%55(?IIORFF;7;;K)-9KWUUKHS%44[^DY]G '2w7M7MQTU\Q]7]GcgF   *v$+C/Ow/O,OQ\$]$S(A(A%A;O=W== A AA060B&,,060B&,,"("46;;$*0*j@} tCtE} | tGjHz} | jj8j,j*jJtLjN|jP| | d } tdjD} t7|jStU| jDcgc]\}}i| ||dc}}d{ t[|jS| Dcgc]}d"|ic}d#$d{ta|Dcic]}|jb|c}te| tfd&tjDjj8j,j*jJ| | ' S7c7W7'cc}}w7#tV$r-}tY|st dd j8d!|d}~wwxYwcc}w7#tV$r"}t]j^d%| |Yd}~d}~wwxYwcc}ww)(ah Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against a second arm, judge the two responses blind, and stratify win rates by tier, by the model that served the real arm, and by key. A forward job answers whether the keys should adopt router_name: it samples the requests the router did not serve and duplicates them through it. A reverse job answers whether a key already on the router still gains from it: it samples the requests the router did serve and duplicates them against baseline_model. A key can hold one active job per direction, so both questions can run at once. Shadow responses are never served to users. Each key samples until its recorded eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's window ends, or the job is stopped, so one key running out of budget does not end sampling for the others; sampling changes propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed key but are excluded from request counts and auto-router adoption metrics. rrwzstart a shadow evalNrrrrz!' is not a configured auto-routerrZrNrc34K|]}|jywrH)rZr>s rLrz$start_shadow_eval..sIpX[#))Ipr?rIzapi_key_ids not on this proxy: rzR; pass each key's token hash, the value the key list and key info endpoints reportc34K|]}|jywrH)r`r>s rLrz$start_shadow_eval..s)R##++)Rr?rr)r(r)r/izAlready in an active z shadow eval job: c3TK|] }|jd|jd"yw)z (job )N)r(r'r>s rLrz$start_shadow_eval..s'"_PScnn%5VCLL>#K"_s&(z. Stop it first.r}) r'rr)rrr*r+r, created_byr-r.c3BK|]}ttywrH)rSr)rrs rLrz$start_shadow_eval..sFs57|Fs)rjr()rnz'A requested key was claimed by another z- shadow eval job concurrently. Stop it first.rT)rnrxz.shadow_eval: funnel seed failed for job %s: %sc 3K|]<}t|tj|j|j>yw))r(r+r,r\r^N)r6r*r,r\r^)rr(rnrQs rLrz$start_shadow_eval..sN   %%0?? ,66 +44     sAA) rrErr)rrr*r-r.)3rrrrr=rrrrrrrflist api_key_idsrrprrrrrrrrr execute_rawrrr)rrrrrSrr duration_daysr*r*r,user_idrpziprrrrrur rZr7)rnrrr token_rowsunknownr requestedclaimedrr'r. shared_configrileg_idrerseed_errr$rQs` @rLstart_shadow_evalrsn:E+-BC4E4\4\4b4bcc!DZQUQaQa!ba8H8H7IIj4kll2=AKKtD$4$4567LJ6)D,<,<"= Ip_i_omoIp@p"pqrG1$))G2D1EFGG  DMM)RAQr)RRSH*d&6&6 xP &j$*=*=?OQYZ&z4B D,,-I    & &'? KKK,];EE+ FG''77IJ))F"_W^"__`a$%  hll+C%'lH9$*<*<==G''^^--''!33+oo'// MFT5E5EFF .::VYZacgcscsVtGRvsB=BcB;   "i!-0<<3:;8V$; =   % %Kciin%KLF    %T%5%56   $$..**$$00% q2LL #A& 9$..9IIvw   <  i""#SU]_ghhi%KsB$R(P )DR7P82R*P+C-RA P"P 5 PPPQ Q* Q6Q7Q; RR A4RRRP Q%(Q  QRQ R%R<RRRz/auto_router/shadow_evalr(z>Filter to jobs that shadow this key, alone or alongside otherslimitrzNewest jobs to return)gelertc l Kddlm}t|d|%tdtj j tj|rt|t||d{nt|t|d{xsd}ttt|td td Dcic]\}}|t!|c}}tfd d }t#||d{ t%|t! fd |Dd{S77cc}}w757w)zList shadow eval jobs, newest first, each key with its attempt count so status is accurate. Judged counts, spend, and results ride the detail endpoint only.rrview shadow evalsNrrrIr'rBc.td|DS)Nc34K|]}|jywrH)r-rDs rLrz:list_shadow_eval_jobs....Js*Xc3>>*Xr?)r4)r'by_groups rLr z'list_shadow_eval_jobs..Jss*XXhEW*X'XrUTr!c3>K|]}t||ywrH)rH)rr'rcountss rLrz(list_shadow_eval_jobs..Ns pW__Xx7I6Rps)rrrr=rrrr7rrr r r rrpr rr;rT) rr(rrr8r'r newest_firstrrs @@rLlist_shadow_eval_jobsr's-9+-@A4E4\4\4b4bcc++],A5*U U U!-GG   D:J$+6$Jz>D4<D* (D4:D0;&D4!D2"D4(D4*D42D4z!/auto_router/shadow_eval/{job_id}rc xKddlm}t|d|%tdtj j tjt|jd|id{xsd }|std d ||Dcgc]}|j}}tjt|t|d{xsd }t|j!d |id dddid{}t#|t%||t'||d{fd{}|dj)|r|dj*nd|r|dj,nd|rt/|dj0dnd|r |j2ndt5||d{dS7=cc}w77777w)zOOne job with derived counts, judge spend, latest error, and stratified results.rrrNrrr'rrINo shadow eval job rNru)routcomer-desc)rEr}r?r)rrr last_errorresultsrk)rrrr=rrrr7rrrfrjrrrrrrTrHr;rmrrrrrurp) rrrr8r:rir8 latest_errorlabeleds rLget_shadow_eval_jobrRs9+-@A4E4\4\4b4bcc++ .88v&9     D 6I&4RSS(,-cff-G-(88(;WEEKF!6m D O OW=V$!P!L,D5!6!6:#0<,,,$1-FF   ! % .E 7eGsyA)F:+F(,"F:F+!%F:F02F:9F2: F:F4F:+F6,A0F:F8 F:+F:2F:4F:6F:8F:z&/auto_router/shadow_eval/{job_id}/stopcKddlm}t|d|%tdtj j tjtj}|jxsd}|jjt|||jdj!d{}t"j%t'|j)d |i d{xsd }|std d |t+||d{}t-|||}|dk(rtdd|d|j.t1||fd{} | dS77}7Q7w)aStop an active shadow eval job, every key it scopes at once. Attempts are kept; sampling halts within ~10s. Keys that already stopped on their own budget keep the stopped_at they earned. The statement is the whole state machine: it claims the job only while a leg still samples inside the window with no stop recorded, so a racing operator, a same-instant budget spend, and a repeat stop all read the same 400 with the status the job actually holds.rrzstop a shadow evalNrroperatorrr'rrIrrrzJob z is already )rrrr=rrrrrrrr}rr{rrrr7rrrfr;rHr?rT) rrrstamprrr8rcurrentrs rLstop_shadow_eval_jobrsc 9+-AB4E4\4\4b4bcc<< -E'//=:H(++77vxd)C)M)M)OG++ .88v&9     D 6I&4RSS-mTBBF$VT6:G!|d6(,w~~N^4_``+MG:FFG 1:! CGsIB0E*2E"33E*&E$'-E*E&AE*E( E*$E*&E*(E*)N2)r4collections.abcrrrrr itertoolsrrr typesr typingr r r ruuidrpydanticrrrrrlitellm._loggingrlitellm.exceptionsr$litellm.litellm_core_utils.llm_judgerlitellm.proxy._typesrrrrrrlitellm.proxy.auth.auth_checksrr$litellm.proxy.auth.user_api_key_authr *litellm.proxy.db.autorouter_session_rollupr!$litellm.proxy.litellm_pre_call_utilsr"r#$litellm.repositories.base_repositoryr$$litellm.repositories.team_repositoryr%)litellm.router_strategy.complexity_routerr&rr'r(r)8litellm.types.management_endpoints.auto_router_endpointsr*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:fastapir;r<r=r>r?rr@litellm.routerrArrBr rDrWrcrhrlrsrwr|rrrrrrSrTrrrrrpost HTTP_200_OKrrrryr rr rrqr#r6r9rYrarrrr_rrrrzrrrrrrrrrrrrrrrrrrrrrrrr r r rrrr#r%r7r;rHrIrTrpHTTP_201_CREATEDrrrrrIrUrLrs .22"<<HH12=CPC?F (HH0% LL  aa )H )fhf  R(R 'H' iXi+h+ /~/*/66;R62^28K25~5:P566.G>.-E((>2~6OVOc3h$0FO,*3E#s(O*D #S ()2 +,./   jn 6 9J1K!KL6 d E&ghh6 d E6e$ffg 6 " 6  6B\^\S\T\X^XSXTX H 3 SW  s t %%(+%9<%HPQTW[Q[H\% %P<#*-<#< AJ6S\J]@^ uY;<=^?uUXZ]U]`cOcId8'=HKd  5c?C  (""'="IQRUX\R\I]" "J 3 3d 3 Y 'tN';<5< U #RTg"g%g#SVi#i5i!8;N!NN #% UUE u"yy ).>)?@U@ u  *$/@*ABeB(s((((>*u_c5I/J"2i2DtG}- 5- ?^ ?8GCT ?Y`adfvavYw ? !'*>XX