tPj UddlZddlZddlZddlZddlmZmZmZmZm Z ddl m Z m Z ddl m Z mZmZddlmZddlmZddlmZmZmZmZmZddlZddlmZdd lmZdd lm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/dd l0m1Z1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ddl;mZ>ddl?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGmHZHmIZImJZJddlKmLZLddlMmNZNmOZOmPZPddlQmRZRddlSmTZTedZUedddidZVeeeWeXfeYd<edddiiZZeeeWeXfeYd<Gd d!eZ[Gd"d#e[eZ\Gd$d%e[eZ]Gd&d'e[eZ^Gd(d)e[eZ_Gd*d+e[eZ`Gd,d-e[eZad.ebfd/Zcd0eddzd.eddzfd1Zededdzd2eddzd.edfd3Zfd4e[d5eeWedfd.edfd6Zgd4e\d.eWfd7Zhd4e\d.eieWd8ffd9Zjd4e]d.eWfd:Zkd4e]d.eieWd8ffd;Zld4e^d.eWfd<Zmd4e^d.eieWd8ffd=Znd4e_d.eWfd>Zod4e_d.eieWd8ffd?Zpd4e`d.eWfd@Zqd4e`d.eieWd8ffdAZrd4ead.eWfdBZsd4ead.eieWd8ffdCZtd4ead5eeWedfd.edfdDZueifdEe eWdFeeWeXfd.eveWeXffdGZweifdHeNdIdJdFeeWeXfd.dfdKZxdHeNdIdJd.dfdLZye dMdMNGdOdJZze dMdMNGdPdQZ{e dMdMNGdRdSZ|ezZ}eeYdT<e dMdMNGdUdVZ~e~ddWZeeYdX<dYe d.e fdZZd[eeXd\e d.efd]Zd^e~d.ebfd_Zd`egee~fd.dfdaZe dMdMNGdbdcZGdddeeZdfe@dgeWdheWd.dfdiZdfe@dgeWdheWd.dfdjZe dMdMNGdkdlZedmdne,jdodpdqeredsdte,jdudvdwerfZeeied8feYdx<dIezd.eveWeXffdyZGdzd{Zy)|N) AwaitableCallableIterableMappingSequence) dataclassfield)datetime timedeltatimezone)EnumMappingProxyType)FinalLiteralProtocolTypeVar assert_never)verbose_proxy_logger) DualCache)GLOBAL_PROXY_SPEND_CACHE_KEYLITELLM_PROXY_BUDGET_NAMERESET_BUDGET_JOB_BATCH_SIZE!RESET_BUDGET_JOB_LOCK_TTL_SECONDS#RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUNRESET_BUDGET_JOB_NAME)duration_in_seconds)DB_RETRY_SAFE_ERROR_TYPESLiteLLM_BudgetTableFullLiteLLM_EndUserTableLitellm_EntityTypeLiteLLM_TeamTableLiteLLM_UserTableLiteLLM_VerificationToken)BudgetResetSettingscompute_budget_reset_atget_budget_reset_settings)end_user_cache_keymodel_access_group_cache_key$model_access_group_spend_counter_key tag_cache_key)roll_window_spend_row)PodLockManager)call_with_db_reconnect_retry) PrismaClient ProxyLogging)OrganizationRepository)SpendLinkedTable)EndUserRepository ModelAccessGroupBudgetRepository TagRepositoryTeamMembershipRepository)TeamRepository)LinkedSpendResetWritesbudget_cascade_unit_of_workspend_reset_unit_of_work)VerificationTokenRepository) ServiceTypes_RowTgt)budget_durationspend_LINKED_KEYS_WHEREr@_SPENT_ROWS_WHEREcDeZdZededzfdZededzfdZy)_BudgetLinkedRowreturnNcyNselfs p/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/proxy/common_utils/reset_budget_job.pyr@z_BudgetLinkedRow.spendJs%(cyrGrHrIs rK budget_idz_BudgetLinkedRow.budget_idMs'*rL)__name__ __module__ __qualname__propertyfloatr@strrNrHrLrKrDrDIs0 (ut|(( *3:**rLrDc8eZdZedefdZedefdZy)_TeamMembershipRowrEcyrGrHrIs rKuser_idz_TeamMembershipRow.user_idR!rLcyrGrHrIs rKteam_idz_TeamMembershipRow.team_idUrYrLN)rOrPrQrRrTrXr[rHrLrKrVrVQs( !!! !!!rLrVc"eZdZedefdZy)_KeyRowrEcyrGrHrIs rKtokenz _KeyRow.tokenZsrLN)rOrPrQrRrTr_rHrLrKr]r]Ys srLr]c"eZdZedefdZy)_OrgRowrEcyrGrHrIs rKorganization_idz_OrgRow.organization_id_s&)rLN)rOrPrQrRrTrcrHrLrKrara^s )))rLrac"eZdZedefdZy)_TagRowrEcyrGrHrIs rKtag_namez_TagRow.tag_nameds"rLN)rOrPrQrRrTrgrHrLrKrerecs "#""rLrec"eZdZedefdZy)_ModelAccessGroupRowrEcyrGrHrIs rKaccess_group_namez&_ModelAccessGroupRow.access_group_nameis(+rLN)rOrPrQrRrTrkrHrLrKririhs +3++rLric"eZdZedefdZy) _EndUserRowrEcyrGrHrIs rKrXz_EndUserRow.user_idnrYrLN)rOrPrQrRrTrXrHrLrKrmrmms !!!rLrmrEc&tjduS)NT)litellmbudget_rolloverrHrLrK_rollover_enabledrrrs  " "d **rL max_budgetc6|tj|sy|SrG)mathisfinite)rss rK _rollover_caprwvsz!: rLcapc.|ytd|xsd|z SN)max)r@rxs rK_carried_spendr}|s  { sU\cS( ))rLrowcapsc|syt|j|j|j|jSdSrz)r}r@rNget)r~rs rK_row_carried_spendrs8  #)) @YTXXcmm%< dd_c ddrLc:d|jd|jS)Nzspend:team_member::)rXr[r~s rK_team_membership_counter_keyrs  }Ackk] ;;rL.c:|jd|jfS)N_)r[rXrs rK_team_membership_cache_keysrs{{m1S[[M * ,,rLc d|jS)N spend:key:r_rs rK_key_counter_keyrs  { ##rLc|jfSrGrrs rK_key_cache_keysrs II<rLc d|jS)Nz spend:org:rcrs rK_org_counter_keyrs ++, --rLc@d|jd|jdfS)Nzorg_id:z :with_budgetrrs rK_org_cache_keysrs0 #%%&' #%%&l3 rLc d|jS)Nz spend:tag:)rgrs rK_tag_counter_keyrs  ~ &&rLc.t|jfSrG)r+rgrs rK_tag_cache_keysrs #,, ' ))rLc,t|jSrG)r*rkrs rK_model_access_group_counter_keyrs /0E0E FFrLc.t|jfSrG)r)rkrs rK_model_access_group_cache_keysrs ()>)> ? AArLc d|jS)Nzspend:end_user:)rXrs rK_enduser_counter_keyrs S[[M **rLc.t|jfSrG)r(rXrs rK_enduser_cache_keysrs s{{ + --rLc|sy|jxstj}t|j||j |SdSrz)rNrpmax_end_user_budget_idr}r@r)r~reffective_budget_ids rK_enduser_carried_spendrsG --I7+I+I #))FYFeTXX.A%B ppko pprL budget_idsextrac$ddt|ii|S)NrNin)list)rrs rK_budget_link_wherers $Z 01 ;U ;;rLwritescascade_BudgetCascadec8jjD]?\}}|jd|i|dd|di|jd|i|dd|ii|At fdj D}|r|jt ||y y ) aReset one linked table's spend for every expiring tier: tiers with a rollover cap keep spend beyond the cap (decrement preserves writes racing the reset), everything else is zeroed as before. Zero the under-cap rows BEFORE decrementing the over-cap ones: the statements run sequentially in one transaction, so the reverse order lets the zero re-match a row the decrement just moved into the (0, cap] range and erase its carried spend.rNr@rr>ltewherer>ramountc3@K|]}|jvs|ywrG) rollover_caps).0bidrs rK z._queue_budget_linked_resets..scS#WMbMbBbScsN)ritemsqueue_spend_zeroqueue_spend_decrementtuplerr)rrrrNrx plain_idss ` rK_queue_budget_linked_resetsrs"//557  3 SUSGAc=RS   $$ IUIGdC[IRU %  cG,>,>ccI&8E&JKrLc:t||ttj}|||jvry|j j |}||jddity|jdd|dd|jdd|id| y) aEnd users reset on the budget link like every other gated table, plus a NULL-budget_id branch: rows created implicitly persist no link and ride the default tier (litellm.max_end_user_budget_id). Matching on the link rather than enumerating user ids keeps a statement's bind count proportional to the expiring tiers instead of the customer population, which past ~32,700 dependents exceeds PostgreSQL's per-statement bind ceiling and wedges the cascade permanently (#40564). rNrNrrrrNr@r>r) rrBrprrrrrr)rrdefault_budget_idrxs rK_queue_enduser_resetsrs 7HI&== $5W=O=O$O&&**+<=C {:(9:     !C+@A    D#;7!rLT)frozenslotsceZdZUdZdZeedfed<dZee dfed<dZ eee e fdfed<dZ ee dfed<dZeee efdfed<dZee dfed <ed Zee efed <y )rzDEverything one budget-tier reset touches, resolved before any write.rH.budgetsr budget_resetsenduserscounter_resets cache_keysctiSrGrrHrLrKz_BudgetCascade.s GWXZG[rL)default_factoryrN)rOrPrQ__doc__rrr__annotations__rrTrr rrmrrSrr rrrHrLrKrrsN35GU*C/ 05"$Jc3h$68M5sH}-s238(*HeK$%*46NE%U +S016"$Jc3h$).?[)\M73:&\rLc"eZdZUeed<eed<y)_BudgetCascadeCommittedradvancedN)rOrPrQrrintrHrLrKrr s MrLrc"eZdZUeed<eed<y)_BudgetCascadeFailedrerrorN)rOrPrQrr ExceptionrHrLrKrrs  rLr_EMPTY_CASCADEc&eZdZUdZeed<eed<y) _ChunkOutcomezOne chunk of a reset phase: rows read, and rows whose new budget_reset_at cleared the due cutoff. Anything else is still due and would come straight back on the next fetch, so it is not progress.fetchedrN)rOrPrQrrrrHrLrKrrs6LMrLrrr _NO_PROGRESSmomentc^|j|S|jtjS)Ntzinfo)rreplacer utc)rs rK_as_utcr%s%]].6WFNN(,,N4WWrL reset_atscutoffcBt|tfd|DS)a$How many rows the write actually moved past the due cutoff. A budget_duration of "0s" (or one the parser cannot read) resolves to the current time, so the row is written and stays due. Counting it as progress would re-read the same chunk until the per-run cap on every tick. c3bK|]&}t|tst|kDs#d(yw)N) isinstancer r)rreset_at utc_cutoffs rKrz"_count_advanced..1s)pXj8.LQXYaQbeoQoqps ///)rsum)rrrs @rK_count_advancedr)s J pp pprLoutcomecJ|jtkxs|jdk(S)zA short chunk means the due rows ran out. A full chunk that advanced nothing would be re-read unchanged forever, so it ends the phase too and those rows wait for the next tick.r)rrr)rs rK_phase_is_drainedr4s% ??8 8 QG  ' - - 4 4&!w ' 5 rczeZdZUdZeed<eed<eed<eed<eed<eed<eeeege dfed <d efd Z y) _WindowSourcez $1 ORDER BY z LIMIT $2)rrrIs rK page_queryz_WindowSource.page_queryusGdnn%%;DJJ<H337>>2BC'y 2 rL) rOrPrQrrTrr!rr/rr"rHrLrKrrisNF JN## \3,io= >>  C  rLrr$r_z spend:keykeyskey)rrrrrrr r"r[z spend:teamteamsteam_WINDOW_SOURCEScXt|jt|jdS)N)num_budgets_foundnum_endusers_found)lenrr)rs rK_budget_cascade_event_metadatar,s& 1!'"2"23 rLc$eZdZdZ dWdedededzdedzfdZded e fd Z d e fd Z d Z d egeefded efdZd egeefded efdZedXdeded dfdZedYdZeded dfdZdeedeeefded eedffdZdeed eedffdZ dee!d e"fdZ#d e"d dfd!Z$d e"d dfd"Z%d e"d dfd#Z&d e'e(zfd$Z)dYd%Z*d e+fd&Z,d e-e.fd'Z/d(e-e0d dfd)Z1d(e-e0d dfd*Z2d+e-e3d dfd,Z4d+e-e3d dfd-Z5d.e-e6d dfd/Z7d.e-e6d dfd0Z8d1ed2e9d3ed4ed5e:eefd df d6Z;dYd7Zd e+fd:Z?dYd;Z@d e+fd<ZAed=e:ded>eBd?eCdeded@eDdAed e fdBZEeded@eDdAedCedDeCd df dEZFed=eeefded>eBd efdFZGdYdGZHdHeId?eCd>eBd dfdIZJdHeIdJed?eCd>eBd edzf dKZKedLe6e3ze0zdMeCdNeLdOdefdPZMedQe6dMeCded e6dzfdRZNedSe3dMeCded e3dzfdTZOedUe0dMeCded e0dzfdVZPy)ZResetBudgetJobzK Resets the budget for all the keys, users, and teams that need it Nproxy_logging_objr reset_settingspod_lock_managercT||_||_|xs t|_||_yrG)r/r r'r0r1)rJr/r r0r1s rK__init__zResetBudgetJob.__init__s.0A+83A3`E^E`7GrL lock_managerrEcK|jy |jt}t|jj |d{S7#t $r }t jd|Yd}~yd}~wwxYww)aTrue only when the lease is readable and someone holds it. An unreadable lock reports as unheld so the caller sweeps rather than skipping; being wrong here costs a duplicate sweep, and the alternative strands every expired budget at its cap. NFz4Reset budget job: could not read the reset lease: %s) redis_cacheget_redis_lock_keyrboolasync_get_cacherrwarning)rJr4lock_keyexcs rK_lease_is_heldzResetBudgetJob._lease_is_heldss  # # + *==>STHl66FFxPPQ QP  ( ()_ad e s@B8A A ABA A=A83B8A==BcK|j}| |jtjS|j t t d{rtjS|j|d{r%tjdtjStjdtjS7w7Pw)amElect one sweeper per tick. Every pod schedules this job, and each one otherwise re-reads the whole due population and writes it back at the same calendar boundary, so a fleet multiplies one sweep's Postgres load by its replica count. A deployment with no Redis-backed lock manager runs unguarded, as it always has. N) cronjob_idttlzGReset budget job: another pod holds the reset lease, skipping this tickz|Reset budget job: could not take the reset lease and no other pod holds it, sweeping unguarded rather than skipping the tick) r1r6rr  acquire_lockrrrr=rdebugr r:)rJr4s rK_acquire_leasezResetBudgetJob._acquire_leases#33  <#;#;#C## #**,1+   == $$\2 2 2 & &'p q?? " $$ ? % 3s%A C C (C4C5ACCcK|jy|jd{}|tjury |j d{|j d{|j d{|jd{|jd{|tjur6|j)|jjtd{yyy77777m7W7#|tjur7|j*|jjtd{7wwwxYww)z Gets all the non-expired keys for a db, which need spend to be reset Resets their spend Updates db Runs on one pod per tick where a Redis lease is available. N)r?) r rCrr reset_budget_for_litellm_keysreset_budget_for_litellm_usersreset_budget_for_litellm_teams%reset_budget_for_litellm_budget_tablereset_budget_windowsrr1 release_lockr)rJleases rK reset_budgetzResetBudgetJob.reset_budgets8    % !0022 FOO #  [446 6 6557 7 7557 7 7<<> > >++- - - %$*?*?*K++88DY8ZZZ+L%3 7 7 7 > -[ %$*?*?*K++88DY8ZZZ+L%s!EC=ED C?D 'D(D ?DD DD /D0D 4AE5D 6E?D D D D D E AE EEE operationreasoncNKt|j||d{S7w)zReconnect and retry once on a transport error, so a dropped connection costs one retry instead of the whole tick. rNN)r.r rJrMrNs rK_with_db_retryzResetBudgetJob._with_db_retrys$2$2D2DiX^____s %#%cXKt|j||td{S7w)zSame, for writes: only replay when the statements provably never reached the database. A reset zeroes spend unconditionally, so replaying an ambiguous commit would erase spend accrued since it landed. )rNretry_safe_error_typesN)r.r rrQs rK_with_db_write_retryz#ResetBudgetJob._with_db_write_retrys1 2    #<     s !*(* counter_key new_spendcvK ddlm}|jj||d|j( |jj ||dd{yy7#t $r!}tjd||Yd}~yd}~wwxYw#t $r!}tjd||Yd}~yd}~wwxYww)aOverwrite a spend counter with the post-reset value (0, or the carried overage when budget rollover is enabled) so a DB-row reset takes effect immediately. Call AFTER the DB write commits. Clearing Redis before the DB commit opens a window where get_current_spend reads 0 from Redis while the DB still holds the pre-reset value, allowing bypass. rspend_counter_cache<)r$rr@NzaFailed to reset spend counter %s in Redis: %s. Budget may be over-enforced until counter expires.z$Failed to reset spend counter %s: %s) litellm.proxy.proxy_serverrZin_memory_cache set_cacher6async_set_cacherrr:)rVrWrZ redis_erres rK_invalidate_spend_counterz(ResetBudgetJob._invalidate_spend_counters a F  / / 9 9kQZ`b 9 c"..:-99IIkajprIsss;s (00M#!  a ( ()OQ\^_ ` ` asiB90B !AAAB9A B (B?B B9B  B B6B1,B91B66B9cRKtjtd{y7w)zDrop the cached global-proxy spend accumulator after the proxy budget aggregate row is reset, so the next auth-time load reads the zeroed row instead of a stale (potentially never-expiring) counter. N)r.$_invalidate_user_api_key_cache_entryrrHrLrK$_invalidate_global_proxy_spend_cachez3ResetBudgetJob._invalidate_global_proxy_spend_cache-s AAB^___s '%' cache_keycK ddlm}|j|d{y7#t$r!}t j d||Yd}~yd}~wwxYww)a8Drop a stale management-cache entry so the next read fetches from DB. Tags and end-users are not reseeded by ``SpendCounterReseed.from_db``; for those, when the spend counter expires the budget check falls back to ``cached_obj.spend``. Keys, orgs, and team memberships are reseeded from the DB, but auth still may consult ``user_api_key_cache`` objects whose ``.spend`` field can lag a cross-pod DB reset. Deleting the cache entry forces the next auth-time fetch to reload the zeroed row from Postgres. r)user_api_key_cacher$Nz4Failed to invalidate user_api_key_cache entry %s: %s)r\rhasync_delete_cacherrr:)rfrhras rKrdz3ResetBudgetJob._invalidate_user_api_key_cache_entry5sP  E$77I7F F F  ( (F   s6A'%'A' AA A AArrr.c K t|jfdd|jdddd{S7#t$r!}t j d||Yd}~y d}~wwxYww) zoRead the rows the cascade will zero, so their counters can be invalidated once the transaction commits.c(jS)Nr find_many)rrsrKrz3ResetBudgetJob._fetch_linked_rows..WsEOO%O8rLreset_budget_read_ r_failurerPNz/Failed to fetch %s for counter invalidation: %srH)rrRrrrr:)rJrrrras `` rK_fetch_linked_rowsz!ResetBudgetJob._fetch_linked_rowsLs| ))8/ 0C0CC0M/NhW*    ( ()Z\gij k s>A13AAAA1A A. A)$A1)A..A1rcKjfddd{}tjtjvrt|xsdSg|xsdj d{S7V7w)NcRjjddtS)Nenduserfind_all) table_name query_typebudget_id_list)r get_datar)rrJsrKrz;ResetBudgetJob._collect_endusers_to_reset..as+D&&//$%#J/0rL"reset_budget_read_endusers_failurerPrH)rRrprr_get_endusers_with_no_budget_id)rJrlinkeds`` rK_collect_endusers_to_resetz)ResetBudgetJob._collect_endusers_to_reset_s<@|Dcic]2}|j$t!|j"x} |j| 4c}ni j%|d{} t't||tfd |D| g fd |D fd |D fd |D fd |D fd|D fd| D gd|Dd|Dd|Dd|Dd|Dd| DS74777h7$cc}w7w)a Resolve every row the expiring budget tiers gate, before any write. Keys carrying their own budget_duration are left out: they run on their own schedule via reset_budget_for_litellm_keys(), so sweeping them here would reset them twice. c3NK|]}|j|jywrG)rN)rbs rKrz9ResetBudgetJob._collect_budget_cascade..ss!c!1;;Kb!++!cs%%zteam memberships)rrrNr#orgstagszmodel access groupsc3K|]I}|j;|j/|jt|jjfKyw)Nr?settings)rNr?r&r0)rrrJs rKrz9ResetBudgetJob._collect_budget_cascade..sP  ;;*q/@/@/L KK+A.s,2#68J3P]8^_!$c3LK|]}t|t|fywrG)rrrs rKrz9ResetBudgetJob._collect_budget_cascade..%bVY#C(*.rrc3LK|]}t|t|fywrG)rrrs rKrz9ResetBudgetJob._collect_budget_cascade..rrc3LK|]}t|t|fywrG)rrrs rKrz9ResetBudgetJob._collect_budget_cascade..s,5S9;McS`;abrc3LK|]}t|t|fywrG)rrrs rKrz9ResetBudgetJob._collect_budget_cascade..s%n^a',.DS-.XYnrc3@K|]}t|D]}|ywrG)rrr~r$s rKrz9ResetBudgetJob._collect_budget_cascade..s#^#=XY\=]^c#^#^c3@K|]}t|D]}|ywrG)rrs rKrz9ResetBudgetJob._collect_budget_cascade..!F#1EF##F#Frc3@K|]}t|D]}|ywrG)rrs rKrz9ResetBudgetJob._collect_budget_cascade..rrc3@K|]}t|D]}|ywrG)rrs rKrz9ResetBudgetJob._collect_budget_cascade..rrc3@K|]}t|D]}|ywrG)rrs rKrz9ResetBudgetJob._collect_budget_cascade..s#d#@^_b@cd#d#drc3@K|]}t|D]}|ywrG)rrs rKrz9ResetBudgetJob._collect_budget_cascade..s"N#5H5MNc#N#Nr)rrrrrrr)rrrrr6r rrr;rAr1rBr5r4rrrrNrwrsr~r) rJrrteam_membershipsr#rrmodel_access_groupsrrxrrs ` @rK_collect_budget_cascadez&ResetBudgetJob._collect_budget_cascadels"!c7G!cc ! !HLH_H_*4+=+=>DD$Z0*I`I C  261H1H-d.@.@AGG$Z1CD2I2 ,  261H1H(););<BB$Z1BC2I2 ,  261H1H 2 2399$Z1BC2I2 ,  NRMdMd243E3EFLL$Z1BC-NeN H  5E !"* ;;*}Q\\7R0R/_ S   5 :>9X9XYc9d3d*+!  *   /  c]ab  c]ab c]ab 2 oemn (^!1^FFGFGF  e!4d  ON 7# # GC , , , H  4es}AJ&JAJ&#J$AJ&*J+AJ&1J2AJ&8J9J&7JJ&$J$%B,J&J&J&J&J&J&rclKjsyjfddd{y7w)a\Zero the gated spend and advance ``budget_reset_at`` in one transaction. Advancing the window on its own hides the tier from every later tick while its dependents stay pinned at the cap for the whole window; batching both means a mid-cascade failure persists nothing and the rows stay due for the next run. Nc&jSrG)_commit_budget_cascade_once)rrJsrKrz7ResetBudgetJob._commit_budget_cascade..sD44W=rL)reset_budget_write_budget_cascade_failurerP)rrU)rJrs``rK_commit_budget_cascadez%ResetBudgetJob._commit_budget_cascades8!! '' =>(   s (424c\Kt|jjj4d{}t |j |t |j |tt |j|tt |j|tt |j|tt|j||jD]"\}}|jj!||$dddd{y77#1d{7swYyxYww)Nr)rNbudget_reset_at)r9r dbbatch_rrr#rA organizationsrBrrrrrrqueue_window_advance)rJruowrNrs rKrz*ResetBudgetJob._commit_budget_cascade_onces.t/A/A/D/D/K/KL g gPS '(<(ResetBudgetJob._reset_expired_budget_cascade..s)**33') 5 4rL!reset_budget_read_budgets_failurerPrHrrc3&K|] \}}| ywrGrH)rrrs rKrz?ResetBudgetJob._reset_expired_budget_cascade..sCkaCsrrr)r rr rrRrrrrrrrrr)rJrrrars` @rK_reset_expired_budget_cascadez,ResetBudgetJob._reset_expired_budget_cascades\\(,,/ IVZViVi ;WjWQ $(#?#?@P@VTV#WWG B--g6 6 644W===&$CW-B-BC||HLL1  'QX I'aH H I 7 B'qA A B >s$D4C&C"C& C$!C&&D:D ;D?D4D2AD4"C&$C&& D /D?D D4D  D4 D D/ D*$D/%D4*D//D4cJKt|jd{y7w)aV Resets the spend a budget tier gates (end users, team members, keys, orgs, tags, model access groups) and advances the tier's budget_reset_at, atomically. Caches are invalidated only after the transaction commits, so a failed run cannot leave a zeroed counter in front of an un-reset DB row. N)r,_reset_budget_for_litellm_budget_table_chunkrIs rKrHz4ResetBudgetJob.reset_budget_for_litellm_budget_tables#4#T#TUUU #!#c Ktj}|jd{}tj}|xtdx\}}tj|j j jtj||z d||it|t|jddtt|j|Stdx{\}}t!j"d|| tj|j j j%tj||z |d ||t| t&S t)|y7Nw) Nrreset_budget_budget_tabler)num_endusers_updatednum_endusers_failedserviceduration call_type start_timeend_timeevent_metadatarrzFailed to reset the budget table cascade (team member, enduser, org, tag and model access group spend, plus budget_reset_at); nothing was committed and the budgets stay due for the next run: %s)exc_inforeset_budget_endusersrrrrrrr)timerrasyncio create_taskr/service_logging_objasync_service_success_hookr<RESET_BUDGET_JOBr,r+rrrrr exceptionasync_service_failure_hookrr)rJrrrrrrs rKrz;ResetBudgetJob._reset_budget_for_litellm_budget_table_chunk sd IIK #AACC))+L(LL##**>>YY , = =!)J!6"=#-!)(>YY , = =!)J!6#"9#-!)'Eg'NZ $#'D(W%SDs(E;E8EE;cKt|jj|jfddd{}|Dcgc]%}t j |j 'c}S75cc}ww)a Fetch end users that have no explicit budget_id set (NULL) and have accumulated spend > 0. These are implicitly-created end users that rely on the default budget (litellm.max_end_user_budget_id) applied in-memory during auth checks. c2jdddidS)Nr>rrrrm)rsrKrz@ResetBudgetJob._get_endusers_with_no_budget_id..As$EOO!%"AY$rL4reset_budget_read_endusers_without_budget_id_failurerPN)r3r rrRr model_validate model_dump)rJrowsr~rs @rKr|z.ResetBudgetJob._get_endusers_with_no_budget_id6sw)););<BB //  J0  RVV#$33CNN4DEVV Ws!9A9A2A9*A4/A94A9 updated_keyscRKjfddd{y7w)u Write per-row {spend, budget_reset_at} updates for keys. Avoids the batched full-model update path, which trips prisma.errors.DataError on any row carrying object_permission_id or budget_limits (see #27730). Both fields are rejected by Prisma's update input type for LiteLLM_VerificationToken, and the failure aborts the entire batch — silently leaving spend over the cap and budget_reset_at unchanged forever. c&jSrG)_write_key_reset_updates_oncerJrsrKrz9ResetBudgetJob._write_key_reset_updates..WsD66|DrLreset_budget_write_keys_failurerPNrUrs``rK_write_key_reset_updatesz'ResetBudgetJob._write_key_reset_updatesKs,'' D4(   '%'cKt|jjj4d{}|D]`}|j|j j |j|j|jxsddkDr |jndbdddd{y7z7#1d{7swYyxYww)Nr{)r_rspend_decrement) r:r rrr_r#queue_spend_resetrr@rs)rJrrks rKrz,ResetBudgetJob._write_key_reset_updates_once[s+D,>,>,A,A,H,HI  S! 77?**''$%$5$556WW^s4JALLPT+        sE.CB+CA&B/ C%B-&C-C/C5B8 6C=C updated_userscRKjfddd{y7w)u Write per-row {spend, budget_reset_at} updates for users. Mirrors _write_key_reset_updates — avoids the full-model update path that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). c&jSrG)_write_user_reset_updates_oncerJrsrKrz:ResetBudgetJob._write_user_reset_updates..oD77 FrL reset_budget_write_users_failurerPNrrs``rK_write_user_reset_updatesz(ResetBudgetJob._write_user_reset_updatesf,'' F5(   rcrKt|jjj4d{}|D]S}|jj |j |j|jxsddkDr |jndUdddd{y7m7#1d{7swYyxYww)Nr{)rXrr) r:r rrusersrrXrr@rs)rJrrus rKrz-ResetBudgetJob._write_user_reset_updates_onces+D,>,>,A,A,H,HI  S"  ++II$%$5$556WW^s4JALLPT,        E.B7BB7AB" B7B B7 B7"B4(B+ )B40B7 updated_teamscRKjfddd{y7w)u Write per-row {spend, budget_reset_at} updates for teams. Mirrors _write_key_reset_updates — avoids the full-model update path that trips Prisma's DataError on rows carrying unrecognised fields (see #27730). c&jSrG)_write_team_reset_updates_oncerJrsrKrz:ResetBudgetJob._write_team_reset_updates..rrL reset_budget_write_teams_failurerPNrrs``rK_write_team_reset_updatesz(ResetBudgetJob._write_team_reset_updates|rrcrKt|jjj4d{}|D]S}|jj |j |j|jxsddkDr |jndUdddd{y7m7#1d{7swYyxYww)Nr{)r[rr) r:r rrr%rr[rr@rs)rJrrts rKrz-ResetBudgetJob._write_team_reset_updates_oncerrrrrrrc tjd||tj|jj j tj||z |||||y)zReport rows that could not be reset without failing the chunk: the rows that did reset are already committed, and raising here would cost the phase every remaining chunk this tick. z%s: %srN) rrrrr/rrr<r)rJrrrrrs rK_emit_phase_failurez"ResetBudgetJob._emit_phase_failuresd ""8Y>  " " 6 6 Q Q$55!J.#%!- R  rLcJKt|jd{y7w)zf Resets the budget for all the litellm keys Catches Exceptions and logs them N)r$_reset_budget_for_litellm_keys_chunkrIs rKrEz,ResetBudgetJob.reset_budget_for_litellm_keyss #4#L#LMMMrc$ Ktj tj}d} j fddd{}t j dt |g}g}|t|dkDr|D]U} tj| jd{}||j|n|j|ddWt j d t ||r]j| d{|D]>}t!|d d} | sj#d | |j$xsdd{@tj} t'|r t|ndt)d|Dtj*t,j.} t|dkDrXj1dtdt|dt3j4|t|| d|r t|ndi| St7j8j:j<j?t@jB| |z d|| |r t|ndt|t|d| S7K7#t$r>}|j|t|dt jd |Yd}~Yd}~wwxYw77#t$r}tj} t7j8j:j<jEt@jB| |z |d|| d|r t|ndit jd|tFcYd}~Sd}~wwxYww)NcLjjddtS)Nr$rv)rwrxexpiresrrrrsrKrzEResetBudgetJob._reset_budget_for_litellm_keys_chunk..s,**33$) 5 4rLreset_budget_read_keys_failurerPzKeys to reset %srr$ current_timer0Returned None without exception)r$rz"Failed to reset budget for key: %szUpdated keys %s)rr_rr{rc34K|]}|jywrGr)rrs rKrzFResetBudgetJob._reset_budget_for_litellm_keys_chunk..s=1Q&&=rrreset_budget_keysFailed to reset z keys: rnum_keys_foundrrrrr)rnum_keys_updatednum_keys_failedrrz#Failed to reset budget for keys: %s)$r utcnowrrRrrBrr+r._reset_budget_for_keyr0appendrrTrrgetattrrbr@rrrr rrrrrrr/rrr<rrr) rJr keys_to_resetr failed_keysr$ updated_keyrarr_rrrs ` @rKrz3ResetBudgetJob._reset_budget_for_litellm_keys_chunks__& IIK @D [ "&"5"58#6 # M ! & &'99];S TCEL!#K(S-?!-C( bC b,:,P,P #),+/+>+>-Q-' '2(// <'..sEf/gh b%**+UV77\7RRR)q '7D 9 "&"@"@:eWAUabahahaolo"@"pppq yy{H*.;M*(= =#<< 5G;!#((1#&6s;7G6HPTPZPZ[fpsPtOu$vw)%( #m*j@jCtDjF| |z d|| |r t |ndt |t |d| S7N7 #t$r>}|j|t|dtjd|Yd}~zd}~wwxYw777#t$r}tj} t;j<j>j@jItDjF| |z |d|| d|r t |nditjd|tJcYd}~Sd}~wwxYww)NcJjjddtS)NuserrvrrrsrKrzFResetBudgetJob._reset_budget_for_litellm_users_chunk.. )**33%) 5 4rLreset_budget_read_users_failurerPrrrr0r)rrz#Failed to reset budget for user: %szUpdated users %s)rrXz spend:user:r{rc34K|]}|jywrGr )rrs rKrzGResetBudgetJob._reset_budget_for_litellm_users_chunk..M>1Q&&>r rrreset_budget_usersr z users: r num_users_foundr)r%num_users_updatednum_users_failedrrz$Failed to reset budget for users: %s)&r rrrRr+r._reset_budget_for_userr0rrrTrrrBrrrrbr@rrerrrr rrrrrrr/rrr<rrr) rJrusers_to_resetr failed_usersr updated_userrarrXrrrs ` @rKrz4ResetBudgetJob._reset_budget_for_litellm_users_chunks__& IIK 9=a #'#6#6 9$7$N=?M"$L)c..AA.E*dDd-;-R-R!%),+/+>+>.S.( (3)00>(//,0-N!"d(%**+=y?WX 88}8UUU*N")!Y"=""&"@"@;wiAXdedkdkdror"@"sss"&??"&"K"K"MMM Nyy{H*/=N+1(> >#<< 5G< 1$((2#*3|+<*=XdjjQ]gjFkElm *%).3~+>VW$)    &&::UU(99% 22)%BP3~+>VW-0-?,/ ,=$ V  @NA(%d$++TCF,KL,667\^bccd VtML yy{H   &&::UU(99% 22)%).3~+>VW$V   ! * *+QST U ! s+NK1 J K1*%JJ,J<8K14K(5?K14K+5 K1K1K.C K1#N$A2K1NK1J K%'3K K1 K%%K1+K1.K11 N:BNN NNNcJKt|jd{y7w)z^ Resets the budget for all LiteLLM Internal Teams if their budget has expired N)r%_reset_budget_for_litellm_teams_chunkrIs rKrGz-ResetBudgetJob.reset_budget_for_litellm_teamsrrc Ktj tj}d} j fddd{}g}g}|t |dkDr|D]U} t j | jd{}||j|n|j|ddWtjd t||r]j| d{|D]>}t!|d d} | sj#d | |j$xsd d{@tj} t'|r t |ndt)d|Dtj*t,j.} t |dkDrXj1dtdt |dt3j4|t|| d|r t |ndi| St7j8j:j<j?t@jB| |z d|| |r t |ndt |t |d| S7,7#t$r>}|j|t|dtjd|Yd}~Yd}~wwxYw77#t$r}tj} t7j8j:j<jEt@jB| |z |d|| d|r t |nditjd|tFcYd}~Sd}~wwxYww)NcJjjddtS)Nr&rvrrrsrKrzFResetBudgetJob._reset_budget_for_litellm_teams_chunk..rrLreset_budget_read_teams_failurerPrr&rr0r)r&rz#Failed to reset budget for team: %szUpdated teams %s)rr[z spend:team:r{rc34K|]}|jywrGr )rrs rKrzGResetBudgetJob._reset_budget_for_litellm_teams_chunk..r#r rrreset_budget_teamsr z teams: r num_teams_foundr)r4num_teams_updatednum_teams_failedrrz$Failed to reset budget for teams: %s)$r rrrRr+r._reset_budget_for_teamr0rrrTrrrBrrrrbr@rrrr rrrrrrr/rrr<rrr) rJrteams_to_resetr failed_teamsr& updated_teamrarr[rrrs ` @rKr-z4ResetBudgetJob._reset_budget_for_litellm_teams_chunksu__& IIK 9=_ #'#6#6 9$7$N=?M"$L)c..AA.E*dDd-;-R-R!%),+/+>+>.S.( (3)00>(//,0-N!"d(%**+=y?WX 88}8UUU*t")!Y"=""&"@"@;wiAXdedkdkdror"@"ssst yy{H*/=N+1(> >#<< 5G< 1$((2#*3|+<*=XdjjQ]gjFkElm *%).3~+>VW$)    &&::UU(99% 22)%BP3~+>VW-0-?,/ ,=$ V  @N}(%d$++TCF,KL,667\^bccd VtL yy{H   &&::UU(99% 22)%).3~+>VW$V   ! * *+QST U ! s+M1K I6 K )%I<I9,I<;8K 3K4K 'K 4K 5C K M1A2K 4M16K 9I<< K3J>8K >KK K M.BM)#M.$M1)M..M1windowrZrr entity_idczK|jd}|sytj|jddjd} | |kDrytj |||d{} |j j|| |j& |jj|| d{|d } t| | } | j|d<tj|||| | d{y 77N#t$r!} tjd|| Yd} ~ qd} ~ wwxYw76w) zNReset a single budget window if expired. Returns True if the window was reset.rFZz+00:00Nr)r$rz$Failed to reset Redis counter %s: %sr?r)r rr<r? next_reset_atT)rr fromisoformatrr._window_carried_spendr]r^r6r_rrr:r& isoformat_roll_window_spend_row)r;rVrZrr0r rr< reset_at_strr new_valuer`r?r?s rK_reset_expired_windowz$ResetBudgetJob._reset_expired_windowsO%jj4 "001E1Ec81TU]]ei]j c>!/!E!Efk[n!oo ++55+Y5W  * * 6 m)55EE+]fEggg"((9!:6aop *446z33'#+' 4   #ph m$,,-SU`bkll m  s[A*D;,D--D; D ;D <D AD;D9D; D D6D1,D;1D66D;r?r?cTK |tt|z } t ||j |||d{y#t$r!}tjd||Yd}~yd}~wwxYw72#t$r-}tjd|j |||Yd}~yd}~wwxYww)a|Move this window's LiteLLM_BudgetWindowSpend row onto the window that just started, so the maintained total the read path uses starts from zero alongside the counter. Best effort: the row is an optimization over aggregating LiteLLM_SpendLogs, so a failure here must not stop the remaining windows from having their counters reset. )secondsz"Unparseable budget_duration %s: %sN)r rr<window_durationnew_window_startz>Failed to roll budget window spend row for %s=%s window=%s: %s)r rrrr:r,r)r rr<r?r? window_startras rKrCz%ResetBudgetJob._roll_window_spend_rows  "/)DWXgDh2i"iL '+'--# /!-      ( ()M`a b     ( (P!!    s`B(AA/A-A/B( A* A% B(%A**B(-A// B%8#B B( B%%B(czKtsy|jd}t|ttfr t |nd}|y |j |d{}t|ttfsytt ||S70#t$r!}tjd||Yd}~yd}~wwxYww)zzPer-window spend lives only in the counter, so the carried overage is read from it before the reset overwrites it.r{rsNriz0Failed to read spend counter %s for rollover: %s) rrrrrrSrwr9rrr:r})r;rVrZ window_maxrxcurrentras rKrAz$ResetBudgetJob._window_carried_spend7s !""JJ|4 2$A?AAAA?A A<!A72A?7A<<A?rPcZKd} |j||||d{}|y|}#7 w)aWalk one table's windowed rows a page at a time, to the end. Paging is what bounds the memory: the previous form pulled every row carrying budget_limits into one result set on every tick, which grows with the deployment's key count and is paid on the event loop. The walk deliberately has no per-run page cap. A cap has to remember where it stopped, and that position cannot live in the process: the lease is released after each sweep, so the next tick can elect a different pod whose own position is unset. It would restart at the first row and never reach the tail, pinning those windows at their cap for good. The cursor strictly advances, so the walk terminates on its own without needing a bound. )rPcursorrrZN)_reset_window_page)rJrPrrZrT next_cursors rKrQz!ResetBudgetJob._reset_windows_for[sP( $ 7 7$7 !8!K " Fs +) +rTc  Kjfddjdd{}|D]}|d}|s |j t|tr|nt j | d} D]_} jd d | d } tj| | ||jjj d{s^d }a|sj  fd djdd{t|tkry|djS77e70w)zHReset one page of windows; return the next cursor, or None when drained.cvjjjjtSrG)r r query_rawr"r)rTrJrPsrKrz3ResetBudgetJob._reset_window_page..s,D&&))33F4E4E4GQlmrLro_windows_failurerPNrFrz:window:r?)r rr<TcdjjtjSrG)r r rr)r rJrPwindowssrKrz3ResetBudgetJob._reset_window_page..s$FLL););VTZZPWEXYrLreset_budget_write_)rRrrrrrloadsrr.rFr0r rrUr+r) rJrPrTrrZrr~rawchangedr;rVr r\s ``` @@rKrUz!ResetBudgetJob._reset_window_page{s!// m'(<(<'==MN0   Co&Cf../F6@d6KsQUQ[Q[\_Q`GG! #!'!6!6 7qPaIbHcd '=='''"&"4"4 & 2 2$>   #G #//Y01E1E0FFVW0+ 4 t92 2Bx(())A  s@*D?D8B#D?D;D?D?"*D? D= ,D?;D?=D?itemr item_type)r$r&rcBK tr)t|jt|jnd|_t |dr(|j t|j ||_|S#t$r}tjd||||d}~wwxYww)a In-place, updates spend=0, and sets budget_reset_at to current_time + budget_duration Common logic for resetting budget for a team, user, or key. Spend-counter invalidation happens in the caller, AFTER the DB write commits. Zeroing the counter here would open a bypass window when the DB write fails: get_current_spend reads 0 from Redis while the DB still holds the pre-reset value, admitting requests past the cap. r{r?Nrz+Error resetting budget for %s: %s. Item: %s) rrr}r@rwrshasattrr?r&rrrr)rbrrcr0ras rK_reset_budget_commonz#ResetBudgetJob._reset_budget_commons" WhWj M$//4RSpsDJt./D4H4H4T'>$($8$8>($K  * *+XZcefhl mG s)BA/A43B4 B=BBBr&cTKtj||d|d{|S7w)Nr&rbrrcr0r.rfr1s rKr7z%ResetBudgetJob._reset_budget_for_team= 11%) 2     (&(rcTKtj||d|d{|S7w)Nrrhrir!s rKr(z%ResetBudgetJob._reset_budget_for_userrjrkr$cTKtj||d|d{|S7w)Nr$rhrirs rKrz$ResetBudgetJob._reset_budget_for_keys= 11%) 2     rk)NN)r{)rEN)QrOrPrQrr0r/r%r-r3r8r=rrCrLrrr=rTrRrU staticmethodrSrbrerdr2rrrrrrrmr~rrrrrrrrrrHrrrr r|r$rrr#rrr"rrrdictrrErrFrrGr-rr r!rFrCrArIrrQrUrrfr7r(rrHrLrKr.r.s6:26 H' H$ H,d2 H )4/ HD  f B[:`hr9U;K7K.L`Y\`af`  HR5AQ=Q4R  _b  gl  aSaUaUYaa4``cd,&sF{#  ucz  & P8C= PUS^`cScMd PQ hG^>_Q drQ f N t g gD gG~GRVG  5LOc5c < V+&M+&ZW " #W* 4@Y;Z _c E^@_ dh   TBS=T  Y]  $GXBY^b  TBS=T  Y]  $GXBY^b       S&[)    2N_M_BO e]eNO c]cJ"""'" " , " $ "("" ""H###'## #  #  ##J3V $3363MV3 33( s!!!' !  !@(*(*(* (* ' (* t (*T"336OO01, 6   ,  T !     ,  T !    &  ,  #T )  rLr.)rrrurcollections.abcrrrrr dataclassesrr r r r enumr typesrtypingrrrrrrplitellm._loggingrlitellm.caching.dual_cacherlitellm.constantsrrrrrr*litellm.litellm_core_utils.duration_parserrlitellm.proxy._typesrrr r!r"r#r$)litellm.proxy.common_utils.timezone_utilsr%r&r'-litellm.proxy.common_utils.user_api_key_cacher(r)r*r++litellm.proxy.db.budget_window_spend_writerr,6litellm.proxy.db.db_transaction_queue.pod_lock_managerr-"litellm.proxy.db.exception_handlerr.litellm.proxy.utilsr/r0,litellm.repositories.organization_repositoryr1%litellm.repositories.prisma_protocolsr2'litellm.repositories.table_repositoriesr3r4r5r6$litellm.repositories.team_repositoryr7!litellm.repositories.unit_of_workr8r9r:2litellm.repositories.verification_token_repositoryr;litellm.types.servicesr<r=rArTrrrBrDrVr]rarerirmr8rrrSrwr}rrrrrrrrrrrrrrrrorrrrrrrrrrrrrrrrrrrKEYTEAMr'r,r.rHrLrKrs LL(22"BB10K  NQK:OB @ 02BW[gkmnfoCp2qE'#v+./q1A7TSTIBV1W5f-.W+x+")8"  ** ## ,+X, ""H" +4+edlut| *%$,*UT\*e* e,eGCJ4GeEe <&8