tPjyUddlZddlZddlZddlZddlZddlZddlmZddlm Z m Z ddl m Z m Z mZmZddlmZddlmZddlmZddlmZdd lmZe Ze ed <d Zeej;d e eg dZej>j;ddjAdk(r ddl!Z!e"de!jFe"de!jHe!jJZ&e ed<e"de!jHe&ejPdej;de egddZ)ej;dde efdefd Z*ej;d!de efdefd"Z+ej;d#de efded$e,e-e ffd%Z.d$e,e-e ffd&Z/d'e0d$e1e0e2e,e-e fffd(Z3d$e,e-e ffd)Z4d$e,e-e ffd*Z5d$e,e-e ffd+Z6d,e0d-e7d$e,e-e fdzfd.Z8ej;d/de eed0d12ed3d42fded'e0d-e7d$e,e-e ffd5Z9ejud6de eed7d82edd92edd:2fded;e0dZ;ej;d?e egdd@Z"#2 XD MMT^^222<=R D@P?QQUV W X&v..sB Bz/memory-usage-in-mem-cache)rC_c&Kddlm}m}m}|d}nSt |j j jt |j j jz}t |j jt |j jz}t |jjj jt |jjj jz}|||dSw)j 1. user_api_key_cache 2. router_cache 3. proxy_logging_cache 4. internal_usage_cache r llm_routerproxy_logging_objuser_api_key_cache)num_items_in_user_api_key_cachenum_items_in_llm_router_cache$num_items_in_proxy_logging_obj_cache) litellm.proxy.proxy_serverr[r\r]rcachein_memory_cache cache_dictttl_dictinternal_usage_cache dual_cache)rWr[r\r]r_r^r`s r$memory_usage_in_mem_cacherh|s ()%(+J,<,<,L,L,W,W(X[^    , , 5 5\ ) %.11C1S1S1^1^-_be**33c.#36..99IITT3  2 2 = = M M V VW3X( ,K)F0T sDDz /memory-usage-in-mem-cache-itemscKddlm}m}m}|i}i}n@|jj j }|jj j}|j j |j j|||jjj j |jjj jdSw)rYrrZ)r]user_api_key_ttlllm_router_cachellm_router_ttlproxy_logging_obj_cacheproxy_logging_obj_ttl) rar[r\r]rbrcrdrerfrg)rWr[r\r]llm_router_in_memory_cache_dictllm_router_in_memory_ttl_dicts r$memory_usage_in_mem_cache_itemsrqs *,'(*%*4*:*:*J*J*U*U'(2(8(8(H(H(Q(Q%1@@KK.>>GG;7#4#I#I#T#T#d#d#o#o!2!G!G!R!R!b!b!k!k  sCCz/debug/memory/summaryreturncKddlm}m}m}i}d} ddl}|j }|j }|jdz } |j} | dd| ddt| d t| d d }| d kDrd }n | d kDrd}nd}i} d} t|jj}| |z } ||ddd| d<|9t|j jj}| |z } ||ddd| d<t|j"j$jj}| |z } ||ddd| d<t'j(}t'j*d}tt&j,}|rdnd|d}|dkDr|d|d<t/j0||| | d|dS#t$r d|d<Y:t$r} t| |d<Yd} ~ Vd} ~ wwxYw#t$r} t| | d<Yd} ~ d} ~ wwxYww)a Get simplified memory usage summary for the proxy. Returns: - worker_pid: Process ID - status: Overall health based on memory usage - memory: Process memory usage and RAM info - caches: Cache item counts and descriptions - garbage_collector: GC status and pending object counts Example usage: curl http://localhost:4000/debug/memory/summary -H "Authorization: Bearer sk-1234" For detailed analysis, call GET /debug/memory/details For cache management, use the cache management endpoints rrZhealthyN.1fz MB (% of system memory)r)summary ram_usage_mbsystem_memory_percentPcritical<rz8Install psutil for memory monitoring: pip install psutilerrorrz,Validated API keys for faster authentication)countcount_readablewhat_it_stores user_api_keysz$LLM responses for identical requests llm_responsesz#Usage metrics before database writeusage_trackingenableddisabled)statusobjects_awaiting_collectionz- uncollectable objects (possible memory leak)) total_items breakdown) worker_pidrmemorycachesgarbage_collector)rar[r\r]psutilProcess memory_inforssmemory_percentround ImportError Exceptionstrrrcrdrbrfrgr isenabled get_countgarbageosgetpid)rWr[r\r]process_memory health_statusrprocessr memory_mbrr"rtotal_cache_itemsuser_cache_itemsrouter_cache_itemslogging_cache_items gc_enabledobjects_pending uncollectablegc_infos r$get_memory_summaryrsl(NM))$002 &??k: ' 6 6 8$CnS-AATU!)Q/%*>1%=  B &M b %M%M%'F!"%&8&H&H&S&S"T--%!1! 4L#   !(+J,<,<,L,L,W,W(X  !3 3 +%7$:"H'F? #&)):)O)O)Z)Z)j)j)u)u%v00(!4Q 7C$   J\\^A.Orzz?M *)z'6G q -.[\ iik , %  o ]"\w )"%a&w)H !a&w!sgH A7F2 H B-G';A7H 2G$H G$ GH G$$H ' H0H>H HH c tjtjdtjdtjdddtjdtjdtjdddt tj Dcgc]\}}||d|d|d d c}}d Scc}}w) z!Get garbage collector statistics.rrrzENumber of allocations before automatic collection for each generation) generation_0 generation_1 generation_2 explanationz6Current number of allocated objects in each generation collections collectedr) generationtotal_collectionstotal_collectedr)rr!current_countscollection_history)rrrrr/ get_stats)irUs r$_get_gc_statisticsr<s<<>,,.q1,,.q1,,.q1b  LLN1-LLN1-LLN1-S  %R\\^4 4  %)-%8#' #4!%o!6     s>C top_nct}d}tjD])}|dz }t|j}||xxdz cc<+|j |Dcgc] \}}|||dd}}}||fScc}}w)z=Count objects by type and return total count and top N types.rrr)typerr)rr get_objectsrr) most_common)r type_counts total_objectsobjobj_typertop_object_typess r$_get_object_type_countsrXs"))KM~~# 9%%H"# +66u= HeEayJ * ** s!A:ctj}t||ddDcgc]}t|jc}t|dkDrddSddScc}w)zEGet information about uncollectable objects (potential memory leaks).NrBrzIIf count > 0, you may have reference cycles preventing garbage collection)r sample_typesr)rrrrr))rrs r$_get_uncollectable_objects_inforjsc::M]#7DSb7IJc++J=!A% X     JsAc,i} tj|jj}tj|jj}t |jj||t ||zdz dd|d<|tj|jjj}tj|jjj}t |jjj||t ||zdz dd|d<tj|jjjj} tj|jjjj} t |jjjj| | t | | zdz dd|d<|dt|jd |d < t|d r}|jrqt|jd r[|jj} t| d r | jndt| dr| j jndd|d d <|Sddi|d < |S#t"$r!} t%j&d| Yd} ~ |Sd} ~ wwxYw#t"$r/} t%j&d| t)| |d<Yd} ~ |Sd} ~ wwxYw)z&Calculate memory usage for all caches.rur) num_itemscache_dict_size_bytesttl_dict_size_bytes total_size_mbr]Nrkproxy_logging_cacheT)r cache_typeredis_usage_cache redis_clientconnection_poolmax_connectionsconnection_class)rrz!Error getting Redis pool info: %srFz!Error calculating cache stats: %sr~)sys getsizeofrcrdrerrrbrfrgrr)hasattrrrrrrr rLr) r]r[r\r cache_statsuser_cache_size user_ttl_sizerouter_cache_sizerouter_ttl_sizelogging_cache_sizelogging_ttl_size pool_infor"s r$_get_cache_memory_statsrxs)+K:&!$/A/Q/Q/\/\!]"}}-?-O-O-X-XY /??JJK%4#0"Om$C #TVWX - ()  !'*}}Z5E5E5U5U5`5`'a %(]]:3C3C3S3S3\3\%]O !1!1!A!A!L!LM):'6!&(9O(KP['\^_!` /K* +!]]+<+Q+Q+\+\+l+l+w+wx==):)O)O)Z)Z)j)j)s)st.CCNN^^iij%7#3"$69I$Ik#Z\]^ . )*  ("#45>>0K+ , S,n=BSB`B`0==?PQ+<+I+I+Y+Y >EYPa=b 9 9hlHOyZlGm : : C Csw O $789JK"  1:5/AK+ ,  S$**+NPQRR  S &""#FJ"1v G  &sCHKB J.$K. K7K KKK L$$LLci} | t|drT|jrHtj|j}t |j|t |dz dd|d<t|drT|j rHtj|j }t |j |t |dz dd|d<t|d rT|jrHtj|j}t |j|t |dz dd |d <t|d rT|jrHtj|j}t |j|t |dz dd |d <t|d rT|jrHtj|j}t |j|t |dz dd|d <tj|}|t |dz dd|d<|Sddi} |S#t$r.}tjd|dt|i}Yd}~|Sd}~wwxYw)z/Get memory usage statistics for LiteLLM router.N model_listru) num_models size_bytessize_mb model_names)num_model_groupsrrmodel_names_setdeployment_names)num_deploymentsrrdeployment_latency_map)num_tracked_deploymentsrr fallbacks)num_fallback_configsrr)rr router_objectnotezRouter not initializedz$Error getting router memory info: %sr~)rrrrrrrrrrrr rLr) r[litellm_router_memorymodel_list_sizemodel_names_sizedeployment_names_sizelatency_map_sizefallbacks_sizerouter_obj_sizer"s r$_get_router_memory_statsrsK,.:2  !z<0Z5J5J),z7L7L)M"%j&;&;"<"1$_ %DaH7%l3z=1j6L6L*--- 8N8N*O (+J,B,B(C"2$%5%EqI<%&78z#56:;V;V/2}}Z=X=X/Y%'*:+F+F'G"7$%:k%JAN=%&89z#;<AbAb*--- 8Y8Y*Z /2:3T3T/U"2$%5%EqIC%&>?z;/J4H4H(+ j6J6J(K,/ 0D0D,E"0$^{%CQG6%k2&)]]:%>O- K!@!D6 !/ 2 ! &,-E$F ! ! 2""#I1M!(#a& 1  2sH HH I##I  Irinclude_process_infoc |sy ddl}|j}|j}t|jdz d}t|j dz d}t|j d}|d|d|dd|dd |d d |d d |d dt|dr|jnddd|jdddS#t$r|ddcYSt$r.}tjd||t|dcYd}~Sd}~wwxYw)z2Get process-level memory information using psutil.Nrrurz Worker PID z using rvz MB of RAM (rwz(Actual physical RAM used by this process) megabytes descriptionz8Total virtual memory allocated (includes swapped memory)z)Percentage of total system RAM being used)percentrnum_fdsz N/A (Windows)z'Number of open file descriptors/handles)rrz(Number of active threads in this process)pidrx ram_usagevirtual_memoryrzopen_file_handlesthreadsz6psutil not installed. Install with: pip install psutil)rr~zError getting process info: %s)rrrrrvmsrrr num_threadsrrr rLr) rrrrrryvirtual_memory_mbrr"s r$_get_process_memory_inforsA (4)$002 #KOO{$CQG #(K)H!#L %g&<&<&> B$ZL S7IVdehUii|})I /Y *J& 07w /J'//+P_H" !,,.I'  0  M   4""#CQG!CF334s$B>CD D #D?D D z/debug/memory/detailsz$Number of top object types to return)rTzInclude process memory infocKddlm}m}m}m}t j }t}t|\} } t} t||||} t|} t||}|||| | d| d| | | dSw)a Get detailed memory diagnostics for deep debugging. Returns: - worker_pid: Process ID - process_memory: RAM usage, virtual memory, file handles, threads - garbage_collector: GC thresholds, counts, collection history - objects: Total tracked objects and top object types - uncollectable: Objects that can't be garbage collected (potential leaks) - cache_memory: Memory usage of user_api_key, router, and logging caches - router_memory: Memory usage of router components (model_list, deployment_names, etc.) Query Parameters: - top_n: Number of top object types to return (default: 20) - include_process_info: Include process-level memory info using psutil (default: true) Example usage: curl "http://localhost:4000/debug/memory/details?top_n=30" -H "Authorization: Bearer sk-1234" All memory sizes are reported in both bytes and MB. r)r[r\rr]r) total_trackedtotal_tracked_readable top_types)rrrrAr cache_memory router_memory) rar[r\rr]rrrrrrrr)rWrrr[r\rr]rgc_statsrruncollectable_inforr process_infos r$get_memory_detailsr,s6 J)*H&=e&D#M# ? A01CZQbduvK#;J#G2:?STL!&%*)6q(9) ,#.  sA8A:z/debug/memory/gc/configureiz%Generation 0 threshold (default: 700)z$Generation 1 threshold (default: 10)z$Generation 2 threshold (default: 10)rrrcKtj} tj|||tjd||||tjd}d|dd |d d |d |d |d ||d ||z d dS#t $r+}tj d|tdd|d}~wwxYww)a Configure Python garbage collection thresholds. Lower thresholds mean more frequent GC cycles (less memory, more CPU overhead). Higher thresholds mean less frequent GC cycles (more memory, less CPU overhead). Returns: - message: Confirmation message - previous_thresholds: Old threshold values - new_thresholds: New threshold values - objects_awaiting_collection: Current object count in gen-0 - tip: Hint about when next collection will occur Query Parameters: - generation_0: Number of allocations before gen-0 collection (default: 700) - generation_1: Number of gen-0 collections before gen-1 collection (default: 10) - generation_2: Number of gen-1 collections before gen-2 collection (default: 10) Example for more aggressive collection: curl -X POST "http://localhost:4000/debug/memory/gc/configure?generation_0=500" -H "Authorization: Bearer sk-1234" Example for less aggressive collection: curl -X POST "http://localhost:4000/debug/memory/gc/configure?generation_0=1000" -H "Authorization: Bearer sk-1234" Monitor memory usage with GET /debug/memory/summary after changes. z-GC thresholds updated from %s to (%s, %s, %s)zFailed to set GC thresholds: %sizFailed to set GC thresholds: ) status_codedetailNrzGC thresholds updatedz, rrzNext collection will run after z more allocations)messageprevious_thresholdsnew_thresholdsrtip) rrrr rrr~rr)rWrrrold_thresholdsr" current_counts r$ configure_gc_thresholds_endpointrgsD,,.NY |\B!! ;^\[giu <<>!,M+"0"3!4B~a7H6IN[\L]K^_)N"\N"\NK'40 1M0NN_`   Y""#DaH6STUSV4WXXYs(B>0B?B> B;&B66B;;B>z /otel-spanscKddlm}|giddS|j}t|dr|j }ng}t d|d}d}i}|D]j}|j |j j}||vrg||<||j|j|j|kDs]|}|j}l|Dcgc]}|j}}|||dScc}ww)Nr)open_telemetry_logger) otel_spansspans_grouped_by_parentmost_recent_parentget_finished_spanszSpans: i@B) rar OTEL_EXPORTERrr!printparenttrace_idrMr; start_time) r otel_exporterrecorded_spansr most_recent_start_timerspanparent_trace_id span_namess r$get_otel_spansr-s A$')"&  1>>M}23&99; )^$$%' 9 ;; ""kk22O&==;='8 #O 4 ; ;DII F!77%4")-& 90>>t>J> #:0 ?sAC! A C!-C!C C!c td}|ytjj|ryt j |}t |tsy|jdd}|jdd}|durcddl }ddl m }m }m }|j|j|j|j|j|j|durdddl }ddl m }m }m }|j|j |j|j |j|j y|dur|durtj"jd d }||j%d k(rFddl }dd l m }m }|j|j|j|jy|j%d k(rFddl }dd l m }m }|j|j |j|j yyyyy#t&$r } ddl }|j)d| Yd} ~ yd} ~ wwxYw)N WORKER_CONFIGrLdetailed_debugTr)verbose_loggerr verbose_router_logger)levelF LITELLM_LOGINFO)r r2DEBUGz"Failed to init verbose loggers: %s)r rpathisfilejsonloads isinstancer3getlogginglitellm._loggingr1r r2setLevelr6r7environupperrr) worker_config _settingsrLr0r>r1r r2litellm_log_settingr"s r$init_verbose_loggersrFsCA-o>   77>>- ( ::m4 )T*  }}Wd3 ) .> E D=     # #',, # 7 ! * * * > ) ) ) = T !     # #'-- # 8 ! * * * ? ) ) ) > e^% 7)+ r)J ".&,,.&8"*222F1(11 E(..0G;" *222G1(11 F</!8^4 Ar,rr:rrrIrrtypingrrfastapirrrr litellmr r?r litellm.constantsr litellm.proxy._typesr $litellm.proxy.auth.user_api_key_authrr__annotations__r%r=r<rAlowerobjgraphr# show_growthshow_most_common_typesget_leaking_objectsr@rstartrVrhrqr3rrrrtuplelistrrrrboolrrpostrr-rFr$rZs <<"11/B 8+,-  <::>>#W-335? ^ !" %&''')3x335u5 #$'''6Kb ZZ/01 /  / (EB 12##C#L .%H 12IB #u= 12ss #s(^s>slDcN8+3+5d4S>6J1J+K+$ c3h ?eijmorjres?D?!DcN?!D-4-4D-4TRUWZRZ^^bMb-4` #u= 12r'MN!&t9V!W77 77 #s(^ 7>7t )UC 12c/VWb.TUb.TU 6666 6  #s(^ 6D6r+,-  '  'VDA] ^\]]^s 6A'KK