uPjjdZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z m Z ddl Z ddlmZmZddlmZddlmZddlmZd d lmZmZmZmZmZd d lmZe rdd lm Z Gd deZ!y)a Redis Semantic Cache implementation for LiteLLM The RedisSemanticCache provides semantic caching functionality using Redis as a backend. This cache stores responses based on the semantic similarity of prompts rather than exact matching, allowing for more flexible caching of LLM responses. This implementation uses RedisVL's SemanticCache to find semantically similar prompts and their cached responses. N)CallableMapping) TYPE_CHECKINGAnyFinalcast) print_verboseverbose_logger)(SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS)get_str_from_messages)EmbeddingResponse)build_router_embedding_metadata"resolve_embedding_max_input_tokensresolve_embedding_routerresolve_embedding_timeouttruncate_embedding_input) BaseCache)Routerc eZdZUdZdZeed<dZeed<dZe dzed<e Z e ed< d6d edzd edzd edzd edzd e dzdededzde dzde dzde fdZede fdZej"de ddfdZde fdZedeeeffdZdede fded ede de f dZdedeeeffdZdede fdZdeee fdedefd Zde dzfd!Zededzfd"Zeded#e eddfd$Z!e"de de fd%Z#d&ed'd(defd)Z$d7d&ed*eeefdzde e fd+Z%d,ede fd-Z&dede ddfd.Z'dede fd/Z(d7d&ed*eeefdzde e fd0Z)dede ddfd1Z*dede fd2Z+deee ffd3Z,d4e e-eefde ddfd5Z.y)8RedisSemanticCachea Redis-backed semantic cache for LLM responses. This cache uses vector similarity to find semantically similar prompts that have been previously sent to the LLM, allowing for cache hits even when prompts are not identical but carry similar meaning. litellm_semantic_cache_indexDEFAULT_REDIS_INDEX_NAMElitellm_cache_keyCACHE_KEY_FIELD_NAMENembedding_max_input_tokensembedding_timeouthostportpassword redis_urlsimilarity_thresholdembedding_model index_namekwargsc  | |j}td|| td||_d|z |_||_||_t| |_|Q |xstjd}|xstjd}|xstjd}d |d |d|}td|||_ ||_d|_y#t$r'} | jd} td | d | d | d} ~ wwxYw)a} Initialize the Redis Semantic Cache. Args: host: Redis host address port: Redis port password: Redis password redis_url: Full Redis URL (alternative to separate host/port/password) similarity_threshold: Threshold for semantic similarity (0.0 to 1.0) where 1.0 requires exact matches and 0.0 accepts any match embedding_model: Model to use for generating embeddings index_name: Name for the Redis index embedding_max_input_tokens: Truncate prompts to this many tokens before embedding; defaults to the Router deployment's configured max_input_tokens embedding_timeout: Seconds a cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM ttl: Default time-to-live for cache entries in seconds **kwargs: Additional arguments passed to the Redis client Raises: Exception: If similarity_threshold is not provided or required Redis connection information is missing Nz*Redis semantic-cache initializing index - z2similarity_threshold must be provided, passed Noner REDIS_HOST REDIS_PORTREDIS_PASSWORDrz&Missing required Redis configuration: z . Provide z or redis_url.z redis://:@:z Redis semantic-cache redis_url: )rr ValueErrorr"distance_thresholdr#rrrosenvironKeyErrorargs _index_name _redis_url _llmcache) selfrrr r!r"r#r$rrr%e missing_vars i/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/caching/redis_semantic_cache.py__init__zRedisSemanticCache.__init__6s?H  66JB:,OP 'QR R%9! #$&:":.*D'!:;L!M   7rzz,77rzz,7#Crzz2B'C$H:QtfAdVzRedisSemanticCache.llmcaches r?cddlm}ddlm} ||j}|j ||j |j|S#t$r}tjd|d}~wwxYw)Nr) SemanticCache)CustomTextVectorizer)semantic_cache_clsr$r!cache_vectorizerz+Redis semantic-cache index build failed: %s) redisvl.extensions.llmcacherDredisvl.utils.vectorizerE_get_embedding_init_semantic_cacher2r3 Exceptionr error)r5rDrErGr6s r8r=z"RedisSemanticCache._build_llmcachesu >@ &:4;N;N&O ,,#0++//!1 -     !NPQ R  s:A A.A))A.c |jddS)Ntag)nametype)r)clss r8_cache_key_filterable_fieldz.RedisSemanticCache._cache_key_filterable_fields,,  r?rF.rGc dtdtfd} |||||j|jgdS#t$r}||s|d}t d| |||||j|jgdcYd}~S#t$rM}||st d||||||j|jgd cYd}~cYd}~Sd}~wwxYwd}~wwxYw) Nexcr:c^t|jtfddDS)Nc3&K|]}|v ywrB).0phrase error_messages r8 zWRedisSemanticCache._init_semantic_cache.._is_schema_mismatch..sg6v.gs)zschema does not matchz index schema)strlowerany)rUr[s @r8_is_schema_mismatchzDRedisSemanticCache._init_semantic_cache.._is_schema_mismatchs%#&s8>>#3Mg=fgg gr?F)rPr! vectorizerr-filterable_fields overwrite _isolatedzSRedis semantic-cache existing index schema is not isolated; using isolated index - zQRedis semantic-cache isolated index schema is stale; recreating isolated index - T)r,boolr-rSr ) r5rFr$r!rGr`rUisolated_index_name isolated_excs r8rKz'RedisSemanticCache._init_semantic_caches4 hZ hD h* %#+#'#:#:#'#C#C#E"F  ! &s+,6@  ),'/'+'>'>'+'G'G'I&J#  *<833F2GI*,'/'+'>'>'+'G'G'I&J"  %! sK&9 C)C$&B C) C!=CC!C$C)C!!C$$C)keyc0|jt|iSrB)rr])r5rhs r8_get_cache_filtersz%RedisSemanticCache._get_cache_filterss))3s844r?cJddlm}||jt|k(S)Nr)Tag)redisvl.query.filterrlrr])r5rhrls r8 _get_cache_key_filter_expressionz3RedisSemanticCache._get_cache_key_filter_expressions,4,,-S99r? cache_hitc|j|j}t|tr|j d}|duxrt |t |k(S)Nutf-8)getr isinstancebytesdecoder])r5rorh cached_keys r8_cache_hit_matches_keyz)RedisSemanticCache._cache_hit_matches_keysO]]4#<#<= j% (#**73J%E#j/SX*EEr?c B|jd}| t|}|S)a Get the TTL (time-to-live) value for cache entries. Args: **kwargs: Keyword arguments that may contain a custom TTL Returns: Optional[int]: The TTL value in seconds, or None if no TTL should be applied ttl)rrint)r5r%rys r8_get_ttlzRedisSemanticCache._get_ttls%jj ?c(C r?c |jd}|r t|Sd|vryg}|j|jd|dj|j }|xsdS)z\ Extract a semantic-cache prompt from chat or Responses API request kwargs. messagesinputN )rrr _collect_responses_input_textjoinstrip)rRr%r} prompt_partsprompts r8_get_prompt_from_kwargsz*RedisSemanticCache._get_prompt_from_kwargssj !**Z0 (2 2 & )+  ))&**W*=|L ,/557~r?rc|j|}|yt|tr$|j}|r|j |yt|t t fr|D]}|j||yt|tru|jd}||j||ydD]I}|j|}t|ts%|j}|s8|j |yyt|dd}||j||ydD]E}t||d}t|ts!|j}|s4|j |yy)Ncontent)textoutput input_text output_text) _coerce_response_input_valuersr]rappendlisttuplerdictrrr<) rRr@rstripped_valueitemrtext_key text_value stripped_texts r8rz0RedisSemanticCache._collect_responses_input_textsd007 =  eS !$)KKMN##N3  edE] + F11$ E F  eT "ii *G"11'<HK "YYx0 j#.$.$4$4$6M$$++M:   %D1    - -g| D G H $7J*c* * 0 0 2   '' 6  r?ct|dd}t|r|St|dd}t|r|S|S)N model_dumpr)r<callable)r@r dict_methods r8rz/RedisSemanticCache._coerce_response_input_value@sC#E<> J < $UFD9 K =  r?rrouterz Router | Nonec nt||jt|j|j|SrB)rr#rr)r5rrs r8_embedding_inputz#RedisSemanticCache._embedding_inputJs5'   .t/N/NPTPdPdfl m  r?metadatac  ddlm}m}t |j ||}|j ||}|Ftt|j|j |dddt||jd}n?tttj|j |ddd|jd}|ddd S#t$rd}d}YwxYw) z Routes through the proxy Router when the embedding model is a Router deployment so per-deployment auth (e.g. Bedrock aws_role_name) applies, mirroring ``_get_async_embedding``; otherwise embeds directly. rllm_model_list llm_routerNTzno-storezno-cachemodelr~cachertimeout num_retriesrr~rrrdata embedding) litellm.proxy.proxy_serverrr ImportErrorrr#rrr rrrlitellm)r5rrrrrembedding_inputembedding_responses r8rJz!RedisSemanticCache._get_embeddingQs   M 11E1EzSab!%!6!6vv!F  !%!  ..)'+> 22 !  " "&)!,[99; !NJ sC CCcached_responsec2||St|tr|jd} tj|}|S#tj $rE t j|}Y|S#ttf$r}td|Yd}~Yyd}~wwxYwwxYw)z Process the cached response to prepare it for use. Args: cached_response: The raw cached response Returns: The processed cache response, or None if input was None NrqzError parsing cached response: ) rsrtrujsonloadsJSONDecodeErrorast literal_evalr, SyntaxErrorr )r5rr6s r8_get_cache_logicz#RedisSemanticCache._get_cache_logicxs  "" " ou --44W=O "jj9O##  "%"2"2?"C   ,  ?sCD  s->BA**B9B B BBc td|d} |jd i|}| tdyt|}|j||j d}||j |d}|j d i|}|t||d<|jj||fi|y#t$r} td|xs|d | Yd} ~ yd} ~ wwxYw) a= Store a value in the semantic cache. Args: key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live z(Redis semantic-cache set_cache, kwargs: N'No prompt provided for semantic cachingrrvectorfiltersryzError setting z in the Redis semantic cache: rX) r rr]rJrrrjr{rzr>storerL) r5rhr@r% value_strrprompt_embedding store_kwargsryr6s r8 set_cachezRedisSemanticCache.set_caches @IJ $  b8D88B6BF~GHE I&*&9&9&6::V`Ka&9&b +22373L '00C&)#h U# DMM   B\ B b N9+=*>>\]^\_` a a bsB4BB44 C=CCc td| |jdi|}|!tdd|jdid<y|j||j d}|||j |d}|j jdi|}|sd|jdid<y|d }|j|| s!td d|jdid<yt|d }d |z } |d} |d} | |jdid<td|jd| d|d| |j| S#t$r-} td| d|jdid<Yd} ~ yd} ~ wwxYw)aM Retrieve a semantically similar cached response. Args: key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: The cached response if a semantically similar prompt is found, else None z(Redis semantic-cache get_cache, kwargs: N,No prompt provided for semantic cache lookuprsemantic-similarityrrrfilter_expressionrrorh6Redis semantic-cache hit did not match cache key scopevector_distancerrresponse!Cache hit: similarity threshold: , actual similarity: , current prompt: , cached prompt: rz,Error retrieving from Redis semantic cache: rX) r r setdefaultrJrrrnr>checkrwfloatr"rrL r5rhr%rr check_kwargsresultsror similarity cached_promptrr6s r8 get_cachezRedisSemanticCache.get_caches @IJ3 K8D88B6BF~LMKN!!*b12GH'+&9&9&6::V`Ka&9&b  *%)%J%J3%O9L 1T]]00@<@GKN!!*b12GH 'qzI...LVWKN!!*b12GH%*95F+G%HO !"O 3J#,X#6M%.z%:OHRF  j" -.C D 3D4M4M3NO&&0\2##)(+""/2 (((I I K HL MGJF  j" -.C D Ks+4EA*E08E)A$E F#E??Fc0K ddlm}m}t |j ||}|j ||}|7|j|j |dddt||jdn0tj|j |ddd|jd} tj||jd{}|ddd S#t$rd}d}YwxYw7"#t$r"} td | td | | d} ~ wwxYww) a# Asynchronously generate an embedding for the given prompt. Args: prompt: The text to generate an embedding for metadata: Request metadata forwarded to the Router embedding call Returns: List[float]: The embedding vector rrNTrrrrrz"Error generating async embedding: zFailed to generate embedding: )rrrrrr#r aembeddingrrrasynciowait_forrLr r,) r5rrrrrrembedding_callrr6s r8_get_async_embeddingz'RedisSemanticCache._get_async_embeddings?  M 11E1EzSab!%!6!6vv!F!   **%#'T:8B..  ##**%#'T:..  $ J.5.>.>~tOeOe.f(f %f-a0= =5 !NJ 2)g J >qcB C=aSAB I Js^DCBD #C(C&C(D C# D"C##D&C(( D1DDDcKtd| |jd i|}| tdyt|}|j||j dd{}||j |d}|j d i|}|||d<|jj||fi|d{y7X7#t$r} td| Yd} ~ yd} ~ wwxYww) aL Asynchronously store a value in the semantic cache. Args: key: The cache key used to isolate semantic cache entries value: The response value to cache **kwargs: Additional arguments including 'messages' for the prompt and optional 'ttl' for time-to-live z.Async Redis semantic-cache set_cache, kwargs: NrrrrryzError in async_set_cache: rX) r rr]rrrrjr{r>astorerL) r5rhr@r%rrrrryr6s r8async_set_cachez"RedisSemanticCache.async_set_cache's FvhOP <8D88B6BF~GH"5zI-1,E,EfW]WaWablWm,E,n&n +22373L '00C&) U#&$--&&  'o   < 6qc: ; ; Ktd| |jdi|}|!tdd|jdid<y|j||j dd{}|||j |d}|j jdi|d{}|sd|jdid<y|d }|j|| s!td d|jdid<yt|d }d |z } |d} |d} | |jdid<td|jd| d|d| |j| S77#t$r-} td| d|jdid<Yd} ~ yd} ~ wwxYww)a\ Asynchronously retrieve a semantically similar cached response. Args: key: The cache key used to isolate semantic cache entries **kwargs: Additional arguments including 'messages' for the prompt Returns: The cached response if a semantically similar prompt is found, else None z.Async Redis semantic-cache get_cache, kwargs: NrrrrrrrrrrrrrrrrrrzError in async_get_cache: rX) r rrrrrrnr>acheckrwrr"rrLrs r8async_get_cachez"RedisSemanticCache.async_get_cacheOs FvhOP4 K8D88B6BF~LMKN!!*b12GH-1,E,EfW]WaWablWm,E,n&n  !*%)%J%J3%O9L $84==#7#7#G,#GGGKN!!*b12GH&qzI...LVWKN!!*b12GH%*95F+G%HO !"O 3J#,X#6M%.z%:OHRF  j" -.C D 3D4M4M3NO&&0\2##)(+""/2 (((I IS'oHB K 6qc: ;GJF  j" -.C D KswF4E$F%E$,E -8E$%E"&E$F8E$:F;A$E$F E$"E$$ F-#FFFFcK|jjd{}|jd{S77w)z Get information about the Redis index. Returns: Dict[str, Any]: Information about the Redis index N)r>_get_async_indexinfo)r5aindexs r8 _index_infozRedisSemanticCache._index_infos5#mm<<>>[[]""?"sA=A?AA cache_listcK g}|D]+}|j|j|d|dfi|-tj|d{y7#t$r}t d|Yd}~yd}~wwxYww)z Asynchronously store multiple values in the semantic cache. Args: cache_list: List of (key, value) tuples to cache **kwargs: Additional arguments rrNz#Error in async_set_cache_pipeline: )rrrgatherrLr )r5rr%tasksvalr6s r8async_set_cache_pipelinez+RedisSemanticCache.async_set_cache_pipelines EE! M 1T11#a&#a&KFKL M..%( ( ( E ?sC D D EsAA7AA A AA7A A4A/*A7/A44A7) NNNNNztext-embedding-ada-002NNNrB)/__name__ __module__ __qualname____doc__rr]__annotations__rrrzr rrobjectr9propertyr>setterr= classmethodrrSrrKrjrnrrerwr{rrrr staticmethodrrrJrrrrrrrrrrXr?r8rr(s%CcB 3#3-1d 1GuG # $-17!%15*.PDjPDjP* P : P $dl PP$JP%($JP!4<PPd& __f& DcN  5$S&[155 5 ! 5  5n5c5d38n5:C:F: FV 0DF3FSWF C$J #* *#*T#Y*SW**XFv s O  %:S%:DcNT4I%:UYZ_U`%:N:"bS"b"bd"bH@KS@Kv@KD*J*JS#XQU@U*Jaefkal*JX&<&rsY   -22:F2"%B EB Er?