Ë
    uPêj Z  ã                  óD  — d Z ddlmZ ddlZddlZddlmZmZ ddlm	Z	m
Z
mZmZ ddlZddlmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ e	rddlZddlmZ ddlm Z   G d„ de«      Z! G d„ de«      Z" G d„ ded¬«      Z# G d„ ded¬«      Z$ G d„ dee«      Z%y)a+  
S3 Vectors-specific RAG Ingestion implementation.

S3 Vectors is AWS's native vector storage service that provides:
- Purpose-built vector buckets for storing and querying vectors
- Vector indexes with configurable dimensions and distance metrics
- Metadata filtering for semantic search

This implementation:
1. Auto-creates vector buckets and indexes if not provided
2. Uses LiteLLM's embedding API (supports any provider)
3. Uses httpx + AWS SigV4 signing (no boto3 dependency for S3 Vectors APIs)
4. Stores vectors with metadata using PutVectors API
é    )ÚannotationsN)ÚMappingÚSequence)ÚTYPE_CHECKINGÚAnyÚFinalÚ	TypedDict)Úverbose_logger)ÚS3_VECTORS_DEFAULT_DIMENSIONÚ"S3_VECTORS_DEFAULT_DISTANCE_METRICÚ/S3_VECTORS_DEFAULT_NON_FILTERABLE_METADATA_KEYS)Ú
safe_dumps)Ú
BaseAWSLLM)Úget_async_httpx_clientÚhttpxSpecialProvider)ÚBaseRAGIngestion)ÚRouter)ÚRAGIngestOptionsc                  ó   — e Zd ZU ded<   y)ÚS3VectorDataPayloadzSequence[float]Úfloat32N©Ú__name__Ú
__module__Ú__qualname__Ú__annotations__© ó    úo/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/rag/ingestion/s3_vectors_ingestion.pyr   r   -   s   … ØÔr   r   c                  ó,   — e Zd ZU ded<   ded<   ded<   y)ÚS3VectorEntryÚstrÚkeyr   ÚdataúMapping[str, str]ÚmetadataNr   r   r   r   r!   r!   1   s   … Ø	ƒHØ
ÓØÔr   r!   c                  ó,   — e Zd ZU ded<   ded<   ded<   y)ÚS3VectorsQueryMatchr"   r#   ÚfloatÚdistancer%   r&   Nr   r   r   r   r(   r(   7   s   … Ø	ƒHØƒOØÔr   r(   F)Útotalc                  ó   — e Zd ZU ded<   y)ÚS3VectorsQueryResponsezSequence[S3VectorsQueryMatch]ÚvectorsNr   r   r   r   r-   r-   =   s   … Ø*Ô*r   r-   c                  ó¼   — e Zd ZdZ	 d	 	 	 dd„Zdd„Zdd„Zd„ Z	 	 d	 	 	 	 	 	 	 	 	 dd„Zd„ Z	d	„ Z
dd
„Z	 	 	 	 dd„Z	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 d	 	 	 	 	 	 	 dd„Zy)ÚS3VectorsRAGIngestionaÏ  
    S3 Vectors RAG ingestion using httpx + AWS SigV4 signing.

    Workflow:
    1. Auto-create vector bucket if needed (CreateVectorBucket API)
    2. Auto-create vector index if needed (CreateVectorIndex API)
    3. Generate embeddings using LiteLLM (supports any provider)
    4. Store vectors with PutVectors API

    Configuration:
    - vector_bucket_name: S3 vector bucket name (required)
    - index_name: Vector index name (auto-creates if not provided)
    - dimension: Vector dimension (default: S3_VECTORS_DEFAULT_DIMENSION)
    - distance_metric: "cosine" or "euclidean" (default: S3_VECTORS_DEFAULT_DISTANCE_METRIC)
    - non_filterable_metadata_keys: List of metadata keys to exclude from filtering
    Nc                ó²  — t        j                  | ||¬«       t        j                  | «       | j                  d   | _        | j                  j                  d«      | _        | j                  j                  dt        «      | _        | j                  j                  dt        «      | _
        | j                  «       | _        | j                  j                  d«      }| j                  |rt        |«      nd ¬«      | _        | j!                  | j                  j                  d«      ¬	«      }t#        t$        j&                  |�d|ind ¬
«      | _        d| _        y )N)Úingest_optionsÚrouterÚvector_bucket_nameÚ
index_nameÚdistance_metricÚnon_filterable_metadata_keysÚaws_region_name)r8   Ú
ssl_verify)r9   )Úllm_providerÚparamsF)r   Ú__init__r   Úvector_store_configr4   Úgetr5   r   r6   r   r7   Ú_get_dimension_from_configÚ	dimensionÚ)get_aws_region_name_for_non_llm_api_callsr"   r8   Ú_get_ssl_verifyr   r   ÚRAGÚasync_httpx_clientÚ_config_initialized)Úselfr2   r3   Ú_aws_regionr9   s        r   r<   zS3VectorsRAGIngestion.__init__S   s4  € ô
 	×!Ñ! $°~ÈfÕUÜ×Ñ˜DÔ!ð (,×'?Ñ'?Ð@TÑ'UˆÔØ&*×&>Ñ&>×&BÑ&BÀ<Ó&PˆŒØ$(×$<Ñ$<×$@Ñ$@ÐARÔTvÓ$wˆÔØ;?×;SÑ;S×;WÑ;WØ*Ü;ó<
ˆÔ)ð ×8Ñ8Ó:ˆŒð *.×)AÑ)A×)EÑ)EÐFWÓ)XˆØ#×MÑMÙ0;œC Ô,Àð  Nó  
ˆÔð
 !×0Ñ0¸D×<TÑ<T×<XÑ<XÐYeÓ<fÐ0Ógˆ
Ü"8Ü-×1Ñ1Ø1;Ð1G�L *Ñ-ÈTô#
ˆÔð $)ˆÕ r   c              ƒ  óp  K  — | j                   rd| j                   vrt        S 	 | j                   d   }t        j                  d|«       d}| j                  r'| j                  j                  ||g¬«      ƒ d{  –—† }n t        j
                  ||g¬«      ƒ d{  –—† }|j                  rLt        |j                  «      dkD  r4t        |j                  d   d   «      }t        j                  d||«       |S t        S 7 Œƒ7 Œd# t        $ r*}t        j                  d	|t        «       Y d}~t        S d}~ww xY w­w)
zÊ
        Auto-detect dimension by making a test embedding request.

        Makes a single embedding request with a test string to determine
        the output dimension of the embedding model.
        Úmodelz?Auto-detecting dimension by making test embedding request to %sÚtest©rI   ÚinputNr   Ú	embeddingz1Auto-detected dimension %s for embedding model %szXCould not auto-detect dimension from embedding model: %s. Using default dimension of %s.)Úembedding_configr   r
   Údebugr3   Ú
aembeddingÚlitellmr$   ÚlenÚ	ExceptionÚwarning)rF   Ú
model_nameÚ
test_inputÚresponser@   Úes         r   Ú%_get_dimension_from_embedding_requestz;S3VectorsRAGIngestion._get_dimension_from_embedding_requestw   s#  è ø€ ð ×$Ò$¨°t×7LÑ7LÑ(LÜ/Ð/ð	Ø $× 5Ñ 5°gÑ >ˆJÜ× Ñ Ð!bÐdnÔoð !'ˆJØ�{Š{Ø!%§¡×!7Ñ!7¸jÐQ[ÐP\Ð!7Ó!]×]‘ä!(×!3Ñ!3¸*ÈZÈLÔ!Y×Y�ð �}Š}¤ X§]¡]Ó!3°aÒ!7Ü#& x§}¡}°QÑ'7¸Ñ'DÓ#E�	Ü×$Ñ$Ð%XÐZcÐeoÔpØ Ð ô ,Ð+ð! ^øàYùô ò 	Ü×"Ñ"ØjØÜ,÷ð ô ,Ð+ûð	üsT   ‚!D6¤AD  Á8C<Á9 D  ÂC>ÂAD  Ã5D6Ã<D  Ã>D  Ä 	D3Ä	D.Ä$
D6Ä.D3Ä3D6c                óT   — d| j                   v r| j                   d   }t        |«      S y)z†
        Get vector dimension from config if explicitly provided.

        Returns None if dimension should be auto-detected.
        r@   N)r=   Úint)rF   Úconfigured_dimensions     r   r?   z0S3VectorsRAGIngestion._get_dimension_from_configš   s2   € ð ˜$×2Ñ2Ñ2Ø/3×/GÑ/GÈÑ/TÐ ÜÐ+Ó,Ð,Ør   c              ƒ  ó`  K  — | j                   ry| j                  €| j                  «       ƒ d{  –—† | _        | j                  «       ƒ d{  –—†  | j                  s+t        j                  «       j                  dd }d|› �| _        | j                  «       ƒ d{  –—†  d| _         y7 Œx7 Œ]7 Œ­w)z,Lazily initialize S3 Vectors infrastructure.Né   zlitellm-index-T)	rE   r@   rY   Ú_ensure_vector_bucket_existsr5   ÚuuidÚuuid4ÚhexÚ_ensure_vector_index_exists)rF   Ú	unique_ids     r   Ú_ensure_config_initializedz0S3VectorsRAGIngestion._ensure_config_initialized¥   s    è ø€ à×#Ò#Øð �>‰>Ð!Ø#'×#MÑ#MÓ#O×OˆDŒNð ×/Ñ/Ó1×1Ð1ð �Šä#Ÿz™z›|×/Ñ/°°Ð3ˆIØ .¨y¨kÐ:ˆDŒOà×.Ñ.Ó0×0Ð0à#'ˆÕ ð Pøð 	2øð 	1ús4   ‚-B.¯B(°B.ÁB*ÁAB.ÂB,ÂB.Â*B.Â,B.c              ƒ  óÌ  K  — 	 ddl }ddlm} ddlm} | j                  | j                  j                  d«      | j                  j                  d«      | j                  j                  d«      | j                  | j                  j                  d	«      | j                  j                  d
«      | j                  j                  d«      | j                  j                  d«      | j                  j                  d«      | j                  j                  d«      ¬«
      }|€i }|r=d|d<   t        j                  |j                  d«      «      j                  «       }	|	|d<   n&t        j                  d«      j                  «       |d<   |j                  ||||¬«      }
|
j                  «       } ||j                   |j"                  |j$                  |j&                  ¬«      } ||d| j                  «      j)                  |«       t+        |j&                  j-                  «       «      }|j/                  «       }|dk(  r(| j0                  j3                  |||¬«      ƒ d{  –—† }|S |dk(  r(| j0                  j5                  |||¬«      ƒ d{  –—† }|S |dk(  r'| j0                  j                  ||¬«      ƒ d{  –—† }|S t7        d|› �«      ‚# t
        $ r t        d«      ‚w xY w7 Œ…7 ŒZ7 Œ0­w)z†
        Helper to sign and execute AWS API requests using httpx + SigV4.

        Pattern from litellm/integrations/s3_v2.py
        r   N)Ú	SigV4Auth)Ú
AWSRequestz=Missing botocore to call S3 Vectors. Run 'pip install boto3'.Úaws_access_key_idÚaws_secret_access_keyÚaws_session_tokenÚaws_session_nameÚaws_profile_nameÚaws_role_nameÚaws_web_identity_tokenÚaws_sts_endpointÚaws_external_id)
ri   rj   rk   r8   rl   rm   rn   ro   rp   rq   zapplication/jsonzContent-Typezutf-8zx-amz-content-sha256r   )r$   Úheaders)ÚmethodÚurlr$   rr   Ú	s3vectorsÚPUTÚPOSTÚGET)rr   zUnsupported HTTP method: )ÚrequestsÚbotocore.authrg   Úbotocore.awsrequestrh   ÚImportErrorÚget_credentialsr=   r>   r8   ÚhashlibÚsha256ÚencodeÚ	hexdigestÚRequestÚpreparers   rt   Úbodyrr   Úadd_authÚdictÚitemsÚupperrD   ÚputÚpostÚ
ValueError)rF   rs   rt   r$   rr   ry   rg   rh   ÚcredentialsÚcontent_hashÚreqÚpreppedÚaws_requestÚsigned_headersÚmethod_upperrW   s                   r   Ú_sign_and_execute_requestz/S3VectorsRAGIngestion._sign_and_execute_request»   s¾  è ø€ ð	_ÛÝ/Ý6ð
 "×1Ñ1Ø"×6Ñ6×:Ñ:Ð;NÓOØ"&×":Ñ":×">Ñ">Ð?VÓ"WØ"×6Ñ6×:Ñ:Ð;NÓOØ ×0Ñ0Ø!×5Ñ5×9Ñ9Ð:LÓMØ!×5Ñ5×9Ñ9Ð:LÓMØ×2Ñ2×6Ñ6°ÓGØ#'×#;Ñ#;×#?Ñ#?Ð@XÓ#YØ!×5Ñ5×9Ñ9Ð:LÓMØ ×4Ñ4×8Ñ8Ð9JÓKð 2ó 
ˆð ˆ?ØˆGáØ&8ˆG�NÑ#ä")§.¡.°·±¸WÓ1EÓ"F×"PÑ"PÓ"RˆLØ.:ˆGÐ*Ò+ô /6¯n©n¸SÓ.A×.KÑ.KÓ.MˆGÐ*Ñ+ð ×%Ñ% f¨c¸ÀgÐ%ÓNˆØŸ™›ˆñ (Ø—>‘>Ø—‘Ø—‘Ø—O‘Oô	
ˆñ 	�+˜{¨D×,@Ñ,@ÓA×JÑJÈ;ÔWô !% [×%8Ñ%8×%>Ñ%>Ó%@Ó Aˆð %Ÿl™l›nˆØ˜5Ò Ø!×4Ñ4×8Ñ8¸À4ÐQ_Ð8Ó`×`ˆHð ˆð ˜VÒ#Ø!×4Ñ4×9Ñ9¸#ÀDÐR`Ð9Óa×aˆHð ˆð ˜UÒ"Ø!×4Ñ4×8Ñ8¸ÀnÐ8ÓU×UˆHð ˆô Ð8¸¸ÐAÓBÐBøôo ò 	_ÜÐ]Ó^Ð^ð	_úðb aøàaøàUúsL   ‚K$„K ”IK$ÉKÉ,K$ÊK Ê+K$Ê1K"Ê2K$ËKËK$Ë K$Ë"K$c              ƒ  óî  K  — t        j                  d| j                  «       t        | j                  «      dk  rt	        d| j                  › d�«      ‚| j                  j                  dd«      j                  dd«      j                  «       st	        d| j                  › d�«      ‚d	| j                  › d
�}t        d| j                  i«      }	 | j                  d||¬«      ƒ d{  –—† }|j                  dk(  r!t        j                  d| j                  «       y	 	 t        j                  d| j                  «       d	| j                  › d�}t        d| j                  i«      }| j                  d||¬«      ƒ d{  –—† }|j                  dv r!t        j                  d| j                  «       y|j                  dk(  r!t        j                  d| j                  «       yt        j                  d|j                  |j                  «       |j                  «        y7 �Œ5# t        $ r!}t        j                  d|«       Y d}~�Œ'd}~ww xY w7 ŒÏ# t        $ r}t        j                   d|«       ‚ d}~ww xY w­w)z[Create vector bucket if it doesn't exist using GetVectorBucket and CreateVectorBucket APIs.z$Ensuring S3 vector bucket exists: %sé   zInvalid vector_bucket_name 'zy': AWS S3 bucket names must be at least 3 characters long. Please provide a valid bucket name (e.g., 'my-vector-bucket').ú-Ú ú.z˜': AWS S3 bucket names can only contain lowercase letters, numbers, hyphens, and periods. Please provide a valid bucket name (e.g., 'my-vector-bucket').úhttps://s3vectors.z.api.aws/GetVectorBucketÚvectorBucketNamerw   ©r$   NéÈ   zVector bucket %s existsz=Bucket check failed (may not exist): %s, attempting to createzCreating vector bucket: %sz.api.aws/CreateVectorBucket©rœ   éÉ   zCreated vector bucket: %sé™  zVector bucket %s already existsz"CreateVectorBucket failed: %s - %sz Error creating vector bucket: %s)r
   rO   r4   rR   r‹   ÚreplaceÚisalnumr8   r   r“   Ústatus_coderS   ÚinfoÚerrorÚtextÚraise_for_statusÚ	exception)rF   Úget_urlÚget_bodyrW   rX   Ú
create_urlÚcreate_bodys          r   r_   z2S3VectorsRAGIngestion._ensure_vector_bucket_exists  s`  è ø€ ä×ÑÐCÀT×E\ÑE\Ô]ô ˆt×&Ñ&Ó'¨!Ò+ÜØ.¨t×/FÑ/FÐ.Gð HQð Róð ð
 ×&Ñ&×.Ñ.¨s°BÓ7×?Ñ?ÀÀRÓH×PÑPÔRÜØ.¨t×/FÑ/FÐ.Gð HQð Róð ð .¨d×.BÑ.BÐ-CÐC[Ð\ˆÜ$Ð&8¸$×:QÑ:QÐ%RÓSˆð	eØ-1×-KÑ-KÈFÐT[ÐbjÐ-KÓ-k×'kˆHØ×#Ñ# sÒ*Ü×$Ñ$Ð%>À×@WÑ@WÔXØð +ð	Ü× Ñ Ð!=¸t×?VÑ?VÔWØ"4°T×5IÑ5IÐ4JÐJeÐ fˆJÜ!+Ð-?À×AXÑAXÐ,YÓ!ZˆKà!×;Ñ;¸FÀJÐU`Ð;Óa×aˆHà×#Ñ# zÑ1Ü×#Ñ#Ð$?À×AXÑAXÕYØ×%Ñ%¨Ò,ä×$Ñ$Ð%FÈ×H_ÑH_Õ`ä×$Ñ$Ð%IÈ8×K_ÑK_Ðai×anÑanÔoØ×)Ñ)Õ+ð- (lúô ò 	eÜ× Ñ Ð!`Ðbc×dÒdûð	eúð bùô ò 	Ü×$Ñ$Ð%GÈÔKØûð	üs‘   ‚CI5ÃH Ã&HÃ'3H ÄI5ÄAI Å;IÅ<2I Æ.I5Æ//I ÇI5Ç;I ÈI5ÈH È	IÈ'IÈ=I5ÉIÉI5ÉI É	I2ÉI-É-I2É2I5c              ƒ  ó¼  K  — t        j                  d| j                  | j                  «       d| j                  › d�}t        | j                  | j                  dœ«      }	 | j                  d||¬«      ƒ d{  –—† }|j                  dk(  r!t        j                  d	| j                  «       y	 	 t        j                  d| j                  | j                  | j                  «       | j                  | j                  d| j                  | j                  dœ}| j                  rd| j                  i|d<   d| j                  › d�}| j                  d|t        |«      ¬«      ƒ d{  –—† }|j                  dv r!t        j                  d| j                  «       y|j                  dk(  r!t        j                  d| j                  «       yt        j                  d|j                  |j                  «       |j                  «        y7 �ŒŠ# t        $ r!}t        j                  d
|«       Y d}~�Œ|d}~ww xY w7 ŒÏ# t        $ r}t        j                   d|«       ‚ d}~ww xY w­w)zLCreate vector index if it doesn't exist using GetIndex and CreateIndex APIs.z#Ensuring vector index exists: %s/%sr™   z.api.aws/GetIndex)rš   Ú	indexNamerw   r›   Nrœ   zVector index %s existsz<Index check failed (may not exist): %s, attempting to createz6Creating vector index: %s with dimension=%s, metric=%sr   )rš   r­   ÚdataTyper@   ÚdistanceMetricÚnonFilterableMetadataKeysÚmetadataConfigurationz.api.aws/CreateIndexr�   zCreated vector index: %srŸ   zVector index %s already existszCreateIndex failed: %s - %szError creating vector index: %s)r
   rO   r4   r5   r8   r   r“   r¢   rS   r@   r6   r7   r£   r¤   r¥   r¦   r§   )rF   r¨   r©   rW   rX   Úindex_configrª   s          r   rc   z1S3VectorsRAGIngestion._ensure_vector_index_exists8  s&  è ø€ ä×ÑÐBÀD×D[ÑD[Ð]a×]lÑ]lÔmð .¨d×.BÑ.BÐ-CÐCTÐUˆÜ$¸$×:QÑ:QÐ`d×`oÑ`oÑ%pÓqˆð	dØ-1×-KÑ-KÈFÐT[ÐbjÐ-KÓ-k×'kˆHØ×#Ñ# sÒ*Ü×$Ñ$Ð%=¸t¿¹ÔOØð +ð 	Ü× Ñ ØHØ—‘Ø—‘Ø×$Ñ$ô	ð %)×$;Ñ$;Ø!Ÿ_™_Ø%Ø!Ÿ^™^Ø"&×"6Ñ"6ñ6ˆLð ×0Ò0Ø9TÐVZ×VwÑVwÐ8x�Ð4Ñ5à"4°T×5IÑ5IÐ4JÐJ^Ð _ˆJØ!×;Ñ;¸FÀJÔU_Ð`lÓUmÐ;Ón×nˆHà×#Ñ# zÑ1Ü×#Ñ#Ð$>ÀÇÁÕPØ×%Ñ%¨Ò,Ü×$Ñ$Ð%EÀtÇÁÕWä×$Ñ$Ð%BÀH×DXÑDXÐZb×ZgÑZgÔhØ×)Ñ)Õ+ðK (lúô ò 	dÜ× Ñ Ð!_Ðab×cÒcûð	dúð2 oùô ò 	Ü×$Ñ$Ð%FÈÔJØûð	üs‘   ‚AIÁ!H Á8HÁ93H Â,IÂ/B3H4 Å"H2Å#2H4 ÆIÆ/H4 ÇIÇ;H4 ÈIÈH È	H/ÈH*È$IÈ*H/È/IÈ2H4 È4	IÈ=IÉIÉIc              ƒ  óR  K  — t        j                  dt        |«      | j                  | j                  «       d| j
                  › d�}| j                  | j                  |dœ}	 | j                  d|t        |«      ¬«      ƒ d{  –—† }|j                  dv r+t        j                  d	t        |«      | j                  «       yt        j                  d
|j                  |j                  «       |j                  «        y7 Œy# t        $ r}t        j                  d|«       ‚ d}~ww xY w­w)z¥
        Call PutVectors API to store vectors in S3 Vectors.

        Args:
            vectors: List of vector objects with keys: "key", "data", "metadata"
        zStoring %s vectors in %s/%sr™   z.api.aws/PutVectors)rš   r­   r.   rw   r›   Nr�   z*Successfully stored %s vectors in index %sz$PutVectors failed with status %s: %szError storing vectors: %s)r
   rO   rR   r4   r5   r8   r“   r   r¢   r£   r¤   r¥   r¦   rS   r§   )rF   r.   rt   Úrequest_bodyrW   rX   s         r   Ú_put_vectorsz"S3VectorsRAGIngestion._put_vectorsk  s  è ø€ ô 	×ÑÐ:¼CÀ»LÈ$×JaÑJaÐcg×crÑcrÔsà)¨$×*>Ñ*>Ð)?Ð?RÐSˆð !%× 7Ñ 7ØŸ™Øñ
ˆð	Ø48×4RÑ4RØ˜¤*¨\Ó":ð 5Só 5÷ /ˆHð ×#Ñ# zÑ1Ü×#Ñ#Ð$PÔRUÐV]ÓR^Ð`d×`oÑ`oÕpä×$Ñ$Ð%KÈX×MaÑMaÐck×cpÑcpÔqØ×)Ñ)Õ+ð/ùô ò 	Ü×$Ñ$Ð%@À!ÔDØûð	üsH   ‚A D'Á# C? ÂC=Â<C? Ã D'Ã;C? Ã<D'Ã=C? Ã?	D$ÄDÄD$Ä$D'c              ƒ  óà  K  — |sy| j                   st        j                  d«       ddi| _         | j                   j                  dd«      }t        j                  dt        |«      |«       t        |«      }| j                  r&| j                  j                  ||¬«      ƒ d{  –—† }nt        j                  ||¬«      ƒ d{  –—† }|j                  D �cg c]  }|d   ‘Œ	 c}S 7 ŒA7 Œ#c c}w ­w)z�
        Generate embeddings using LiteLLM's embedding API.

        Supports any embedding provider (OpenAI, Bedrock, Cohere, etc.)
        NzBNo embedding config provided, using default text-embedding-3-smallrI   útext-embedding-3-smallz,Generating embeddings for %s chunks using %srK   rM   )rN   r
   rT   r>   rO   rR   Úlistr3   rP   rQ   r$   )rF   ÚchunksÚembedding_modelÚinput_chunksrW   Úitems         r   ÚembedzS3VectorsRAGIngestion.embed‹  sÓ   è ø€ ñ Øð ×$Ò$Ü×"Ñ"Ð#gÔhØ%,Ð.FÐ$GˆDÔ!à!%×!6Ñ!6×!:Ñ!:¸7ÐD\Ó!]ˆä×ÑÐKÌSÐQWË[ÐZiÔjô *.¨f«ˆà�;Š;Ø!Ÿ[™[×3Ñ3¸/ÐQ]Ð3Ó^×^‰Hä$×/Ñ/°oÈ\ÔZ×ZˆHà.6¯m©mÖ< d��[Ó!Ò<Ð<ð	 _øàZúâ<ùs6   ‚B!C.Â#C%Â$C.ÃC'ÃC.ÃC)Ã"C.Ã'C.Ã)C.c              ƒ  óâ  K  — | j                  «       ƒ d{  –—†  |r|s"d}t        j                  |«       t        |«      ‚t	        t        ||«      «      D ��	�
cg c]I  \  }\  }	}
t        |r|› d|› �nd|› �t        |
¬«      |r|	t        |«      |dœn|	t        |«      dœ¬«      ‘ŒK }}	}}
| j                  |«      ƒ d{  –—†  | j                  › d	| j                  › �}||fS 7 ŒÍc c}
}	}w 7 Œ,­w)
aÊ  
        Store vectors in S3 Vectors using PutVectors API.

        Steps:
        1. Ensure vector bucket exists (auto-create if needed)
        2. Ensure vector index exists (auto-create if needed)
        3. Prepare vector data with metadata
        4. Call PutVectors API with httpx + SigV4 signing

        Args:
            file_content: Raw file bytes (not used for S3 Vectors)
            filename: Name of the file
            content_type: MIME type (not used for S3 Vectors)
            chunks: Text chunks
            embeddings: Vector embeddings
            existing_file_id: Existing provider file ID, unsupported for S3 Vectors

        Returns:
            Tuple of (index_name, filename)
        Na“  No text content could be extracted from the file for embedding. Possible causes:
  1. PDF files require OCR - add 'ocr' config with a vision model (e.g., 'anthropic/claude-3-5-sonnet-20241022')
  2. Binary files cannot be processed - convert to text first
  3. File is empty or contains no extractable text
For PDFs, either enable OCR or use a PDF extraction library to convert to text before ingestion.Ú_Úchunk_)r   )Úsource_textÚchunk_indexÚfilename)rÁ   rÂ   )r#   r$   r&   ú:)re   r
   r¤   r‹   Ú	enumerateÚzipr!   r   r"   rµ   r4   r5   )rF   Úfile_contentrÃ   Úcontent_typer¹   Ú
embeddingsÚexisting_file_idÚ	error_msgÚiÚchunkrM   r.   Úvector_store_ids                r   ÚstorezS3VectorsRAGIngestion.storeª  s  è ø€ ð< ×-Ñ-Ó/×/Ð/á¡ðsð ô × Ñ  Ô+Ü˜YÓ'Ð'ô *3´3°v¸zÓ3JÓ)K÷
ð 
ñ &�Ñ%�E˜9ô Ù)1�x�j  ! Ñ%¸À¸s°|Ü(°Ô;ñ  ð %*¼#¸a»&ÈhÒWà).¼sÀ1»vÑFöð
ˆò 
ð ×Ñ Ó(×(Ð(ð %)×$;Ñ$;Ð#<¸A¸d¿o¹oÐ=NÐ!OˆØ Ð(Ð(ðC 	0úô
ð 	)ús/   ‚C/–C$—AC/ÁAC&Â(C/Ã C-Ã$C/Ã&C/c           	   ƒ  ó²  K  — t        j                  d||«       | j                  s	ddi| _        | j                  j                  dd«      }t	        j
                  ||g¬«      ƒ d{  –—† }|j                  d   d   }d| j                  › d	�}| j                  |d
|i|dddœ}	 | j                  d|t        |«      ¬«      ƒ d{  –—† }	|	j                  dk(  r£|	j                  «       }
|
j                  d«      }t        j                  dt        |
j                  dg «      «      «       |rO|D ]J  }|j                  di «      }|j                  dd«      }|j                  «       |j                  «       v sŒH|
c S  |
S t        j                  d|	j                  |	j                   «       y7 �Œ@7 Œå# t"        $ r }t        j$                  d|«       Y d}~yd}~ww xY w­w)a  
        Query S3 Vectors using QueryVectors API.

        Args:
            vector_store_id: Index name
            query: Query text
            top_k: Number of results to return

        Returns:
            Query results with vectors and metadata
        z Querying index %s with query: %srI   r·   rK   Nr   rM   r™   z.api.aws/QueryVectorsr   T)rš   r­   ÚqueryVectorÚtopKÚreturnDistanceÚreturnMetadatarw   r›   rœ   r.   zQuery returned %s resultsr&   rÁ   r—   z&QueryVectors failed with status %s: %szError querying vectors: %s)r
   rO   rN   r>   rQ   rP   r$   r8   r4   r“   r   r¢   ÚjsonrR   Úlowerr¤   r¥   rS   r§   )rF   rÎ   ÚqueryÚtop_krº   rW   Úquery_embeddingrt   r´   Úquery_responseÚresultsÚmatchesÚresultr&   rÁ   rX   s                   r   Úquery_vector_storez(S3VectorsRAGIngestion.query_vector_storeë  sá  è ø€ ô 	×ÑÐ?ÀÐRWÔXð ×$Ò$Ø%,Ð.FÐ$GˆDÔ!à!%×!6Ñ!6×!:Ñ!:¸7ÐD\Ó!]ˆä ×+Ñ+°/È%ÈÔQ×QˆØ2:·-±-ÀÑ2BÀ;Ñ2Oˆð *¨$×*>Ñ*>Ð)?Ð?TÐUˆð !%× 7Ñ 7Ø(Ø% Ð7ØØ"Ø"ñ
ˆð	Ø:>×:XÑ:XØ˜¤*¨\Ó":ð ;Yó ;÷ 5ˆNð ×)Ñ)¨SÒ0Ø9G×9LÑ9LÓ9N�Ø!(§¡¨YÓ!7�Ü×$Ñ$Ð%@Ä#ÀgÇkÁkÐR[Ð]_ÓF`ÓBaÔbñ Ø")ò +˜Ø#)§:¡:¨j¸"Ó#=˜Ø&.§l¡l°=À"Ó&E˜Ø Ÿ;™;›=¨K×,=Ñ,=Ó,?Ò?Ø#*šNð	+ð �ä×$Ñ$Ø<¸n×>XÑ>XÐZh×ZmÑZmôð ðO Rùð 5ùô0 ò 	Ü×$Ñ$Ð%AÀ1ÔEÜûð	üsm   ‚A$GÁ&F&Á';GÂ# F+ ÃF)ÃB.F+ Å3F+ Å6GÅ7F+ Å9GÅ:+F+ Æ%GÆ)F+ Æ+	GÆ4GÇ
GÇGÇG)N)r2   r   r3   zRouter | None)Úreturnr[   )rß   z
int | None)NN)
rs   r"   rt   r"   r$   ú
str | Nonerr   zdict[str, str] | Nonerß   r   )r.   zSequence[S3VectorEntry])r¹   ú	list[str]rß   úlist[list[float]] | None)rÇ   zbytes | NonerÃ   rà   rÈ   rà   r¹   rá   rÉ   râ   rÊ   rà   rß   ztuple[str | None, str | None])é   )rÎ   r"   r×   r"   rØ   r[   rß   zS3VectorsQueryResponse | None)r   r   r   Ú__doc__r<   rY   r?   re   r“   r_   rc   rµ   r½   rÏ   rÞ   r   r   r   r0   r0   A   s   „ ñð( !%ð")à(ð")ð ó")óH!,óF	ò(ð4  Ø)-ðIàðIð ðIð ð	Ið
 'ðIð 
óIòV0òd1ófð@=àð=ð 
"ó=ðL (,ð?)à"ð?)ð ð?)ð !ð	?)ð
 ð?)ð -ð?)ð %ð?)ð 
'ó?)ðD >?ð@Ø"ð@Ø+.ð@Ø7:ð@à	&ô@r   r0   )&rä   Ú
__future__r   r~   r`   Úcollections.abcr   r   Útypingr   r   r   r	   rQ   Úlitellm._loggingr
   Úlitellm.constantsr   r   r   Ú*litellm.litellm_core_utils.safe_json_dumpsr   Ú!litellm.llms.bedrock.base_aws_llmr   Ú&litellm.llms.custom_httpx.http_handlerr   r   Ú$litellm.rag.ingestion.base_ingestionr   Úhttpxr   Úlitellm.types.ragr   r   r!   r(   r-   r0   r   r   r   ú<module>rð      s“   ðñõ #ã Û ß -ß 7Ó 7ã Ý +÷ñ õ
 BÝ 8÷õ BáÛåÝ2ô˜)ô ô �Iô  ô ˜)¨5õ  ô+˜Y¨eõ +ôjÐ,¨jõ jr   