Ë
    uPêjm-  ã                  ó’   — d Z ddlmZ ddlmZmZmZmZ ddlm	Z	 ddl
mZmZ ddlmZ ddlmZ erddlmZ dd	lmZ  G d
„ de«      Zy)zÜ
Gemini-specific RAG Ingestion implementation.

Gemini handles embedding and chunking internally when files are uploaded to File Search stores,
so this implementation skips the embedding step and directly uploads files.
é    )Úannotations)ÚTYPE_CHECKINGÚAnyÚFinalÚcast)Úverbose_logger)Úget_async_httpx_clientÚhttpxSpecialProvider)ÚGeminiModelInfo)ÚBaseRAGIngestion)ÚRouter)ÚRAGIngestOptionsc                  óØ   ‡ — e Zd ZdZ	 d		 	 	 d
ˆ fd„Z	 	 	 	 dd„Z	 d		 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z		 	 	 	 	 	 dd„Z
ˆ xZS )ÚGeminiRAGIngestionaj  
    Gemini-specific RAG ingestion using File Search API.

    Key differences from base:
    - Embedding is handled by Gemini when files are uploaded to File Search stores
    - Files are uploaded using uploadToFileSearchStore API
    - Chunking is done by Gemini's File Search (supports custom white_space_config)
    - Supports custom metadata attachment
    c                óF   •— t         ‰| �  ||¬«       t        «       | _        y )N)Úingest_optionsÚrouter)ÚsuperÚ__init__r   Ú
model_info)Úselfr   r   Ú	__class__s      €úk/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/litellm/rag/ingestion/gemini_ingestion.pyr   zGeminiRAGIngestion.__init__$   s!   ø€ ô
 	‰Ñ¨¸vÐÔFÜ)Ó+ˆ�ó    c              ƒ  ó   K  — y­w)z§
        Gemini handles embedding internally - skip this step.

        Returns:
            None (Gemini embeds when files are uploaded to File Search store)
        N© )r   Úchunkss     r   ÚembedzGeminiRAGIngestion.embed,   s   è ø€ ð ùs   ‚c              ƒ  ój  K  — | j                   j                  d«      }t        t        t        t
        f   | j                   «      }t        t        dz  |j                  d«      «      xs t        j                  «       }	t        t        dz  |j                  d«      «      xs t        j                  «       }
|	st        d«      ‚|
st        d«      ‚d}|
› d|› �}|s*| j                  |	|| j                  xs d	¬
«      ƒ d{  –—† }d}|r#|r!|r| j                  |	|||||¬«      ƒ d{  –—† }||fS 7 Œ/7 Œ
­w)ak  
        Store content in Gemini File Search store.

        Gemini workflow:
        1. Create File Search store (if not provided)
        2. Upload file using uploadToFileSearchStore (Gemini handles chunking/embedding)

        Args:
            file_content: Raw file bytes
            filename: Name of the file
            content_type: MIME type
            chunks: Ignored - Gemini handles chunking
            embeddings: Ignored - Gemini handles embedding
            existing_file_id: Existing provider file ID, unsupported for Gemini

        Returns:
            Tuple of (vector_store_id, file_id)
        Úvector_store_idNÚapi_keyÚapi_basezCGEMINI_API_KEY or GOOGLE_API_KEY is required for Gemini File SearchzGEMINI_API_BASE is requiredÚv1betaú/zlitellm-rag-ingest)r!   Úbase_urlÚdisplay_name)r!   r%   r    ÚfilenameÚfile_contentÚcontent_type)Úvector_store_configÚgetr   ÚdictÚstrr   r   Úget_api_keyÚget_api_baseÚ
ValueErrorÚ_create_file_search_storeÚingest_nameÚ_upload_to_file_search_store)r   r(   r'   r)   r   Ú
embeddingsÚexisting_file_idr    r*   r!   r"   Úapi_versionr%   Úresult_file_ids                 r   ÚstorezGeminiRAGIngestion.store9   sM  è ø€ ð6 ×2Ñ2×6Ñ6Ð7HÓIˆä%)¬$¬s´C¨x©.¸$×:RÑ:RÓ%SÐô œc D™jÐ*=×*AÑ*AÀ)Ó*LÓMÒnÔQ`×QlÑQlÓQnˆÜœs T™zÐ+>×+BÑ+BÀ:Ó+NÓOÒqÔSb×SoÑSoÓSqˆáÜÐbÓcÐcáÜÐ:Ó;Ð;à%ˆØ%˜J a¨ }Ð5ˆñ Ø$(×$BÑ$BØØ!Ø!×-Ñ-ÒEÐ1Eð %Có %÷ ˆOð ˆÙ™H©Ø#'×#DÑ#DØØ!Ø /Ø!Ø)Ø)ð $Eó $÷ ˆNð  Ð.Ð.ð%øðús$   ‚C=D3Ã?D/Ä &D3Ä&D1Ä'	D3Ä1D3c              ƒ  óˆ  K  — |› d�}d|i}t        t        j                  ddi¬«      }|j                  ||d|dœ¬«      ƒ d	{  –—† }|j                  d
k7  r/d|j
                  › �}t        j                  |«       t        |«      ‚|j                  «       }	|	j                  dd«      }
t        j                  d|
«       |
S 7 Œ|­w)a  
        Create a Gemini File Search store.

        Args:
            api_key: Gemini API key
            base_url: Base URL for Gemini API
            display_name: Display name for the store

        Returns:
            Store name (format: fileSearchStores/xxxxxxx)
        z/fileSearchStoresÚdisplayNameÚtimeoutç      N@©Úllm_providerÚparamsúapplication/json)úContent-Typeúx-goog-api-key©ÚjsonÚheadersNéÈ   z$Failed to create File Search store: ÚnameÚ zCreated File Search store: %s)r	   r
   ÚRAGÚpostÚstatus_codeÚtextr   ÚerrorÚ	ExceptionrD   r+   Údebug)r   r!   r%   r&   ÚurlÚrequest_bodyÚclientÚresponseÚ	error_msgÚresponse_dataÚ
store_names              r   r1   z,GeminiRAGIngestion._create_file_search_store{   s×   è ø€ ð" !�zÐ!2Ð3ˆà,¨lÐ;ˆä.Ü-×1Ñ1Ø˜tÐ$ô
ˆð !'§¡ØØà 2Ø")ñð !,ó !
÷ 
ˆð ×Ñ 3Ò&Ø!EÀhÇmÁmÀ_ÐUˆIÜ× Ñ  Ô+Ü˜IÓ&Ð&à'Ÿ}™}›ˆØ)×-Ñ-¨f°bÓ9ˆ
ä×ÑÐ<¸jÔIØÐð%
ús   ‚ACÁC ÁA=Cc           	   ƒ  ó¤   K  — | j                  ||||t        |«      |xs d¬«      ƒ d{  –—† }| j                  ||¬«      ƒ d{  –—† }|S 7 Œ!7 Œ­w)a—  
        Upload a file to Gemini File Search store using resumable upload.

        Args:
            api_key: Gemini API key
            base_url: Base URL for Gemini API
            vector_store_id: File Search store name
            filename: Name of the file
            file_content: File content bytes
            content_type: MIME type

        Returns:
            File ID or document name
        zapplication/octet-stream)r!   r%   r    r'   Ú	file_sizer)   N)Ú
upload_urlr(   )Ú_initiate_resumable_uploadÚlenÚ_upload_file_content)	r   r!   r%   r    r'   r(   r)   rY   Úfile_ids	            r   r3   z/GeminiRAGIngestion._upload_to_file_search_store¨   st   è ø€ ð0 #'×"AÑ"AØØØ+ØÜ˜,Ó'Ø%ÒCÐ)Cð #Bó #
÷ 
ˆ
ð  $×8Ñ8Ø!Ø%ð  9ó  
÷ 
ˆð
 ˆð
øð
ús!   ‚(AªA«AÁAÁAÁAc              ƒ  óŠ  K  — |j                  dd«      }|› d|› d�}d|i}	| j                  }
|
rMt        |
t        «      r=|
j	                  d«      }|r*d|j	                  dd	«      |j	                  d
d«      dœi|	d<   t        t        t        t        t        f      dz  | j                  j	                  d«      «      }|r||	d<   ddt        |«      |d|dœ}t        j                  d|«       t        t        j                  ddi¬«      }|j                  ||	|¬«      ƒ d{  –—† }|j                   dvr/d|j"                  › �}t        j$                  |«       t'        |«      ‚t        j                  d|j(                  «       |j(                  j	                  d«      }|st'        d«      ‚t        j                  d|«       |S 7 Œ¡­w) zy
        Initiate a resumable upload session.

        Returns:
            Upload URL for the resumable session
        z/v1betarH   z/upload/v1beta/z:uploadToFileSearchStorer:   Úwhite_space_configÚwhiteSpaceConfigÚmax_tokens_per_chunki   Úmax_overlap_tokensi�  )ÚmaxTokensPerChunkÚmaxOverlapTokensÚchunkingConfigNÚcustom_metadataÚcustomMetadataÚ	resumableÚstartr@   )zX-Goog-Upload-ProtocolúX-Goog-Upload-Commandz#X-Goog-Upload-Header-Content-Lengthz!X-Goog-Upload-Header-Content-TyperA   rB   zInitiating resumable upload: %sr;   r<   r=   rC   ©rF   éÉ   zFailed to initiate upload: z&Initiate resumable upload response: %szx-goog-upload-urlz*No upload URL returned in response headerszGot upload URL: %s)ÚreplaceÚchunking_strategyÚ
isinstancer,   r+   r   Úlistr-   r   r*   r   rO   r	   r
   rI   rJ   rK   rL   rM   rN   rE   )r   r!   r%   r    r'   rX   r)   r"   rP   rQ   rn   r_   rf   rE   rR   rS   rT   rY   s                     r   rZ   z-GeminiRAGIngestion._initiate_resumable_uploadÑ   së  è ø€ ð$ #×*Ñ*¨9°bÓ9ˆØ �z °Ð0AÐAYÐZˆð 0=¸hÐ.Gˆð $(×#9Ñ#9ÐÙ¤Ð,=¼tÔ!DØ(9×(=Ñ(=Ð>RÓ(SÐÙ!à&Ø-?×-CÑ-CÐDZÐ\_Ó-`Ø,>×,BÑ,BÐCWÐY\Ó,]ñ)ð2�Ð-Ñ.ô "&Ü””cœ3�h‘Ñ  4Ñ'Ø×$Ñ$×(Ñ(Ð):Ó;ó"
ˆñ Ø-<ˆLÐ)Ñ*ð '2Ø%,Ü36°y³>Ø1=Ø.Ø%ñ
ˆô 	×ÑÐ>ÀÔDä.Ü-×1Ñ1Ø˜tÐ$ô
ˆð !'§¡ØØØð !,ó !
÷ 
ˆð ×Ñ zÑ1Ø!<¸X¿]¹]¸OÐLˆIÜ× Ñ  Ô+Ü˜IÓ&Ð&Ü×ÑÐEÀx×GWÑGWÔXà$×,Ñ,×0Ñ0Ð1DÓEˆ
ÙÜÐHÓIÐIä×ÑÐ1°:Ô>ØÐð#
ús   ‚DGÄGÄ B"Gc              ƒ  ó~  K  — t        t        |«      «      dddœ}t        j                  dt        |«      «       t	        t
        j                  ddi¬«      }|j                  |||¬«      ƒ d	{  –—† }|j                  d
vr/d|j                  › �}t        j                  |«       t        |«      ‚	 |j                  «       }|j                  dd«      xs" |j                  di «      j                  dd«      }t        j                  d|«       |S 7 Œ # t        $ r }	t        j                  d|	«       Y d	}	~	yd	}	~	ww xY w­w)z‹
        Upload file content to the resumable upload URL.

        Returns:
            File ID or document name from the response
        Ú0zupload, finalize)zContent-LengthzX-Goog-Upload-Offsetrj   z!Uploading file content (%s bytes)r;   g     Àr@r=   )ÚcontentrE   Nrk   zFailed to upload file: rG   rH   ÚfilezUpload complete. File ID: %sz#Could not parse upload response: %sÚuploaded)r-   r[   r   rO   r	   r
   rI   ÚputrK   rL   rM   rN   rD   r+   Úwarning)
r   rY   r(   rE   rR   rS   rT   rU   r]   Úes
             r   r\   z'GeminiRAGIngestion._upload_file_content  s6  è ø€ ô "¤# lÓ"3Ó4Ø$'Ø%7ñ
ˆô 	×ÑÐ@Ä#ÀlÓBSÔTä.Ü-×1Ñ1Ø˜uÐ%ô
ˆð !'§
¡
ØØ Øð !+ó !
÷ 
ˆð ×Ñ zÑ1Ø!8¸¿¹¸ÐHˆIÜ× Ñ  Ô+Ü˜IÓ&Ð&ð		Ø#+§=¡=£?ˆMà*×.Ñ.¨v°rÓ:Òk¸m×>OÑ>OÐPVÐXZÓ>[×>_Ñ>_Ð`fÐhjÓ>kˆGÜ× Ñ Ð!?ÀÔIØˆNð#
ùô$ ò 	Ü×"Ñ"Ð#HÈ!ÔLäûð	üs>   ‚A,D=Á.DÁ/AD=Â1AD ÄD=Ä	D:ÄD5Ä0D=Ä5D:Ä:D=)N)r   r   r   zRouter | None)r   ú	list[str]Úreturnúlist[list[float]] | None)r(   zbytes | Noner'   ú
str | Noner)   r|   r   ry   r4   r{   r5   r|   rz   ztuple[str | None, str | None])r!   r-   r%   r-   r&   r-   rz   r-   )r!   r-   r%   r-   r    r-   r'   r-   r(   Úbytesr)   r|   rz   r-   )r!   r-   r%   r-   r    r-   r'   r-   rX   Úintr)   r-   rz   r-   )rY   r-   r(   r}   rz   r-   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r8   r1   r3   rZ   r\   Ú__classcell__)r   s   @r   r   r      s{  ø„ ñð !%ð,à(ð,ð õ,ðàðð 
"óð( (,ð@/à"ð@/ð ð@/ð !ð	@/ð
 ð@/ð -ð@/ð %ð@/ð 
'ó@/ðD+àð+ð ð+ð ð	+ð
 
ó+ðZ'àð'ð ð'ð ð	'ð
 ð'ð ð'ð !ð'ð 
ó'ðRLàðLð ðLð ð	Lð
 ðLð ðLð ðLð 
óLð\,àð,ð ð,ð 
÷	,r   r   N)r‚   Ú
__future__r   Útypingr   r   r   r   Úlitellm._loggingr   Ú&litellm.llms.custom_httpx.http_handlerr	   r
   Ú litellm.llms.gemini.common_utilsr   Ú$litellm.rag.ingestion.base_ingestionr   Úlitellmr   Úlitellm.types.ragr   r   r   r   r   ú<module>rŒ      s=   ðñõ #ç 2Ó 2å +÷õ =Ý AáÝÝ2ôrÐ)õ rr   