Ë
    ÛJêjµ  ã                  óâ   — d Z ddlmZ ddlZddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej                  e«      Z ed¬	«       G d
„ d«      «       Ze G d„ d«      «       Z G d„ d«      Zg d¢Zy)zõDeduplication engine for the knowledge plane.

Provides multi-level deduplication:
1. Content-hash deduplication (SHA-256 exact match)
2. Semantic deduplication (cosine similarity > threshold)
3. URL deduplication (same URL + same content hash)
é    )ÚannotationsN)Ú	dataclassÚfield)ÚKnowledgeDocument)ÚEmbeddingService)ÚKnowledgeVectorStoreT)Úfrozenc                  ó:   — e Zd ZU dZded<   ded<   ded<   ded<   y)	ÚDuplicateCandidatez1A potential duplicate found during deduplication.ÚstrÚdocument_idÚ
source_uriÚfloatÚ
similarityÚ
match_typeN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    úP/home/oem/CyberBrain26-unified/src/cyberbrain26/knowledge_plane/deduplication.pyr   r      s   … á;àÓØƒOØÓØ„Or   r   c                  óF   — e Zd ZU dZded<    ee¬«      Zded<   dZded	<   y
)ÚDeduplicationResultz Result of a deduplication check.ÚboolÚis_duplicate)Údefault_factoryzlist[DuplicateCandidate]Ú
candidatesÚregisterr   Úrecommended_actionN)	r   r   r   r   r   r   Úlistr   r!   r   r   r   r   r       s&   … á*àÓÙ+0ÀÔ+F€JÐ(ÓFØ(Ð˜Ô(r   r   c                  ó\   — e Zd ZdZddddœ	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 d	d„Z	 	 	 	 	 	 d
d„Zy)ÚDeduplicationEnginez2Multi-level deduplication for knowledge documents.Ngffffffî?©Úembedding_serviceÚvector_storeÚsemantic_thresholdc               ó.   — || _         || _        || _        y )Nr%   )Úselfr&   r'   r(   s       r   Ú__init__zDeduplicationEngine.__init__,   s   € ð "3ˆÔØ(ˆÔØ"4ˆÕr   c           
     ó  — t        j                  |«      j                  «       }g }|D ]D  }|j                  |k(  sŒ|j	                  t        |j                  |j                  dd¬«      «       ŒF |rt        d|d¬«      S |D ]T  }|j                  |k(  sŒ|j                  |k(  sŒ#|j	                  t        |j                  |j                  dd¬«      «       ŒV |rt        d|d¬«      S | j                  �Ê| j                  �¾	 |j                  d	d
¬«      }| j                  j                  |g«      }|rŒ| j                  j                  |d   d¬«      }	|	D ]g  }
|
j                  dd«      }d|z
  }|| j                  k\  sŒ*|j	                  t        |
j                  dd«      |
j                  dd«      |d¬«      «       Œi |rt        d|d¬«      S t        dd¬«      S # t         $ r }t"        j%                  d|«       Y d}~ŒAd}~ww xY w)z6Check if content is a duplicate of existing documents.ç      ð?Ú
exact_hash©r   r   r   r   TÚskip©r   r   r!   ÚurlNzutf-8Úreplace)Úerrorsr   é   )ÚlimitÚ	_distanceÚidÚ ÚpathÚsemanticz%Semantic similarity check skipped: %sÚmergeFr    )r   r!   )ÚhashlibÚsha256Ú	hexdigestÚcontent_sha256Úappendr   r   r   r   r&   r'   ÚdecodeÚembed_textsÚsearchÚgetr(   Ú	ExceptionÚloggerÚdebug)r*   Úcontentr   Úexisting_documentsÚcontent_hashr   ÚdocÚtextÚvectorsÚsimilarÚresultÚdistancer   Úexcs                 r   Úcheck_duplicatez#DeduplicationEngine.check_duplicate7   s  € ô —~‘~ gÓ.×8Ñ8Ó:ˆØ/1ˆ
ð &ò 		ˆCØ×!Ñ! \Ó1Ø×!Ñ!Ü&Ø$'§O¡OØ#&§>¡>Ø#&Ø#/ô	õð		ñ Ü&Ø!Ø%Ø#)ôð ð &ò 		ˆCØ�~‰~ Ó+°×0BÑ0BÀlÓ0RØ×!Ñ!Ü&Ø$'§O¡OØ#&§>¡>Ø#&Ø#(ô	õð		ñ Ü&Ø!Ø%Ø#)ôð ð ×!Ñ!Ð-°$×2CÑ2CÐ2OðKØ—~‘~ g°i�~Ó@�Ø×0Ñ0×<Ñ<¸d¸VÓD�ÙØ"×/Ñ/×6Ñ6°w¸q±zÈÐ6ÓK�GØ")ò ˜à#)§:¡:¨k¸3Ó#?˜Ø%(¨8¡^˜
Ø%¨×)@Ñ)@Ó@Ø&×-Ñ-Ü 2Ø06·
±
¸4ÀÓ0DØ/5¯z©z¸&À"Ó/EØ/9Ø/9ô	!"õðñ  Ü&Ø!Ø%Ø#*ôð ô #°È*ÔUÐUøô ò KÜ—‘ÐDÀc×JÑJûðKús    ÄA<G Å>A G Ç	HÇ$G?Ç?Hc                ó¢  — g }|D �ch c]  }|j                   ’Œ }}t        |«      D ]   \  }\  }}t        j                  |«      j	                  «       }	|	|v r.|j                  |t        dt        d|dd¬«      gd¬«      f«       Œ^| j                  |||«      }
|
j                  s|j                  |	«       |j                  ||
f«       Œ¢ |S c c}w )z&Check a batch of items for duplicates.Tr9   r-   r.   r/   r0   r1   )r@   Ú	enumerater=   r>   r?   rA   r   r   rS   r   Úadd)r*   ÚitemsrJ   ÚresultsrL   Úseen_hashesÚidxrI   r   rK   rP   s              r   Údeduplicate_batchz%DeduplicationEngine.deduplicate_batch‡   sè   € ð ˆØ?QÖ R¸ ×!3Ó!3Ð RˆÐ RÜ*3°EÓ*:ò 	*Ñ&ˆCÑ&�'˜:Ü"Ÿ>™>¨'Ó2×<Ñ<Ó>ˆLØ˜{Ñ*Ø—‘àÜ+Ø)-ä 2Ø02Ø/9Ø/2Ø/;ô	!"ð(ð 06ôðôð" Ø×)Ñ)¨'°:Ð?QÓRˆFØ×&Ò&Ø—‘ Ô-Ø�N‰N˜C ˜=Õ)ð1	*ð2 ˆùò5 !Ss   ‡C)r&   zEmbeddingService | Noner'   zKnowledgeVectorStore | Noner(   r   ÚreturnÚNone)rI   Úbytesr   r   rJ   úlist[KnowledgeDocument]r\   r   )rW   zlist[tuple[bytes, str]]rJ   r_   r\   z%list[tuple[int, DeduplicationResult]])r   r   r   r   r+   rS   r[   r   r   r   r$   r$   )   s”   „ Ù<ð
 6:Ø48Ø$(ñ	5ð 3ð	5ð 2ð		5ð
 "ð	5ð 
ó	5ðNVàðNVð ðNVð 4ð	NVð
 
óNVð`!à&ð!ð 4ð!ð 
/ô	!r   r$   )r$   r   r   )r   Ú
__future__r   r=   ÚloggingÚdataclassesr   r   Ú&cyberbrain26.knowledge_plane.contractsr   Ú.cyberbrain26.knowledge_plane.embedding_servicer   Ú)cyberbrain26.knowledge_plane.vector_storer   Ú	getLoggerr   rG   r   r   r$   Ú__all__r   r   r   ú<module>rh      s   ðñõ #ã Û ß (å DÝ KÝ Jà	ˆ×	Ñ	˜8Ó	$€ñ �$Ô÷ð ó ðð ÷)ð )ó ð)÷ñ òD O�r   