Ë
    ÛJêjÐ&  ã                  ó$  — d Z ddlmZ ddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlmZ ddlZddlmZmZmZmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZ  ed¬«       G d„ d«      «       Z ed¬«       G d„ d«      «       Z G d„ d«      Z g d¢Z!y)zÛNetwork source adapter for deepweb/clearnet content ingestion.

Fetches content from approved URLs, extracts text, and registers it in the
knowledge plane.  All network access goes through the EgressGate (fail-closed).
é    )ÚannotationsN)Ú	dataclassÚfield)ÚAny)Úurlsplit)ÚDocumentRegistrationÚDocumentTrustÚKnowledgeDocumentÚKnowledgeSourceÚSensitivity)ÚSQLiteKnowledgeRepository)ÚAuditEventTypeÚAuditLog)ÚDataClassificationÚ
EgressGateÚEgressRequestT)Úfrozenc                  ó^   — e Zd ZU dZded<   ded<   ded<   ded<    ed	¬
«      Zded<   dd„Zy)ÚFetchResultzResult of a single URL fetch.ÚstrÚurlÚbytesÚcontentÚ
media_typeÚintÚstatus_codeF)ÚinitÚcontent_sha256c                óˆ   — t         j                  | dt        j                  | j                  «      j                  «       «       y )Nr   )ÚobjectÚ__setattr__ÚhashlibÚsha256r   Ú	hexdigest)Úselfs    úQ/home/oem/CyberBrain26-unified/src/cyberbrain26/knowledge_plane/network_source.pyÚ__post_init__zFetchResult.__post_init__-   s+   € Ü×Ñ˜4Ð!1´7·>±>À$Ç,Á,Ó3O×3YÑ3YÓ3[Õ\ó    N)ÚreturnÚNone)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r   r   r'   © r(   r&   r   r   #   s/   … á'à	ƒHØƒNØƒOØÓÙ UÔ+€N�CÓ+ô]r(   r   c                  ó˜   — e Zd ZU dZded<    ee¬«      Zded<   ej                  Z
ded<   ej                  Zd	ed
<   dZded<   dZded<   y)ÚNetworkSourceConfigz#Configuration for a network source.r   Ú	source_id)Údefault_factoryzfrozenset[str]Úallowed_hostsr	   Útrustr   Úsensitivityg      >@ÚfloatÚtimeouti€–˜ r   Úmax_content_bytesN)r+   r,   r-   r.   r/   r   Ú	frozensetr5   r	   ÚEXTERNALr6   r   ÚPERSONALr7   r9   r:   r0   r(   r&   r2   r2   1   sN   … á-àƒNÙ$)¸)Ô$D€M�>ÓDØ(×1Ñ1€Eˆ=Ó1Ø*×3Ñ3€K�Ó3Ø€GˆUÓØ'Ð�sÔ'r(   r2   c                  óœ   — e Zd ZdZdddœ	 	 	 	 	 	 	 	 	 dd„Zedd„«       Zddœdd„Zddd	œ	 	 	 	 	 	 	 dd
„Zdd„Z	edd„«       Z
edd„«       Zy)ÚNetworkSourceAdapteru  Fetch and register content from approved network URLs.

    All outbound requests are authorized through the EgressGate before any
    bytes are transferred.  The adapter never persists raw bytes to the
    knowledge plane â€” only metadata and a SHA-256 digest are stored.
    N)Ú	audit_logÚconfigc               óX   — || _         || _        || _        |xs t        d¬«      | _        y )NÚnetwork)r3   )Ú
repositoryÚegress_gater@   r2   rA   )r%   rD   rE   r@   rA   s        r&   Ú__init__zNetworkSourceAdapter.__init__E   s,   € ð %ˆŒØ&ˆÔØ"ˆŒØÒHÔ 3¸iÔ Hˆ�r(   c                óÔ   — 	 t        j                  | «      }|j                  xs6 |j                  xs( |j                  xs |j
                  xs |j                  S # t        $ r Y yw xY w)zÂReturn True if host is a private/reserved/link-local IP address.

        This prevents SSRF via redirects to cloud metadata endpoints,
        localhost, or internal RFC1918 addresses.
        F)Ú	ipaddressÚ
ip_addressÚ
is_privateÚis_loopbackÚis_link_localÚis_reservedÚis_multicastÚ
ValueError)ÚhostÚips     r&   Ú_is_private_hostz%NetworkSourceAdapter._is_private_hostR   sm   € ð
	Ü×%Ñ% dÓ+ˆBà—‘ò #Ø—>‘>ò#à×#Ñ#ò#ð —>‘>ò#ð —?‘?ðøô ò 	Ùð	ús   ‚AA Á	A'Á&A'zknowledge-ingester©Úagent_idc               ó€  — | j                  |«       t        |«      }|j                  xs dj                  «       }| j                  j
                  r'|| j                  j
                  vrt        d|› d�«      ‚| j                  |«      rt        d|› d�«      ‚t        j                  |j                  «       «      j                  «       }| j                  j                  t        j                  k(  rt        j                  nt        j                   }t#        |d|t%        |h«      |¬«      }| j&                  j)                  |«       |}d}	d}
t+        |	d	z   «      D �]‘  }t-        j.                  | j                  j0                  d
¬«      5 }|j3                  |«      }
|
j5                  «        ddd«       |
j6                  �r,|
j8                  j3                  d«      �r|
j8                  d   }| j                  |«       t        |«      }|j                  xs dj                  «       }| j                  j
                  r'|| j                  j
                  vrt        d|› d�«      ‚| j                  |«      rt        d|› d�«      ‚t        j                  |j                  «       «      j                  «       }t#        |d|t%        |h«      |¬«      }| j&                  j)                  |«       |}�Œ’ n t;        d|	› d�«      ‚|
j<                  }t?        |«      | j                  j@                  kD  r.tC        dt?        |«      › d| j                  j@                  › �«      ‚|
j8                  j3                  dd«      }|jE                  d«      d   jG                  «       }|r|dk(  rtI        jJ                  |«      \  }}|r|}tM        ||||
jN                  ¬«      }| jP                  �R| jP                  jS                  tT        jV                  ||d|d|jX                  ||
jN                  t?        |«      dœ¬«       |S # 1 sw Y   �Œ‹xY w)zÚFetch content from a URL through the egress gate.

        Automatic redirects are disabled. Every redirect target is re-validated
        against the allowlist and blocked if it resolves to a private address.
        Ú zhost 'z"' is not in the allowed hosts listz.' is a private/internal address; access deniedznetwork-source)rT   Úprovider_idÚpayload_sha256ÚclassificationsÚdestinationé   Né   F)r9   Úfollow_redirectsÚlocationzredirect target 'ztoo many redirects (max ú)zcontent size z exceeds max zcontent-typezapplication/octet-streamú;r   )r   r   r   r   Únetwork_source_fetchedÚallowed)r   r   r   Úcontent_length)ÚactorrT   ÚactionÚresourceÚresultÚmetadata)-Ú_validate_urlr   ÚhostnameÚlowerrA   r5   ÚPermissionErrorrR   r"   r#   Úencoder$   r7   r   ÚPUBLICr   r=   r   r;   rE   Ú	authorizeÚrangeÚhttpxÚClientr9   ÚgetÚraise_for_statusÚis_redirectÚheadersÚRuntimeErrorr   Úlenr:   rO   ÚsplitÚstripÚ	mimetypesÚ
guess_typer   r   r@   Úrecordr   ÚDATA_ACCESSr   )r%   r   rT   ÚparsedrP   rX   ÚclassificationÚrequestÚcurrent_urlÚmax_redirectsÚresponseÚ_ÚclientÚredirect_urlÚr_parsedÚr_hostr   r   Úguessedrg   s                       r&   ÚfetchzNetworkSourceAdapter.fetche   sß  € ð 	×Ñ˜3Ôä˜#“ˆØ—‘Ò% 2×,Ñ,Ó.ˆØ�;‰;×$Ò$¨°T·[±[×5NÑ5NÑ)NÜ! F¨4¨&Ð0RÐ"SÓTÐTà× Ñ  Ô&Ü! F¨4¨&Ð0^Ð"_Ó`Ð`ô !Ÿ™¨¯
©
«Ó5×?Ñ?ÓAˆð �{‰{×&Ñ&¬+×*<Ñ*<Ò<ô ×%Ò%ä#×,Ñ,ð 	ô
  ØØ(Ø)Ü% ~Ð&6Ó7Øô
ˆð 	×Ñ×"Ñ" 7Ô+ð ˆØˆØˆä�} qÑ(Ó)ó  	LˆAÜ—‘ d§k¡k×&9Ñ&9ÈEÔRð ,ÐV\Ø!Ÿ:™: kÓ2�Ø×)Ñ)Ô+÷,ð ×#Ó#¨×(8Ñ(8×(<Ñ(<¸ZÕ(Hà'×/Ñ/°
Ñ;�Ø×"Ñ" <Ô0ä# LÓ1�Ø"×+Ñ+Ò1¨r×8Ñ8Ó:�Ø—;‘;×,Ò,°¸t¿{¹{×?XÑ?XÑ1XÜ)Ð,=¸f¸XÐEgÐ*hÓiÐiØ×(Ñ(¨Ô0Ü)Ð,=¸f¸XÐEsÐ*tÓuÐuô ")§¡°×0CÑ0CÓ0EÓ!F×!PÑ!PÓ!R�Ü'Ø%Ø 0Ø#1Ü$-¨~Ð.>Ó$?Ø ,ô�ð × Ñ ×*Ñ*¨7Ô3à*’áð= 	Lô@ Ð!9¸-¸ÈÐJÓKÐKà×"Ñ"ˆÜˆw‹<˜$Ÿ+™+×7Ñ7Ò7ÜØ¤ G£˜~¨]¸4¿;¹;×;XÑ;XÐ:YÐZóð ð ×%Ñ%×)Ñ)¨.Ð:TÓUˆ
Ø×%Ñ% cÓ*¨1Ñ-×3Ñ3Ó5ˆ
á˜ZÐ+EÒEÜ"×-Ñ-¨cÓ2‰JˆG�QÙØ$�
ä °gÈ*Ðbj×bvÑbvÔwˆà�>‰>Ð%Ø�N‰N×!Ñ!Ü×*Ñ*ØØ!Ø/Ø$Ø à&,×&;Ñ&;Ø",Ø#+×#7Ñ#7Ü&)¨'£lñ	ð "ô ð ˆ÷C,ñ ,ús   Æ"P3Ð3P=	)rT   rh   c               óÞ  — | j                  ||¬«      }t        | j                  j                  || j                  j                  | j                  j
                  d|t        |«      j                  xs ddœ¬«      }t        ||j                  |j                  |t        d«      j                  «       |j                  |j                  dœ|xs i ¥¬«      }| j                  j!                  |«      S )	z3Fetch a URL and register it in the knowledge plane.rS   TrV   )r   rP   )r3   Ú
source_urir6   r7   Úapprovedrh   Útime)r   Ú
fetched_atr   r   )Úsourcer   r   rh   )r‹   r   rA   r3   r6   r7   r   rj   r   r   r   Ú
__import__r�   r   r   rD   Úregister)r%   r   rT   rh   rg   r‘   Úregistrations          r&   Úregister_urlz!NetworkSourceAdapter.register_urlÐ   sÓ   € ð —‘˜C¨(�Ó3ˆä Ø—k‘k×+Ñ+ØØ—+‘+×#Ñ#ØŸ™×/Ñ/ØØ ¬(°3«-×*@Ñ*@Ò*FÀBÑGô
ˆô ,ØØ—N‘NØ×(Ñ(àÜ(¨Ó0×5Ñ5Ó7Ø%×1Ñ1Ø"(×"7Ñ"7ñ	ð
 ’>˜rðô	
ˆð �‰×'Ñ'¨Ó5Ð5r(   c                ó:  — d|j                   v r| j                  |j                  «      S d|j                   v r|j                  j                  dd¬«      S d|j                   v r|j                  j                  dd¬«      S |j                  j                  dd¬«      S )z<Extract plain text from fetched content based on media type.z	text/htmlzapplication/jsonúutf-8Úreplace©Úerrorsz
text/plain)r   Ú_extract_htmlr   Údecode)r%   rg   s     r&   Úextract_textz!NetworkSourceAdapter.extract_textò   s�   € à˜&×+Ñ+Ñ+Ø×%Ñ% f§n¡nÓ5Ð5Ø ×!2Ñ!2Ñ2Ø—>‘>×(Ñ(¨¸Ð(ÓCÐCØ˜6×,Ñ,Ñ,Ø—>‘>×(Ñ(¨¸Ð(ÓCÐCà�~‰~×$Ñ$ W°YÐ$Ó?Ð?r(   c                ó„  — | j                  dd¬«      }t        j                  dd|t        j                  t        j                  z  ¬«      }t        j                  dd|t        j                  t        j                  z  ¬«      }t        j                  dd|«      }t        j                  d	d|«      j                  «       }|S )
z:Simple HTML to text extraction (no external dependencies).r—   r˜   r™   z<script[^>]*>.*?</script>ú )Úflagsz<style[^>]*>.*?</style>z<[^>]+>z\s+)rœ   ÚreÚsubÚDOTALLÚ
IGNORECASErz   )r   Útexts     r&   r›   z"NetworkSourceAdapter._extract_htmlý   s�   € ð �~‰~˜g¨iˆ~Ó8ˆä�v‰vÐ2°C¸ÄRÇYÁYÔQS×Q^ÑQ^ÑE^Ô_ˆÜ�v‰vÐ0°#°tÄ2Ç9Á9ÌrÏ}É}ÑC\Ô]ˆä�v‰v�j # tÓ,ˆä�v‰v�f˜c 4Ó(×.Ñ.Ó0ˆØˆr(   c                ó”   — t        | «      }|j                  dvrt        d|j                  › �«      ‚|j                  st        d«      ‚y )N)ÚhttpÚhttpszunsupported URL scheme: zURL must have a hostname)r   ÚschemerO   rj   )r   r   s     r&   ri   z"NetworkSourceAdapter._validate_url
  sG   € ä˜#“ˆØ�=‰=Ð 1Ñ1ÜÐ7¸¿¹°ÐGÓHÐHØ�ŠÜÐ7Ó8Ð8ð r(   )
rD   r   rE   r   r@   zAuditLog | NonerA   zNetworkSourceConfig | Noner)   r*   )rP   r   r)   Úbool)r   r   rT   r   r)   r   )r   r   rT   r   rh   zdict[str, Any] | Noner)   ztuple[KnowledgeDocument, bool])rg   r   r)   r   )r   r   r)   r   )r   r   r)   r*   )r+   r,   r-   r.   rF   ÚstaticmethodrR   r‹   r•   r�   r›   ri   r0   r(   r&   r?   r?   =   sÍ   „ ñð &*Ø-1ñIà-ðIð  ð	Ið
 #ðIð +ðIð 
óIð òó ðð$ 2Fõ ið^ -Ø*.ñ 6àð 6ð ð	 6ð
 (ð 6ð 
(ó 6óD	@ð ò
ó ð
ð ò9ó ñ9r(   r?   )r   r?   r2   )"r.   Ú
__future__r   r"   rH   r{   r¡   Údataclassesr   r   Útypingr   Úurllib.parser   rq   Ú&cyberbrain26.knowledge_plane.contractsr   r	   r
   r   r   Ú'cyberbrain26.knowledge_plane.repositoryr   Ú!cyberbrain26.security_plane.auditr   r   Ú"cyberbrain26.security_plane.egressr   r   r   r   r2   r?   Ú__all__r0   r(   r&   ú<module>rµ      s˜   ðñõ #ã Û Û Û 	ß (Ý Ý !ã ÷õ õ Nß F÷ñ ñ �$Ô÷
]ð 
]ó ð
]ñ �$Ô÷(ð (ó ð(÷S9ñ S9òl I�r(   