Ë
    FìjI5  ã                  óè   — d Z ddlmZ ddlZddlZddlZddlmZmZ ddl	m
Z
 ddlmZ 	 ddlZddlmZ  ej"                  e«      Z ed¬	«       G d
„ d«      «       Z G d„ d«      Zy# e$ r dZY ŒBw xY w)uÈ  Markdown-based hierarchical knowledge store.

Stores classified documents as human-readable Markdown files with YAML frontmatter.
Maintains hierarchy.md tree for agent navigation (per ByteRover and NeuRAG research).

Storage layout:
    knowledge_docs/
    â”œâ”€â”€ _index.md              # root overview tree
    â”œâ”€â”€ ai/
    â”‚   â”œâ”€â”€ security/
    â”‚   â”‚   â””â”€â”€ audit_protocols.md
    â”‚   â””â”€â”€ architecture/
    â”‚       â””â”€â”€ design.md
    â”œâ”€â”€ research/
    â”œâ”€â”€ notes/                 # uncategorized / low-confidence
    â””â”€â”€ reports/

Frontmatter fields:
    topic, subtopic, tags[], summary, confidence (0-1), entities[],
    intent ("actionable"|"reference"|"idea"), created, updated, sha256

Research references:
- ByteRover (arXiv:2604.01599) â€” Context Tree as markdown hierarchy
- NeuRAG â€” trigger-navigable hierarchy via markdown structure
- secondmem â€” markdown files with tags for agent consumption
é    )ÚannotationsN)Ú	dataclassÚfield)ÚPath)ÚAny)ÚClassificationResultT)Úfrozenc                  óV   — e Zd ZU dZded<   ded<   ded<   ded<    ee¬«      Zd	ed
<   y)ÚHierarchyNodezaA node in the knowledge hierarchy tree.

    Used by hierarchy.md generation and navigation.
    ÚstrÚnameÚpathÚintÚlevelÚdocument_count)Údefault_factoryz	list[str]ÚtagsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__r   Úlistr   © ó    úQ/home/oem/CyberBrain26-unified/src/cyberbrain26/knowledge_plane/markdown_store.pyr   r   3   s-   … ñð
 ƒIØ
ƒIØƒJØÓÙ¨DÔ1€Dˆ)Ô1r   r   c                  ó¶   — e Zd ZdZddd„Zdd„Zdd„Z	 	 	 	 	 	 	 	 dd„Zdd„Zdd„Z		 d	 	 	 	 	 	 	 	 	 	 	 dd	„Z
dd
„Zddd„Zdd„Zdd„Zdd„Zdd„Zdd„Zd d„Zy)!ÚMarkdownStorea±  Hierarchical Markdown knowledge store.

    Writes classified documents as Markdown with YAML frontmatter.
    Maintains hierarchy.md tree for agent navigation.

    Research-backed design:
    - Hierarchical folder structure (per ByteRover Context Tree)
    - YAML frontmatter for structured metadata
    - SHA-256 dedup gate (per Grimore-MD two-tier change detection)
    - hierarchy.md maintained in sync with filesystem tree
    c                óú   — t        |«      j                  «       j                  «       | _        | j                  j	                  dd¬«       || _        | j                  dz  | _        i | _        | j                  «        y )NT©ÚparentsÚexist_okú	_index.md)	r   Ú
expanduserÚresolveÚrootÚmkdirÚauto_classifyÚ_index_pathÚ_document_indexÚ_rebuild_index)Úselfr&   r(   s      r   Ú__init__zMarkdownStore.__init__S   s`   € ä˜“J×)Ñ)Ó+×3Ñ3Ó5ˆŒ	Ø�	‰	�‰ ¨tˆÔ4Ø*ˆÔØŸ9™9 {Ñ2ˆÔØ:<ˆÔØ×ÑÕr   c                óÌ  — | j                   j                  «        | j                  j                  «       sy| j                  j	                  d«      D ]d  }|j
                  dk(  rŒ	 | j                  |«      }|r>t        |j                  | j                  «      «      |dœ| j                   |j                  <   Œf y# t        $ r!}t        j                  d||«       Y d}~Œ�d}~ww xY w)z=Rebuild internal document index from existing markdown files.Nú*.mdr#   ©r   Úmetadataz$Skipping broken markdown file %s: %s)r*   Úclearr&   ÚexistsÚrglobr   Ú_read_frontmatterr   Úrelative_toÚstemÚ	ExceptionÚloggerÚwarning)r,   Úmd_fileÚdoc_metaÚexcs       r   r+   zMarkdownStore._rebuild_index`   sÇ   € à×Ñ×"Ñ"Ô$Ø�y‰y×ÑÔ!ØØ—y‘y—‘ vÓ.ò 	UˆGØ�|‰|˜{Ò*ØðUØ×1Ñ1°'Ó:�Ùä # G×$7Ñ$7¸¿	¹	Ó$BÓ CØ$,ñ:�D×(Ñ(¨¯©Ñ6øñ	Uøô ò UÜ—‘ÐEÀwÐPS×TÑTûðUús   Á%AB9Â9	C#ÃCÃC#c                ó@  — |j                  d¬«      }|j                  d«      sy|j                  dd«      }|dk(  ry|d| }t        �	 t        j                  |«      xs i S i }|j                  «       D ]ë  }d	|v sŒ|j                  d	«      \  }}	}
|j                  «       }|
j                  «       j                  d
«      j                  d«      }
|
j                  d«      r{|
j                  d«      rj|
dd j                  «       }|rO|j                  d«      D �cg c]0  }|j                  «       j                  d
«      j                  d«      ‘Œ2 c}||<   Œág ||<   Œç|
||<   Œí |S # t        j
                  $ r!}t        j                  d||«       Y d}~yd}~ww xY wc c}w )z+Read YAML frontmatter from a Markdown file.úutf-8©Úencodingú---
Nú
---
é   éÿÿÿÿzYAML parse error in %s: %sú:ú"ú'ú[ú]é   ú,)Ú	read_textÚ
startswithÚfindÚyamlÚ	safe_loadÚ	YAMLErrorr9   r:   Ú
splitlinesÚ	partitionÚstripÚendswithÚsplit)r,   r   ÚcontentÚend_idxÚfrontmatter_textr=   ÚresultÚlineÚkeyÚ_ÚvalueÚinnerÚxs                r   r5   zMarkdownStore._read_frontmatterv   s‚  € à—.‘.¨'�.Ó2ˆØ×!Ñ! 'Ô*ØØ—,‘,˜y¨!Ó,ˆØ�bŠ=ØØ" 1 WÐ-ÐÜÐðÜ—~‘~Ð&6Ó7Ò=¸2Ð=ð
 "$ˆØ$×/Ñ/Ó1ò 	(ˆDØ�dŠ{Ø $§¡¨sÓ 3‘��Q˜Ø—i‘i“k�ØŸ™›×+Ñ+¨CÓ0×6Ñ6°sÓ;�Ø×#Ñ# CÔ(¨U¯^©^¸CÔ-@à! ! B˜K×-Ñ-Ó/�EÙØPU×P[ÑP[Ð\_ÓP`Ö&aÈ1 q§w¡w£y§¡°sÓ';×'AÑ'AÀ#Õ'FÒ&a˜˜sšà&(˜˜sšà"'�F˜3’Kð	(ð ˆøô' —>‘>ò Ü—‘Ð;¸TÀ3ÔGÜûðüò 'bs   Á	E$ Ä5FÅ$FÅ7FÆFc                óz  — |j                   j                  dd¬«       dg}t        �8t        j                  |dd¬«      j	                  «       }|j                  |«       no|j                  «       D ]\  \  }}t        |t        «      r0dj                  d„ |D «       «      }|j                  |› d	|› d
�«       ŒF|j                  |› d|› d�«       Œ^ |j                  d«       |j                  d«       |j                  |«       |j                  dj                  |«      dz   d¬«       t        j                  d|«       y)z0Write a Markdown document with YAML frontmatter.Tr    z---NF)Údefault_flow_styleÚallow_unicodez, c              3  ó(   K  — | ]
  }d |› d �–— Œ y­w)rG   Nr   )Ú.0Úvs     r   ú	<genexpr>z0MarkdownStore._write_document.<locals>.<genexpr>¦   s   è ø€ Ò%>°1¨¨!¨¨A¤hÑ%>ùs   ‚z: [rJ   z: "rG   Ú ú
r?   r@   zWrote markdown document: %s)Úparentr'   rP   ÚdumprU   ÚappendÚitemsÚ
isinstancer   ÚjoinÚ
write_textr9   Údebug)	r,   r   rX   ÚfrontmatterÚlinesÚ	yaml_textr]   r_   rn   s	            r   Ú_write_documentzMarkdownStore._write_document—   s  € ð 	�‰×Ñ $°ÐÔ6Ø!˜7ˆÜÐÜŸ	™	 +À%ÐW[Ô\×bÑbÓdˆIØ�L‰L˜Õ#à)×/Ñ/Ó1ò 6‘
��UÜ˜e¤TÔ*Ø ŸI™IÑ%>¸Ô%>Ó>�EØ—L‘L C 5¨¨E¨7°!Ð!4Õ5à—L‘L C 5¨¨E¨7°!Ð!4Õ5ð6ð 	�‰�UÔØ�‰�RÔØ�‰�WÔØ�‰˜Ÿ	™	 %Ó(¨4Ñ/¸'ˆÔBÜ�‰Ð2°DÕ9r   c                ó  ‡— t        j                  «       }|j                  d«      5 Št        ˆfd„d«      D ]  }|j	                  |«       Œ 	 ddd«       |j                  «       S # 1 sw Y   |j                  «       S xY w)z+Return SHA-256 hex digest of file contents.Úrbc                 ó&   •— ‰ j                  d«      S )Ni   )Úread)Úfs   €r   ú<lambda>z,MarkdownStore._file_sha256.<locals>.<lambda>¸   s   ø€  a§f¡f¨U£m€ r   r   N)ÚhashlibÚsha256ÚopenÚiterÚupdateÚ	hexdigest)r,   r   ÚhÚchunkr{   s       @r   Ú_file_sha256zMarkdownStore._file_sha256´   sj   ø€ ä�N‰NÓˆØ�Y‰Y�t‹_ð 	  ÜÓ3°SÓ9ò  �Ø—‘˜•ñ ÷	 ð �{‰{‹}Ð÷	 ð �{‰{‹}Ðús   §&A&Á&A>c                óø   — | j                  |«      }| j                  j                  d«      D ]<  }|j                  dk(  rŒ	 | j	                  |«      }|r|j                  d«      |k(  r yŒ> y# t        $ r Y ŒKw xY w)zHCheck if a file should be processed (not already stored with same hash).r/   r#   r~   FT)r…   r&   r4   r   r5   Úgetr8   )r,   Úfilepathrƒ   Údoc_pathr<   s        r   Úshould_processzMarkdownStore.should_process¼   s‚   € à×Ñ˜hÓ'ˆØŸ	™	Ÿ™¨Ó/ò 	ˆHØ�}‰} Ò+ØðØ×1Ñ1°(Ó;�Ù §¡¨XÓ 6¸!Ò ;Ù øð	ð øô ò Ùðús   Á'A-Á-	A9Á8A9Nc                ó  — |rt        |«      j                  }n-|j                  dd j                  «       j	                  dd«      }dj                  d„ |D «       «      j                  d«      }| j                  |j                  z  }||j                  z  }	|	|› d�z  }
t        j                  «       }|j                  |j                  |j                  |j                  |j                  |j                  |j                  ||t        j                  |j!                  d	«      «      j#                  «       ||d
œ}| j%                  |
||«       t'        |
j)                  | j                  «      «      |dœ| j*                  |<   | j-                  «        |
S )zµStore a classified document as Markdown with frontmatter.

        Writes to:
            {root}/{topic}/{subtopic}/{slug}.md

        Returns the path to the written file.
        Né   ú r^   ri   c              3  óh   K  — | ]*  }|j                  «       s|d v r|j                  «       nd–— Œ, y­w)z-_ú-N)ÚisalnumÚlower)rf   Úcs     r   rh   z1MarkdownStore.store_classified.<locals>.<genexpr>ã   s)   è ø€ ÒTÈ! A§I¡I¤K°1¸±9�q—w‘w”yÀ#ÓEÑTùs   ‚02r�   z.mdr?   )ÚtopicÚsubtopicr   ÚsummaryÚ
confidenceÚentitiesÚintentÚcreatedÚupdatedr~   Ú	source_idÚ
source_urir0   )r   r7   r•   rU   Úreplacerp   r&   r“   r”   Útimer   r–   r—   r˜   r}   r~   Úencoder‚   rv   r   r6   r*   Ú_rebuild_hierarchy)r,   ÚclassificationrX   rœ   r›   Úfilenamer7   ÚslugÚ	topic_dirÚsubtopic_dirÚtarget_pathÚnowrs   s                r   Ústore_classifiedzMarkdownStore.store_classifiedÎ   sg  € ñ  Ü˜“>×&Ñ&‰Dà!×)Ñ)¨#¨2Ð.×4Ñ4Ó6×>Ñ>¸sÀCÓHˆDà�w‰wÑTÈtÔTÓT×ZÑZÐ[^Ó_ˆð —I‘I × 4Ñ 4Ñ4ˆ	Ø  >×#:Ñ#:Ñ:ˆØ"¨ v¨S \Ñ1ˆô �i‰i‹kˆà#×)Ñ)Ø&×/Ñ/Ø"×'Ñ'Ø%×-Ñ-Ø(×3Ñ3Ø&×/Ñ/Ø$×+Ñ+ØØÜ—n‘n W§^¡^°GÓ%<Ó=×GÑGÓIØ"Ø$ñ'
ˆð 	×Ñ˜[¨'°;Ô?ô ˜×/Ñ/°·	±	Ó:Ó;Ø#ñ&
ˆ×Ñ˜TÑ"ð 	×ÑÔ!àÐr   c                óx  — ddg}|j                  d| j                  j                  › d�«       |j                  d«       i }| j                  j                  d«      D ]‡  }|j                  dk(  rŒ| j	                  |«      }|sŒ'|j                  dd«      }|j                  d	d«      }|j                  }|j                  |i «      j                  |g «      j                  |«       Œ‰ t        |«      D ]y  }|j                  d
|› �«       t        ||   «      D ]A  }|j                  d|› d�«       t        ||   |   «      D ]  }|j                  d|› �«       Œ ŒC |j                  d«       Œ{ | j                  j                  dj                  |«      dz   d¬«       t        j                  dt        |«      «       y)z8Rebuild _index.md hierarchy tree from current documents.z# Knowledge Hierarchyri   zRoot: `ú`r/   r#   r“   Úgeneralr”   z## z  - **z**z    - rj   r?   r@   z#Rebuilt hierarchy.md with %d topicsN)rm   r&   r   r4   r5   r‡   Ú
setdefaultÚsortedr)   rq   rp   r9   rr   Úlen)	r,   rt   Útopicsr;   r<   r“   r”   Údoc_nameÚdocs	            r   r    z MarkdownStore._rebuild_hierarchy  sž  € à3°RÐ8ˆØ�‰�w˜tŸy™yŸ~™~Ð.¨aÐ0Ô1Ø�‰�RÔð 35ˆØ—y‘y—‘ vÓ.ò 		SˆGØ�|‰|˜{Ò*ØØ×-Ñ-¨gÓ6ˆHÙØØ—L‘L ¨)Ó4ˆEØ—|‘| J°	Ó:ˆHØ—|‘|ˆHØ×Ñ˜e RÓ(×3Ñ3°H¸bÓA×HÑHÈÕRð		Sô ˜F“^ò 	ˆEØ�L‰L˜3˜u˜g˜Ô'Ü" 6¨%¡=Ó1ò 1�Ø—‘˜v h Z¨rÐ2Ô3Ü! &¨¡-°Ñ"9Ó:ò 1�CØ—L‘L 6¨#¨ Õ0ñ1ð1ð �L‰L˜Õð	ð 	×Ñ×#Ñ# D§I¡I¨eÓ$4°tÑ$;ÀgÐ#ÔNÜ�‰Ð:¼CÀ»KÕHr   c                ó,  — g }| j                   j                  d«      D ]s  }|j                  dk(  rŒ| j                  |«      }|sŒ'|r|j	                  d«      |k7  rŒ>|j                  t        |j                  | j                   «      «      |dœ«       Œu |S )z1List all documents, optionally filtered by topic.r/   r#   r“   r0   )r&   r4   r   r5   r‡   rm   r   r6   )r,   r“   Úresultsr;   r<   s        r   Úlist_documentszMarkdownStore.list_documents/  s�   € àˆØ—y‘y—‘ vÓ.ò 	ˆGØ�|‰|˜{Ò*ØØ×-Ñ-¨gÓ6ˆHÙØÙ˜Ÿ™ gÓ.°%Ò7ØØ�N‰Nä × 3Ñ 3°D·I±IÓ >Ó?Ø (ñõð	ð ˆr   c                ó  — | j                   |z  }|j                  «       sy|j                  d¬«      }| j                  |«      xs i }|}|j	                  d«      r|j                  dd«      }|dk7  r||dz   d }|j                  «       ||d	œS )
z/Read a document by its relative path from root.Nr?   r@   rB   rC   rD   rE   é   )rX   r1   r   )r&   r3   rM   r5   rN   rO   rU   )r,   Úrelative_pathÚ	full_pathrX   rs   ÚbodyrY   s          r   Úget_documentzMarkdownStore.get_documentB  s”   € à—I‘I Ñ-ˆ	Ø×ÑÔ!ØØ×%Ñ%¨wÐ%Ó7ˆØ×,Ñ,¨YÓ7Ò=¸2ˆàˆØ×Ñ˜gÔ&Ø—l‘l 9¨aÓ0ˆGØ˜"Š}Ø˜w¨™{˜|Ð,�ØŸ:™:›<°[È-ÑXÐXr   c                ó  — g }| j                   j                  d«      D ]Ù  }|j                  dk(  rŒ| j                  |«      }|sŒ'|j	                  dg «      }t        |t        «      r,|j                  d«      D �cg c]  }|j                  «       ‘Œ }}|j                  «       |D �cg c]  }|j                  «       ‘Œ c}v sŒ¤|j                  t        |j                  | j                   «      «      |dœ«       ŒÛ |S c c}w c c}w )z3Search documents by tag (simple containment match).r/   r#   r   rL   r0   )r&   r4   r   r5   r‡   ro   r   rW   rU   r‘   rm   r6   )r,   Útagr³   r;   r<   r   Úts          r   Úsearch_by_tagzMarkdownStore.search_by_tagQ  sÙ   € àˆØ—y‘y—‘ vÓ.ò 
	dˆGØ�|‰|˜{Ò*ØØ×-Ñ-¨gÓ6ˆHÙØØ—<‘< ¨Ó+ˆDÜ˜$¤Ô$Ø+/¯:©:°c«?Ö; a˜Ÿ™�	Ð;�Ð;Ø�y‰y‹{°$Ö7¨Q˜qŸw™w�yÒ7Ò7Ø—‘¬¨G×,?Ñ,?ÀÇ	Á	Ó,JÓ(KÐYaÑbÕcð
	dð ˆùò <ùÚ7s   Á:C<Â&Dc                ó&   — | j                  |¬«      S )zSearch documents by topic.)r“   )r´   )r,   r“   s     r   Úsearch_by_topiczMarkdownStore.search_by_topica  s   € à×"Ñ"¨Ð"Ó/Ð/r   c                 ó   — y)u8   No-op â€” MarkdownStore holds no persistent connections.Nr   ©r,   s    r   ÚclosezMarkdownStore.closee  s   € àr   c                ó   — | S ©Nr   rÂ   s    r   Ú	__enter__zMarkdownStore.__enter__i  s   € Øˆr   c                ó$   — | j                  «        y rÅ   )rÃ   )r,   Ú_excs     r   Ú__exit__zMarkdownStore.__exit__l  s   € Ø�
‰
�r   )T)r&   z
Path | strr(   ÚboolÚreturnÚNone)rË   rÌ   )r   r   rË   údict[str, Any] | None)r   r   rX   r   rs   zdict[str, Any]rË   rÌ   )r   r   rË   r   )rˆ   r   rË   rÊ   rÅ   )r¡   r   rX   r   rœ   r   r›   r   r¢   ú
str | NonerË   r   )r“   rÎ   rË   úlist[dict[str, Any]])r·   r   rË   rÍ   )r¼   r   rË   rÏ   )r“   r   rË   rÏ   )rË   r   )rÈ   ÚobjectrË   rÌ   )r   r   r   r   r-   r+   r5   rv   r…   rŠ   r¨   r    r´   rº   r¾   rÀ   rÃ   rÆ   rÉ   r   r   r   r   r   F   s½   „ ñ
ôóUó,ðB:àð:ð ð:ð $ð	:ð
 
ó:ó:óð0  $ð9à,ð9ð ð9ð ð	9ð
 ð9ð ð9ð 
ó9ó~IôDó&Yóó 0óóôr   r   )r   Ú
__future__r   r}   Úloggingrž   Údataclassesr   r   Úpathlibr   Útypingr   rP   ÚImportErrorÚ'cyberbrain26.knowledge_plane.classifierr   Ú	getLoggerr   r9   r   r   r   r   r   ú<module>rÙ      s�   ðñõ6 #ã Û Û ß (Ý Ý ðÛõ Ià	ˆ×	Ñ	˜8Ó	$€ñ �$Ô÷
2ð 
2ó ð
2÷$gò gøð? ò Ø‚Dðús   ªA' Á'A1Á0A1