_Qj ddlZddlmZddlZ ddlmZddlmcm Z ddl m Z m Z mZmZmZdZdZdZdZdZd Zd Zd Zd Zd ZdZej:j<ej:j gZej@ddZ!ej@ddZ"ej@ddZ#dZ$dZ%dZ&dZ'dZ( d;dZ)dZ*dZ+dZ,dZ-dZ.d Z/d!Z0d"Z1d#Z2d$Z3d%Z4d&Z5d'Z6d(Z7d)Z8ej:jsd*ejtddd+,ejtd-dd.,ejtdd-d/,ejtd-d-d0,gd1Z;dd6Z?d7Z@d8ZAGd9d:ZBy#e$rdZdZ YwxYw)=N) timedelta)InMemoryKmsClientMockVersioningKmsClientverify_file_encryptedread_external_keys_to_dictparse_wrapped_keyzencrypted_table.in_mem.parquets0123456789112345 footer_keys1234567890123450col_keys0123456789abcdefs0123456789abcdef01234567s 0123456789abcdef0123456789abcdefstest_aad_prefixzkms-123zhttps://example.com/kmsmodule)scopectjjtjgdtjgdtjgdd}|S)Nabcxyz)paTable from_pydictarray) data_tables j/home/oem/CyberBrain26-unified/.venv/lib/python3.12/site-packages/pyarrow/tests/parquet/test_encryption.pyrr9sG%% XXi XXo & XXo &'J cLtjttddgi}|S)Nrr)r column_keyspeEncryptionConfigurationFOOTER_KEY_NAME COL_KEY_NAME)basic_encryption_configs rr(r(Cs, 88" 3*   #"r cNtjttddgid}|S)NrrFr r"internal_key_materialr#)external_encryption_configs rr,r,Ms1!#!;!;" 3* $ "% &%r cftj|}d}tj|}||fS)z Sets up and returns the KMS connection configuration and crypto factory based on provided KMS configuration parameters. custom_kms_confct|SNrkms_connection_configurations r kms_factoryz1setup_encryption_environment..kms_factory_s !=>>r )r$KmsConnectionConfig CryptoFactory)r/kms_connection_configr5crypto_factorys rsetup_encryption_environmentr:Xs7 22?S?%%k2N . 00r c||jd||jdi}t|\}} t||||| || fS)zL Writes an encrypted parquet file based on the provided parameters. UTF-8)decoder:write_encrypted_parquet) pathrfooter_key_name col_key_namer r encryption_configr/r8r9s rwrite_encrypted_filerChsc **73gnnW-O -I-)>D*.?1>C !. 00r c v|tz }tjttddgidt dd}|j dusJt||tttt|\}}t|tjt d }t||||}|j|sJy ) DWrite an encrypted parquet, verify it's encrypted, and then read it.rr AES_GCM_V1@minutesr r"encryption_algorithmcache_lifetimedata_key_length_bitsFrMN) PARQUET_NAMEr$r%r&r'runiform_encryptionrC FOOTER_KEYCOL_KEYrDecryptionConfigurationread_encrypted_parquetequalstempdirrr?rBr8r9decryption_config result_tables r!test_encrypted_parquet_write_readr[~s \ !D 22" 3* * - "  / /5 88 8,@ j/<W-)>$22 -/) !6HL   \ ** *r c ^|tz }tjtddt dd}|j dusJt ||tttd|\}}t|tjt d}t||||}|j|sJy ) rETrFrGrHrJ)r rQrLrMrNr rON) rPr$r%r&rrQrCr'rRrrTrUrVrWs r)test_uniform_encrypted_parquet_write_readr]s \ !D22") - "  / /4 77 7,@ j/<S-)>$22 -/) !6HL   \ ** *r c|jr|j||}n|j|||}|Jtj||j|5}|j |dddy#1swYyxYw)Nencryption_properties)r+file_encryption_propertiespq ParquetWriterschema write_table)r?tablerBr8r9rawriters rr>r>s..%3%N%N !#4&6"&4%N%N !#4d&<" % 11 1   %,,"< >"AG5!"""s A44A=Tc^|r|j||}n|j|||}|Jtj||}|jdk(sJtj||}t |j dk(sJtj||}|jdS)Ndecryption_propertiesrT use_threads) file_decryption_propertiesrb read_metadata num_columns read_schemalennames ParquetFileread) r?rYr8r9r+rmmetardresults rrUrUs%3%N%N !#4&6"&4%N%N !#4d&<" & 11 1   $> @D   q  ^^ $>@F v||  !! ! ^^ $>@F ;;4; ((r c  |tz }tjttddgidt dd}t ||tttt|t|tttjdttjdi\}}tjt d }tjtd 5t!||||d d d y #1swYy xYw) zYWrite an encrypted parquet, verify it's encrypted, and then read it using wrong keys.rrrFrGrHrJrKr<rOzIncorrect master key usedmatchN)rPr$r%r&r'rrCrRrSrr:r=rTpytestraises ValueErrorrU)rXrr?rBwrong_kms_connection_configwrong_crypto_factoryrYs r+test_encrypted_parquet_write_read_wrong_keyrs \ !D 22" 3* * - "z?L#W.?A$8T0j''0V95!5 22 -/ z)E F" #%@  """"s !C99Dct||tjtd5t j |t z jdddy#1swYyxYw)zmWrite an encrypted parquet, verify it's encrypted, but then try to read it without decryption properties. no decryptionrxN)r[rzr{IOErrorrbrsrPrtrXrs r0test_encrypted_parquet_read_no_decryption_configrsL&gz: w&6 76 w-.335666s +AA%ct||tjtd5t j |t z dddy#1swYyxYw)zwWrite an encrypted parquet, verify it's encrypted, but then try to read its metadata without decryption properties.rrxN)r[rzr{rrbrnrPrs r9test_encrypted_parquet_read_metadata_no_decryption_configr sE&gz: w&6 71 </0111 AAct||tjtd5t j |t z dddy#1swYyxYw)zuWrite an encrypted parquet, verify it's encrypted, but then try to read its schema without decryption properties.rrxN)r[rzr{rrbrprPrs r7test_encrypted_parquet_read_schema_no_decryption_configrsC&gz: w&6 7/ w-.///rc |dz }tjt}tjt d5t ||tttd|dddy#1swYyxYw)MWrite an encrypted parquet, but give only footer key, without column key.z)encrypted_table_no_col_key.in_mem.parquetr z4Either column_keys or uniform_encryption must be setrxr N) r$r%r&rzr{OSErrorrCr'rRrXrr?rBs r'test_encrypted_parquet_write_no_col_keyrsn @ @D22"$ w% &A T: '.? A AAAs A""A+c |dz }tjttddgid}t j t d5t||tttd|d d d y #1swYy xYw) rz=encrypted_table_col_key_and_uniform_encryption.in_mem.parquetrrT)r r"rQz2Cannot set both column_keys and uniform_encryptionrxr N) r$r%r&r'rzr{rrCrRrs r;test_encrypted_parquet_write_col_key_and_uniform_encryptionr.s T TD22" 3*  ! wR TA T: '.? AAAAs A,,A5c|dz }|}tj}d}tj|}tjt d5t |||||dddy#1swYyxYw).kms_factoryKs!!=>>r r rxN)r$r6r7rzr{KeyErrorr>rXrr(r?rBr8r5r9s r&test_encrypted_parquet_write_kms_errorrBsy ? ?D/224? %%k2N x| 4Gj2C 5~ GGGG A((A1c |dz }|}tj}Gddtjfd}tj|}t j t d5t|||||dddy#1swYyxYw)rrc"eZdZdZdZdZdZy)Jtest_encrypted_parquet_write_kms_specific_error..ThrowingKmsClientzVA KmsClient implementation that throws exception in wrap/unwrap calls cPtjj|||_y)z%Create an InMemoryKmsClient instance.N)r$ KmsClient__init__configselfrs rrzStest_encrypted_parquet_write_kms_specific_error..ThrowingKmsClient.__init__es LL ! !$ ' DKr ctd)NCannot Wrap Keyr|r key_bytesmaster_key_identifiers rwrap_keyzStest_encrypted_parquet_write_kms_specific_error..ThrowingKmsClient.wrap_keyjs./ /r ctd)NzCannot Unwrap Keyrr wrapped_keyrs r unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error..ThrowingKmsClient.unwrap_keyms01 1r N__name__ __module__ __qualname____doc__rrrr rThrowingKmsClientr`s  !  0 2r rc|Sr1r)r4rs rr5zDtest_encrypted_parquet_write_kms_specific_error..kms_factoryps !=>>r rrxN)r$r6rr7rzr{r|r>) rXrr(r?rBr8r5r9rs @r/test_encrypted_parquet_write_kms_specific_errorrWs ? ?D/2242BLL2 ?%%k2N z): ;Gj2C 5~ GGGGs +BB c|dz }|}tj}d}tj|}tjt d5t |||||dddy#1swYyxYw)z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetctd)NCannot create KmsClientrr3s rr5zCtest_encrypted_parquet_write_kms_factory_error..kms_factorys233r rrxN)r$r6r7rzr{r|r>rs r.test_encrypted_parquet_write_kms_factory_errorr{s~ G GD/2244%%k2N z6 8G j2C 5~ GGGGrc|dz }|}tj}Gddfd}tj|}tjt 5t |||||dddy#1swYyxYw)z_Write an encrypted parquet, but use wrong KMS client type that doesn't implement KmsClient.rc"eZdZdZdZdZdZy)Otest_encrypted_parquet_write_kms_factory_type_error..WrongTypeKmsClientz4This is not an implementation of KmsClient. c&|j|_yr1)r/master_keys_maprs rrzXtest_encrypted_parquet_write_kms_factory_type_error..WrongTypeKmsClient.__init__s#)#9#9D r cyr1rrs rrzXtest_encrypted_parquet_write_kms_factory_type_error..WrongTypeKmsClient.wrap_keyr cyr1rrs rrzZtest_encrypted_parquet_write_kms_factory_type_error..WrongTypeKmsClient.unwrap_keyrr Nrrr rWrongTypeKmsClientrs  :  r rc|Sr1r)r4rs rr5zHtest_encrypted_parquet_write_kms_factory_type_error..kms_factorys!">??r N)r$r6r7rzr{ TypeErrorr>) rXrr(r?rBr8r5r9rs @r3test_encrypted_parquet_write_kms_factory_type_errorrs G GD/224  @%%k2N y !Gj2C 5~ GGGGs A33A<c Jd}tjttddgidddt ddd }||tjt }tddgi|_d|_d|_d|_t d|_ d|_ d |_ ||y) Nc0t|jk(sJddg|jtk(sJd|jk(sJ|j sJ|j rJtd|jk(sJ|jrJd|jk(sJy)NrrAES_GCM_CTR_V1$@rH) r&r r"r'rLplaintext_footerdouble_wrappingrrMr+rN)rBs r!validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration..validate_encryption_configurations"3">">>>>Sz.::<HHHH#4#I#IIII 1111$4444&*;*J*JJJJ$::::'<<<<.validate_kms_connection_configsb3CCCCC.?????1BBBBB)3CD%556 76r rrrrrrr)r$r6rrrr/)rr8kms_connection_config_1s r(test_encrypted_parquet_kms_configurationrs722#"$$  ##89 446.9+/5,/8,  /+##:;r c |tz }dD]}|rtjjtjgdtjdddddddd dgtj d tj fd tj fg tjgd d}n|}tjttddgidd}tjttjdttjdi}d}tj|}t!|||||t#j$|5} | j'dd} |j(| j(k(sJ|j+dgj-| sJ dddy#1swYxYw)a Write an encrypted parquet, with plaintext footer and with single wrapping, verify it's encrypted, and then read plaintext columns. Runs once with a flat schema and once where the encrypted column `b` is itself a nested (struct) field. )FTrrr)rrrrr)typerrrrTF)r r"rrr<r.ct|Sr1r2r3s rr5zStest_encrypted_parquet_write_read_plain_footer_single_wrapping..kms_factory0s$%AB Br r)columnsrlN)rPrrrrstructint32r$r%r&r'r6rRr=rSr7r>rbrsrtnum_rowsselectrV) rXrr?nestedrfrBr8r5r9rvrZs r>test_encrypted_parquet_write_read_plain_footer_single_wrappingr s \ !D)< HH((XXi(XX1%QQ'7qq9IJS"((*$5RXXZ7H#IJLXXo. *EE66&sCj"! #!# 6 6!2!27!;gnnW5!  C))+6e-> 5~ G^^D ! >\%:%:: ::<<&--l; ;; < $224) !6#%L  " " = = F Ft LE %..00w 1 & 2 2< @AAE GG G wG!&&q)5G HH H   \ ** *Hs D)double_wrap_initialdouble_wrap_rotatedzdouble wrapping)idFzsingle to double wrappedzdouble to singe wrappedzsingle wrappingcz |tz }tjttddgid}tj d}d}tj |}t|||||t| |jd|j|| t| t|t|tj||d } t vsJt vsJt vsJt vsJd td d f fd } | t| t|j| sJy )aTests CryptoFactory.rotate_master_keys Note: The CryptoFactory.rotate_master_keys() double_wrapping keword arg may be either True (the default) or False regardless of whether EncryptionConfig.double_wrapping was set to true (also the default) when the external key material store was written. This means double wrapping may be set one way initially and then applied or removed during rotation. rrF)r r"r+r1rct|Sr1)rr3s rr5z8test_external_key_material_rotation..kms_factoryus&'CDDr 2)rr master_key_idreturnNc |} r |j}n |j}t|\}}}|} r |j}n |j}t|\}}}||ksJyr1) wrapped_kek wrapped_dekr) r before_key_matbefore_key_wrapped_ before_ver after_key_matafter_key_wrapped after_ver after_keys before_keysrrs rcheck_rotated_external_keyszHtest_external_key_material_rotation..check_rotated_external_keyss$]3 !/!;!; !/!;!; ,-?@:q"=1  - 9 9  - 9 9 +,=>9aI%%%r )rPr$r%r&r'r6r7r>rrefresh_key_access_tokenrotate_master_keysrrUrTstrrV) reusable_tempdirrrrr?rBr8r5r9table_read_after_rotationrrrs `` @@r#test_external_key_material_rotationrXs]& l *D22"!C:.#+ -22CHE%%k2N   -T2K2237%% +&- ,D1J$ 6  ""$# !% k )) ) ; && & j (( ( : %% %&3&4&&" 0 -   6 77 7r c&tfd}|S)zCreate a KMS client factory that appends the KMS instance ID and URL of each connection configuration it is given to created_configscbj|j|jf|Sr1)appendrr)r4 client_classcreated_configss rr5z*recording_kms_factory..kms_factorys6 ) 9 9 ) : : < =899r r2)r#r"r5s`` rrecording_kms_factoryr$s( : r rcD|tz }ttjdtt jdi}t jtt|}t jt}t|||||t|t j|}g} t jt| } t j|} t!|| || } |j#| sJ|r| ttfgk(sJy| tdfgk(sJy)zxRead a file written with KMS connection properties configured, using a KmsConnectionConfig that doesn't specify themr<)rrr/r.rDEFAULTN)rPr&rRr=r'rSr$r6KMS_INSTANCE_IDKMS_INSTANCE_URLr7rr>rr$rTrUrV) rXrr(rr?r/ write_configwrite_crypto_factory read_configr#read_crypto_factoryrYrZs rtest_read_kms_url_from_filer.s  \ !D**73gnnW-O ))')')L++,=>D*.E(*>@$((IKO**o.022!#) .ACL   \ ** *O5E#F"GGGGOY#?"@@@@r c|tz }tjttddgid}tj t td}tjt}t|||||tj d}g}tjt|t} | j||| |r|d t tfgk(sJ|d t d fgk(sJt|} | t} | jd k(sJ| jd k(sJt!|tj"|| d } |j%| sJy )zRotate the keys of a file written with KMS connection properties configured, using a KmsConnectionConfig that doesn't specify themrrFr*r)rrrr rr&)r0r'rN)rPr$r%r&r'r6r(r)r7rr>r$rrrrrUrTrV) rrrr?rBr*r+rotation_configr#rotation_crypto_factory rotated_keysfooter_key_materialrs r)test_key_rotation_reads_kms_url_from_filer5su l *D22"!C:.#% ))')L++,CDD*.?02,,cBOO ..o/FGI..L/:8o?O-P"QQQQ8oy-I"JJJJ.d3L&7  . .) ;; ;  / /9 << < 6  ""$# !%   6 77 7r c x|tz }t||tttt |\}}t |tjtd}tdD]T}|j||}|Jtj||} | jd} |j| rTJy) z`Write an encrypted parquet, verify it's encrypted, and then read it multithreaded in a loop.rGrHrO2NriTrk)rPrCr&r'rRrSrr$rTrrangermrbrsrtrV) rXrr(r?r8r9rYirmrvrZs rtest_encrypted_parquet_loopr:s \ !D -A j/<W-!)>$22 -/2Y /%3%N%N !#4&6")555 (BD{{t{4   ... /r c R|tz }t||tttt |\}}t |tjtd}|j||}~tj||}|jd} |j| sJy)z^ Test that decryption properties can be used if the crypto factory is no longer alive rGrHrOriTrkN)rPrCr&r'rRrSrr$rTrrmrbrsrtrV) rXrr(r?r8r9rYrmrvrZs r%test_read_with_deleted_crypto_factoryr<1s \ !D,@ j/<W-!)>$22 -/!/!J!J0"2 ^^ $>@F;;4;0L   \ ** *r c j|tz }t||tttt |\}}t jtd}|j||}tj||}|j|sJtj||}|j|sJy)z>Write an encrypted parquet then read it back using read_table.rGrHrOriN) rPrCr&r'rRrSr$rTrrmrb read_tablerV) rXrr(r?r8r9rYrmrZs r!test_encrypted_parquet_read_tabler?Is \ !D-A j/<W-!)>22 -/!/!J!J0"2===WXL   \ ** *=='ACL   \ ** *r ceZdZdZej j deee ggddZ dZ dZ dZ d Zd Zd Zd Zd ZdZdZdZdZdZdZdZdZy)TestDirectKeyEncryptionzFTests for create_encryption_properties / create_decryption_properties.key)aes128aes192aes256)idsc |dt|dzdz }tj|}tj|||tj |}tj ||}|j|sJy)Ndirect_z.parquetrr_ri)rqr$create_encryption_propertiesrbrecreate_decryption_propertiesr>rV)rrXrrBr? enc_props dec_propsrvs rtest_roundtrip_key_sizesz0TestDirectKeyEncryption.test_roundtrip_key_sizesfss73s8a<.9933sC  z4yI33sC t9E  (((r c|dz }tjtt}t j |||tj tt}t j||}|j|sJy)Nzdirect_aad.parquetr aad_prefixr_ri) r$rJDIRECT_KEY_128DIRECT_AAD_PREFIXrbrerKr>rVrrXrr?rLrMrvs rtest_roundtrip_with_aad_prefixz6TestDirectKeyEncryption.test_roundtrip_with_aad_prefixssq--33%(  z4yI33%( t9E  (((r c|dz }tjttd}t j |||tj t}tjtd5t j||d d d tj tt }t j||}|j|sJy #1swYSxYw) z;When store_aad_prefix=False, reader must supply aad_prefix.zdirect_aad_not_stored.parquetF)r rQstore_aad_prefixr_rAADrxriNrP) r$rJrRrSrbrerKrzr{rr>rV)rrXrr?rLdec_props_no_aadrMrvs r$test_roundtrip_aad_prefix_not_storedz)rrXrr?rLrMs rtest_wrong_aad_prefix_failsz3TestDirectKeyEncryption.test_wrong_aad_prefix_failss3333%(  z4yI33%& ]]7% 0 A MM$i @ A A As 3BBc|dz }tjt}tj|||t |d5}|j d}ddddk(sJy#1swYxYw)Nzdirect_magic.parquetrr_rbrsPAREr$rJrRrbreopenrtrrXrr?rLfmagics r"test_encrypted_file_has_pare_magicz:TestDirectKeyEncryption.test_encrypted_file_has_pare_magicsl//33%'  z4yI $  FF1IE   s A&&A/c|dz }tjtd}tj|||tj t}tj ||}|j|sJy)Nzdirect_plaintext_footer.parquetTr rr_rrir$rJrRrbrerKr>rVrTs rtest_plaintext_footerz-TestDirectKeyEncryption.test_plaintext_footersl::33%!  z4yI33%' t9E  (((r c|dz }tjtd}tj|||tj t}tj ||}|j|sJy)Nzdirect_ctr.parquetrr rLr_rrirgrTs rtest_aes_gcm_ctr_v1_algorithmz5TestDirectKeyEncryption.test_aes_gcm_ctr_v1_algorithmsl--33%!1  z4yI33%' t9E  (((r c0|dz }tjt}tj|||d}tj |}t jtd5tj||dddy#1swYyxYw)Nzdirect_wrong_key.parquetrr_sfedcba9876543210decryptrxri) r$rJrRrbrerKrzr{rr>)rrXrr?rL wrong_keyrMs rtest_wrong_key_failsz,TestDirectKeyEncryption.test_wrong_key_failssz3333%'  z4yI' 33yI ]]7) 4 A MM$i @ A A As +B  Bc|dz }tjt}tj|||t j td5tj|dddy#1swYyxYw)Nzdirect_no_decrypt.parquetrr_zencrypted metadatarx) r$rJrRrbrerzr{rr>)rrXrr?rLs r%test_reading_without_decryption_failsz=TestDirectKeyEncryption.test_reading_without_decryption_failss`4433%'  z4yI ]]7*> ? MM$    s A22A;c|dz }tj||tjtd}tj ||}|j |sJy)zAPlaintext file reads should work when allow_plaintext_files=True.zplaintext.parquetT)r allow_plaintext_filesriN)rbrer$rKrRr>rV)rrXrr?rMrvs rtest_allow_plaintext_filesz2TestDirectKeyEncryption.test_allow_plaintext_filessW,, z4(33%"& t9E  (((r c|dz }tj||tjt}t j td5tj||dddy#1swYyxYw)zBDefault allow_plaintext_files=False should reject plaintext files.zplaintext_rejected.parquetr plaintextrxriN) rbrer$rKrRrzr{rr>)rrXrr?rMs r'test_plaintext_file_rejected_by_defaultz?TestDirectKeyEncryption.test_plaintext_file_rejected_by_defaultsc55 z4(33%' ]]7+ 6 A MM$i @ A A As A22A;c|dz }tjt}tj|||tj td}tj ||}|j|sJy)z;check_footer_integrity=False should still allow decryption.zdirect_no_footer_check.parquetrr_F)r check_footer_integrityriNrgrTs r!test_check_footer_integrity_falsez9TestDirectKeyEncryption.test_check_footer_integrity_falsesl9933%'  z4yI33%#( t9E  (((r c|dz }tjtd}tj|||t |d5}|j d}ddddk(sJy#1swYxYw) z:plaintext_footer=True should produce PAR1 magic, not PARE.zdirect_plaintext_magic.parquetTrfr_r^rNsPAR1r_ras r$test_plaintext_footer_has_par1_magiczrUrrrrrrrrrrrrrrrrparamrr$r.r5r:r<r?rArr rrsl" E ++ EE0   $,4&,  KK"" KK h h# #h& & 1 1,+>+6 ""26). "F61/A"A(G*!GHG(GB ;FH2<:3? UE&78 5:; J8 ; J8Z %7$A8$AN%7/88/8d/:+0+4ZZY B BsG GG