
    '                     *   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm	Z	 ddl
mZ dd	l
mZ dd
l
mZ ddl
mZ ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z!  G d de      Z"y)z6Unit tests for tracker_file and parallel_tracker_file.    )absolute_import)print_function)division)unicode_literalsN)CommandException)ObjectFromTracker)ReadParallelUploadTrackerFile)$ValidateParallelCompositeTrackerData))WriteComponentToParallelUploadTrackerFile)WriteParallelUploadTrackerFile)StorageUrlFromString)GsUtilUnitTestCase)storage_v1_messages_HashFilename)DeleteTrackerFile)GetRewriteTrackerFilePath)HashRewriteParameters)ReadRewriteTrackerFile)WriteRewriteTrackerFile)parallelism_framework_util)UTF8c                   @    e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zy
)TestTrackerFilez7Unit tests for parallel upload functions in cp command.c                 0    t        d       t        d       y )Ns   file1file1r   )selfs    0platform/gsutil/gslib/tests/test_tracker_file.pytest_HashFilenamez!TestTrackerFile.test_HashFilename/   s     ('    c                    t        dddd| j                        }t        |       t        j                  dddd      }t        j                  dddd	      }t        j                  dd
      }d}| j                  t        ||             t        ||d      }t        |||       | j                  t        ||      |       t        ||d      }| j                  t        ||             t        |       y)z%Tests Rewrite tracker file functions.bk1obj1bk2obj2etag112345)bucketnameetagmd5Hashetag267890)r(   r)   token1fullN)
r   test_apir   apitools_messagesObjectassertIsNoner   r   r   assertEqual)r   tracker_file_namesrc_obj_metadatasrc_obj2_metadatadst_obj_metadatarewrite_tokenrewrite_params_hashrewrite_params_hash2s           r   test_RewriteTrackerFilez'TestTrackerFile.test_RewriteTrackerFile5   s   1%26--A '((//u5;5<8?A *006<6=9@B )//u6JM02BCE/0@0@&J-/B)+02EF 11B1A6K 	02FGI'(r    c           	         d}g d}dj                  |g|z         dz   }| j                  d|j                  t                    }t	        dt        |      dz        D cg c]  }t        |d|z     |d|z  dz             }}t        || j                        \  }}}	| j                  ||       | j                  ||	       y	c c}w )
zCTests the parallel upload tracker file format prior to gsutil 4.17.123)r#   42r%   314159
foo	file_namecontentsr         N)
joinCreateTempFileencoder   rangelenr   r	   loggerr4   )
r   random_prefixobjectsrE   fpathiexpected_objects_actual_prefixactual_objectss
             r   *testReadGsutil416ParallelUploadTrackerFilez:TestTrackerFile.testReadGsutil416ParallelUploadTrackerFileX   s    M.Gyy-723d:H%(//$:OPE q7|q(**A 	'!a%.'!a%!)*<=*   5UDKKHQ]M2%~6s   "Cc                     | j                  dd      }t        || j                        \  }}}| j                  d|       | j                  g |       y)zDTests reading an empty pre-gsutil 4.17 parallel upload tracker file.rB   r    rC   N)rI   r	   rM   r4   )r   rP   rS   rT   rU   s        r   /testReadEmptyGsutil416ParallelUploadTrackerFilez?TestTrackerFile.testReadEmptyGsutil416ParallelUploadTrackerFileh   sQ    %#>E4UDKKHQT=)R(r    c                    | j                  d      }d}t        dd      t        dd      g}t        |||       t        || j                        \  }}}| j                  ||       | j                  d |       | j                  ||       y )NrB   rD   r>   r#   r?   r%   r@   rI   r   r   r	   rM   r4   )r   rP   rN   rO   enc_keyrT   rU   s          r   )testParallelUploadTrackerFileNoEncryptionz9TestTrackerFile.testParallelUploadTrackerFileNoEncryptionp   s    %0EM&$'&(+G #5-A4UDKKHWm]M2T7#Wn-r    c                    | j                  d      }d}d}t        dd      t        dd      g}t        ||||	       t        || j                        \  }}}| j                  ||       | j                  ||       | j                  ||       y )
NrB   rZ   r>   456r#   r?   r%   r@   encryption_key_sha256r[   )r   rP   rN   r\   rO   
actual_keyrT   rU   s           r   +testParallelUploadTrackerFileWithEncryptionz;TestTrackerFile.testParallelUploadTrackerFileWithEncryption~   s    %0EMG&$'&(+G #5#0#*9@B
 5UDKKHZWj)]M2Wn-r    c                 T   t        j                         }| j                  d      }d}d}t        dd      t        dd      g}t	        ||||	       t        d
d      }	 t        |||| j                  d 	       | j                  d       t        |||| j                  d	       t        || j                        \  }}	}
| j                  ||       | j                  ||	       | j                  ||gz   |
       y # t        $ r%}| j                  dt        |             Y d }~d }~ww xY w)NrB   rZ   r>   r_   r#   r?   r%   r@   r`   obj343z9Expected CommandException due to different encryption keyzdoes not match encryption key)r   
CreateLockrI   r   r   r   rM   failr   assertInstrr	   r4   )r   tracker_file_lockrP   rN   r\   rO   
new_objecterb   rT   rU   s              r   -testWriteComponentToParallelUploadTrackerFilez=TestTrackerFile.testWriteComponentToParallelUploadTrackerFile   s4   2==?%0EMG&$'&(+G #5#0#*9@B #640J=/0A0:04FJ	L
 iiKL .e.?.8.2kkDI	K 5UDKKHZWj)]M2W
|+^<  =
mm3SV<<=s   +C9 9	D'D""D'c                 p   | j                         }t        j                  j                  |d      }d}d}t	        d      }t        dd      t        dd      g}t        ||||	       t        j                  d
k(  rVt        t        j                  t        j                  |      j                              }| j                  t        d      |        G d dt              }d }	d }
 |       }t        |||||||| j                  |	|

      \  }}| j                  d|j                          | j                  ||       | j                  ||       d} |       }t        |||||||| j                  |	|

      \  }}| j                  d|j                          | j                  d |       | j                  g |       y )NrB   r>   r_   zgs://foor#   r?   r%   r@   r`   posixi  c                   ,    e Zd ZdZ G d de      Zd Zy)STestTrackerFile.testValidateParallelCompositeTrackerData.<locals>.MockCommandObjectFc                       e Zd ZdZy)jTestTrackerFile.testValidateParallelCompositeTrackerData.<locals>.MockCommandObject.ParallelOverrideReasonspeedN)__name__
__module____qualname__SPEED r    r   ParallelOverrideReasonrt      s    r    r{   c                     d| _         y )NT)delete_called)r   unused_argsunused_kwargss      r   ApplyzYTestTrackerFile.testValidateParallelCompositeTrackerData.<locals>.MockCommandObject.Apply   s
    !r    N)rv   rw   rx   r}   objectr{   r   rz   r    r   MockCommandObjectrr      s    m6 "r    r   c                       y Nrz   rz   r    r   MockDeleteFunczPTestTrackerFile.testValidateParallelCompositeTrackerData.<locals>.MockDeleteFunc       
r    c                       y r   rz   rz   r    r   MockDeleteExceptionHandlerz\TestTrackerFile.testValidateParallelCompositeTrackerData.<locals>.MockDeleteExceptionHandler   r   r    F789T)CreateTempDirospathrH   r   r   r   r)   octstatS_IMODEst_moder4   r   r
   rM   r}   )r   tempdirrP   rN   old_enc_key
bucket_urlrO   moder   r   r   command_objrT   rU   new_enc_keys                  r   (testValidateParallelCompositeTrackerDataz8TestTrackerFile.testValidateParallelCompositeTrackerData   s     "GGGLL%(EMK%j1J&$'&(+G #5#0#*9DF
 
ww'bggen4456d
s5z4(	"F 	" $%K&J{M7KT[[.2L'N#]N 	UK556]M2Wn-K#%K&J{M7KT[[.2L'N#]N 	T;445T=)R(r    N)rv   rw   rx   __doc__r   r<   rV   rX   r]   rc   rn   r   rz   r    r   r   r   ,   s/    ?!)F7 )..$"=H9)r    r   )#r   
__future__r   r   r   r   r   r   gslib.exceptionr   gslib.parallel_tracker_filer   r	   r
   r   r   gslib.storage_urlr   "gslib.tests.testcase.unit_testcaser   "gslib.third_party.storage_apitoolsr   r1   gslib.tracker_filer   r   r   r   r   r   gslib.utilsr   gslib.utils.constantsr   r   rz   r    r   <module>r      s^    = & %  ' 	  , 9 E L Q F 2 A W , 0 8 4 5 6 2 &A)( A)r    