
    V                       d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlm	Z	 ddl
Z
ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmc mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ddlm!Z! ddlm"Z" ddlm#Z# ddlm$Z$ ddlm%Z% ddlm&Z& ddlm'Z' ddlm(Z( ddlm)Z) ddlmZ ddl*m+Z+ ddl,m-Z- dd l.m/Z/ dd!l.m0Z0 dd"l.m1Z1 dd#l.m2Z2 dd$l.m3Z3 dd%l.m4Z4 dd&l5m6Z6 dd'l7m8Z8 dd(l7m9Z9 dd)l:m;Z; e9sdd*l<m=Z= dd+lm>Z> dd,lm?Z? dd-lm@Z@ dd.lmAZA e
j                  reCZDd/ZEe8r	d0eEz   d1z   ZFnd2ZF e+       se-d3z   eFz   ZF G d4 d5ej                        ZH G d6 d7ej                        ZJ G d8 d9ej                        ZLy):$Integration tests for rsync command.    )absolute_import)print_function)division)unicode_literalsN)mock)command)rsync)PopulateProjectId)StorageUrlFromString)	SkipForGS)	SkipForS3)
SkipForXML)"AuthorizeProjectToUseTestingKmsKey)TEST_ENCRYPTION_KEY_S3)TEST_ENCRYPTION_KEY_S3_MD5)BuildErrorRegex)ObjectToURI)ORPHANED_FILE)POSIX_GID_ERROR)POSIX_INSUFFICIENT_ACCESS_ERROR)POSIX_MODE_ERROR)POSIX_UID_ERROR)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)TailSet)unittest)UsingCrcmodExtension)SLOW_CRCMOD_RSYNC_WARNING)ConvertDatetimeToPOSIX)GID_ATTR)	MODE_ATTR)
MTIME_ATTR)NA_TIME)UID_ATTR)Retry)IS_OSX)
IS_WINDOWS)	shim_util)util)DEFAULT_MODE)INVALID_GID)INVALID_UID)USER_IDa_  If you experience problems with multiprocessing on MacOS, they might be related to https://bugs.python.org/issue33725. You can disable multiprocessing by editing your .boto config or by adding the following flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that multithreading is still available even if you disable multiprocessing.

z"Building synchronization state...
zStarting synchronization...
z>Building synchronization state...
Starting synchronization...

c                       e Zd ZdZd Z ej                  d       ej                  d       ej                  d ej                  ej                  j                              d                      Zy	)
TestRsyncUnitz4Unit tests for methods in the commands.rsync module.c                 8   d}d}| j                  t        j                  t        j                  |            t        j
                  |             | j                  t        j                  t        j
                  |            t        j                  |             y )Nu+   gs://bkt/space fslash/plus+tilde~unicodeeèz;gs%3A%2F%2Fbkt%2Fspace+fslash%2Fplus%2Btilde~unicodee%C3%A8)assertEqualr
   
_EncodeUrlsixensure_text
ensure_str
_DecodeUrl)selfdecoded_urlencoded_urls      )platform/gsutil/gslib/tests/test_rsync.pytestUrlEncodeAndDecodez$TestRsyncUnit.testUrlEncodeAndDecodec   ss    ?KE  	U%%cook&BC^^K02 	U%%cnn[&AB__[13    zFgslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessaryz/gslib.command.Command._GetProcessAndThreadCountzgslib.command.Command.Apply)spec)newc                 b   | j                         }| j                         }d|_        | j                  d|t	        |      g       |j                  t        t	        |            t        j                  d       |j                  d d t        j                  j                  j                  d       y )N)      r
      )worker_countF)process_countthread_countparallel_operations_overrideprint_macos_warning)CreateTempDirCreateBucketreturn_value
RunCommandsuriassert_called_once_withr   r   ANYr	   CommandParallelOverrideReasonSPEED)r:   !mock_get_process_and_thread_countmock_trigger_reauthpath
bucket_uris        r=   testRsyncTriggersReauthz%TestRsyncUnit.testRsyncTriggersReauthp   s     D""$J59%2OOGdD$456//T*-. 0  &==%,__%K%K! > r?   N)__name__
__module____qualname____doc__r>   r   patchMockr	   rR   ApplyrY    r?   r=   r2   r2   `   sq    <3 4::NP4::?@4::+$))!6!6799 AP
r?   r2   c                       e Zd Zd Zy)TestRsyncUnitWithShimc                 P   | j                         }| j                         }t        ddg      5  t        ddd      5  | j	                  dddd	t        |      |gd
      }dj                  |j                  d         }| j                  dj                  t        j                  d      t        |      |      |       dj                  |j                  d         }| j                  d|       | j                  d|       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N)GSUtiluse_gcloud_storageTrue)re   hidden_shim_modedry_runrg   fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr
   -apublic-read-PT)return_log_handlerr0   infozpGcloud Storage Command: {} storage rsync --no-ignore-symlinks --predefined-acl publicRead --preserve-posix {} {}warningz'By default, gsutil copies file symlinksz)For preserving POSIX with rsync downloads)rL   rK   r   r   rN   rO   joinmessagesassertInformatr*   _get_gcloud_binary_path)r:   rX   fpathmock_log_handler
info_lines
warn_liness         r=   testShimTranslatesFlagsz-TestRsyncUnitWithShim.testShimTranslatesFlags   s,   ""$J E	GHJ 
K 6<)" 	  ??dM4:&/# + % YY/88@A
&&,f11*=tJ?O'"	$ YY/88CD
?LA:N#	
K 
K	 	
K 
Ks#   DCD?DD	DD%N)rZ   r[   r\   r|   ra   r?   r=   rc   rc      s    Or?   rc   c                   n   e Zd ZdZd Zd Z	 dIdZd Zd Z e	j                  ed       e	j                   e       d      d	               Zd
 Zd Zd Zd Zd Ze e	j                   e       d      d               Ze e	j                   e       d      d               Ze e	j                   e       d      d               Z e	j                   e       d      d        Z e	j                   e       d      d        Zd Z e	j                   e       d      d        Ze e	j                   e       d      d               Z e	j                  ed      d        Ze e	j                  ed       e	j                   e       d      d                      Z e	j                   e       d      d        Z e	j                   e       d      d        Z  e	j                   e       d       e!d      d               Z" e	j                  ed      d        Z# e	j                  ed      d        Z$ e!d       d!        Z%d" Z&d# Z'd$ Z(d% Z)d& Z*d' Z+d( Z,d) Z-d* Z.d+ Z/d, Z0d- Z1d. Z2d/ Z3d0 Z4d1 Z5d2 Z6d3 Z7 e	j                  ed4      d5        Z8 e	j                  ed6      d7        Z9d8 Z:d9 Z;d: Z<d; Z= e!d<       e>d=      ed>                      Z? e!d<       e>d=      ed?                      Z@ e!d<       e>d=      ed@                      ZA e!dA      dB        ZB eCdC      dD        ZDdE ZE eCdF      dG        ZFyH)J	TestRsyncr   c                     | j                   dk(  r| j                  n| j                  }|j                  ||| j                   |g      }t	        ||d      S )ai  Retrieves and returns an attribute from an objects metadata.

    Args:
      bucket_name: The name of the bucket the object is in.
      object_name: The name of the object itself.
      attr_name: The name of the custom metadata attribute.

    Returns:
      The value at the specified attribute name in the metadata. If not present,
      returns None.
    gs)providerfieldsN)default_providerjson_apixml_apiGetObjectMetadatagetattr)r:   bucket_nameobject_name	attr_name
gsutil_apimetadatas         r=   _GetMetadataAttributezTestRsync._GetMetadataAttribute   sa     **d2 --8< ++K,7595J5J4=; , @H 8Y--r?   c                 n    | j                   r| j                  d|       y | j                  t        |       y )N	Completed)_use_gcloud_storageassertNotInr4   
NO_CHANGES)r:   stderrs     r=   _VerifyNoChangeszTestRsync._VerifyNoChanges   s+    
{F+
z6*r?   c                 8    | j                  ||t        ||       y)af  Retrieves the object's mtime.

    Args:
      bucket_name: The name of the bucket the object is in.
      object_name: The name of the object itself.
      expected_mtime: The expected retrieved mtime.
      expected_present: True if the mtime must be present in the
          object metadata, False if it must not be present.


    Returns:
      None
    expected_presentN)VerifyObjectCustomAttributer$   )r:   r   r   expected_mtimer   s        r=   _VerifyObjectMtimezTestRsync._VerifyObjectMtime   s&    $ 	$$[%0%/%36F	 % Hr?   c                 R   | j                         }| j                  |dd      }| j                         }| j                  dt	        |      t	        |      gd       | j                  dt	        |      t	        |      gd       | j                  dt	        |      d| j
                  z   gd       | j                  dt	        |      |gd       | j                  s| j                  d|t	        |      gd       | j                  d|t	        |      d| j
                  z   gd       y)	z%Tests various invalid argument cases.obj1   obj1rX   r   contentsr
      )expected_statuszgs://N)rL   CreateObjectrK   	RunGsUtilrO   nonexistent_bucket_namer   )r:   rX   r   tmpdirs       r=   test_invalid_argszTestRsync.test_invalid_args   s$   ""$J
)/&-  /D !FNNGT$Zj)9:ANNNNGT*-tDz:ANNNN		j	7T%A%AA	C  
 	NNGT$Z0!ND## nngvtDz2AnFNN	&	dWt;;;	=  r?   c                 t     j                          j                          j                  ddd        j                  ddd        j                  dd	t        d
              j                  ddd        j                  ddd       t        t        dd       fd       } |        y)zDTests that an exception is thrown if mtime cannot be cast as a long.r   r   xyzrX   r   r   mtimeobj2   obj2{   obj3   obj3l   Cc}obj4   obj4iobj5   obj5rD   r   triestimeout_secsc                     j                  dt              t              gd      } j                  r[j                  | d       j	                  | d       j                  | d       j                  | d       j                  | d       y j                  d	|        j                  d
|        j                  d|        j                  d|        j                  d|        y )Nr
   Treturn_stderrzNobj1#\d+ metadata did not contain a numeric value for goog-reserved-file-mtimezNobj2#\d+ metadata did not contain a numeric value for goog-reserved-file-mtimezNobj3#\d+ metadata that is more than one day in the future from the system timez)Found negative time value in gs://.*/obj4z)Found negative time value in gs://.*/obj5z)obj1 has an invalid mtime in its metadataz)obj2 has an invalid mtime in its metadataz:obj3 has an mtime more than 1 day from current system timez)obj4 has a negative mtime in its metadataz)obj5 has a negative mtime in its metadata)r   rO   r   assertRegexassertNotRegexru   r   r   bucket1_uribucket2_urir:   s    r=   _Check1z1TestRsync.test_invalid_src_mtime.<locals>._Check1'  s    ~~D%-1  3f 
	!	! )	* 	 )	* 	 	! 	!MN!MNA6JDfM	 	A6JA6Jr?   N)rL   r   longr'   AssertionError)r:   r   r   r   s   ` @@r=   test_invalid_src_mtimez TestRsync.test_invalid_src_mtime  s     ##%K##%K"(&!  # 	"(&  ! 	"(& /  1
 	"(&   " 	"(&    >3K 4K4 Ir?   z*POSIX attributes not available on Windows.zTest requires fast crcmod.c           
      |     j                          j                         t        j                         }t        j                         } j                  ddd        j                  dd|        j                  dd	|        j                  d
d	t               t               d        j                  ddt        |t        t                      j                  dd       t        t        dd       fd       } |         j                  j                  dt        d        j                  j                  dt         t        |              j                  j                  dt         t        |              j                  j                  d
t         t        t                             j                  j                  d
t"        t        t                             j                  j                  d
t        d        j                  j                  dt"        t        t                      j                  j                  dt         t        |              j                  j                  dt        t        t                     t        t        dd       fd       } |        y)z0Tests that rsync -P works with bucket to bucket.r   r   444rX   r   r   moder   r   rX   r   r   gidr   r   r   222rX   r   r   uidr   r   r   r   r   rD   r   r   c                     j                  ddt              t              gd      } t        t              j                              }t        t              j                              }j	                  |t        g d             j	                  |t        g d             j                  rj                  d|        yj                  d|        y)	z?Test bucket to bucket rsync with -P flag and verify attributes.r
   ro   Tr   )/obj1/obj2/obj3/obj4z/obj5Patching,Copying POSIX attributes from src to dst forN)r   rO   r   FlatListBucketr4   setr   ru   )r   listing1listing2
dst_bucketr:   
src_buckets      r=   r   z?TestRsync.test_bucket_to_bucket_preserve_posix.<locals>._Check1r  s     ~~D$z*
  f j)4+>+>z+JKhj)4+>+>z+JKh
xHIK xHIK		!	!j&)DfMr?   c                      j                  ddt              t              gd      } j                  rj                  d|        yj                  d|        y)zBCheck that we are not patching destination metadata a second time.r
   ro   Tr   r   r   N)r   rO   r   r   r   r   r:   r   s    r=   _Check2z?TestRsync.test_bucket_to_bucket_preserve_posix.<locals>._Check2  s`     ~~D$z*
  f 
	!	!V,GPr?   N)rL   osgetgidr+   GetNonPrimaryGidr   r.   r-   r/   strr,   r'   r   r   r   r#   r"   r&   )r:   primary_gidnon_primary_gidr   r   r   r   s   `    @@r=   $test_bucket_to_bucket_preserve_posixz.TestRsync.test_bucket_to_bucket_preserve_posixD  s    ""$J""$J))+K++-O"(&   " 	"(&%  ' 	"(&)  + 	"(&%-%-   " 	"(&!%|,  . 	"(&  (
 >3N 4N& I$$Z%;%;VY%*,$$Z%;%;VX%(%57$$Z%;%;VX%(%9;
 	$$Z%;%;VX%(%79$$Z%;%;VX%(%79$$Z%;%;VY%*, 	$$Z%;%;VX%(\3$$Z%;%;VX%(%57$$Z%;%;VY%(%68 >3	Q 4	Q Ir?   c                      j                          j                          j                  ddd        j                  ddd        j                  dd        j                  dd       t        t        d	d
       fd       } |        t        t        d	d
       fd       } |         j                  rG j                  j                  dt        d        j                  j                  dt        d       y j                  j                  dd        j                  j                  dd       y)zTests bucket to bucket with mtime.

    Each has the same items but only the source has mtime stored in its
    metadata.
    Ensure that destination now also has the mtime of the files in its metadata.
    r   r   r   r   subdir/obj2   subdir/obj2r   r   rD   r   c                      j                  ddt              t              g       t        t              j                              } j	                  | t        ddg             yTests rsync works as expected.r
   -rr   /subdir/obj2Nr   rO   r   r   r4   r   )r   r   r:   r   s    r=   r   zGTestRsync.test_bucket_to_bucket_same_objects_src_mtime.<locals>._Check1  sZ     nngtT*%5tJ7GHIj)4+>+>z+JKh
xg~%>!?@r?   c                  x    j                  dt              t              gd      } j                  |        y Nr
   Tr   r   rO   r   r   s    r=   r   zGTestRsync.test_bucket_to_bucket_same_objects_src_mtime.<locals>._Check2  s@    ~~D$
,0  2f F#r?   Fr   01N)rL   r   r'   r   r   r   r   r%   )r:   r   r   r   r   s   `  @@r=   ,test_bucket_to_bucket_same_objects_src_mtimez6TestRsync.test_bucket_to_bucket_same_objects_src_mtime  sj    ""$J""$J"(&   	"/-   	"(&  ( 	"/-  /
 >3A 4A I >3$ 4$ I j44$%/4  6 j44+%/4  6 j44fcB
j44mSIr?   c                      j                          j                          j                  ddd      } j                  ddd      } j                  |j                  |j                  d        j                  |j                  |j                  d	       t        t        d
d       fd       } |         j                  j                  dd        j                  j                  dd	       y)z7Tests bucket to bucket where source has mtime in files.r   r   r   r   r   r   r   r   r   rD   r   c                  X   j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        ddg             j	                  |t        ddg             yr   r   r   r   r   r:   r   s     r=   r   z:TestRsync.test_bucket_to_bucket_src_mtime.<locals>._Check1  s     nngtT*%5tJ7GHIj)4+>+>z+JKhj)4+>+>z+JKh
xg~%>!?@
xg~%>!?@r?   N)rL   r   r   r   r   r'   r   )r:   r   r   r   r   r   s   `   @@r=   test_bucket_to_bucket_src_mtimez)TestRsync.test_bucket_to_bucket_src_mtime  s     ""$J""$J
)/&-#$  &D 
)6&4#$  &D
 	D,,d.>.>DD,,d.>.>D >3A 4A I 	J22FC@J22M3Gr?   c                 R   	
  j                         
 j                         	 j                  
dd        j                  
dd        j                  
dd        j                  
dd	        j                  
d
dd        j                  	ddd        j                  	ddd        j                  	ddt        d              j                  	ddd        j                  	d
dd       t        t        dd      	 
fd       } |         j                  	j                  dd        j                  r j                  
j                  ddg      j                  }t        t        |            } j                  	j                  d|        j                  
j                  ddg      j                  }t        t        |            } j                  	j                  d|       nF j                  	j                  dt        d        j                  	j                  dt        d       t        t        dd      	 
fd       } |        t        t        dd      	 fd       } |        t        t        dd      	 
fd       } |        y )!z>Tests bucket to bucket where destination has mtime in objects.r      OBJ1r   r   r   .obj3s   .obj3zsubdir/obj4s   subdir/obj4obj6   OBJ6d   r   r   
   s   .OBJ3l    J)subdir/obj5   subdir/obj5   obj6rD   r   r   c            	      `   j                  dddt              t              gd      f t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             y)r   r
   r   -dTr   r   r   z/.obj3z/subdir/obj4/obj6Nr   r   s     r=   r   z:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check1A  s     nngtT:&:&( $(  ) *
 j)4+>+>z+JKhj)4+>+>z+JKh


J
KM 

J
KMr?   10timeCreatedFr   c                  x    j                  dt              t              gd      } j                  |        y r   r   r   s    r=   _Check3z:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check3r  sB     ~~D$
,0  2f F#r?   c            
          j                  dj                  dt         d      gd             j                  dj                  dt         d      gd             j                  dj                  dt         d      gd             y )	NOBJ1catr   Treturn_stdoutr   OBJ6r   )r4   r   rO   )r   r:   s   r=   _Check4z:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check4}  s     

..%j&!9:$.
OQ
 

..%j'!:;'+  -. 

..%j&!9:$.
OQr?   c            
         j                  ddddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             j	                  dj                  dt        d      gd	
             j                  j                  dd       y)!Tests rsync -c works as expected.r
   r   r  -cr  r  r  r   Tr  100Nr   rO   r   r   r4   r   r   r   r   s     r=   _Check5z:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check5  s     nnD$

 j)4+>+>z+JKhj)4+>+>z+JKh


J
KM 

J
KM
 

..%j&!9:$.
OQ j44feDr?   N)rL   r   r   r'   r   r   r   r   GetObjectMetadataWithFieldsr  r   r!   r%   )r:   r   source_o1_time_createdsource_o1_posix_time_createdsource_o4_time_createdsource_o4_posix_time_createdr  r  r  r   r   s   `        @@r=   test_bucket_to_bucket_dst_mtimez)TestRsync.test_bucket_to_bucket_dst_mtime  s    ""$J""$J"(&  ( 	"/-  / 	")'  ) 	"/-  / 	"(&  ! 	"(&    	"/-    	")' /  1 	"/-    	"(&  ! >3M 4M$ I 	J22M4H  $??

 
 &=/ ;;F; %(
 !7
8&:"
j44f:<#??

 
 -- BBM+ %(
 !7
8&:"
j44m:< j44$%/4  6 j44+%/4  6 >3$ 4$ I >3Q 4Q$ I >3E 4E0 Ir?   c                 l     j                          j                          j                  dd        j                  ddd        j                  dd	        j                  d
dd        j                  dd        j                  dd        j                  dd        j                  d
dd       t        t        dd       fd       } |        t        t        dd       fd       } |         j                  d
d        j                  dd        j	                  dt        d      g        j	                  dt        d      g       t        t        dd       fd       } |        t        t        dd       fd       } |        y)FTests that flat and recursive rsync between 2 buckets works correctly.r   r   r   .obj2   .obj2r   r   subdir/obj3   subdir/obj3r   s   obj6_r      .OBJ2r   r   r   r   r   rD   r   r   c            
         j                  dt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             j	                  dj                  dt        d      gd             j	                  dj                  dt        d      gd             j	                  dj                  dt        d	      gd             j                  j                  dd
       y)r   r
   )r   /.obj2/subdir/obj3r  )r   r$  r   /subdir/obj5r  r  r  Tr  obj6_r   r  Nr  r   r   r   r   r:   s     r=   r   z0TestRsync.test_bucket_to_bucket.<locals>._Check1  sI    nngtK0${2CDEk*D,?,?,LMhk*D,?,?,LMh
xGHJ
 
CMNP
 

..%k7!;<'+  -. 

..%k7!;<'+  -. 

..%k6!:;'+  -.
 k55wEr?   c                  x    j                  dt              t              gd      } j                  |        y r   r   r   s    r=   r   z0TestRsync.test_bucket_to_bucket.<locals>._Check2  sB     ~~D%-1  3f F#r?   obj7   obj7rmc                  X   j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             y )Nr
   r   r$  r  r%  )r   r$  r   r  /obj7r%  r&  r   r(  s     r=   r  z0TestRsync.test_bucket_to_bucket.<locals>._Check3  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%H!IJ 

  r?   c                  z    j                  ddt              t              gd      } j                  |        y )Nr
   r   Tr   r   r   s    r=   r  z0TestRsync.test_bucket_to_bucket.<locals>._Check4#  D     ~~D${+  f F#r?   NrL   r   r'   r   r   rO   r:   r   r   r  r  r   r   s   `    @@r=   test_bucket_to_bucketzTestRsync.test_bucket_to_bucket  s    ##%K##%K"(&  ( 	")'    	"/-  / 	"('  !
 	")'  ) 	"(&  ( 	"/-  / 	"(&  ! >3F 4F> I >3$ 4$ I 	"(&  ( 	"(&  ( 	NND${F345NND${G456 >3 4  I >3$ 4$ Ir?   c                      j                          j                          j                  dd        j                  dd        j                  dd        j                  dd        j                  d	d
        j                  dd       t        t        dd       fd       } |        t        t        dd       fd       } |         j                  dd        j                  dd        j	                  dt        d      g        j	                  dt        d      g       t        t        dd       fd       } |        t        t        dd       fd       } |        y)r  r   r   r   r  r  r   r!  r"  r   r   r   r   rD   r   r   c            
         j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             j	                  dj                  dt        d      gd             j	                  dj                  dt        d      gd             y	)
r   r
   r  r   r$  r%  r   r$  r&  r  r  Tr  Nr   r(  s     r=   r   z8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check1K  s     nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%H!IJ x%H!IJ 

..%k7!;<'+  -. 

..%k7!;<'+  -.r?   c                  z    j                  ddt              t              gd      } j                  |        y Nr
   r  Tr   r   r   s    r=   r   z8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check2e  r1  r?   r   r   r*  r+  r,  c            	      Z   j                  dddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             y Nr
   r  r   r.  r   r(  s     r=   r  z8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check3{  s    
nnD$ k*D,?,?,LMhk*D,?,?,LMh
x%H!IJ x%H!IJr?   c            	      |    j                  dddt              t              gd      } j                  |        y Nr
   r  r   Tr   r   r   s    r=   r  z8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check4  I     ~~D$ 	  f
 F#r?   Nr2  r3  s   `    @@r=   test_bucket_to_bucket_minus_dz'TestRsync.test_bucket_to_bucket_minus_d.  s    ##%K##%K"(&  ( 	")'  ) 	"/-  / 	")'  ) 	"(&  ( 	"/-  /
 >3. 4., I >3$ 4$ I 	"(&  ( 	"(&  ( 	NND${F345NND${G456 >3K 4K I >3$ 4$ Ir?   c                 ~     j                         t        j                  j                  d      }t        j                  |        j                  ddd        j                  ddd        j                  |dd	d        j                  |d
dd        j                  ddd        j                  ddd        j                          j                  dd        j                  dd        j                   j                  j                  ddd        j                  dd        j                  ddd        j                  ddd        j                  ddd       t               t        t        dd       fd       } |        t        t        dd       fd       } |         j                  j                  dd         j                  j                  dd         j                  j                  d!d         j                  j                  dd"        j                  j                  dd#       d$t        d      z  }|vr* j                   s j#                  j                  ddd       t        t        dd       fd%       } |        y&)'aK  Tests dir to bucket with mtime.

    Each has the same items, the source has mtime for all objects, whereas dst
    only has mtime for obj5 and obj6 to test for different a later mtime at src
    and the same mtime from src to dst, respectively. Ensure that destination
    now also has the mtime of the files in its metadata.
    subdirr   r   r   r   	file_namer   r   r  r  r   r!  r   r      r   r   r   r*     obj7_r   r   testr   r   r   r   r   r+  rD   r   r   c            
      `   j                  dddt              gd      } j                  | j                         D cg c]  }|s|	 c}       t	        j                              }t	        t              j                              }j                  |t        g d             j                  |t        g d             j                  dj                  dt        d	      gd
             j                  dj                  dt        d      gd
             yc c}w )r   r
   r   r  Tr   r   r$  r%  r&  r  r/  r  r  r   r  obj7_r*  N)	r   rO   update
splitlinesr   FlatListDirr   r4   r   )r   sr   r   rX   cumulative_stderrr:   r   s       r=   r   z3TestRsync.test_dir_to_bucket_mtime.<locals>._Check1  s3    ~~wdF#J/1,0  2f 6+<+<+>D+>a!+>DE!1!1&!9:hj)4+>+>z+JKh


   

   

..%j&!9:$.
OQ 

..%j&!9:$.
OQ/  Es    D+D+c            	      j    j                  dddt              gd      } j                  |        y )Nr
   r   r  Tr   r   r   rX   r:   r   s    r=   r   z3TestRsync.test_dir_to_bucket_mtime.<locals>._Check2   @     ~~wdF#J/1,0  2f F#r?   r  r   15r  zkCopying whole file/object for %s instead of patching because you don't have owner permission on the object.c            
         j                  ddddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             j                  dj                  dt        d      gd	             j                  j                  dd
       y)r  r
   r   r  r  rI  r   r  Tr  r  N)	r   rO   r   rM  r   r4   r   r   r   r   r   rX   r:   r   s     r=   r  z3TestRsync.test_dir_to_bucket_mtime.<locals>._Check4  s     nngtT4j9IJK!1!1&!9:hj)4+>+>z+JKh


   

   

..%j&!9:$.
OQ j44feDr?   N)rK   r   rW   rs   mkdirCreateTempFilerL   r   !_SetObjectCustomMetadataAttributer   r   r   r'   r   r   rO   r   r   )	r:   rB  r   r   copied_over_object_noticer  rX   rO  r   s	   `     @@@r=   test_dir_to_bucket_mtimez"TestRsync.test_dir_to_bucket_mtime  s    !FWW\\&(+FHHVv"(!(   " 	v")!)   " 	v"(!/   " 	v"(!/   " 	v"(!(!  # 	v"(!)!  # ""$J"(&  ( 	")'  ) 	**4+@+@+5+A+A7+16; 	"(&  ( 	"/-    	"(&  ! 	"(&  !
 
>3Q 4Q@ I >3$ 4$ I 	J22FDAJ22GTBJ22M4HJ22M4HJ22FEB	6Z!	"  	")::$$ &&z'='=w'-/
 >3E 4E4 Ir?   c                 8    t        t        dd       fd       } |         j                  d      } j                         }t	        ddg      5   j                  dd	d
d|t        |      gd      } j                  d|       ddd       y# 1 sw Y   yxY w)z5Tests that rsync seek-ahead iterator works correctly.rD   r   r   c                     j                         } t        j                  j                  | d      }t        j                  |       j                  | dd       j                  | dd       j                  |dd       j                         }j                  |dd	
       j                  |dd
       j                  |dd
       j                  |d       t        ddg      5  j                  dddd| t        |      gd      }j                  d|       j                  |d       j                  dddd| t        |      gd      }j                  d|       ddd       y# 1 sw Y   yxY w)z*Test estimating an rsync upload operation.rB  r   r   r   rD  r   r  r  r   r!  r"  r   r   r   r   r   rD   )re   task_estimation_thresholdr   re   task_estimation_forcerg   -mr
   r  r   Tr   z;Estimated work for this command: objects: 5, total size: 20Estimated workN)rK   r   rW   rs   rV  rW  rL   r   AssertNObjectsInBucketr   r   rO   ru   r   )r   rB  rX   r   r:   s       r=   r   z8TestRsync.test_dir_to_bucket_seek_ahead.<locals>._Check1B  s    !!#fww||FH-fhhv
6GL
7XN
$*#1  3 $$&j
:$+!)  + :$*!(  * :$1!/  1
 !!*a0!M!L!N O7D$*     	I	 	##J27D$*     	)62%O O Os   .A;E22E;
test_files)re   r^  r   r_  ra  r
   r  r   Tr   rb  N)r'   r   rK   rL   r   r   rO   r   )r:   r   r   rX   r   s   `    r=   test_dir_to_bucket_seek_aheadz'TestRsync.test_dir_to_bucket_seek_ahead<  s     >3*3 4*3X I1-F""$J 
KJL 
M~~$f
  f '0
M 
M 
Ms   5BBc                 `   	
  j                         
t        j                  j                  
d      }t        j                  |        j                          j                  
dd        j                  
dd      	 j                  |dd        j                  dd	
        j                  dd
        j                  dd
        j                  d       t        t        dd      	 
fd       } |        t        t        dd       
fd       } |        t        t        dd       
fd       } |        t        t        dd       
fd       } |         j                  
dd        j                  dd
       t        j                  t        j                  j                  
d              j                  dt        d      g       t        t        dd       
fd       } |        t        t        dd       
fd       } |        y)zBTests that flat and recursive rsync dir to bucket works correctly.rB  r   r   r]  r  r  r   r!  r"  r   r   r   r   r   rD   r   r   c                  \   j                  dd	t              g       t        	j                  	            } t        t              j	                              }j                  | t        g d             j                  |t        g d             t        t        j                  j                  	d            5 }j                  ddj                  |j                                      ddd       j                  dt        d      gd	
      }j                  rjt        t        j                  j                              }t        j!                  j"                  dd            }j%                  |dk(  xs ||k(         yj                  d|       y# 1 sw Y   xY w)r   r
   r  r7  r8  r  r0   Nr  Tr  r  )r   rO   r   rM  r   r4   r   openr   rW   rs   	readlinesr   intgetmtimer!   r   r   
assertTrue)
r   r   fcloud_obj2_contentlocal_obj2_mtimecloud_obj2_ctimerX   o2_pathr:   r   s
         r=   r   z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check1  sp    nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJ VW-.!$))AKKM":; />>$z7+
,D * B		!	! rww//891&&z'='=w'467 	*g5 =(,<<	> 	"45 /.s   0F""F+c                  h    j                  ddt              gd      } j                  |        y r:  r   rQ  s    r=   r   z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check2  :     ~~D&$z"2
34  If
F#r?   c            
      n   j                  dddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       j                  dj                  d	t        d      gd
             y# 1 sw Y   9xY w)r  r
   r  r  r7  r8  r  r0   Nr  Tr  )r   rO   r   rM  r   r4   r   ri  r   rW   rs   rj  r   r   rn  rX   r:   r   s      r=   r  z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check3  s     nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJ VW-.!$))AKKM":; /


..%j'!:;'+  -. /.s   0D++D4c            	      j    j                  dddt              gd      } j                  |        y Nr
   r  r  Tr   r   rQ  s    r=   r  z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check4  rR  r?   r   r   r*  r+  r,  c            	      6   j                  dddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             y r<  r   rO   r   rM  r   r4   r   rU  s     r=   r  z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check5  s    
nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJr?   c            	      j    j                  dddt              gd      } j                  |        y r>  r   rQ  s    r=   _Check6z5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check6  rR  r?   N)rK   r   rW   rs   rV  rL   rW  r   rc  r'   r   unlinkr   rO   )r:   rB  r   r   r  r  r  r|  rX   rr  r   s   `       @@@r=   test_dir_to_bucket_minus_dz$TestRsync.test_dir_to_bucket_minus_d~  s:    !FWW\\&(+FHHV""$Jv'J!!,3+3 " 5G 	v"(!/  1 	")'  ) 	"(&  ( 	"/-  / 	
A. >36 46< I >3$ 4$ I >3. 4.& I >3$ 4$ I 	v'J"(&  ( IIbggll66*+NND$z7345 >3K 4K I >3$ 4$ Ir?   c                 P   	
  j                         	 j                         
t        j                  j                  	d      }t        j                  j                  
d      }t        j                  |       t        j                  |        j                  	ddd        j                  	ddd        j                  |d	d
d        j                  	ddd        j                  	ddd        j                  
ddd        j                  
ddd        j                  |ddd        j                  
ddd        j                  
ddd        j                  ddd	
g       t        	 j                  	            }t        
 j                  
            } j                  |t        g d              j                  |t        g d             t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd        	
fd } |         j                  dddd!	
g       t        	 j                  	            }t        
 j                  
            } j                  |t        g d              j                  |t        g d             t        t        j                  j                  	d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  	d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd        	
fd"} |        t        j                  t        j                  j                  	d             t        j                  t        j                  j                  
d              j                  	ddd        j                  
ddd       t        j                  t        j                  j                  	d             t        j                  t        j                  j                  
d              j                  ddd	
g       t        	 j                  	            }t        
 j                  
            } j                  |t        g d#              j                  |t        g d#              	
fd$} |        y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   uxY w# 1 sw Y   xY w# 1 sw Y   8xY w# 1 sw Y   xY w)%?Tests that flat and recursive rsync dir to dir works correctly.subdir1subdir2r   r   r   rC  r  r  r      subdir1/obj3r   r   r   r*  rF  r"    r   r   r      subdir2/obj5r   r+  r
   r   r  )r   r$  /subdir1/obj3r  r/  r0   Nr  rJ  c            	      \     j                  t         j                  ddgd             y r:  r4   r   r   r:   tmpdir1tmpdir2s   r=   r   z0TestRsync.test_dir_to_dir_mtime.<locals>._Check1Y  .    


..'4':$.
OQr?   r  c            
      ^     j                  t         j                  dddgd             y rx  r  r  s   r=   r   z0TestRsync.test_dir_to_dir_mtime.<locals>._Check2t  5    


..'4w@'+  -.r?   r$  r  r  c            
      ^     j                  t         j                  dddgd             y r>  r  r  s   r=   r  z0TestRsync.test_dir_to_dir_mtime.<locals>._Check3  r  r?   rK   r   rW   rs   rV  rW  r   r   rM  r4   r   ri  rj  r}  )r:   r  r  r   r   rn  r   r   r  r  r  s   `        @@r=   test_dir_to_dir_mtimezTestRsync.test_dir_to_dir_mtime  s      "G  "Gggll7I.Gggll7I.GHHWHHWw"(!(   " 	w")!)   " 	w"(!0   " 	w"(!(!  # 	w"(!)!  # 	w")!)"  $ 	w"(!(   " 	w"(!0   " 	w"(!(!  # 	w"(!(!  #
 	NNGT4':;w 0 0 9:Hw 0 0 9:H#LMO 	#LMO 
bggll7G,	-
w		!++- 89 
.	bggll7F+	,
vtyy78 
-	bggll7F+	,
w		!++- 89 
-Q I 	NNGT4w@Aw 0 0 9:Hw 0 0 9:H#LMO 	#LMO 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.	bggll7F+	,
vtyy78 
-. I IIbggll7F+,IIbggll7F+,w"(!(   " 	w"(!(!  # IIbggll7F+,IIbggll7G,-NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHIXs#GHI. IQ 
.	-	,	,	,	,. 
.	-	-	-	,	,sH   0Y00Y'0Y40Z(0Z	0ZY$'Y14Y>ZZZ%c                    
  j                         
 j                         t        j                  j                  
d      }t        j                  j                  d      }t        j                  |       t        j                  |        j                  
dd        j                  
dd        j                  |dd	        j                  dd
        j                  dd        j                  |dd        j                  dd
g       t        
 j                  
            }t         j                              } j                  |t        g d              j                  |t        g d             t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  d            5 } j                  ddj                  |j                                      ddd        
fd} |         j                  ddd
g       t        
 j                  
            }t         j                              } j                  |t        g d              j                  |t        g d             t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd       t        t        j                  j                  
d            5 } j                  ddj                  |j                                      ddd        
fd} |         j                  
dd        j                  dd       t        j                  t        j                  j                  
d             t        j                  t        j                  j                  d              j                  ddd
g       t        
 j                  
            }t         j                              } j                  |t        g d              j                  |t        g d              
fd} |         j                         
 j                          j                  
dd        j                  
d d!        j                  d d!        j                  dd
g       t        
 j                  
            }t         j                              } j                  |t        d"d#g              j                  |t        d"d#g              
fd$}	 |	        y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   QxY w)%r  r  r  r   r   r]  r  r  r   r  r"  r   r   r   r  r
   r  )r   r$  r  )r   r$  z/subdir2/obj5r0   Nz.OBJ2c            	      \     j                  t         j                  ddgd             y r:  r  r  s   r=   r   z2TestRsync.test_dir_to_dir_minus_d.<locals>._Check1  r  r?   r  c            
      ^     j                  t         j                  dddgd             y rx  r  r  s   r=   r   z2TestRsync.test_dir_to_dir_minus_d.<locals>._Check2  r  r?   r   r   r*  r+  r   r  c            
      ^     j                  t         j                  dddgd             y r>  r  r  s   r=   r  z2TestRsync.test_dir_to_dir_minus_d.<locals>._Check3  r  r?   r   r   r   r   c            	      \     j                  t         j                  ddgd             y r:  r  r  s   r=   r  z2TestRsync.test_dir_to_dir_minus_d.<locals>._Check4  r  r?   r  )r:   r  r  r   r   rn  r   r   r  r  r  r  s   `         @@r=   test_dir_to_dir_minus_dz!TestRsync.test_dir_to_dir_minus_d  s      "G  "Gggll7I.Gggll7I.GHHWHHWw&7Kw'HMw"(!0  2 	w'HMw&7Kw"(!0  2 	NNGT7G45w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.Q I 	NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.
. I 	w&7Kw&7KIIbggll7F+,IIbggll7G,-NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI. I   "G  "Gw&7Kw&7Kw&7KNNGT7G45w 0 0 9:Hw 0 0 9:HXsGW#567XsGW#567Q IA 
.	-	-	-4 
.	-	-	-s0   40W0W
0W+0W*WWW'*W4c                      j                          j                         t        dd      D ]5  } j                  d|z  d|dz           j                  d|z  d|       7 d	d
t        rdndfg}t	        |      5   j                  ddg       ddd       t         j                              }t         j                              } j                  ||       t        dd      D ]  } j                  |dz   t        t        j                  j                  t        j                  j                  d|z                           t        t        j                  j                  d|z              5 } j                  ddj                  |j                                      ddd         fd} |        y# 1 sw Y   2xY w# 1 sw Y   xY w)zBTests concurrently building listing from multiple tmp file ranges.r   r  zd1-%s   xr   rC  zd2-%s   yre   rsync_buffer_lines502r
   r  Nxr0   c            	      \     j                  t         j                  ddgd             y r:  r  r  s   r=   _CheckzITestRsync.test_dir_to_dir_minus_d_more_files_than_bufsize.<locals>._CheckD  r  r?   )rK   rangerW  r)   r   r   r   rM  r4   r   r   rW   rl  rs   ri  rj  )	r:   irsync_buffer_configr   r   rn  r  r  r  s	   `      @@r=   /test_dir_to_dir_minus_d_more_files_than_bufsizez9TestRsync.test_dir_to_dir_minus_d_more_files_than_bufsize"  s      "G  "G1d^
$+aK#'!"Q  ) $+aK#' !  #  %&:$.DC9 : 
1	2
nngtWg67 
3w 0 0 9:Hw 0 0 9:HXx(1d^

a%bgg&&Wgk(JLMOWgk23qdii67 43 Q H' 
3	2 43s   G>0GGG	c                    | j                  dd      }| j                         }| j                         }| j                  dd|t	        |      g       | j                  dt	        |      |gd      }t        t        j                  j                  |d      d	      5 }| j                  d|j                                d d d        | j                  d
|       y # 1 sw Y   xY w)N   foobar)r   rD  cpz-Zr
   Tr   rbz%bar has a compressed content-encoding)rW  rL   rK   r   rO   ri  r   rW   rs   r4   readru   )r:   	temp_filerX   r   r   fps         r=   &test_bucket_to_dir_compressed_encodingz0TestRsync.test_bucket_to_dir_compressed_encodingL  s    ##Vu#EI""$J!FNND$	4
+;<=^^Wd:&6?*.  0F 
bggll65)4	0B
vrwwy) 
1MM96B 
1	0s   !CC c                      j                          j                         t        j                  j	                  d      }t        j
                  |        j                  ddd        j                  ddd        j                  dd	
        j                  dd
        j                  ddd        j                  ddd        j                  ddd        j                  ddd        j                  dd
       t         j                  j                  dd            } j                  ddd        j                  ddd         j                  dd!d         j                  |d"d#d         j                  ddd         j                  dd$d         j                  ddd         j                  dd%d         j                  dd&|         j                  dd'd        t        t        d(d)*       fd+       } |         fd,} |        t        t        d(d)*       fd-       } |        y.)/z-Tests bucket to dir with mtime at the source.rB  r   r      r   r  r  r   r!  r   r   s   OBJ4r   r   2   r*  r+  obj8   obj8r   obj9   obj9   obj10   obj10r  obj11s   obj11_K   r"  r   rC  r   r   r   s   OBJ7s   OBJ9s   OBJ10   obj11rD   r   r   c                  z   j                  ddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       t        t        j                  j                  d            5 }j                  d	dj                  |j                                      ddd       t        t        j                  j                  d
            5 }j                  ddj                  |j                                      ddd       t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       y# 1 sw Y   xY w# 1 sw Y   ;xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)r   r
   r  
r   r$  r%  r   r  r/  /obj8/obj9/obj10/obj11)
r   r$  r   r&  r  r/  r  r  r  r  r  r0   Nr   OBJ4r  OBJ9r  OBJ10r  obj11_r   rO   r   r   rM  r4   r   ri  r   rW   rs   rj  rv  s      r=   r   z3TestRsync.test_bucket_to_dir_mtime.<locals>._Check1  s    nngtT*%5v>?j)4+>+>z+JKh!1!1&!9:h


   

   VW-.!$))AKKM":; / VV,-1;;=!9: .
 VV,-1;;=!9: . VW-.!$))AKKM":; /VW-.!499Q[[]#;< /. /. .-
 .- /...s<   0I?#0J0J%0J%0J1?J	JJ"%J.1J:c            
      b    j                  t        t        j                  j	                  t        j                  j                  d                  d        j                  t        t        j                  j	                  t        j                  j                  d                  d        j                  t        t        j                  j	                  t        j                  j                  d                  d        j                  t        t        j                  j	                  t        j                  j                  d                  d        j                  t        t        j                  j	                  t        j                  j                  d                  d	       y
)z0Verify mtime was set for objects at destination.r   r  r  r   r  r  r   r  r  N)r4   r   r   rW   rl  rs   )r:   r   s   r=   r   z3TestRsync.test_bucket_to_dir_mtime.<locals>._Check2  s   
tBGG,,RWW\\&&-IJKQO
tBGG,,RWW\\&'-JKLaP
tBGG,,RWW\\&&-IJKRP
tBGG,,RWW\\&&-IJK
tBGG,,RWW\\&&-IJKRPr?   c            
      0   j                  ddddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             j                  dj                  dt        d      gd	             j                  j                  dd
       j                  t        t        j                  j                  t        j                  j                  d                  d       t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       t        t        j                  j                  d            5 }j                  ddj                  |j                                      ddd       y# 1 sw Y   kxY w# 1 sw Y   yxY w)r  r
   r   r  r  r  r*  r  Tr  5r  r  r0   Nr  )r   rO   r   r   rM  r4   r   r   r   r   r   rW   rl  rs   ri  rj  rv  s      r=   r  z3TestRsync.test_bucket_to_dir_mtime.<locals>._Check3  s    nngtT4j1A6JKj)4+>+>z+JKh!1!1&!9:h


   

   

..%j&!9:$.
OQ j44fcB
tBGG,,RWW\\&&-IJKQO VV,-1;;=!9: .VW-.!$))AKKM":; /. .-..s   &0H 0H H	HN)rL   rK   r   rW   rs   rV  r   r!   r   r   rW  r'   r   )r:   rB  time_createdr   r   r  rX   r   s   `     @@r=   test_bucket_to_dir_mtimez"TestRsync.test_bucket_to_dir_mtimeY  s    ""$J!FWW\\&(+FHHV"(&   	")'   	"/-  / 	"(&  ( 	"(&    	"(&   	"(&  ! 	"(&    	")'  ) *"":#9#97#0	23L 	")(    	v")!)   " 	v"(!(!  # 	v"(!/   " 	v"(!(   " 	v"(!(   " 	v"(!(   " 	v"(!(   " 	v")!)*  , 	v")!)   " >3&= 4&=P IQ I >3!< 4!<F Ir?   c                    	
  j                          j                         t        j                        j                  }t        j                         }t        j                  j                  d      }t        j                  |        j                  dddt        j                                j                  ddt               d	       j                  d
d        j                  ddt               d	       j                  dd|d	       j                  ddt                      j                  ddt               d       j                  ddt               t                      j                  ddt               t               d       	 j                  d!d"t               t        #      
 j                  d$d%t               |d&'       j                  d(d)t        t               #       j                  d*d+t        t               d,'       j                  d-d.t        d/       j                  d0d1t        |d'       j                  d2d3t        |d'       j                  d4d5d6       j                  dd78        j                  d9d:8        j                  |d;d<8       t!        t"        d=d>?      	
 fd@       } |         j%                   j&                  j(                  dt*        dA       t!        t"        d=d>?       fdB       } |        yC)DzETests that rsync -P works properly with files that would be orphaned.rB  r   r   r   )rX   r   r   r   r   r  r  540rX   r   r   r   r   r   r!  r   r   r   440r*  r+  333r  r  rX   r   r   r   r  r  777rX   r   r   r   r   r  r  )rX   r   r   r   r   r  r  544)rX   r   r   r   r   r   obj12   obj12rX   r   r   r   r   obj13   obj13644r   obj14   obj14obj15s   obj15655obj16s   obj16244obj17s   obj17obj18s   obj18obj19s   obj19r   r"  r]  r   r   r   r   rD   r   r   c                  .
   j                  dddt              gdd      } j                  r,d}j                  | |       d}d	}d
}j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             j                  | |j	                  d             nj                  t        |        j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        	t                     j                  | t        
t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     j                  | t        t                     t        t              j                              }t        j                              }j                  |t!        g d             j                  rj                  |t!        g d             yj                  |t!        g d             y)zATests that an exception is thrown because files will be orphaned.r
   ro   r   r   Tr   r   zFor preserving POSIX with rsync downloads, gsutil aborts if a single download will result in invalid destination POSIX. However, thisz9{}#\d+\. User \d+ owns file, but owner does not have readz5{}#\d+ metadata doesn't exist on current system\. GIDz5{}#\d+ metadata doesn't exist on current system\. UIDr   r   r   r*  r  r  r  r  r  r  r  r  r  r  r  r  r   r$  r%  r  r/  r  r  r  r  /obj12/obj13/obj14z/obj15z/obj16z/obj17z/obj18z/obj19r$  r   r%  r&  r$  r   r&  N)r   rO   r   r   rv   ru   r   r   r   r   r   r   r   r   rM  r4   r   )r   gcloud_preserve_posix_warning
read_regex	gid_regex	uid_regexr   r   rX   r   r  r  r  r  r  r  r  r  r  r  r   r   r*  r  r  r:   r   s          r=   r   zCTestRsync.test_bucket_to_dir_preserve_posix_errors.<locals>._Check1  s1    ~~D$Z 0&
9  f 
	!	!Q 	& 	!>?Q
L	L	!2!26!:;!1!1&!9:!1!1&!9:!2!26!:;!1!1&!9:!1!1&!9:!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!2!27!;<!2!27!;<!2!27!;<!2!27!;<mV,7G!HI!GH!GH7G!HI!GH!GH!HI!HI!HI!HI!HI!HIOE+JK	M8H!IJ8H!IJ8H!IJj)4+>+>z+JKh!1!1&!9:h


   
	!	! 	cMN	P 	3'J#KLr?   640c                     j                  dddt              gdd      } j                  rj                  d|        nj                  t        |        t        t              j                              }t        j                              }j                  |t        g d             j                  rj                  |t        g d	             yj                  |t        g d
             y)zATests that a file with a valid mode in metadata, nothing changed.r
   ro   r   r   Tr  z%doesn't exist on current system. GID:r  r  r  N)
r   rO   r   ru   r   r   r   rM  r4   r   )r   r   r   rX   r:   r   s      r=   r   zCTestRsync.test_bucket_to_dir_preserve_posix_errors.<locals>._Check2  s     ~~D$Z 0&
9  f 
	!	!=vFmV,j)4+>+>z+JKh!1!1&!9:h


   
	!	!cMN	P 	3'J#KLr?   N)rL   rK   r   statst_gidr+   r   rW   rs   rV  r   getuidr-   r.   r/   rW  r'   r   rX  r   r   r#   )r:   r   r   rB  r   r   rX   r   r  r  r  r  r  r  r  r  r  r  r   r   r*  r  r  r   s   `     @@@@@@@@@@@@@@@@@@r=   (test_bucket_to_dir_preserve_posix_errorsz2TestRsync.test_bucket_to_dir_preserve_posix_errors  s    ""$J!F''&/((K++-OWW\\&(+FHHV
)/&-"'!#	  .D 
)0&.!,"'	  )D
 	"/-  / 
)/&-!,"'	  )D 
)/&-!0"'	  )D 
)/&-!,  0D
 
)/&-!,"'	  )D *1'/"--"--	  1E *1'/"--"--#(  *E *1'/"--")	  +E *1'/"--"-#(  *E *1'/")"--	  1E *1'/")"--#(  *E *1'/")#(	  *E *1'/")"-#(  *E *1'/")"1#(  *E *1'/#(  *E 	v8Lv'Jv"(!/  1
 >3DM DM DM 4DML I 	**4+@+@+5+A+A7+4e=
 >3M 4M8 Ir?   c                 v	     j                          j                         t        j                        j                  }t        j                         }t        j                  j                  d      }t        j                  |        j                  ddd        j                  dd|        j                  d	d
|        j                  dd|d        j                  dd|d        j                  ddt                j                  ddt        d        j                  ddt        |        j                  ddt        |        j                  ddt        |d         j                  d!d"t        |d#         j                  d$d%t        d        j                  dd&'        j                  d(d)'        j                  |d*d+'       t        t        d,d-.       fd/       } |         j                  t        j                  j                  d      t        j                          d01        j                  t        j                  j                  d      |t        j                          t"        2        j                  t        j                  j                  |d3      |t"        4        j                  t        j                  j                  d      |d54        j                  t        j                  j                  d      |d04        j                  t        j                  j                  d      |t"        4        j                  t        j                  j                  d      t        d61        j                  t        j                  j                  d      t        |t"        7        j                  t        j                  j                  d      t        |t"        7        j                  t        j                  j                  d      t        |d87        j                  t        j                  j                  d!      t        |d97        j                  t        j                  j                  d$      t        d01       y:);z@Tests that rsync -P works properly with default file attributes.rB  r   r   r   r   r  r  r   r   r!  r   r   555r  r*  r+  r  r  r  r  r  422r  r  r  r  r  r  r  r  400r   r  r  533r  r  r"  r]  r   r   r   r   rD   r   r   c                  6   j                  dddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             y)zCVerifies that all attributes were copied correctly when -P is used.r
   ro   r   )r   r$  r%  r  r/  r  r  r  r  r  r  r  )r   r$  r%  r   r&  r  r/  r  r  r  r  r  r  r  Nr   rO   r   r   rM  r4   r   rU  s     r=   r   zFTestRsync.test_bucket_to_dir_preserve_posix_no_errors.<locals>._Check1?  s     nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h


   

  r?   i$  )r   r   )r   r   r   r   )r   r   im  i  )r   r   r      i[  N)rL   rK   r   r  r  r+   r   rW   rs   rV  r   r/   rW  r'   r   VerifyLocalPOSIXPermissionsr  r,   )r:   r   r   rB  r   rX   r   s   `    @@r=   +test_bucket_to_dir_preserve_posix_no_errorsz5TestRsync.test_bucket_to_dir_preserve_posix_no_errors  s   
 ""$J!F''&/((K++-OWW\\&(+FHHV"(&   " 	")'%  ' 	"/-)  + 	"(&% 	  "
 	"(&) 	  "
 	"(&!  # 	"(&! 	  "
 	")'!%	  '
 	")'!)	  +
 	")'!%   " 	")'!)   " 	")'! 	  "
 	v8Lv'Jv"(!/  1
 >3 4* I$$RWW\\&&%A)+*/ % 1 	$$RWW\\&'%B)4)+*6 % 8 	$$RWW\\&&%A)8*6 % 8 	$$RWW\\&&%A)4*/ % 1 	$$RWW\\&&%A)8*/ % 1 	$$RWW\\&&%A)4*6 % 8 	$$RWW\\&&%A)0*/ % 1 	$$RWW\\&'%B)0)4*6 % 8 	$$RWW\\&'%B)0)8*6 % 8 	$$RWW\\&'%B)0)4*/ % 1 	$$RWW\\&'%B)0)8*/ % 1 	$$RWW\\&'%B)0*/ % 1r?   c                 :   	  j                          j                         	t        j                  j	                  	d      }t        j
                  |        j                  dd        j                  ddd        j                  d	d
        j                  	dd        j                  	dd        j                  |dd       t        t        dd       	fd       } |        t        t        dd       	fd       } |        t        t        dd       	fd       } |        t        t        dd       	fd       } |         j                  dd        j                  	dd        j                  dt        d      g       t        j                  t        j                  j	                  	d             t        t        dd       	fd       } |        t        t        dd       	fd       } |        y) zBTests that flat and recursive rsync bucket to dir works correctly.rB  r   r   r   r  r  r   r   r   r!  r"  r]  r   r   r   r   rD   r   r   c            
      l   j                  ddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             j                  dj                  dt        d      gd             t        t        j                  j                  d            5 }j                  dd	j                  |j                                      d
d
d
       y
# 1 sw Y   y
xY w)r   r
   r  r7  r8  r  r  Tr  r0   Nr  rv  s      r=   r   z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check1  s     nngtT*%5v>?j)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJ 

..%j'!:;'+  -. VW-.!$))AKKM":; /..s   10D**D3c                  h    j                  ddt              gd      } j                  |        y r:  r   rQ  s    r=   r   z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check2  s:     ~~D$z*F
34  If
F#r?   c            
      n   j                  dddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             j                  dj                  dt        d      gd	             t        t        j                  j                  d            5 }j                  dd
j                  |j                                      ddd       y# 1 sw Y   yxY w)r  r
   r  r  r7  r8  r  r  Tr  r0   Nr  rv  s      r=   r  z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check3  s     nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJ 

..%j'!:;'+  -. VW-.!$))AKKM":; /..s   20D++D4c                  j    j                  dddt              gd      } j                  |        y rx  r   rQ  s    r=   r  z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check4  <     ~~D$Z 0&
9  Of
F#r?   r   r   r*  r+  r,  c                  6   j                  dddt              g       t        t              j                              } t        j	                              }j                  | t        g d             j                  |t        g d             y r<  r  rU  s     r=   r  z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check5  s    
nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJr?   c                  j    j                  dddt              gd      } j                  |        y r>  r   rQ  s    r=   r|  z5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check6  r  r?   N)rL   rK   r   rW   rs   rV  r   rW  r'   r   r   rO   r}  )
r:   rB  r   r   r  r  r  r|  rX   r   s
   `       @@r=   test_bucket_to_dir_minus_dz$TestRsync.test_bucket_to_dir_minus_d  s!    ""$J!FWW\\&(+FHHV"(&  (
 	")'   	"/-  / 	v8Lv'Jv"(!/  1
 >3< 4<( I >3$ 4$ I >3< 4<& I >3$ 4$ I 	"(&  ( 	v'JNND$z6234IIbggll67+, >3K 4K I >3$ 4$ Ir?   c                       j                          j                          j                  dd        j                  dd       t	        t
        dd       fd	       } |        y
)a*  Tests that name case changes work correctly.

    Example:

    Windows filenames are case-preserving in what you wrote, but case-
    insensitive when compared. If you synchronize from FS to cloud and then
    change case-naming in local files, you could end up with this situation:

    Cloud copy is called .../TiVo/...
    FS copy is called      .../Tivo/...

    Then, if you rsync from cloud to FS, if rsync doesn't recognize that on
    Windows these names are identical, each rsync run will cause both a copy
    and a delete to be executed.
    r   r   r   Obj1r]  rD   r   r   c                      j                  dddt              gd      } t        rj                  t        |        yj                  t        |        y)r   r
   r  r   Tr   N)r   rO   r)   r4   r   assertNotEqual)outputrX   r:   r   s    r=   r   zLTestRsync.test_bucket_to_dir_minus_d_with_fname_case_change.<locals>._Check1	  sS     ~~D$Z 0&
9  Of 
V,J/r?   N)rL   rK   r   rW  r'   r   )r:   r   rX   r   s   ` @@r=   1test_bucket_to_dir_minus_d_with_fname_case_changez;TestRsync.test_bucket_to_dir_minus_d_with_fname_case_change	  sw    & ""$J!F"(&  ( 	v'J >30 40 Ir?   z^The boto lib used for S3 does not handle objects starting with slashes if we use V4 signature.c                       j                          j                          j                  dd        j                  d      } j	                  |d       t        t        dd       fd	       } |        y
)zTests that we correctly handle leftover dir placeholders.

    See comments in gslib.commands.rsync._FieldedListingIterator for details.
    r   r   r   / rD   r   r   c                  8   j                  dddt              gd       t        t              j                              } t        j	                              }j                  | t        ddg             j                  |t        dg             y)	r   r
   r  r   Tr   r   z//Nr  rU  s     r=   r   zSTestRsync.test_bucket_to_dir_minus_d_with_leftover_dir_placeholder.<locals>._Check1=	  s     nnD$Z 0&
9  Oj)4+>+>z+JKh!1!1&!9:h
xgt_!56
xgY0r?   N)rL   rK   r   StorageUriCloneReplaceNameStorageUriSetContentsFromStringr'   r   )r:   key_urir   rX   r   s   `  @@r=   8test_bucket_to_dir_minus_d_with_leftover_dir_placeholderzBTestRsync.test_bucket_to_dir_minus_d_with_leftover_dir_placeholder+	  s     ""$J!F"(&  ( --j#>G(("5 >3	1 4	1 Ir?   z)os.symlink() is not available on Windows.c           	      \   | j                         }| j                         }t        j                  j	                  |d      }t        j
                  |       t        j
                  t        j                  j	                  |d             t        j                  t        j                  j	                  |d      t        j                  j	                  |d             t        j                  t        j                  j	                  |d             | j                  ddd|t        |      g       y)z7Tests that rsync -e -r ignores symlinks when recursing.rB  missingr
   r   -eN)
rL   rK   r   rW   rs   rV  symlinkrmdirr   rO   )r:   rX   r   rB  s       r=   test_rsync_minus_r_minus_ez$TestRsync.test_rsync_minus_r_minus_eK	  s     ""$J!FWW\\&(+FHHVHHRWW\\&),- JJrww||FI.VY0OPHHRWW\\&),-NNGT4j1ABCr?   c                    	  j                         	t        j                  j                  	d      }t        j                  |        j                          j                  	dd      } j                  	dd        j                  |dd       t        j                  j                  	d	      }t        j                  ||       t        j                  j                  	d
      }t        j                  t        j                  j                  dd      |        j                  dd        j                  dd        j                  dd       t        t        dd       	fd       } |        t        j                  |       t        t        dd       	fd       } |        t        t        dd       	fd       } |        y)z%Tests that rsync -e ignores symlinks.rB  r   r   r]  r  r  r   r!  symlink1symlink2r  znon-existentr"  r   r   r   r   r   rD   r   r   c            	      6   j                  dddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             y)z1Ensure listings match the commented expectations.r
   r  r  )r   r$  r%  	/symlink1z	/symlink2r8  Nrz  rU  s     r=   r   z5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check1x	  s     nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh


K
LN
 x%H!IJr?   c            
         j                  ddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             j                  dj                  dt        d      gd	             y
)r   r
   r  )r   r$  r%  r!  )r   r$  r&  r!  r   r  r  Tr  Nrz  rU  s     r=   r   z5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check2	  s     nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
xKLN xKLN


..%j*!=>'+  -.r?   c                  h    j                  ddt              gd      } j                  |        y r:  r   rQ  s    r=   r  z5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check3	  rt  r?   N)rK   r   rW   rs   rV  rL   rW  r  r   r'   r   r}  )
r:   rB  fpath1good_symlink_pathbad_symlink_pathr   r   r  rX   r   s
   `       @@r=   test_rsync_minus_d_minus_ez$TestRsync.test_rsync_minus_d_minus_eY	  s    !FWW\\&(+FHHV""$J  +1*1 ! 3F 	v8Lv"(!/  1 VZ8JJv() ww||FJ7JJrww||C02BC")'  ) 	"(&  ( 	"/-  /
 >3K 4K I
 II
>3. 4." I >3$ 4$ Ir?   z%S3 does not support composite objectsc           	           j                          j                          j                  dd        j                  dd        j                  dt        d      t        d      t        d      g        j                  dd        j                  d	d
       t	        t
        dd       fd       } |        t	        t
        dd       fd       } |        y)zFTests that rsync works with composite objects (which don't have MD5s).r   r   r   r  r  composer   r"  r   r   rD   r   r   c                  X   j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        g d             j	                  |t        g d             y )Nr
   r  r   r$  r   r   r(  s     r=   r   zHTestRsync.test_bucket_to_bucket_minus_d_with_composites.<locals>._Check1	  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%A!BC x%A!BCr?   c                  z    j                  ddt              t              gd      } j                  |        y r:  r   r   s    r=   r   zHTestRsync.test_bucket_to_bucket_minus_d_with_composites.<locals>._Check2	  r1  r?   NrL   r   r   rO   r'   r   r:   r   r   r   r   s   `  @@r=   -test_bucket_to_bucket_minus_d_with_compositesz7TestRsync.test_bucket_to_bucket_minus_d_with_composites	  s    ##%K##%K"(&  ( 	")'  ) 	NN[&!['"[&!	  	")'  ) 	"(&  (
 >3D 4D I >3$ 4$ Ir?   c                 &     j                          j                          j                  dd        j                  dd       t        t        dd       fd	       } |        t        t        dd       fd
       } |        y)zETests working with empty dest bucket (iter runs out before src iter).r   r   r   r  r  rD   r   r   c                  X   j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        ddg             j	                  |t        ddg             y )Nr
   r  r   r$  r   r(  s     r=   r   zCTestRsync.test_bucket_to_bucket_minus_d_empty_dest.<locals>._Check1	  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
xgx%8!9:
xgx%8!9:r?   c                  z    j                  ddt              t              gd      } j                  |        y r:  r   r   s    r=   r   zCTestRsync.test_bucket_to_bucket_minus_d_empty_dest.<locals>._Check2	  r1  r?   NrL   r   r'   r   r.  s   `  @@r=   (test_bucket_to_bucket_minus_d_empty_destz2TestRsync.test_bucket_to_bucket_minus_d_empty_dest	  s    ##%K##%K"(&  ( 	")'  )
 >3; 4; I >3$ 4$ Ir?   c                 &     j                          j                          j                  dd        j                  dd       t        t        dd       fd	       } |        t        t        dd       fd
       } |        y)zDTests working with empty src bucket (iter runs out before dst iter).r   r   r   r  r  rD   r   r   c                     j                  ddt              t              g       j                  dt        d      gdd      } j                  d|        j                  dt        d      gdd      } j                  d|        y )	Nr
   r  ls**r   Tr  #One or more URLs matched no objects)r   rO   ru   r   s    r=   r   zBTestRsync.test_bucket_to_bucket_minus_d_empty_src.<locals>._Check1
  s    
nngtT+%6[8IJK~~tT+t%<=./,0  2f mm96B~~tT+t%<=./,0  2f mm96Br?   c                  z    j                  ddt              t              gd      } j                  |        y r:  r   r   s    r=   r   zBTestRsync.test_bucket_to_bucket_minus_d_empty_src.<locals>._Check2
  r1  r?   Nr3  r.  s   `  @@r=   'test_bucket_to_bucket_minus_d_empty_srcz1TestRsync.test_bucket_to_bucket_minus_d_empty_src	  s    ##%K##%K"(&  ( 	")'  )
 >3	C 4	C I >3$ 4$ Ir?   c           	      <     j                          j                          j                  dd        j                  dddt        d      g       t	        t
        dd	       fd
       } |        t	        t
        dd	       fd       } |        y)z#Tests that rsync -p preserves ACLs.r   r   r   aclr   rn   rD   r   r   c            	         j                  dddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        dg             j	                  |t        dg             j                  ddt        d      gd	      }j                  ddt        d      gd	      }j	                  ||       y
)z!Tests rsync -p works as expected.r
   r  -pr   r=  getr   Tr  Nr   r   r   	acl1_json	acl2_jsonr   r   r:   s       r=   r   z5TestRsync.test_rsync_minus_d_minus_p.<locals>._Check10
  s     nnD$ k*D,?,?,LMhk*D,?,?,LMh
xgY0
xgY0..%k62
34 ! Ii..%k62
34 ! Ii
y),r?   c            	      |    j                  dddt              t              gd      } j                  |        y )Nr
   r  r?  Tr   r   r   s    r=   r   z5TestRsync.test_rsync_minus_d_minus_p.<locals>._Check2D
  r?  r?   Nr-  r.  s   `  @@r=   test_rsync_minus_d_minus_pz$TestRsync.test_rsync_minus_d_minus_p%
  s    ##%K##%K"(&  ( 	NNE5-k61JKL >3- 4-  I >3$ 4$ Ir?   c                       j                          j                          j                  dd        j                  ddt        d      g       t	        t
        dd       fd	       } |        y
)z!Tests that rsync -a applies ACLs.r   r   r   r=  r@  rD   r   r   c            
      :   j                  ddddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        dg             j	                  |t        dg             j                  dddt        d      g       j                  dd	t        d      gd
      }j                  dd	t        d      gd
      }j	                  ||       y)z!Tests rsync -a works as expected.r
   r  rm   rn   r   r=  r   r   r@  Tr  Nr   rA  s       r=   r  z/TestRsync.test_rsync_canned_acl.<locals>._CheckZ
  s    nn
4}
{

{
 	
 k*D,?,?,LMhk*D,?,?,LMh
xgY0
xgY0 nneUM4V3LMN..%k62
34 ! Ii..%k62
34 ! Ii
y),r?   Nr-  )r:   r  r   r   s   ` @@r=   test_rsync_canned_aclzTestRsync.test_rsync_canned_aclP
  sz    ##%K##%K"(&  ( 	NNE5${F";<= >3- 4-( Hr?   c                      j                         t        j                  j                  d      }t        j                  |        j                          j                  dd        j                  dd        j                  |dd       t        t        d	d
       fd       } |        t        t        d	d
       fd       } |        y)z5Tests that rsync to non-existent bucket subdir works.rB  r   r   r]  r  r  r   r!  rD   r   r   c            	      X   j                  ddt        d      g       t        j                              } t        t        d      j	                  j                  d                  }j                  | t        g d             j                  |t        g d             y)r   r
   r   rB  r7  Nr   rO   r   rM  r   r  r4   r   r   r   
bucket_urlr:   r   s     r=   r   zBTestRsync.test_rsync_to_nonexistent_bucket_subdir.<locals>._Check1
  s     nngtVT*h-GHI!1!1&!9:h
z8
$


--j(CEFh
 x%H!IJ
x%H!IJr?   c            	      j    j                  ddt        d      gd      } j                  |        y )Nr
   r   rB  Tr   r   r   rM  r:   r   s    r=   r   zBTestRsync.test_rsync_to_nonexistent_bucket_subdir.<locals>._Check2
  sA     ~~D&
H%'  f F#r?   N)	rK   r   rW   rs   rV  rL   rW  r'   r   )r:   rB  r   r   rM  r   s   `   @@r=   'test_rsync_to_nonexistent_bucket_subdirz1TestRsync.test_rsync_to_nonexistent_bucket_subdirq
  s     !FWW\\&(+FHHV""$Jv'Jv8Lv"(!/  1
 >3K 4K I >3$ 4$ Ir?   c                 N     j                          j                          j                  dd        j                  dd        j                  dd       t	        t
        dd	
       fd       } |        t	        t
        dd	
       fd       } |        y)a`  Tests that rsync with destination url as a prefix of existing obj works.

    Test to make sure that a dir/subdir gets created if it does not exist
    even when the new dir is a prefix of an existing dir.
    e.g if gs://some_bucket/foobar exists, and we run the command
    rsync some_dir gs://some_bucket/foo
    this should create a subdir foo
    foobarr   r   r   r]  r  r  rD   r   r   c            	      X   j                  ddt        d      g       t        j                              } t        t        d      j	                  j                  d                  }j                  | t        ddg             j                  |t        ddg             y)r   r
   r   foor   r$  NrK  rL  s     r=   r   zYTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj.<locals>._Check1
  s     nngtVT*e-DEF!1!1&!9:h
z5
!


d==%! "#h
 xgx%8!9:
xgx%8!9:r?   c            	      j    j                  ddt        d      gd      } j                  |        y )Nr
   r   rT  Tr   r   rO  s    r=   r   zYTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj.<locals>._Check2
  s@     ~~wf#J68,0  2f F#r?   NrK   rL   r   rW  r'   r   )r:   r   r   rM  r   s   `  @@r=   >test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_objzHTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj
  s     !F""$J"*&  ( 	v'Jv8L >3; 4; I >3$ 4$ Ir?   c                       j                          j                  dd        j                  dd        j                  d j                  t	        t
        dd	       fd
       } |        y)DTests that rsync from a non-existent bucket subdir fails gracefully.r   r   r]  r  r  ://rD   r   r   c                     j                  ddgdd      } j                  rj                  d|        nj                  d|        t        j	                              }j                  |t        dd	g             y 
Nr
   r  r   Tr  znot found: 404zCaught non-retryable exceptionr   r$  r   r   ru   r   rM  r4   r   r   listingbucket_url_strr:   r   s     r=   r  z<TestRsync.test_rsync_from_nonexistent_bucket.<locals>._Check
      ~~wnfE./,0  2f 
	!	!&/6? 0 0 89g
wWh$7 89r?   NrK   rW  r   r   r'   r   r:   r  r`  r   s   ` @@r=   "test_rsync_from_nonexistent_bucketz,TestRsync.test_rsync_from_nonexistent_bucket
  y    !Fv'Jv8L"&"7"7"&">">@N >3
: 4
: Hr?   c                       j                          j                  dd        j                  dd        j                  d j                  t	        t
        dd	       fd
       } |        y)rY  r   r   r]  r  r  rZ  rD   r   r   c                     j                  ddgdd      } j                  rj                  d|        nj                  d|        t        j	                              }j                  |t        dd	g             y r\  r]  r^  s     r=   r  z:TestRsync.test_rsync_to_nonexistent_bucket.<locals>._Check
  ra  r?   Nrb  rc  s   ` @@r=    test_rsync_to_nonexistent_bucketz*TestRsync.test_rsync_to_nonexistent_bucket
  re  r?   c                      j                          j                          j                  dd        j                  dd        j                  dd        j                  dd        j                  d        j                  d       t        t        dd	
       fd       } |        t        t        dd	
       fd       } |        y)z<Tests that punc chars in filenames don't confuse sort order.e/obj1r   r   	e-1/.obj2r  r   rC   rD   r   r   c            
         j                  ddt              t              g       t        t              j                              } t        t              j                              }j	                  | t        ddg             j	                  |t        ddg             j	                  dj                  dt        d      gd	             j	                  d
j                  dt        d      gd	             y)r   r
   z-rdz/e/obj1z
/e-1/.obj2r   r  rj  Tr  r  rk  Nr   r(  s     r=   r   zVTestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars.<locals>._Check1  s     nngud;&7k9JKLk*D,?,?,LMhk*D,?,?,LMh
xi%>!?@
xi%>!?@


..%k8!<='+  -. 

..%k;!?@'+  -.r?   c                  z    j                  ddt              t              gd      } j                  |        y r:  r   r   s    r=   r   zVTestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars.<locals>._Check2&  r1  r?   N)rL   r   rc  r'   r   r.  s   `  @@r=   ;test_bucket_to_bucket_minus_d_with_overwrite_and_punc_charszETestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars
  s    ##%K##%K 	"*&  ( 	"-'  ) 	"*&  ( 	"-'  )
 	Q/Q/ >3. 4.$ I >3$ 4$ Ir?   c                      j                          j                          j                  dd        j                  dd        j                  dd        j                  dd        j                  d	d
        j                  dd        j	                  d       t        t        dd       fd       } |        t        t        dd       fd       } |        y)z1Tests that rsync regex exclusions work correctly.r   r   r]  r  r  r   r   r   r   r   r   r   rD   r   r   c            
      8   j                  dddt              g       t        j                              } t        t              j	                              }j                  | t        g d             j                  |t        g d             y)r   r
   r  obj[34]r+  )r   r$  r   Nrz  )r   r   rX   flagr:   r   s     r=   r   zBTestRsync._test_dir_to_bucket_regex_paramaterized.<locals>._Check1H  s     nngtT9fd:>NOP!1!1&!9:hj)4+>+>z+JKh
x%A!BC x%A!BCr?   c            
      l    j                  dddt              gd      } j                  |        y )Nr
   r  rq  Tr   r   )r   rX   rr  r:   r   s    r=   r   zBTestRsync._test_dir_to_bucket_regex_paramaterized.<locals>._Check2W  sC     ~~D$	6
  f F#r?   N)rK   rL   rW  r   rc  r'   r   )r:   rr  r   r   rX   r   s   ``  @@r=   '_test_dir_to_bucket_regex_paramaterizedz1TestRsync._test_dir_to_bucket_regex_paramaterized1  s    !F""$Jv'Jv8Lv'J")'  ) 	"(&  ( 	"(&  ( 	
A. >3	D 4	D I >3$ 4$ Ir?   c                 &    | j                  d       y)z8Tests that rsync regex exclusions work correctly for -x.-xNrt  r:   s    r=   test_dir_to_bucket_minus_xz$TestRsync.test_dir_to_bucket_minus_xb      006r?   c                 &    | j                  d       y)z8Tests that rsync regex exclusions work correctly for -y.-yNrw  rx  s    r=   test_dir_to_bucket_minus_yz$TestRsync.test_dir_to_bucket_minus_yf  rz  r?   c           
         | j                         }| j                  g d      }| j                  dd|d|t        |      gd       t	        || j                  |            }| j                  |t        g d             |rBt	        t        |      | j                  |            }| j                  |t        g d	             y| j                  d
t        |d      gdd      }| j                  d|       y)z3Tests if negative lookahead includes files/objects.abc)data1za.txt)r  ok)data2zb.txt)data3data4zc.txtrd  r
   r   z^(?!.*\.txt$).*Tr   /a/b/c/data1/a.txt	/data1/ok/data2/b.txt/data3/data4/c.txt)r  r  r  r7  r8  r   r  r9  N)
rL   rK   r   rO   r   rM  r4   r   r   ru   )r:   rr  includesrX   r   r_  actualr   s           r=   ,_test_dir_to_bucket_regex_negative_lookaheadz6TestRsync._test_dir_to_bucket_regex_negative_lookaheadj  s   ""$J , F 	NN	$0&	j		   fd..v67G 
 	 tJ')<)<Z)HIf

#LMO ~~tT*d%;<./,0  2f mm96Br?   c                 *    | j                  dd       y)z=Test that rsync -x negative lookahead includes objects/files.rv  Tr  Nr  rx  s    r=   6test_dir_to_bucket_negative_lookahead_works_in_minus_xz@TestRsync.test_dir_to_bucket_negative_lookahead_works_in_minus_x  s    55dT5Jr?   c                 *    | j                  dd       y)zFTest that rsync -y nevative lookahead does not includes objects/files.r|  Fr  Nr  rx  s    r=   7test_dir_to_bucket_negative_lookahead_breaks_in_minux_yzATestRsync.test_dir_to_bucket_negative_lookahead_breaks_in_minux_y  s    55dU5Kr?   c                       j                  g d      t        t        dd       fd       fd} |         fd} |         fd	}|r |        y
y
)zFTest that rsync regex options work with a relative regex per the docs.r  rd  rD   r   r   c                 l   j                         }d}	t        rdndz   }|j                  dd| |t        |      gd      z  }t	        	j                  	            }j                  |t        g d             t	        t        |      j                  |            }j                  ||       |S )	z'Tests rsync skips the excluded pattern.r  \r  r
   r   Tr   r  )	rL   r)   r   rO   r   rM  r4   r   r   )
exclude_regexexpectedrX   r   localr_  r  rr  r:   r   s
          r=   _check_exclude_regexzXTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._check_exclude_regex  s     $$&jf
4eD$u
   f
  0 0 89g


   tJ')<)<Z)HIf
vx(mr?   c                  .      dt        g d             y)zCEnsure the example exclude pattern from the docs works as expected.zdata.[/\\].*\.txt$)r  r  r  r  N)r   )r  s   r=   r   zKTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check1  s    2>?Ar?   c                  x    j                   rt        r  dt        dg             y  dt        dg             y)z1Tests that a regex with a pipe works as expected.z^data^^^|[bc]$r  z^data|[bc]$N)r   r)   r   )r  r:   s   r=   r   zKTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check2  s1    		!	!j 	-sD6{;]CK8r?   c            	      ,    dt        g d            } j                  dj                  t        j                  j                  d            |        j                  dj                  t        j                  j                  dd            |        y)z>Tests that directories are skipped from iteration as expected.r  )r  r  r  r  r  r  z!Skipping excluded directory {}...r  N)r   ru   rv   r   rW   rs   r   )r   r  r:   r   s    r=   r  zKTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check3  s    #

M
NPf mm
-
4
4ggll67+-.46 
-
4
4ggll67G467=?r?   N)rK   r'   r   )r:   rr  	skip_dirsr   r   r  r  r   s   ``    @@r=   0_test_dir_to_bucket_relative_regex_paramaterizedz:TestRsync._test_dir_to_bucket_relative_regex_paramaterized  se     , F >3 4,A
 I9 I
? i r?   c                 *    | j                  dd       y)zCTest that rsync -x option works with a relative regex per the docs.rv  Fr  Nr  rx  s    r=   #test_dir_to_bucket_relative_minus_xz-TestRsync.test_dir_to_bucket_relative_minus_x  s    99$%9Pr?   c                 *    | j                  dd       y)zCTest that rsync -y option works with a relative regex per the docs.r|  Tr  Nr  rx  s    r=   #test_dir_to_bucket_relative_minus_yz-TestRsync.test_dir_to_bucket_relative_minus_y  s    99$$9Or?   z1os.chmod() won't make file unreadable on Windows.c                 8     j                          j                          j                  dd        j                  dd      t        j                  d        j                  dd       t        t        d	d
       fd       } |        y)z+Tests that rsync -C option works correctly.r   r   r]  r   r   r   r   r   rD   r   r   c                     j                  ddt              gdd      } j                  r"j                  dj	                        |        nj                  d|        t        j                              }t        t              j                              }j                  |t        g d             j                  |t        d	d
g             y)r   r
   z-Cr   Tr  zPermission denied: '{}'z,1 files/objects could not be copied/removed.)r   r   r   r   r   N)
r   rO   r   ru   rv   r   rM  r   r4   r   )r   r   r   rX   rW   r:   r   s      r=   r  z4TestRsync.test_dir_to_bucket_minus_C.<locals>._Check  s     ~~D&$z"2
3  f 
	!	!/66t<fEDfM!1!1&!9:hj)4+>+>z+JKh
x%@!AB
xgw%7!89r?   N)rK   rL   rW  r   chmodr'   r   )r:   r  rX   rW   r   s   ` @@@r=   test_dir_to_bucket_minus_Cz$TestRsync.test_dir_to_bucket_minus_C  s    
 !F""$Jv'Jf)/(/  1D HHT1v'J >3: 4:" Hr?   z5Windows Unicode support is problematic in Python 2.x.c                      j                          j                         g d}|D ]  } j                  |        t        rt	        ddg      nt	        g d      t        t        dd       fd	       } |        y
)z;Tests that rsync -r works correctly with unicode filenames.)u   morales_suenÌƒos.jpgu   morales_suenÌƒos.jpgu   fooꝾoo)r   rD     /morales_suenÌƒos.jpg	   /fooꝾoo)u   /morales_suenÌƒos.jpgr  r  rD   r   r   c                  ,   j                  ddt              g       t        j                              } t        t              j	                              }j                  t        |              j                  t        |             y)r   r
   r   Nrz  )r   r   rX   expected_list_resultsr:   r   s     r=   r  z?TestRsync.test_dir_to_bucket_with_unicode_chars.<locals>._Check  s|     nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
s8}&;<
s8}&;<r?   N)rK   rL   rW  r(   	frozensetr'   r   )r:   	file_listfilenamer  rX   r  r   s   `   @@@r=   %test_dir_to_bucket_with_unicode_charsz/TestRsync.test_dir_to_bucket_with_unicode_chars  s    
 !F""$J
I
 
8<  7= '	#[13BKPCR 
 >3= 4= Hr?   c                      j                          j                         d j                  dd        j                  dd        j                  dd        j                  d	d
        j                  dd        j                  dd        j                  dddz
          j                  dddz
          j                  dddz
          j                  d	d        j                  dd        j                  dddz          t	        t
        dd       fd       } |        y)z$Tests that rsync -u works correctly.r   r      obj1-1r   r      obj2-1r      obj3-1r   s   obj4-1r   s   obj5-1r   s   obj6-1   obj1-2r   rC  
   obj3-newers   obj4-2s   obj5-biggerrD   r   c            
      j   j                  ddt              g       j                  dj                  dt        d      gd             j                  dj                  dt        d	      gd             j                  d
j                  dt        d      gd             j                  dj                  dt        d      gd             j                  dj                  dt        d      gd             j                  j                  dt         dz                y )Nr
   z-uobj1-1r  r   Tr  obj2-1r   obj3-1r   zobj4-1r   zobj5-biggerr   r   r   )r   rO   r4   r   r   r   )
ORIG_MTIMEr   r:   r   s   r=   r  z4TestRsync.test_dir_to_bucket_minus_u.<locals>._Checke  s1   
nngtVT*-=>?


..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ j44f!*q.13r?   NrV  )r:   r  r  r   r   s   ` @@@r=   test_dir_to_bucket_minus_uz$TestRsync.test_dir_to_bucket_minus_u!  s   !F""$JJ"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	v"(!*(1n  .
 	v"(!*(1n  .
 	v"(!.(1n  . 	v"(!*(  *
 	v"(!/(  * 	v"(!*(1n  .
 >33 43. Hr?   c                      j                          j                         d} j                  dd        j                  dd        j                  dd|	        j                  dd
        j                  dd|        j                  dd|dz
         t	        t
        dd       fd       } |        y)z$Tests that rsync -i works correctly.r   r   r  r   r   r  r   r  r   r  r]  rC  r  r   rD   r   c            
      \   j                  ddt               g       j                  dj                  dt         d      gd             j                  dj                  dt         d	      gd             j                  d
j                  dt         d      gd             y )Nr
   z-ir  r  r   Tr  r  r   r  r   )r   rO   r4   )r   r:   r   s   r=   r  z4TestRsync.test_dir_to_bucket_minus_i.<locals>._Check  s    
nngtVT*-=>?


..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQr?   NrV  )r:   r  r  r   r   s   `  @@r=   test_dir_to_bucket_minus_iz$TestRsync.test_dir_to_bucket_minus_i  s    !F""$JJ"((  * 	"((  * 	"((&  ( 	v)Lv"(!*(  *
 	v"(!.(1n  .
 >3Q 4Q Hr?   c                    d}g }| j                         }| j                         }d}| j                  |||      }t        dg      }| j	                  dd|t        |      g       t        || j                  |            }t        t        |      | j                  |            }	| j                  t        |      |       | j                  t        |	      |       y)z7Test to ensure filenames with whitespace can be rsyncedzfoo bar baz.txtz6File from rsync test: test_rsync_files_with_whitespacez/foo bar baz.txtr
   r   NrL   rK   rW  r  r   rO   r   rM  r   r4   r   
r:   r  
local_urisrX   r   r   
local_filer  r   r   s
             r=    test_rsync_files_with_whitespacez*TestRsync.test_rsync_files_with_whitespace  s     HJ""$J!FGH$$VXx@J%'9&:; 	NNGT64
+;<=vt//78HtJ')<)<Z)HIHS]$9:S]$9:r?   c                    d}g }| j                         }| j                         }d}| j                  |||      }t        dg      }| j	                  dd|t        |      g       t        || j                  |            }t        t        |      | j                  |            }	| j                  t        |      |       | j                  t        |	      |       y)z?Test to ensure filenames with special characters can be rsyncedu   Æ.txtz>File from rsync test: test_rsync_files_with_special_charactersu   /Æ.txtr
   r   Nr  r  s
             r=   (test_rsync_files_with_special_charactersz2TestRsync.test_rsync_files_with_special_characters  s    HJ""$J!FOH$$VXx@J%yk2 	NNGT64
+;<=vt//78HtJ')<)<Z)HIHS]$9:S]$9:r?   z0No compressed transport encoding support for S3.z9No compressed transport encoding support for the XML API.c           
         d}g }| j                         }| j                         }ddz  }|D ]$  }|j                  | j                  |||             & | j	                  dddd|t        |      gd	      }| j                  |t        |             |D ]  }| j                  d
|z  |        | j                  s| j                  d|       yy))Test gzip encoded files upload correctly.rG  test.txttest.xmlr  '  -Dr
   -Jr   Tr   2Using compressed transport encoding for file://%s.4send: Using gzip transport encoding for the request.N
rL   rK   appendrW  r   rO   rc  lenru   r   	r:   
file_namesr  rX   r   r   rD  r   	local_uris	            r=   &test_gzip_transport_encoded_all_uploadz0TestRsync.test_gzip_transport_encoded_all_upload  s     2JJ""$J!Fe|H	++FHiHI   ^^	wdF	j		  F 	
C
O<	
mm
>)
L
   ##
mmJ $r?   c                 D   d}d}g }g }| j                         }| j                         }ddz  }|D ]$  }|j                  | j                  |||             & |D ]$  }|j                  | j                  |||             & | j	                  ddddd	|t        |      gd
      }	| j                  |t        |      t        |      z          |D ]  }
| j                  d|
z  |	        |D ]  }
| j                  d|
z  |	        | j                  s| j                  d|	       yy)r  )r  z	photo.txt)fileztest.pngr  r  r  r  r
   z-jtxtr   Tr   r  r  N)rL   rK   r  rW  r   rO   rc  r  ru   r   r   )r:   file_names_validfile_names_invalidlocal_uris_validlocal_uris_invalidrX   r   r   rD  r   r  s              r=   +test_gzip_transport_encoded_filtered_uploadz5TestRsync.test_gzip_transport_encoded_filtered_upload  sX    19""$J!Fe|H%	d11&(INO &'	 3 3FH4=!? @ ( ^^	weT6	j		  F 	
 #$4 5<N8O OQ &	
mm
>)
L
 & (	

>)
L
 ( ##
mmJ $r?   c                    d}g }| j                         }| j                         }ddz  }|D ]$  }|j                  | j                  |||             & | j	                  ddddd|t        |      gd	
      }| j                  |t        |             |D ]  }| j                  d|z  |        | j                  s| j                  d|       yy)r  r  r  r  r  ra  r
   r  r   Tr   r  r  Nr  r  s	            r=   /test_gzip_transport_encoded_all_upload_parallelz9TestRsync.test_gzip_transport_encoded_all_upload_parallel  s     2JJ""$J!Fe|H	++FHiHI   ^^	tWdD&	j		  F 	
C
O<	
mm
>)
L
   ##
mmJ $r?   z$Test uses gs-specific KMS encryptionc                    | j                         }t        |      dz   }d}d}| j                         }| j                  |||       t	               }t        dd|fg      5  | j                  d||g       d d d        t        dg      5  | j                  d	d
|d|gd      }d d d        | j                  d|z         y # 1 sw Y   OxY w# 1 sw Y   +xY w)Nz/foor  s   barr]  re   encryption_keyr
   re   
prefer_apijsonr7  z-Lr  Tr  zKMS key:\s+%s)rL   rO   rK   rW  r   r   r   r   )r:   rX   cloud_container_suriobj_nameobj_contentstmp_dirkey_fqnstdouts           r=   $test_kms_key_applied_to_dest_objectsz.TestRsync.test_kms_key_applied_to_dest_objects5  s    ""$J
+f4HL  "Gw"*!-  / 12G 
*:GDE	F
nngw(<=> 
G 
?@	A~~"6A
B  f 
B 	V-78 
G	F 
B	As    C	CCCz>Tests that gs-specific encryption settings are skipped for s3.c                     | j                         }| j                  |d       | j                         }d}t        dg      5  | j	                  d|t        |      g       d d d        y # 1 sw Y   y xY w)Nr  r   r   zGprojects/myproject/locations/global/keyRings/mykeyring/cryptoKeys/mykeyr  r
   )rK   rW  rL   r   r   rO   )r:   r  rX   	dummy_keys       r=   :test_kms_key_specified_will_not_prevent_non_kms_copy_to_s3zDTestRsync.test_kms_key_specified_will_not_prevent_non_kms_copy_to_s3L  sk      "Gw8""$J$I
 
?@	A
nngwZ(89: 
B	A	As   A))A2c           
         | j                         }| j                         }| j                  |dd       | j                  ddddt        |      t        |      gd	      }| j                  rQt        t        j                  d
|            }t        t        j                  d|            }| j                  ||       y t        j                  d|      }| j                  |       |D ]  }| j                  d|        y )Nr   r   r   z-DD-hzarbitrary:headerr
   Tr   z= request start =zb'arbitrary': b'header'zHeaders: \{([\s\S]*?)\}z'arbitrary': 'header')rL   r   r   rO   r   r  refindallr4   rm  ru   )r:   r   r   r   request_counttarget_header_countheaders_for_all_requestsheaderss           r=   0test_bucket_to_bucket_includes_arbitrary_headersz:TestRsync.test_bucket_to_bucket_includes_arbitrary_headersY  s    ##%K##%K"(&  ( ^^t'[[
 +/  0F "**%96BCm

+Ev NO
}&9:!#,F!O
oo./-'-w7 .r?   zTests that S3 SSE-C is handled.c           
      $   | j                         }| j                  |d      }| j                         }| j                         }ddddj                  t              ddj                  t
              g}t        dg      5  | j                  |d|t        |d	      gz          | j                  |d
t        |      t        |      gz          | j                  |dt        |d	      gz   d      }d d d        | j                  d       y # 1 sw Y   xY w)Nr  r  r  z8"x-amz-server-side-encryption-customer-algorithm:AES256"z."x-amz-server-side-encryption-customer-key:{}"z2"x-amz-server-side-encryption-customer-key-md5:{}")re   check_hashesneverr  rG  r
   r  Tr  rT  )
rK   rW  rL   rv   r   r   r   r   rO   r4   )r:   r  tmp_filebucket_uri1bucket_uri2header_flagsr   s          r=   -test_s3_sse_is_handled_with_arbitrary_headersz7TestRsync.test_s3_sse_is_handled_with_arbitrary_headersp  s      "G""'F"CH##%K##%K 	B8??"	$<CC&	(	L 
BC	D
nn\T8T+v5N$OOP
nn\tK0;')) * !&[&(A B!C.2   4h 
E 	Xu% 
E	Ds   >A-DDN)T)GrZ   r[   r\   r]   r   r   r   r   r   r   skipIfr)   
skipUnlessr   r   r   r   r  r4  r@  r   rZ  rf  r~  r  r  r  r  r  r  r  r  r  r   r  r  r'  r/  r4  r;  rE  rH  rP  rW  rd  rh  rn  rt  ry  r}  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r   r  r  r
  ra   r?   r=   r~   r~      s   ,.(+ +/	H0H<| 8??:KL8+-/KLf M MfP>J@"HH[z}~hV !8+-/KL^ M !^B !8+-/KL=1 M !=1@ !8+-/KLN M !N` 8+-/KLI MIV 8+-/KLC MCJ(T 8+-/KL
C M
C !8+-/KLr M !rh 8??:KLa MaF !8??:KL8+-/KLI1 M M !I1V 8+-/KL MB 8+-/KL% M%N 8+-/KL > ?? M: 8??:JKD LD 8??:JKO LOb 45/ 6/b D$L)VB(T,\..8t/b77C<KLAFQP 8??:FHHB 8??:JLLB\|-^;$;$ ?@IJ  ! K A6 ?@IJ # ! K A#J ?@IJ  ! K A4 349 59, MN
; O
;8. ./& 0&r?   r~   )Mr]   
__future__r   r   r   r   r   r  r   r   r6   gslibr	   gslib.commandsr
   gslib.project_idr   gslib.storage_urlr   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   r   r   gslib.tests.utilr   r   r   r   r   rO   r   r   r   r   r   r   r   r   r   gslib.utils.boto_utilr   gslib.utils.hashing_helperr    gslib.utils.posix_utilr!   r"   r#   r$   r%   r&   gslib.utils.retry_utilr'   gslib.utils.system_utilr(   r)   gslib.utilsr*   gslib.testsr+   r,   r-   r.   r/   PY3rk  r   MACOS_WARNINGr   GsUtilUnitTestCaser2   ShimUnitTestBaserc   GsUtilIntegrationTestCaser~   ra   r?   r=   <module>r#     s0   + & %  ' 	 	  
    . 2 ' ' ? ? @ ? 3 7 , 0 * , < - , : 1 2 $ % 6 @ 9 + , - * + ( * . ! +**& 77	$Q  
5E/0* I  (4/*<*)H// )XOH55 OB^3&22 ^3&r?   