
    O4                     x   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 dd	l	mZ dd
l	mZ ddlmc mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z   G d dejB                        Z" G d dejF                        Z$ G d dejJ                        Z&y)!Integration tests for mv command.    )absolute_import)print_function)division)unicode_literalsN)ApiSelector) TestCpMvPOSIXBucketToLocalErrors)"TestCpMvPOSIXBucketToLocalNoErrors)"TestCpMvPOSIXLocalToBucketNoErrors)	SkipForS3)ObjectToURI)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)unittest)UsingCrcmodExtension)Retry)
IS_WINDOWS)	shim_utilc                   "    e Zd ZdZd Zd Zd Zy)TestMvUnitTestszUnit tests for mv command.c           	         | j                         }| j                  |dd       | j                  |dd       | j                         }| j                  dt        |d      t        |      g       t	        d | j                  t        |d            j                  d	
      D              }t	        t        |dd      t        |dd      g      }| j                  ||       y)6Tests moving multiple top-level items between buckets.dir1/file.txt   data
bucket_uriobject_namecontentsdir2/file.txtmv*c              3   2   K   | ]  }t        |        y wNstr.0us     &platform/gsutil/gslib/tests/test_mv.py	<genexpr>z]TestMvUnitTests.test_move_bucket_objects_with_duplicate_names_inter_bucket.<locals>.<genexpr>9          ,+A 	A+   **Texpand_top_level_bucketsdir1file.txtdir2NCreateBucketCreateObject
RunCommandsuriset_test_wildcard_iteratorIterAllassertEqualselfbucket1_uribucket2_uriactualexpecteds        r*   :test_move_bucket_objects_with_duplicate_names_inter_bucketzJTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_inter_bucket,   s    ##%K"1&  ( 	"1&  ( ##%KOOD4S143DEF ,--d;.EFNN%) O +, ,F [&*-[&*- H 	VX&    c           	         | j                         }| j                  |dd       | j                  |dd       | j                         }| j                  dt        |d      t        |d      g       t	        d | j                  t        |d	            j                  d
      D              }t	        t        |ddd      t        |ddd      g      }| j                  ||       y)r   r   r   r   r    r!   r"   dirc              3   2   K   | ]  }t        |        y wr$   r%   r'   s     r*   r+   zaTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_to_bucket_subdir.<locals>.<genexpr>P   r,   r-   r.   Tr/   r1   r2   r3   Nr4   r=   s        r*   >test_move_bucket_objects_with_duplicate_names_to_bucket_subdirzNTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_to_bucket_subdirC   s    ##%K"1&  ( 	"1&  ( ##%KOOD4S14U3KLM ,--d;.EFNN%) O +, ,F [%4[%4 H 	VX&rD   c           	      l   | j                         }| j                  ddg      }| j                  d|dz   t        |      g       t	        d | j                  t        |d            j                  d	      D              }t	        t        |d
d      t        |dd      g      }| j                  ||       y)2Tests moving multiple top-level items to a bucket.)r1   r2   )r3   r2   
test_filesr!   z/*c              3   2   K   | ]  }t        |        y wr$   r%   r'   s     r*   r+   zUTestMvUnitTests.test_move_dirs_with_duplicate_file_names_to_bucket.<locals>.<genexpr>d   r,   r-   r.   Tr/   r1   r2   r3   N)r5   CreateTempDirr7   r8   r9   r:   r;   r<   )r>   r   dir_pathrA   rB   s        r*   2test_move_dirs_with_duplicate_file_names_to_bucketzBTestMvUnitTests.test_move_dirs_with_duplicate_file_names_to_bucketZ   s    ""$J!!.! H
 	OOD8d?D,<=> ,--d:t.DEMM%) N +, ,F Z,Z, H 	VX&rD   N)__name__
__module____qualname____doc__rC   rH   rP    rD   r*   r   r   )   s    "'.'.'rD   r   c                       e Zd ZdZd Zy)TestMvUnitTestsWithShimz$Unit tests for mv command with shim.c                    | j                         }| j                  d      }t        ddg      5  t        ddd      5  | j	                  dd	d
ddd|t        |      gd      }dj                  |j                  d         }| j                  dj                  t        j                  d      |t        |            |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Ns   abcdr   )GSUtiluse_gcloud_storageTrue)rZ   hidden_shim_modedry_runr\   fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr!   z-azpublic-readz-Uz-rz-RT)return_log_handler
infozZGcloud Storage Command: {} storage mv --predefined-acl publicRead --skip-unsupported {} {})r5   CreateTempFiler   r   r7   r8   joinmessagesassertInformatr   _get_gcloud_binary_path)r>   r   fpathmock_log_handler
info_liness        r*   test_shim_translates_flagsz2TestMvUnitTestsWithShim.test_shim_translates_flagsr   s    ""$J1E	GHJ 
K 6<)" 	  ??4dD%
#%# + % YY/88@A
DDJF11*=uZ E"#-		/	
K 
K	 	
K 
Ks#   CBCCC	CC'N)rQ   rR   rS   rT   rn   rU   rD   r*   rW   rW   o   s
    ,/rD   rW   c                   &   e Zd ZdZd Zd Zd Zed        Zd Z	 e
j                  ed       e
j                   e       d      d	               Z e
j                  ed      d
        Z e
j                  ed      d        Z ed      d        Zd Zy)TestMvE2ETestsr   c           	         | j                  d      }| j                  |d       | j                         }| j                  |d       |j                         D cg c]  }| j                  ||      j                    }}ddg|z   t        |      gz   }| j                  |d      }| j                  |j                  d      dd	|z         | j                  |j                  d      d
d|z         | j                  |j                  d      dz  dd|z         | j                  |j                  d      dd|z         | j                  |d       | j                  |d       |j                         D cg c]  }| j                  ||      j                    }}|d   }| j                  d|g       | j                  |d       | j                  |d       |j                         D cg c]  }t        | j                  ||             }}ddg|z   t        |      gz   }| j                  |d      }| j                  |j                  d      dd|z         | j                  |j                  d      dd|z         | j                  |j                  d      d       | j                  |d       | j                  |d       yc c}w c c}w c c}w )zDTests moving two buckets, one with 2 objects and one with 0 objects.   test_objectsr   z-mr!   Treturn_stderrCopyingz,stderr did not contain 2 "Copying" lines:
%s   z/stderr did not contain <= 4 "Copying" lines:
%sz9stderr did not contain even number of "Copying" lines:
%sRemovingz-stderr did not contain 2 "Removing" lines:
%srm   z/stderr did not contain >= 1 "Copying" lines:
%sz/stderr did not contain <= 2 "Copying" lines:
%sN)r5   AssertNObjectsInBucketlist_bucketStorageUriCloneReplaceKeyversionless_urir8   	RunGsUtilassertGreaterEqualcountassertLessEqualr<   )r>   r?   r@   keyobjscmdstderrobj1s           r*   test_movingzTestMvE2ETests.test_moving   s   ###3KQ/##%KQ/
 **,,C 	&&{C8HH, 	  $<$${"3!44C^^Ct^4FY7&@B 	Y:VCE 	Y!#QDvMO 	V\\*-qENP 	Q/Q/
 **,,C 	&&{C8HH, 	  7DNND$< Q/Q/
 **,,C 	T++K=>, 	  $<$${"3!44C^^Ct^4FY:VCE 	Y:VCE 	V\\*-q1Q/Q/e.s   #K 2#K%"K
c                    | j                  d      }| j                  |d       | j                         }| j                  dt	        |d      |g       g }t        j                  |      D ]<  \  }}}|D ]1  }|j                  t
        j                  j                  ||             3 > | j                  t        |      d       | j                  |d       y)+Tests moving a local directory to a bucket.rr   rs   r!   r"   r   N)r5   r|   rN   r   r8   oswalkappendpathrf   r<   len)r>   r   tmpdirdir_listdirname_	filenamesfilenames           r*   test_move_bucket_to_dirz&TestMvE2ETests.test_move_bucket_to_dir   s    """2J
A.!FNND$z3/89H!#I(Wh78   "1 	S]A&
A.rD   c                     | j                         }| j                  d      }| j                  d|t        |      g       | j	                  |d       y)r   rr   rK   r!   N)r5   rN   r   r8   r|   )r>   r   dir_to_moves      r*   test_move_dir_to_bucketz&TestMvE2ETests.test_move_dir_to_bucket   sK    ""$J$$$2KNND+tJ'789
A.rD   c                 @     j                         } j                  |d       j                  |d       j                          j                  ddt	              gdj                  f             t        t        dd	
       fd       } |        y)zTests mv with the -I option.   data1)r   r      data2r!   z-Irc   )stdin   r{   )triestimeout_secsc                  "   j                  dt              gd      } j                  t        j                  j                        |        j                  t        j                  j                        |        j                  | d       y )NlsTreturn_stdoutrr   )r   r8   rh   r   r   basenameassertNumLines)stdoutr   fpath1fpath2r>   s    r*   _Check1z/TestMvE2ETests.test_stdin_args.<locals>._Check1   si    ~~tT*%56d~Kf
mmBGG$$V,f5
mmBGG$$V,f5
&!$rD   N)rN   re   r5   r   r8   rf   r   AssertionError)r>   r   r   r   r   r   s   `  @@@r*   test_stdin_argszTestMvE2ETests.test_stdin_args   s     !F   BF   BF""$JNND$Z 01FF#34  6 >3% 4% IrD   c                    | j                  d      }| j                         }| j                  |d      }| j                  dd|t	        |      gd      }| j
                  r| j                  d	t	        |      z  |       n| j                  d
t	        |      z  |       | j                  dt	        |      z  |       | j                  dt	        |      gd      }| j                  |d       y)zTests mv with the -n option.r   rY   r   r   r   r!   z-nTru   z3Skipping existing destination item (no-clobber): %szSkipping existing item: %szRemoving %scatr   data2N)	re   r5   r6   r   r8   _use_gcloud_storagerh   assertNotInr<   )r>   r   r   
object_urir   r   s         r*   test_mv_no_clobberz!TestMvE2ETests.test_mv_no_clobber   s      ( 3F""$J""j8"LJ^^	tVT*-.d  DF 
mm
?
z
"$ mm04
3CCVL]T&\16:~~ud:&67t~LHXw'rD   z*POSIX attributes not available on Windows.zTest requires fast crcmod.c                 b    | j                         }| j                         }t        | ||d       y)zTests use of the -P flag with mv from a bucket to a local dir.

    Specifically tests combinations of POSIX attributes in metadata that will
    pass validation.
    Fis_cpN)r5   rN   r
   )r>   r   r   s      r*   .test_mv_preserve_posix_bucket_to_dir_no_errorsz=TestMvE2ETests.test_mv_preserve_posix_bucket_to_dir_no_errors  s.     ""$J!F&tZuMrD   c                     | j                         }| j                         }| j                  |dd      }t        | |||d       y)zTests use of the -P flag with mv from a bucket to a local dir.

    Specifically, combinations of POSIX attributes in metadata that will fail
    validation.
    obj   objr   Fr   N)r5   rN   r6   r	   )r>   r   r   r   s       r*   +test_mv_preserve_posix_bucket_to_dir_errorsz:TestMvE2ETests.test_mv_preserve_posix_bucket_to_dir_errors  sO     ""$J!F


z(-%+  -C %T:sF%PrD   c                 @    | j                         }t        | |d       y)z>Tests use of the -P flag with mv from a local dir to a bucket.Fr   N)r5   r   )r>   r   s     r*   -test_mv_preseve_posix_dir_to_bucket_no_errorsz<TestMvE2ETests.test_mv_preseve_posix_dir_to_bucket_no_errors  s     ""$J&tZuErD   z-Test is only relevant for gs storage classes.c                 @   | j                   t        j                  k(  rt        j                  d      S | j                  d      }| j                  |d      }| j                  dt        |      t        |d      gd	      }| j                  d
t        |      z  |       y)zETests that mv on a recent nearline object warns about early deletion.z)boto does not return object storage classNEARLINE)storage_classr   r   r!   fooTru   zWarning: moving nearline object %s may incur an early deletion charge, because the original object is less than 30 days old according to the local system time.N)
test_apir   XMLr   skipr5   r6   r   r8   rh   )r>   r   r   r   s       r*   test_mv_early_deletion_warningz-TestMvE2ETests.test_mv_early_deletion_warning"  s     }}']]FGG"""<J""j6"JJ^^	tJ	j%	 	"15  7F 	MM	.04Z0@	ABHJrD   c                    | j                         }| j                  |dd       | j                  |dd       | j                  |d       | j                         }| j	                  dt        |d      |g       g }t        j                  |      D ]<  \  }}}|D ]1  }|j                  t        j                  j                  ||             3 > | j                  t        |      d       | j                  dj                  |t        j                  t        j                        |       | j                  d	j                  |t        j                  t        j                        |       | j                  |d
       y)rJ   r   r   r   r    rr   r!   r"   z{}{}dir1{}file.txtz{}{}dir2{}file.txtr   N)r5   r6   r|   rN   r   r8   r   r   r   r   rf   r<   r   rh   ri   sep)r>   r   r   	file_listr   r   r   r   s           r*   4test_move_bucket_objects_with_duplicate_names_to_dirzCTestMvE2ETests.test_move_bucket_objects_with_duplicate_names_to_dir2  s?   ""$J"1&  ( 	"1&  ( 	
A.!FNND$z3/89I!#I(gx89   "1 	S^Q'MM&--fbffbffEMM&--fbffbffE
A.rD   N)rQ   rR   rS   rT   r   r   r   r   r   r   r   skipIfr   
skipUnlessr   r   r   r   r   r   r   rU   rD   r*   rp   rp      s    ):0x// ! !&(& 8??:KL8+-/KLN M MN 8??:KLQ MQ 8??:KLF MF
 <=J >J/rD   rp   )'rT   
__future__r   r   r   r   r   gslib.cs_api_mapr   gslib.tests.test_cpr	   r
   r   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   gslib.tests.utilr   r8   r   r   r   r   gslib.utils.boto_utilr   gslib.utils.retry_utilr   gslib.utils.system_utilr   gslib.utilsr   GsUtilUnitTestCaser   ShimUnitTestBaserW   GsUtilIntegrationTestCaserp   rU   rD   r*   <module>r      s    ( & %  ' 	 ( @ B B ' ' ? 0 : 1 2 % 6 ( . !C'h11 C'L/h77 /0B/X77 B/rD   