
    *]                     R   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	Z	ddl
mc mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dZ dZ! G d dejD                        Z# G d dejH                        Z%y)!Integration tests for mb command.    )absolute_import)print_function)division)unicode_literalsN)randint)PopulateProjectId)	SkipForS3)
SkipForXML)SkipForJSON)ObjectToURI)SECONDS_IN_DAY)SECONDS_IN_MONTH)SECONDS_IN_YEAR)SetBotoConfigForTest)SetEnvironmentForTest)Retry)	shim_utilz>gsutil does not support bucket lock operations for S3 buckets.z/gsutil KMS operations only run on GCS JSON API.c                   p   e Zd ZdZd+dZ ed      d        Zd Z ee      d        Z	 ee      d        Z
 ee      d        Z ee      d	        Z ee      d
        Zd Z ed      d        Z ed      d        Z ed      d        Z ed      d        Z ed      d        Z ed      d        Z ed      d        Z ee       ee      d               Z ee       ee      d               Z ee       ee       eedd      d                      Z ed       ed      d               Z ed       ed      d               Z ed      d         Z ed      d!        Z  ed      d"        Z! e"d#      d$        Z# e"d#      d%        Z$ ed&      d'        Z% ed&      d(        Z&d) Z'y*),TestMbr   c                 4   | j                   j                  t        d       t        j                  j
                  t        j                  j                        }t        j                  j                  }|r>t        j                  j                  t        dd      t        dd      t        dd      fz  }| j                   j                  ||      }| j                   j                  |      }|j                  r#g |_        | j                   j                  ||       |S )N)locationr   	   )kms_apiCreateKeyRingr	   testcaseKmsTestingResourcesKEYRING_NAMEKEYRING_LOCATION"CONSTANT_KEY_NAME_DO_NOT_AUTHORIZEMUTABLE_KEY_NAME_TEMPLATEr   CreateCryptoKeyGetKeyIamPolicybindingsSetKeyIamPolicy)selfmutablekeyring_fqnkey_namekey_fqn
key_policys         &platform/gsutil/gslib/tests/test_mb.pyGetKeyzTestMb.GetKey0   s     ,,,,$$$11-->> - @K ++NNH--GG
!Q-A1K7 7h ll**;AG --g6Jj
ll""7J7N    z.S3 returns success when bucket already exists.c                     | j                         }| j                  dt        |      gdd      }| j                  r| j	                  d|       y | j	                  d|       y )Nmb   Texpected_statusreturn_stderrz:HTTPError 409: The requested bucket name is not available.zalready exists)CreateBucket	RunGsUtilsuri_use_gcloud_storageassertIn)r&   
bucket_uristderrs      r,   test_mb_bucket_existszTestMb.test_mb_bucket_existsG   sc    ""$J^^T4
#34,-*.  0F 
mm
FP mm$f-r.   c                     | j                  g ddd      }| j                  r| j                  d|       y | j                  d|       y )N)r0   -p   ãzgs://fobarbazr1   Tr2   z6The project property must be set to a valid project IDzInvalid non-ASCII)r6   r8   r9   )r&   r;   s     r,   test_non_ascii_project_failsz#TestMb.test_non_ascii_project_failsS   sL    ^^?,-*.  0F 
mmL mm'0r.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j                  |d	       y )
Nbucketgs://%sFsuppress_consec_slashesr0   --retention60s<   $expected_retention_period_in_seconds)MakeTempNamebotostorage_urilowerr6   r7   VerifyRetentionPolicyr&   bucket_namer:   s      r,   "test_create_with_retention_secondsz)TestMb.test_create_with_retention_seconds]   si    ##H-K!!){/@/@/B"C:?AJNND-Z0@ABzDF  Hr.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j                  |t               y )	NrB   rC   FrD   r0   rF   1drI   )rK   rL   rM   rN   r6   r7   rO   r   rP   s      r,   test_create_with_retention_daysz&TestMb.test_create_with_retention_daysf   si    ##H-K!!){/@/@/B"C:?AJNND-tJ/?@A  Ir.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j                  |t               y )	NrB   rC   FrD   r0   rF   1mrI   )rK   rL   rM   rN   r6   r7   rO   r   rP   s      r,   !test_create_with_retention_monthsz(TestMb.test_create_with_retention_monthso   sj    ##H-K!!){/@/@/B"C:?AJNND-tJ/?@A9I  Kr.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j                  |t               y )	NrB   rC   FrD   r0   rF   1yrI   )rK   rL   rM   rN   r6   r7   rO   r   rP   s      r,    test_create_with_retention_yearsz'TestMb.test_create_with_retention_yearsx   si    ##H-K!!){/@/@/B"C:?AJNND-tJ/?@A  Jr.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  |d       y )NrB   rC   FrD   r0   rF   1secondr1   Tr2   z$Incorrect retention period specifiedrK   rL   rM   rN   r6   r7   assertRegexr&   rQ   r:   r;   s       r,   &test_create_with_retention_invalid_argz-TestMb.test_create_with_retention_invalid_arg   su    ##H-K!!){/@/@/B"C:?AJ^^T=)!*-/,-*.  0F 	VDEr.   c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  r| j                  d|       y | j                  |d       y )NrB   zs3://%sFrD   r0   rF   rZ   r1   Tr2   z4Features disallowed for S3: Setting Retention Periodz7Retention policy can only be specified for GCS buckets.	rK   rL   rM   rN   r6   r7   r8   r9   r_   r`   s       r,   +test_create_with_retention_on_s3_urls_failsz2TestMb.test_create_with_retention_on_s3_urls_fails   s    ##H-K!!){/@/@/B"C:?AJ^^	}dD$45  F 
mmJ 
LNr.   z3Public access prevention only runs on GCS JSON API.c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j                  |d       y )NrB   rC   FrD   r0   --papenforced)rK   rL   rM   rN   r6   r7   !VerifyPublicAccessPreventionValuerP   s      r,   test_create_with_pap_enforcedz$TestMb.test_create_with_pap_enforced   s`    ##H-K!!){/@/@/B"C:?AJNND':tJ/?@A**:zBr.   c                 ,   | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j	                  dd	t        |      gd
      }| j                  |dt        |      z         y )NrB   rC   FrD   r0   rf   	inheritedpublicaccesspreventiongetTreturn_stdoutz%s:\s+inheritedr^   r&   rQ   r:   stdouts       r,   test_create_with_pap_inheritedz%TestMb.test_create_with_pap_inherited   s    ##H-K!!){/@/@/B"C:?AJNND';Z0@AB^^5u!*-/*.  0F 	V/$z2BBCr.   c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  r| j                  d|       y | j                  |d       y )NrB   rC   FrD   r0   rf   invalid_argr1   Tr2   z:Flag value not in translation map for "--pap": invalid_argz invalid_arg is not a valid valuerc   r`   s       r,    test_create_with_pap_invalid_argz'TestMb.test_create_with_pap_invalid_arg   s    ##H-K!!){/@/@/B"C:?AJ^^T7M!*-/,-*.  0F 
mm
FP vBCr.   z%RPO flag only works for GCS JSON API.c           
          | j                  d      }t        j                  d|j                         z  d      }| j	                  ddddd	t        |      g       | j                  |d
d	       y )NrB   rC   FrD   r0   -lnam4--rpoASYNC_TURBOrpo)rK   rL   rM   rN   r6   r7   VerifyCommandGetrP   s      r,    test_create_with_rpo_async_turboz'TestMb.test_create_with_rpo_async_turbo   sn    ##H-K!!){/@/@/B"C:?AJNN	tVWm	j		 	*e];r.   c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       	 | j                  |dd	       y # t        $ r | j                  |dd
       Y y w xY w)NrB   rC   FrD   r0   rw   rx   r{   DEFAULTNone)rK   rL   rM   rN   r6   r7   r|   AssertionErrorrP   s      r,   test_create_sets_rpo_to_defaultz&TestMb.test_create_sets_rpo_to_default   s    ##H-K!!){/@/@/B"C:?AJNND$Z(89:7
Jy9 7 Jv67s   A- -BBc           
          | j                  d      }t        j                  d|j                         z  d      }| j	                  ddddd	t        |      gd
d      }| j                  d|       y)z0Turbo replication is only meant for dual-region.rB   rC   FrD   r0   rw   zus-central1ry   rz   Tr1   r4   r3   z.ASYNC_TURBO cannot be enabled on REGION bucketNrK   rL   rM   rN   r6   r7   r9   r`   s       r,   :test_create_with_rpo_async_turbo_fails_for_regional_bucketzATestMb.test_create_with_rpo_async_turbo_fails_for_regional_bucket   sz     ##H-K!!){/@/@/B"C:?AJ^^	t]G]	j			  F
 	MMBFKr.   c           
      >   | j                  d      }t        j                  d|j                         z  d      }| j                  rdnd}| j                  ddd	d
dt        |      gd|      }| j                  r| j                  d|       y | j                  d|       y )NrB   rC   FrD      r1   r0   rw   rx   ry   incorrect_valueTr   z=--recovery-point-objective: Invalid choice: 'incorrect_value'zYInvalid value for --rpo. Must be one of: (ASYNC_TURBO|DEFAULT), provided: incorrect_valuerK   rL   rM   rN   r8   r6   r7   r9   r&   rQ   r:   r3   r;   s        r,   1test_create_with_rpo_incorrect_value_raises_errorz8TestMb.test_create_with_rpo_incorrect_value_raises_error   s    ##H-K!!){/@/@/B"C:?AJ33aO^^	tVW&7	j		'	  )F 
mm
K
 mm'(.0r.   c           
         | j                  d      }t        j                  d|j                         z  d      }| j	                         }| j                  ddt        j                  j                  d|t        |      gdd	
      }| j                  r| j                  d|       y | j                  d|       | j                  d|z  |       y )NrB   rC   FrD   r0   rw   -kTr1   r   2HTTPError 403: Permission denied on Cloud KMS key.To authorize, run:z-k %s)rK   rL   rM   rN   r-   r6   r   r   r   r7   r8   r9   r&   rQ   r:   keyr;   s        r,   &test_create_with_k_flag_not_authorizedz-TestMb.test_create_with_k_flag_not_authorized   s     ##H-K!!){/@/@/B"C:?AJ
++-C^^dH00AA4Z +/,-  /F 
mmH mm(&1
mmGcM6*r.   c                    | j                  d      }t        j                  d|j                         z  d      }| j	                         }| j                  ddt        j                  j                  d|dt               t        |      gd	d
      }| j                  r| j                  d|       y | j                  d|       | j                  dt               z  |       y )NrB   rC   FrD   r0   rw   r   r>   Tr1   r   r   r   z-p %s)rK   rL   rM   rN   r-   r6   r   r   r   r	   r7   r8   r9   r   s        r,   -test_create_with_k_flag_p_flag_not_authorizedz4TestMb.test_create_with_k_flag_p_flag_not_authorized  s     ##H-K!!){/@/@/B"C:?AJ
++-C^^dH00AA4Z	 +/,-  /F 
mmH mm(&1
mmG/116:r.      r1   )triestimeout_secsc           
      8   | j                  d      }t        j                  d|j                         z  d      }| j	                  d      }| j                  ddd	|g       | j                  d
dt        j                  j                  d	|t        |      gd       y )NrB   rC   FrD   T)r'   kms	authorizer   r0   rw   r   )r3   )
rK   rL   rM   rN   r-   r6   r   r   r   r7   )r&   rQ   r:   r   s       r,   "test_create_with_k_flag_authorizedz)TestMb.test_create_with_k_flag_authorized#  s     ##H-K!!){/@/@/B"C:?AJ
++d+
#CNNE;c23NNdH00AA4Z $%	  &r.   z4Custom Dual Region is not supported for the XML API.z3Custom Dual Region is not supported for S3 buckets.c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j	                  dd	t        |      gd
      }| j                  |d       y )NrB   rC   FrD   r0   rw   zus-east1+us-east4ls-LbTrn   z(ocations:\s*\[\s*.US-EAST1.,\s*.US-EAST4r^   rp   s       r,   /test_create_with_custom_dual_regions_via_l_flagz6TestMb.test_create_with_custom_dual_regions_via_l_flag2  s     ##H-K!!){/@/@/B"C:?AJNND$ 3T*5EFG^^T5$z*:;4^PFVHIr.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  d|       y )NrB   rC   FrD   r0   rw   zinvalid_reg1+invalid_reg2Tr1   r   z.The specified location constraint is not validr   r`   s       r,   =test_create_with_invalid_dual_regions_via_l_flag_raises_errorzDTestMb.test_create_with_invalid_dual_regions_via_l_flag_raises_error<  sw     ##H-K!!){/@/@/B"C:?AJ^^	t0	j			  F
 	MMBFKr.   z1The --placement flag only works for GCS JSON API.c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j	                  dd	t        |      gd
      }| j                  |d       y )NrB   rC   FrD   r0   --placementzus-central1,us-west1r   r   Trn   z+ocations:\s*\[\s*.US-CENTRAL1.,\s*.US-WEST1r^   rp   s       r,   test_create_with_placement_flagz&TestMb.test_create_with_placement_flagI  s    ##H-K!!){/@/@/B"C:?AJNN	}4	j		 ^^T5$z*:;4^PFVKLr.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  |d       y )NrB   rC   FrD   r0   r   zinvalid_reg1,invalid_reg2Tr1   r   z.*400.*(Invalid custom placement config|One or more unrecognized regions in dual-region, received: INVALID_REG1, INVALID_REG2).*r^   r`   s       r,   4test_create_with_invalid_placement_flag_raises_errorz;TestMb.test_create_with_invalid_placement_flag_raises_errorT  s|    ##H-K!!){/@/@/B"C:?AJ^^	}9	j			  F
 	 *+r.   c                 :   | j                  d      }t        j                  d|j                         z  d      }| j                  rdnd}| j                  ddd	t        |      gd
|      }| j                  r| j                  d|       y | j                  d|       y )NrB   rC   FrD   r   r1   r0   r   zval1,val2,val3Tr   z--placement: too many argszhCommandException: Please specify two regions separated by comma without space. Specified: val1,val2,val3r   r   s        r,   Btest_create_with_incorrect_number_of_placement_values_raises_errorzITestMb.test_create_with_incorrect_number_of_placement_values_raises_errorc  s    ##H-K!!){/@/@/B"C:?AJ  33aO^^	}.	j		'	  )F
 
mm0&9
mm67=?r.   zTesting XML only behavior.c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      gdd	
      }| j                  d|       y )NrB   rC   FrD   r0   ry   rz   Tr1   r   zXCommandException: The --rpo option(s) can only be used for GCS Buckets with the JSON APIr   r`   s       r,   4test_single_json_only_flag_raises_error_with_xml_apiz;TestMb.test_single_json_only_flag_raises_error_with_xml_apiv  sx    ##H-K!!){/@/@/B"C:?AJ^^T7M!*-/*.,-  /F 	MM	-.46r.   c                     | j                  d      }t        j                  d|j                         z  d      }| j	                  ddddd	d
ddddt        |      gdd      }| j                  d|       y )NrB   rC   FrD   r0   --autoclassrf   enabledr   zuscentral-1,us-asia1ry   rz   z-bonTr1   r   z}CommandException: The --autoclass, --pap, --placement, --rpo, -b option(s) can only be used for GCS Buckets with the JSON APIr   r`   s       r,   6test_multiple_json_only_flags_raise_error_with_xml_apiz=TestMb.test_multiple_json_only_flags_raise_error_with_xml_api  s    ##H-K!!){/@/@/B"C:?AJ^^mWidZ
 +/,-  /F 	MM	Kr.   1The --autoclass flag only works for GCS JSON API.c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  ddt        |      g       | j	                  ddt        |      gd	
      }| j                  |d       y )NrB   rC   FrD   r0   r   r   r   Trn   zAutoclass:.*Enabled on *.r^   rp   s       r,   test_create_with_autoclass_flagz&TestMb.test_create_with_autoclass_flag  s}    ##H-K!!){/@/@/B"C:?AJNND-j)9:;^^T5$z*:;4^PFV9:r.   c           	         | j                  d      }t        j                  d|j                         z  d      }| j	                  ddddt        |      gd	d
      }| j                  r| j                  d|       y | j                  d|       y )NrB   rC   FrD   r0   r   z-carchiveTr1   r   z/HTTPError 400: Cannot set default storage classz9BadRequestException: 400 Cannot set default storage class)rK   rL   rM   rN   r6   r7   r8   r9   r`   s       r,   :test_create_with_invalid_storage_class_with_autoclass_flagzATestMb.test_create_with_invalid_storage_class_with_autoclass_flag  s    ##H-K!!){/@/@/B"C:?AJ^^T=$	4
CSTdhz{^|F
mmEvN
mmOQWXr.   c                    | j                  d      }t        j                  d|j                         z  d      }| j	                  dddt        |      g       | j	                  dd	t        |      gd
      }| j                  |d       y )NrB   rC   FrD   r0   -snearliner   r   Trn   zStorage\sclass:\s*NEARLINEr^   rp   s       r,   test_create_with_storage_classz%TestMb.test_create_with_storage_class  s    ##H-K!!){/@/@/B"C:?AJNND$
D,<=>^^T5$z*:;4^PFV:;r.   N)F)(__name__
__module____qualname____doc__r-   r
   r<   r@   BUCKET_LOCK_SKIP_MSGrR   rU   rX   r[   ra   rd   r   ri   rr   ru   r}   r   r   r   KMS_SKIP_MSGr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r.   r,   r   r   -   s   ). =>	. ?	.1 !"H #H !"I #I !"K #K !"J #J !"F #FN CDC EC CDD ED CDD ED 56< 7< 56
7 7
7 56
L 7
L 560 70* l\+  +& l\;  ;* l\qq1
& 2  
& DEBCJ D FJ DEBC	L D F	L ABM CM AB+ C+ AB? C?$ +,
6 -
6 +, -  AB; C; ABY CY<r.   r   c                   T    e Zd ZdZd Z ed      d        Z ed      d        Zd Zy)	TestMbUnitTestsWithShimz#Unit tests for gsutil mb with shim.c           	      ^   t        ddg      5  t        ddd      5  | j                  dg dd	      }d
j                  |j                  d         }| j                  dj                  t        j                  d            |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)NGSUtiluse_gcloud_storageTruer   hidden_shim_modedry_runr   fake_dir(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr0   )rF   rZ   zgs://fake-bucketTargsreturn_log_handler
infoz_Gcloud Storage Command: {} storage buckets create --retention-period 31557600s gs://fake-bucket	r   r   
RunCommandjoinmessagesr9   formatr   _get_gcloud_binary_pathr&   mock_log_handler
info_liness      r,   ,test_shim_translates_retention_seconds_flagszDTestMbUnitTestsWithShim.test_shim_translates_retention_seconds_flags  s    	GHJ 
K 6<)" 	  ??41-
 ?C + D YY/88@A
 HIO$<<ZHJJ 	"	
K 
K	 	
K 
K#   B#A)BB#B 	B##B,z+The --rpo flag only works for GCS JSON API.c           	      b   d}t        ddg      5  t        d|d      5  | j                  dg dd	      }d
j                  |j                  d         }| j                  dj                  t        j                  d            |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   r   r   r   r0   )ry   r   zgs://fake-bucket-1Tr   r   r   zTGcloud Storage Command: {} storage buckets create --recovery-point-objective DEFAULTr   )r&   fake_cloudsdk_dirr   r   s       r,   2test_shim_translates_recovery_point_objective_flagzJTestMbUnitTestsWithShim.test_shim_translates_recovery_point_objective_flag  s    "	GHJ 
K 6<0" 	  ??;# + %
 YY/88@A
BCI622:>D@AK	M	
K 
K	 	
K 
Ks#   B%A)BB%B"	B%%B.r   c           	      ^   t        ddg      5  t        ddd      5  | j                  dddgd	
      }dj                  |j                  d         }| j                  dj                  t        j                  d            |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   r   r   r   r0   r   zgs://fake-bucket-2Tr   r   r   zWGcloud Storage Command: {} storage buckets create --enable-autoclass gs://fake-bucket-2r   r   s      r,   #test_shim_translates_autoclass_flagz;TestMbUnitTestsWithShim.test_shim_translates_autoclass_flag  s    	GHJ 
K 6<)" 	  ??4/</C1- ?C + D YY/88@A
 @AG$<<ZHBJ 	"	
K 
K	 	
K 
Kr   c           	      ^   t        ddg      5  t        ddd      5  | j                  dg dd	      }d
j                  |j                  d         }| j                  dj                  t        j                  d            |       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   r   r   r   r   r0   )r   r   zgs://fake-bucket-3Tr   r   r   zeGcloud Storage Command: {} storage buckets create --default-storage-class nearline gs://fake-bucket-3r   r   s      r,   test_shim_translates_class_flagz7TestMbUnitTestsWithShim.test_shim_translates_class_flag  s    	GHJ 
K 6<)" 	  ??41-
 ?C + D YY/88@A
 NOUv$<<ZHPJ 	"	
K 
K	 	
K 
Kr   N)	r   r   r   r   r   r   r   r   r   r   r.   r,   r   r     sF    +"( ;<M =M& AB" C"&"r.   r   )&r   
__future__r   r   r   r   osrandomr   rL   gslib.tests.testcasetestsr   gslib.project_idr	   )gslib.tests.testcase.integration_testcaser
   r   r   gslib.tests.utilr   r7   gslib.utils.retention_utilr   r   r   r   r   gslib.utils.retry_utilr   gslib.utilsr   r   r   GsUtilIntegrationTestCaser   ShimUnitTestBaser   r   r.   r,   <module>r      s    ( & %  ' 	   ' ' . ? @ A 0 5 7 6 1 2 ( !& @A<X// A<HQ"h77 Q"r.   