
    u_                    x   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZ ddlmZ ddlZdd	lmZ dd
lmZ ddlmZ ddlm Z  ddl!m"Z" ddl!m#Z# ddl!m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6 ddl5m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z= ddl<m>Z> ddl<m?Z? ddl@mAc mBZB ddlCmDZD dd lEmFZF dd!lEmGZG dd"lEmHZH dd#lEmIZI dd$lJmKZK dd%lJmLZL dd&lJmMZN dd'lJmOZO dd(lJmPZP dd)lJmQZQ dd*lJmRZR dd+lJmSZS dd,lJmTZU dd-lJmVZV dd.lJmWZW dd/lJmXZX dd0lJmYZY dd1lJmZZZ dd2lJm[Z[ dd3lJm\Z\ dd4lJm]Z] dd5lJm^Z^ dd6lJm_Z_ dd7lJm`Z` dd8lJmaZa dd9lJmbZb dd:lJmZ dd;lcmdZe dd<lfmgZg dd=lfmhZh dd>lfmiZi dd?ljmkZk dd@llmmZm ddAlnmoZo ddBlpmqZq ddClpmrZr ddDlsmtZt ddElsmuZu ddFlsmvZv ddGlsmwZw ddHlxmyZy ddIlxmzZz ddJlxm{Z{ ddKl|m}Z} ddLl~mZ ddMl~mZ ddNl~mZ ddOl~mZ ddPl~mZ ddQl~mZ ddRl~mZ ddSl~mZ ddTlmZ ddUlmZ ddVlmZ ddWlmZ ddXlmZ ddYlmZ ddZlmZ dd[lmZ dd\llmZ ddlZdd]lmZ dd^lmZ dd^lmZ ej0                  reZes*dd_lmZ dd`lJmZ ddalJmZ ddblJmZ ddclJmZ dddlJmZ ddelJmZ dfZdzdgZdzdhZdzdiZdj Zdk Z G dl dme      Z G dn doe      Z G dp dqe      Z G dr dse      Z G dt dueBj\                        Z G dv dweBj`                        Z G dx dyeBjd                        Zy){!Integration tests for cp command.    )absolute_import)division)print_function)unicode_literalsN)mock)
exceptions)storage_uri)ResumableTransferDisposition)StorageResponseError)BucketStorageUri)command)	exception)name_expansion)!ResumableUploadStartOverException)(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)'ShimTranslatePredefinedAclSubOptForCopy)ApiSelector)_DEFAULT_DOWNLOAD_CHUNK_SIZE)DiscardMessagesQueue)InvalidUrlError)
GcsJsonApi)ObjectFromTracker)WriteParallelUploadTrackerFile)PopulateProjectId)StorageUrlFromString)"EnsureRewriteResumeCallbackHandler)HaltingRewriteCallbackHandler)RewriteHaltException)NotParallelizable)	SkipForGS)	SkipForS3)
SkipForXML)SkipForJSON)"AuthorizeProjectToUseTestingKmsKey)BuildErrorRegex)GenerationFromURI)HaltingCopyCallbackHandler)#HaltOneComponentCopyCallbackHandler)HAS_GS_PORT)HAS_S3_CREDS)KmsTestingResources)ObjectToURI)ORPHANED_FILE)POSIX_GID_ERROR)POSIX_INSUFFICIENT_ACCESS_ERROR)POSIX_MODE_ERROR)POSIX_UID_ERROR)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)TailSet)TEST_ENCRYPTION_KEY1)TEST_ENCRYPTION_KEY1_SHA256_B64)TEST_ENCRYPTION_KEY2)TEST_ENCRYPTION_KEY3)unittest)storage_v1_messages)DeleteTrackerFile)GetRewriteTrackerFilePath)!GetSlicedDownloadTrackerFilePaths)BytesToFixedWidthString)hashing_helper)UsingCrcmodExtension)START_CALLBACK_PER_BYTES)UTF8)GetTrackerFilePath)PARALLEL_UPLOAD_STATIC_SALT)PARALLEL_UPLOAD_TEMP_NAMESPACE)TrackerFileType)"CalculateB64EncodedMd5FromContents)CalculateMd5FromContents)GetMd5)CreateCustomMetadata)GID_ATTR)	MODE_ATTR)NA_ID)NA_MODE)UID_ATTR)ParseAndSetPOSIXAttributes)ValidateFilePermissionAccess)ValidatePOSIXMode)Retry)
IS_WINDOWS)get_random_ascii_chars)	EIGHT_MIB)HumanReadableToBytes)MakeHumanReadable)ONE_KIB)ONE_MIB)	shim_util)http_client)range)util)DEFAULT_MODE)GetInvalidGid)GetNonPrimaryGid)GetPrimaryGid)INVALID_UID)USER_ID)   ERRORziAt most one of --gzip-in-flight | --gzip-in-flight-all | --gzip-local | --gzip-local-all can be specifiedc                 V   d}| j                   rBt        j                  d      x}}t        j                  d      }t        j                  d      }	n@t        |t              }t        |t
              }t        |t              }	t        |t              }t        d||it        t        ||it        t        t        d||it        t        ||	it        t        t        d||	it        t        t        t        ||	it        t        t        t        t        d||	it        t        t        t        ||	it        t        t        t        ||	it        t        t        t        t        d||	it        t        t        t        t        d||	it        t        t        t        ||it        t        t        t        t        d||it        t        t        d	||id
}
t!        j"                  |
      D ]  \  }}| j%                  |       |j'                  t              }|t)        |      r |       }|j'                  t              }|t)        |      r |       }|j'                  t              }| j+                  | j,                  |j.                  |j0                  |||       | j3                  |rdnddt5        ||j0                        |gdd      }| j                   rd}nt6        }| j9                  ||d|d|d|       ||   }| j;                  |j=                  |      d|d|j>                  d|       tA        t5        |      | jC                  |            }tA        || jE                  |            }| jG                  |tI        d|j0                  z  g             | jG                  |tI        dg              y)aR  Helper function for preserve_posix_errors tests in test_cp and test_mv.

  Args:
    cls: An instance of either TestCp or TestMv.
    bucket_uri: The uri of the bucket that the object is in.
    obj: The object to run the tests on.
    tmpdir: The local file path to cp to.
    is_cp: Whether or not the calling test suite is cp or mv.
  error_regexz;User \d+ owns file, but owner does not have read permissionz2GID in .* metadata doesn't exist on current systemz2UID in .* metadata doesn't exist on current system333420530640240)test1test2test3test4test5test6test7test8test9test10test11test12test13test14Nuidgidmodecpmv-P   Texpected_statusreturn_stderrrh   zError during test "z": z not found in stderr:
zTest z: did not match expected error; could not find a match for z

in stderr:
z/%s )%_use_gcloud_storagerecompiler&   r0   r/   r2   r1   rN   rM   rb   rQ   re   rd   rc   rf   six	iteritemsClearPOSIXMetadatagetcallableSetPOSIXMetadatadefault_providerbucket_nameobject_name	RunGsUtilsurir.   assertIn
assertTruesearchpatternr6   FlatListBucketFlatListDirassertEqualset)cls
bucket_uriobjtmpdiris_cp	error_keyinsufficient_access_errorno_read_access_errormissing_gid_errormissing_uid_errortest_params	test_name
attrs_dictr   r   r   stderrgeneral_posix_errorrj   listing1listing2s                        &platform/gsutil/gslib/tests/test_cp.py TestCpMvPOSIXBucketToLocalErrorsr      s    )79zzF8H H 4

=?

=? !0,!.'_='_=*30@A U
)
 M
&
 M
U
& K
&
 K
U
& K
M
& K
M
U
&	 K
M
& K
$
& K
M
U
&	 K
$
U
&	 G
M
& G
M
U
&	 M
U
.GH+V  #}}[9i3 ..
"C
8C=Ec
..
"C
8C=Ec>>)$D--#//  "  $ ]]4Z)6 ,-)-  /F #)LLV	'	12
 Y'KNN6""+[-@-@&	JK tJ'););J)GHHvsv67HOOHc53??#:";<=OOHc2$i(_  :    c                    t        j                  |      j                  }t        j                         }t
        |it
        |it
        |t        dit
        |t        dit        t        it        t        t        dit        t        t
        |it        t        t
        |it        t        t
        |t        dit        t        t
        |t        did
}t        j                  |      D ]n  \  }}|j                  t              }	|j                  t
              }
|j                  t              }| j                  |||j                  t              |	|
|       p t        j                  |      D ]%  }| j!                  |rdnd	d
t#        ||      |g       ' t%        || j'                  |            }| j)                  |t+        g d             | j-                  t         j.                  j1                  |d      |t2               | j-                  t         j.                  j1                  |d      |t2               | j-                  t         j.                  j1                  |d      |d       | j-                  t         j.                  j1                  |d      |d       | j-                  t         j.                  j1                  |d      t        |t2               | j-                  t         j.                  j1                  |d      t        |d       | j-                  t         j.                  j1                  |d      t        |t2               | j-                  t         j.                  j1                  |d      t        |t2               | j-                  t         j.                  j1                  |d      t        |d       | j-                  t         j.                  j1                  |d      t        |d       y)a,  Helper function for preserve_posix_no_errors tests in test_cp and test_mv.

  Args:
    cls: An instance of either TestCp or TestMv.
    bucket_uri: The uri of the bucket that the object is in.
    tmpdir: The local file path to cp to.
    is_cp: Whether or not the calling test suite is cp or mv.
  440444rl   433442
obj1obj2obj3obj4obj5obj6obj7obj8obj9obj10)r   r   contentsr   r   r   r   r   r   )
z/obj1z/obj2z/obj3z/obj4z/obj5z/obj6z/obj7z/obj8z/obj9z/obj10r   )r   r   r   r   i   r   i$  r   r~   r   i  r   r   r   i  r   i"  N)osstatst_gidr`   rc   rM   rN   rQ   rf   r   r   r   CreateObjectencoderD   iterkeysr   r   r6   r   r   r   VerifyLocalPOSIXPermissionspathjoinra   )r   r   r   r   primary_gidnon_primary_gidr   obj_namer   r   r   r   listings                r   "TestCpMvPOSIXBucketToLocalNoErrorsr   7  s|    &&+))+/ K O K
U
 O
U
 G G
U
 G
K
 G
O
 G
K
U G
O
UG(+R "mmK8h

..
"C
..
"C>>)$D
!)&ood3   	 9 ,,{+hMMD$	j(	#V	-. ,
 FCOOF34'//	  	
 !!"'',,vv">&1'3 " 5 !!"'',,vv">&5'3 " 5 !!"'',,vv">&1', " . !!"'',,vv">&5', " . !!"'',,vv">&-&1'3 " 5 !!"'',,vv">&-&1', " . !!"'',,vv">&-&1'3 " 5 !!"'',,vv">&-&5'3 " 5 !!"'',,vv">&-&1', " . !!"'',,vw"?&-&5', " .r   c                    t        j                         }t        j                         }t        |it        |it        |t
        dit        |t
        dit        t        it        t        t
        dit        t        t        |it        t        t        |it        t        t        |t
        dit        t        t        |t
        did
}t        j                  |      D ]i  \  }}|j                  t        t              }|j                  t        t              }	|j                  t
        t              }
|
t        k7  rt        t        |
d             t        ||t        |	      t        |
             | j!                  d	||	|

      }| j#                  |rdndd|t%        ||      g       |t        k7  r+| j'                  |j(                  |t        t+        |             |	t        k7  r+| j'                  |j(                  |t        t+        |	             |
t        k7  s?| j'                  |j(                  |t
        t+        |
             l y)zHelper function for testing local to bucket POSIX preservation.

  Args:
    cls: An instance of either TestCp or TestMv.
    bucket_uri: The uri of the bucket to cp/mv to.
    is_cp: Whether or not the calling test suite is cp or mv.
  r   r   rl   r   r   r      r~      foo)r   r   r   r   r   r   r   N)r   getgidr`   rc   rM   rN   rQ   rf   r   r   r   rO   rP   rT   intrS   CreateTempFiler   r   VerifyObjectCustomAttributer   str)r   r   r   r   r   r   r   r   r   r   r   fpaths               r   "TestCpMvPOSIXLocalToBucketNoErrorsr     s     		+))+/ K O K
U
 O
U
 G G
U
 G
K
 G
O
 G
K
U G
O
UG(+R "mmK8h

..5
)C
..5
)C>>)W-DwD!% %(%(X&)$i1 CStLEMMD$	j(	#	%& e|	%%j&<&<h&.C:
e|	%%j&<&<h&.C:w	%%j&<&<h&/T<+ 9r   c                     t        | d      5 }|j                  |j                                d d d        y # 1 sw Y   y xY w)Nrb)openappendread)	fifo_pathlist_for_outputfs      r   _ReadContentsFromFifor     s.    It1668$ s    6?c                 h    t        |d      5 }|j                  |        d d d        y # 1 sw Y   y xY w)Nwb)r   write)r   r   r   s      r   _WriteContentsToFifor     s&    ItGGH s   (1c                       e Zd ZdZd Zd Zy)&_JSONForceHTTPErrorCopyCallbackHandlerzDTest callback handler that raises an arbitrary HTTP error exception.c                 .    || _         || _        d| _        y NF)_startover_at_byte_http_error_numstarted_over_once)selfstartover_at_bytehttp_error_nums      r   __init__z/_JSONForceHTTPErrorCopyCallbackHandler.__init__   s    /D)D"Dr   c                 4   || j                   k\  r| j                  s|t        j                  j	                  d| j
                  d| j                   dt        |      dt        |      d	       d| _        t        j                  d| j
                  idd      yy)	<Forcibly exits if the transfer has passed the halting point.zForcing HTTP error z after byte . / transferred.
TstatusN)	r   r   sysr   r   r   rZ   apitools_exceptions	HttpErrorr   total_bytes_transferred
total_sizes      r   callz+_JSONForceHTTPErrorCopyCallbackHandler.call  s    4#:#::""	jj,,d.E.E)*AB)*5	7 8
  $d))8T5I5I*J*.6 6 # 	;r   N__name__
__module____qualname____doc__r   r   r   r   r   r     s    L#6r   r   c                       e Zd ZdZd Zd Zy)/_XMLResumableUploadStartOverCopyCallbackHandlerzETest callback handler that raises start-over exception during upload.c                      || _         d| _        y r   )r   r   )r   r   s     r   r   z8_XMLResumableUploadStartOverCopyCallbackHandler.__init__  s    /D"Dr   c           
      0   || j                   k\  r| j                  szt        j                  j	                  d| j                   dt        |      dt        |      d       d| _        t        j                  j                  dt        j                        yy)r   4Forcing ResumableUpload start over error after byte r   r   r   TzForcing upload start overN)r   r   r   r   r   rZ   botor   ResumableUploadExceptionr   
START_OVERr   s      r   r  z4_XMLResumableUploadStartOverCopyCallbackHandler.call  s    4#:#::""	jj ""$56M$NZ(*+
  $dNN33
%'C'N'NP P # 	;r   Nr  r  r   r   r	  r	    s    M#
Pr   r	  c                       e Zd ZdZd Zd Zy)-_DeleteBucketThenStartOverCopyCallbackHandlerzATest callback handler that deletes bucket then raises start-over.c                 .    || _         || _        d| _        y r   )r   _bucket_urir   )r   r   r   s      r   r   z6_DeleteBucketThenStartOverCopyCallbackHandler.__init__-  s    /D!D"Dr   c           
          | j                   k\  r j                  st        j                  j	                  d j
                  j                  z         t        t        dd       fd       } |        t        j                  j	                  d j                   dt        |      dt        |      d	       d
 _        t        d      yy)r   zDeleting bucket (%s)   r   triestimeout_secsc                     t        j                  j                  d            } | D ]A  }j                  j                         j	                  |j
                  |j                         C j                  j                          y )NT)all_versions)
version_id)listr  list_bucket
get_bucket
delete_keynamer  delete_bucket)bucket_listkr   s     r   DeleteBucketzH_DeleteBucketThenStartOverCopyCallbackHandler.call.<locals>.DeleteBucket9  so    4++77T7JKA



%
%
'
2
2166>?ll 3 L  	&&(r   r  r   r   r   TzArtificially forcing start-overN)r   r   r   r   r   r  r   rU   r   rZ   r   )r   r   r   r$  s   `   r   r  z2_DeleteBucketThenStartOverCopyCallbackHandler.call3  s    4#:#::""	jj-1A1A1M1MNO!;) <) n	jj ""$56M$NZ(*+
  $d-.OPP% # 	;r   Nr  r  r   r   r  r  *  s    I#Qr   r  c                        e Zd ZdZ	 ddZd Zy)_ResumableUploadRetryHandlerzFTest callback handler for causing retries during a resumable transfer.c                 J    || _         || _        || _        || _        d| _        y )Nr   )_retry_at_byte_exception_to_raise_exception_args_num_retries_retries_made)r   retry_at_byteexception_to_raiseexc_argsnum_retriess        r   r   z%_ResumableUploadRetryHandler.__init__N  s,    
 (D1D#D#DDr   c                     || j                   k\  rH| j                  | j                  k  r.| xj                  dz  c_         | j                  | j                   yy)z(Cause a single retry at the retry point.r   N)r(  r,  r+  r)  r*  )r   r   unused_total_sizes      r   r  z!_ResumableUploadRetryHandler.call[  sX    4#6#66T...
A$D$$d&:&:;; 	/ 	7r   N)r   r  r  r   r   r&  r&  K  s    N 	
<r   r&  c                   J   e Zd ZdZedz  Zd Zd Zed        Z	ed        Z
d Zd Ze ed	      d
               Ze ed	      d               Ze ed	       ej"                  ed      d                      Z ej"                  ed      ed               Z ej"                  ed      ed               Z ej"                  ed      ed               Zed        Z ej"                  ed      ed               Z ej"                  ed      ed               Z ej"                  ed      ed               Zd Zed        Zd Zd Z ej"                  ed      ed               Zed        Z ed        Z!ed        Z"ed        Z#e ed       d!               Z$e e%d"      d#               Z&ed$        Z'd% Z( e)d&      d'        Z*e e)d(      d)               Z+e e)d(      d*               Z,ed+        Z-e e)d&      d,               Z.d- Z/ed.        Z0d/ Z1 ejd                  e3d0      d1        Z4 ejd                  e3d0       ejj                  d2      d3               Z6 ejd                  e3d0      d4        Z7 ejj                  d5      d6        Z8d7 Z9 ejd                  e: d8      d9        Z;d: Z<d; Z=d< Z>d= Z?ed>        Z@d? ZAd@ ZBedA        ZCdB ZDdC ZEdD ZF eGdE      dF        ZHdG ZI ejd                  e3d0      dH        ZJ e)dI      dJ        ZKdK ZLdL ZMdM ZN ej"                  edN      edO               ZOedP        ZPdQ ZQedR        ZRedS        ZSedT        ZT e)dU       edV      edW                      ZU e)dU       edV      edX                      ZV e)dU       edV      edY                      ZWedZ        ZX e)dU       edV      ed[                      ZYd\ ZZd] Z[d^ Z\d_ Z]d` Z^eda        Z_db Z` ej"                  ed      dc        Za ej"                  ed      dd        Zbde Zcedf        Zd e)dg      edh               Ze e)dg      edi               Zf e)dg      edj               Zg e)dk      dl        Zh e)dk      dm        Zi e)dk      dn        Zjdo Zk e)dp      dq        Zl e)dU       edV      edr                      Zm e)dp      ds        Zn e)dp      dt        Zo e)du      dv        Zp e)dp      dw        Zq e)dp      dx        Zr e)dp      dy        Zs e)dp      dz        Zt e)dp      d{        Zu e)dp      d|        Zv e)dp      d}        Zw e)dp      d~        Zx e)d      d        Zy e)d      d        Zze{ e)dp       ej"                  ed      ed                             Z|e{ ej"                  ed      ed                      Z}	 ddZ~d Z e)dg      d        Z e)dg      d        Zed        Z ejd                   e       d       e)d      d               Zd Zd Zd Zd Zd Z e)dU       edV      ed                      Z ej"                  ed      ed               Zd Zd Zed        Z e)dp      d        Z e)d      d        Z ejd                   e       d       e)d      d               Z ejd                   e       d       e)d      d               Z ejd                   e       d       e)d      d               Z ejd                   e       d       e)d      d               Z ejd                   e       d       e)d      d               Z ejd                   e       d       e)d      d               Z e)dp      d        Zd Zd Zd Zd Zd Zd Z ej"                  ed       ejd                   e       d      d               Z ej"                  ed      d        Z ej"                  ed      d        Zd Z ed      d        Z e)d      d        Z e)d      d        Z e)d      d        Z e)d      d        Z e)d      d        Z e)d       ed      d               Z ejd                  ed      d        Zd Zed        Zed        Zy)TestCpr   rg   c                 Z    t        j                  dd|z        }| j                  ||      S )Ngslibztests/test_data/%s	file_namer   )pkgutilget_datar   )r   r   r   s      r   _get_test_filezTestCp._get_test_filej  s0    )=)DEHAAr   c                     dg}|j                  |       |j                  |       |j                   | j                  |fi |       y )Nr   )extendr   r   )r   src_path_tupledst_pathlist_for_return_valuekwargsarg_lists         r   )_CpWithFifoViaGsUtilAndAppendOutputToListz0TestCp._CpWithFifoViaGsUtilAndAppendOutputToListn  sC     vHOON#OOH   !CF!CDr   c                    | j                  d      }| j                  d      }| j                  dd|t        |      gd      }| j	                  |dj                  t        j                  t        |                         | j                  |j                         d       | j                  ddt        |      |gd      }t        |d	      5 }| j	                  |dj                  t        j                  t        |                         | j                  |j                         d       d d d        y # 1 sw Y   y xY w)
Nr   r      barr   -nTr   Skipping.*: {}r   r   r   r   r   assertRegexformatr   escaper   get_contents_as_stringr   r   r   key_urir   r   r   s        r   test_noclobberzTestCp.test_noclobberx  s   0G0E^^	tUDM*$  @FV.55biiW6NOPW335v>^^T4g>*.  0F	eT	a
v077		$q'8JKL
qvvx( 
		   AD55D>c                    | j                  d      }| j                  d      }| j                  dd|t        |      gd      }| j	                  |dj                  t        j                  t        |                         | j                  |j                         d       | j                  ddt        |      |gd      }t        |d	      5 }| j	                  |dj                  t        j                  t        |                         | j                  |j                         d       d d d        y # 1 sw Y   y xY w)
Nr   rE  s   quuxr   rG  TrH  rI  r   rJ  rO  s        r   test_noclobber_different_sizez$TestCp.test_noclobber_different_size  s   0G1E^^	tUDM*$  @FV.55biiW6NOPW335v>^^T4g>*.  0F	eT	a
v077		$q'8JKL
qvvx) 
		rR  c                       j                  d       j                  d j                  t        t        dd       fd       } |        y )Nr   rE  z://   r   r  c                      j                  dgdd      } j                  rj                  d|        y j                  d|        y )Nr   r   Tr   znot found: 404zdoes not existr   r   r   )r   r   invalid_bucket_urir   s    r   _Checkz1TestCp.test_dest_bucket_not_exist.<locals>._Check  sO    ~~tU,>?./,0  2f 
	!	!&/&/r   )r   r   nonexistent_bucket_namerU   AssertionError)r   rZ  r   rY  s   ` @@r   test_dest_bucket_not_existz!TestCp.test_dest_bucket_not_exist  sT    0E00$2N2NP >30 40 Hr   c           	         | j                         }| j                         }| j                  |d      }| j                  dt        |      t        |      gd      }| j	                  |j                  d      d       | j                  |j                  d      d       | j                  dd	t        |      t        |      gd      }| j                  |d
j                  t        ||j                                     y )Nr   r   r   r   TrH  Copyingr   rg   rG  rI  )
CreateBucketr   r   r   assertGreaterEqualcountassertLessEqualrK  rL  r   )r   bucket1_uribucket2_urirP  r   s        r   test_copy_in_cloud_noclobberz#TestCp.test_copy_in_cloud_noclobber  s    ##%K##%K;HG^^	tG}d;/0  FF 	FLL3Q7i0!4^^	tT']	k		+/  1F 	!((k.5.A.A*C DEr   z5Boto library does not handle objects with .. in them.c           	         | j                         }| j                  |ddd      }| j                  |dd       | j                         }| j                  ddt	        |      |gd	      }| j
                  j                  |j                  |j                         | j                  d
t	        |      dt        j                  j                  |      d|       | j                  t        j                  j                  t        j                  j                  |d                   | j!                  t        j                  j                  t        j                  j                  ||j                  d                   y )Ndir/../../../file   dataTr   r   r   prefer_json_apifile2r   r   r   r   -rrH  Skipping copy of source URL H because it would be copied outside the expected destination directory: .file)ra  r   CreateTempDirr   r   json_apiDeleteObjectr   r   r   r   r   abspathassertFalseexistsr   r   )r   r   rP  	directoryr   s        r   5test_skip_object_with_parent_directory_symbol_in_namez<TestCp.test_skip_object_with_parent_directory_symbol_in_name  s8    ""$J:,?)004   6G 	")&  ( ""$I^^	tT*%y1  GF 	MMz55w7J7JKMM 
g	2	45;= 	RWW^^BGGLLF$CDEOO
rww||Iz/E/E$+- 	./r   c           	      t   | j                         }| j                  |ddd      }| j                         }t        j                  j                  |d      }| j                  ddd|t        |      |gd	      }| j                  j                  |j                  |j                         | j                  d
t        |      dt        j                  j                  |      d|       | j                  t        j                  j                  t        j                  j                  |d                   t!        |d      5 }|j#                         }|d   j%                         j'                  d      }| j)                  |d   t        |             | j)                  |d   d       d d d        y # 1 sw Y   y xY w)Nri  rj  Trk  zlog.csvr   ro  -LrH  rp  rq  rr  rs  rr   ,r   r   skip)ra  r   rt  r   r   r   r   r   ru  rv  r   r   r   rw  rx  ry  r   	readlinesstripsplitr   )	r   r   rP  rz  log_pathr   r   linesresultss	            r   Btest_skip_parent_directory_symbol_in_name_is_reflected_in_manifestzITestCp.test_skip_parent_directory_symbol_in_name_is_reflected_in_manifest  sh    ""$J:,?)004   6G ""$Iww||Iy1H^^	tT8	j	9	&  F 	MMz55w7J7JKMM 
g	2	45;= 	RWW^^BGGLLF$CDE	h	kkmea &&s+g
wqz4=1
wqz6*	 
		s   ?A&F..F7z)os.symlink() is not available on Windows.c           	         | j                         }| j                  |ddd      }| j                  |dd      }| j                         }t        j                  j                  |d      }t        j                  j                  |d      }t        j                  |       t        j                  ||       | j                  d	d
dt        |      t        |      |gd      }| j                  j                  |j                  |j                         | j                  dt        |      d|d|       | j                  t        j                  j!                  t        j                  j                  |d                   | j#                  t        j                  j!                  t        j                  j                  |d                   y )Nri  rj  Trk  rm  rn  linked_destinationdestination-Dr   ro  rH  rp  rq  rr  rs  )ra  r   rt  r   r   r   mkdirsymlinkr   r   ru  rv  r   r   r   rx  ry  r   )r   r   rP  second_key_urirz  r  r  r   s           r   Atest_skip_parent_directory_symbol_object_with_symlink_destinationzHTestCp.test_skip_parent_directory_symbol_object_with_symlink_destination  su    ""$J:,?)004   6G &&*3:07 ' 9N ""$Ii1EF'',,y-8KHH[JJ{./^^dDZ^0
 +/  0F 	MMz55w7J7JKMM 
g*	,-35 	RWW^^BGGLL1CV$LMNOOBGGNN277<<0BG#LMNr   z#os.mkfifo not available on Windows.c                    d}| j                         }| j                  |      }g }t        j                  t        ||f      }|j                          t        j                  | j                  |f|g f      }|j                          |j                  d       |j                  d       |s| j                  d       | j                  |d   j                         |       y )NrF  rE  targetargsx   &Reading/writing to the fifo timed out.r   )CreateTempFifor   	threadingThreadr   startrC  r   failr   r  )r   r   r   	file_pathr   read_threadwrite_threads          r   test_cp_from_local_file_to_fifoz&TestCp.test_cp_from_local_file_to_fifo  s     H##%I##X#6IO""*?)2O(DFK##==lIr*,L cS
ii89_Q'--/:r   c                    | j                         }| j                         }d}| j                  ||      }g }t        j                  t
        ||f      }|j                          t        j                  | j                  t        |      f|g f      }|j                          |j                  d       |j                  d       |s| j                  d       | j                  |d   j                         |       y )NrF  r_  r  r  r  r   )r  ra  r   r  r  r   r  rC  r   r   r  r   r  )r   r   r   r   obj_urir   r  r  s           r   test_cp_from_one_object_to_fifoz&TestCp.test_cp_from_one_object_to_fifo1  s     ##%I""$JH:IGO""*?)2O(DFK##==G}	2.0L cS
ii89_Q'--/:r   c                 D   | j                         }| j                         }d}d}| j                  ||      }| j                  ||      }g }t        j                  t
        ||f      }|j                          t        j                  | j                  t        |      t        |      f|g f      }	|	j                          |	j                  d       |j                  d       |s| j                  d       | j                  ||d          | j                  ||d          y )Ns   foo and bars   baz and quxr_  r  r  r  r   )r  ra  r   r  r  r   r  rC  r   r   r  r   )
r   r   r   	contents1	contents2obj1_uriobj2_urir   r  r  s
             r   %test_cp_from_multiple_objects_to_fifoz,TestCp.test_cp_from_multiple_objects_to_fifoG  s    ##%I""$JII  J KH  J KHO""*?)2O(DFK##==H~tH~.	2>@L cS
ii89MM)_Q/0MM)_Q/0r   c           	      L   | j                         }| j                  dddt        |d      z  gdd      }| j                  r | j	                  dt        |d      z   |       n| j	                  d	|       | j                  |d      }| j                  |j                         d
       y )Nr   -z%sfoobarTstdinr   zCopying file://- to zCopying from <STDIN>rF  )ra  r   r   r   r   StorageUriCloneReplaceNamer   rN  )r   r   r   rP  s       r   test_streamingzTestCp.test_streaming`  s    ""$J^^	sD4
E223  F 
mm*T*e-DDfM
mm*F3--j%@GW335v>r   c           	         | j                         }| j                         }d}d}g }t        j                  t        ||f      }|j                          t        j                  | j                  |ft        ||      |fddi      }|j                          |j                  d       |j                  d       |s| j                  d       | j                  r0| j                  d	j                  |t        ||            |d
          n| j                  d|d
          | j                  ||      }| j                  |j                         |       y )Nr  rF  r  r   Tr  r  rA  r  r  zCopying file://{} to {}r   zCopying from named pipe)ra  r  r  r  r   r  rC  r   r   r  r   r   rL  r  r   rN  )	r   r   r   r   object_contentsr   r  r  rP  s	            r   "test_streaming_from_fifo_to_objectz)TestCp.test_streaming_from_fifo_to_objectn  sB    ""$J##%IKOO ##+?*99)EGL ""==lD[9?K&(K Sc
ii89
mm
#
*
*9+/
K+HJ
!

 mm-q/AB--j+FGW335Gr   c                    | j                         }d}g }t        j                  t        ||f      }|j	                          t        j                  | j
                  |fd|fddi      }|j	                          |j                  d       |j                  d       |s| j                  d       | j                  |d	   j                         j                  d
      |       y )NrF  r  r  return_stdoutTr  r  r  r   ascii)r  r  r  r   r  rC  r   r  r   r  r   )r   r   r   r   r  r  s         r   "test_streaming_from_fifo_to_stdoutz)TestCp.test_streaming_from_fifo_to_stdout  s     ##%IHO##+?*2I)>@L""==lC1&(K Sc
ii89_Q'--/66w?Jr   c                    | j                         }d}g }g }t        j                  t        ||f      }|j	                          t        j                  | j
                  d||fd|d      }|j	                          |j                  d       |j                  d       |s| j                  d       | j                  |d	   j                         |       y )
NrF  r  )r  T)r   r  r  r  r  r   )
r  r  r  r   r  rC  r   r  r   r  )r   r   r   r   list_for_gsutil_outputr  r  s          r   "test_streaming_from_stdout_to_fifoz)TestCp.test_streaming_from_stdout_to_fifo  s     ##%IHO""*?)2O(DFK##==i!78!
L cS
ii89_Q'--/:r   c                     | j                         }| j                  dddt        |      gddd      }| j                  r| j	                  d|       y | j	                  d|       y )	Nr   r  r  Tr   )r  r   r   zDMultiple URL strings are not supported when transferring from stdin.z5Multiple URL strings are not supported with streamingra  r   r   r   r   r   r   r   s      r   !test_streaming_multiple_argumentsz(TestCp.test_streaming_multiple_arguments  so    ""$J^^T3T*-=>"'*.,-  /F 
mm " mmKr   c                 T     j                         }t        |d       j                  d j                  d      g       t	        t
        dd       fd       } |         j                  d j                  d      g       t	        t
        dd       fd	       } |        y
)z&Tests local detection of content type.r  r   test.mp3rV  r   r  c                      j                  ddgd      } t        r>j                  t        j                  d|       xs t        j                  d|              y j                  | d       y )Nlsr}  Tr  zContent-Type:\s+audio/x-mpgzContent-Type:\s+audio/mpeg)r   rV   r   r   r   rK  stdoutdsturir   s    r   _Check1z0TestCp.test_detect_content_type.<locals>._Check1  sa    ~~tT62$~Gf	II4f= =II3V<	> 	!>?r   test.gifc                  V    j                  ddgd      } j                  | d       y Nr  r}  Tr  Content-Type:\s+image/gifr   rK  r  s    r   _Check2z0TestCp.test_detect_content_type.<locals>._Check2  -    ~~tT62$~Gf
v;<r   Nra  r   r   r;  rU   r\  r   r   r  r  r  s   `   @r   test_detect_content_typezTestCp.test_detect_content_type  s     ""$J*e$FNND$--j96BC >3@ 4@ INND$--j96BC >3= 4= Ir   c                 \     j                         }t        |d       j                  ddd j                  d      g       t	        t
        dd       fd	       } |         j                  ddd j                  d
      g       t	        t
        dd       fd       } |        y)z5Tests overriding content type with the default value.r  -hzContent-Type:r   r  rV  r   r  c                  V    j                  ddgd      } j                  | d       y Nr  r}  Tr  z(Content-Type:\s+application/octet-streamr  r  s    r   r  z:TestCp.test_content_type_override_default.<locals>._Check1  -    ~~tT62$~Gf
vJKr   r  c                  V    j                  ddgd      } j                  | d       y r  r  r  s    r   r  z:TestCp.test_content_type_override_default.<locals>._Check2
  r  r   Nr  r  s   `   @r   "test_content_type_override_defaultz)TestCp.test_content_type_override_default  s    ""$J*e$FNN				Z	(&	23
 >3L 4L INN				Z	(&	23
 >3L 4L Ir   c                 \     j                         }t        |d       j                  ddd j                  d      g       t	        t
        dd       fd	       } |         j                  ddd j                  d
      g       t	        t
        dd       fd       } |        y)z+Tests overriding content type with a value.r  r  zContent-Type:text/plainr   r  rV  r   r  c                  V    j                  ddgd      } j                  | d       y Nr  r}  Tr  Content-Type:\s+text/plainr  r  s    r   r  z2TestCp.test_content_type_override.<locals>._Check1  -    ~~tT62$~Gf
v<=r   r  c                  V    j                  ddgd      } j                  | d       y r  r  r  s    r   r  z2TestCp.test_content_type_override.<locals>._Check2)  r  r   Nr  r  s   `   @r   test_content_type_overridez!TestCp.test_content_type_override  s    ""$J*e$FNN'J'  >3> 4> INN'J'  >3> 4> Ir   z&magicfile is not available on Windows.c                       j                         }t        |d       j                  d      } j                  d|g       t	        t
        dd       fd       } |        y	)
z1Tests content type override with magicfile value.r     foo/bar
rE  r   rV  r   r  c                      j                  ddgd      } t        j                  j                  ddd      }|rdnd	}j	                  | d
|z         y )Nr  r}  Tr  GSUtiluse_magicfileFz
text/plainzapplication/octet-streamzContent-Type:\s+%s)r   r  configgetboolrK  )r  r  content_typer  r   s      r   r  z/TestCp.test_magicfile_override.<locals>._Check1:  sZ    ~~tT62$~Gfkk))(OUKm& #,F 
v4|CDr   N)ra  r   r   r   rU   r\  )r   r   r   r  r  s   `   @r   test_magicfile_overridezTestCp.test_magicfile_override0  sk     ""$J*e$F6ENND%() >3E 4E Ir   c                      j                         }t        |d       j                  d      } j                  ddd j	                  d      g       t        t        dd	
       fd       } |         j                  ddd j	                  d      g       t        t        dd	
       fd       } |         j                  ddd|g       t        t        dd	
       fd       } |        y)zCTests overriding content type when it does not match the file type.r  r  rE  r  Content-Type:image/gifr   r  rV  r   r  c                  V    j                  ddgd      } j                  | d       y r  r  r  s    r   r  z4TestCp.test_content_type_mismatches.<locals>._Check1Q  r  r   r  c                  V    j                  ddgd      } j                  | d       y r  r  r  s    r   r  z4TestCp.test_content_type_mismatches.<locals>._Check2^  r  r   c                  V    j                  ddgd      } j                  | d       y r  r  r  s    r   _Check3z4TestCp.test_content_type_mismatches.<locals>._Check3h  r  r   N)ra  r   r   r   r;  rU   r\  )r   r   r   r  r  r  r  s   `     @r   test_content_type_mismatchesz#TestCp.test_content_type_mismatchesD  s    ""$J*e$F6ENN&J'  >3= 4= INN&J'  >3= 4= INND2D%HI >3= 4= Ir   c           	      F     j                         }t        |d       j                  d      } j                  ddd|g       t	        t
        dd       fd	       } |         j                  dd
ddd|g       t	        t
        dd       fd       } |        y)zBTests that content type header is treated with case insensitivity.r  r  r  zcontent-Type:text/plainr   rV  r   r  c                  z    j                  ddgd      } j                  | d       j                  | d       y )Nr  r}  Tr  r  z	image/gifr   rK  assertNotRegexr  s    r   r  zATestCp.test_content_type_header_case_insensitive.<locals>._Check1y  s=    ~~tT62$~Gf
v<=
&,/r   zCONTENT-TYPE:image/gifzcontent-type:image/gifc                  z    j                  ddgd      } j                  | d       j                  | d       y )Nr  r}  Tr  r  zimage/gif,\s*image/gifr  r  s    r   r  zATestCp.test_content_type_header_case_insensitive.<locals>._Check2  s>    ~~tT62$~Gf
v;<
&";<r   N)ra  r   r;  r   rU   r\  )r   r   r   r  r  r  s   `    @r   )test_content_type_header_case_insensitivez0TestCp.test_content_type_header_case_insensitiveo  s     ""$J*e$F
+ENND3T5&IJ >30 40
 INN&.Fv  >3= 4=
 Ir   c           	         | j                         }t        |d      }| j                  d      }| j                  dddd| j                  z  d||g       | j                  dd|gd	
      }| j                  |d       | j                  |d       t        |d      }| j                  d||g       | j                  dd|gd	
      }| j                  |d       | j                  |d       y)zETests that non-content-type headers are applied successfully on copy.r  r  r  Cache-Control:public,max-age=12x-%s-meta-1:abcdr   r  r}  Tr  z%Cache-Control\s*:\s*public,max-age=12zMetadata:\s*1:\s*abcdr  N)ra  r   r;  r   provider_custom_metarK  )r   r   dst_urir   r  dst_uri2s         r   test_other_headerszTestCp.test_other_headers  s     ""$J:u%G
+ENN/T666eW 
 ^^T41^FFVEFV56J&HNND'8,-^^T42$^GFVEFV56r   c                 J   dt         j                  d<   | j                         }t        |d      }| j	                  d      }| j                  dd||gd      }| j                  rd	}nd
}| j                  ||       | j                  ddd|gd      }| j                  ||       y)zJTest that x-goog-request-header can be set using the environment variable.b/this_is_env_reasonCLOUDSDK_CORE_REQUEST_REASONr  r  -DDr   TrH  z1b'X-Goog-Request-Reason': b'b/this_is_env_reason'z/'x-goog-request-reason': 'b/this_is_env_reason'r  r}  N)r   environra  r   r;  r   r   rK  )r   r   r  r   r   reason_regexs         r   test_request_reason_headerz!TestCp.test_request_reason_header  s     2HBJJ-.""$J:u%G
+E^^UD%9^NFIlGlV\*^^UD$8^MFV\*r   z*XML APIs use a different debug log format.c                 P   dt         j                  d<   | j                         }t        |d      }| j	                  d      }d}t        |g      5  | j                  dd||gd	      }d
d
d
       | j                  rd}nd}| j                  d|z   dz   |z          y
# 1 sw Y   6xY w)FTest that x-goog-request-header works when cp sends multiple requests.r  r  r  r  r  resumable_threshold0r   r   TrH  Nz0X-Goog-Request-Reason\': b\'b/this_is_env_reasonz/x-goog-request-reason\': \'b/this_is_env_reasonz
GET[\s\S]*z[\s\S]*POST[\s\S]*)	r   r  ra  r   r;  r4   r   r   rK  r   r   r  r   boto_config_for_testr   r  s          r   :test_request_reason_header_persists_multiple_requests_jsonzATestCp.test_request_reason_header_persists_multiple_requests_json  s     2HBJJ-.""$J:u%G
+EA	34	5~~udE7;4~Pf 
6 HlGl$'<<|KM 
6	5s   BB%z+JSON API uses a different debug log format.c                 ,   dt         j                  d<   | j                         }t        |d      }| j	                  d      }d}t        |g      5  | j                  dd||gd	      }d
d
d
       d}| j                  |dz   |z          y
# 1 sw Y   $xY w)r  r  r  r  r  r  r  r   TrH  NzSFinal headers: \{[\s\S]*\'x-goog-request-reason\': \'b/this_is_env_reason\'[\s\S]*}z[\s\S]*)r   r  ra  r   r;  r4   r   rK  r	  s          r   9test_request_reason_header_persists_multiple_requests_xmlz@TestCp.test_request_reason_header_persists_multiple_requests_xml  s     2HBJJ-.""$J:u%G
+EA	34	5~~tT5':$~Of 
6	E 
 	V\J6EF 
6	5s   B

Bc                    | j                         }| j                  |d      }| j                  |d      }t        |      }| j                  dt	        |      t	        |      g       | j                  ||j                        }| j                  ||j                               }t        |      }| j                  |d       t        |      }| j                         }| j                  d|j                  |g       t        |d      5 }| j                  |j                         d       ddd       | j                  d|j                  d	||g       t        |d      5 }| j                  |j                         d       ddd       | j                  d|j                  d	||g       t        |d      5 }| j                  |j                         d       ddd       | j                  d|j                  d	||g       t        |d      5 }| j                  |j                         d       ddd       | j                  d|j                  d	||j                  g       | j                  d|j                  |g       t        |d      5 }| j                  |j                         d       ddd       | j                  d||j                  gd
d      }	| j                   r| j#                  d|	       y| j#                  d|	       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   OxY w# 1 sw Y   xY w# 1 sw Y   xY w)zTests copy with versioning.   data2r_     data1r   data3r   s   data3N#Tr   r   r   zGdestination argument of the cp command cannot be a version-specific URLz'cannot be the destination for gsutil cp)CreateVersionedBucketr   urigenr   r   r  r   StorageUriCloneReplaceKeyget_keyStorageUriSetContentsFromStringr   versionless_urir   r   r   urir   r   )
r   r   k1_urik2_urig1g2g3r   r   r   s
             r   test_versioningzTestCp.test_versioning  s    ++-J*xHF*xHF	BNND$v,V56,,Z9K9KLF++J8HIF	B((9	B!ENND&00%89	eT	a
qvvx* 
 	NNDV%;%;R@%HI	eT	a
qvvx* 
NNDV%;%;R@%HI	eT	a
qvvx* 
NNDV%;%;R@%HI	eT	a
qvvx* 
 	NN	**B	/1G1G	IJ 	NND&00%89	eT	a
qvvx* 
 ^^T5&**5*.,-  /F 
mm'(.0 mm=vF? 
	
 
	 
	 
	 
	s<   !L
!L6!L$!L1!L>
LL!$L.1L;>Mc                 H     t        t        dd       fd       } |        y)z@Tests that copy all-versions errors when parallelism is enabled.rV  r   r  c            
          j                  dddt        j                  d      t        j                  d      gdd      } j                  rj	                  d	|        y j	                  d
|        y )N-mr   -Ar  r  r   Tr   z-sequential instead of parallel task executionz.-m option is not supported with the cp -A flag)r   r   r[  r   r   )r   r   s    r   rZ  z5TestCp.test_versioning_no_parallelism.<locals>._Check  ss    ~~
d
t++U
3
t++U
3
 /0,0  2f 
	!	!EvNFOr   N)rU   r\  )r   rZ  s   ` r   test_versioning_no_parallelismz%TestCp.test_versioning_no_parallelism  s)     >3P 4P Hr   z6S3 lists versioned objects in reverse timestamp order.c           	      v     j                          j                          j                          j                  dd      } j                  ddt        |              j                  dd        j                  d	d        j                  d	d        j	                  d
ddt        d      t              g       t        t        dd       fd       } |         j	                  d
dt        d      t              g       t        t        dd       fd       } |        y)z#Tests cp -R with versioned buckets.r#  s   data0rn  s   longer_data1r   r   r   gs_idempotent_generationrg   T)	versionedr   r   -Rr$  *rV  r   r  c                  2   	j                  ddt              gd      j                  d      } 	j                  ddt              gd      j                  d      }	j                  t	        |       d       	j                  t	        |      d       | d   j                         \  }}}}	j                  |t        t	        d                   	j                  t        |      j                  d	       |d   j                         \  }}}}	j                  |t        t	        d                   	j                  t        |      j                  d	       | d
   j                         \  }}}}	j                  |t        t	        d                   	j                  t        |      j                  d	       |d
   j                         \  }}}}	j                  |t        t	        d                   	j                  t        |      j                  d	       y)#Validates the results of the cp -R.r  -laTr  
   r   data0r#  r   longer_data1Nr   r   r  r   lenr   r
   r   )
r   r   size1_uri_str1size2uri_str2re  rf  r   s
          r   r  z?TestCp.test_recursive_copying_versioned_bucket.<locals>._Check2G  s    ud;.? @.2   449E$K ud;.? @.2   449E$K  s8}a(
s8}a( 'qk//1eQ!
uc#g,/0
{8,88#>&qk//1eQ!
uc#g,/0
{8,88#> 'qk//1eQ!
uc#n"567
{8,88#>&qk//1eQ!
uc#n"567
{8,88#>r   c                     j                  ddt              gd      j                  d      } j                  ddt              gd      j                  d      }j                  t	        |       d       j                  t	        |      d       |d   j                         \  }}}}j                  |t        t	        d	                   j                  t        |      j                  d
       y)r-  r  r.  Tr  r/  r0  rV  r   r2  r#  Nr3  )r   r   r5  r6  r7  re  bucket3_urir   s        r   r  z?TestCp.test_recursive_copying_versioned_bucket.<locals>._Check3i  s     ud;.? @.2   449E$K ud;.? @.2   449E$K  s8}a(
s8}a( 'qk//1eQ!
uc#n"567
{8,88#>r   N)r  r   r  AssertNObjectsInBucketr   r   rU   r\  )r   v1_urir  r  re  rf  r;  s   `   @@@r   'test_recursive_copying_versioned_bucketz.TestCp.test_recursive_copying_versioned_bucket+  sJ    ,,.K,,.K,,.K ++.(0  2F 	"%./5f~  ?
 	Q$?Q$?Q$? 	NN	tT	k3		k		 >3? 4?4 I 	NND$[# 6[8IJK >3? 4?  Ir   z#Preconditions not supported for S3.c           	      0   | j                         }| j                  d      }d}| j                  d|d|t        |      g       | j                  d|d|t        |      gdd      }| j                  r| j                  d	|       y| j                  d
|       y)z?Tests that cp handles an object-not-exists precondition header.r  rE  zx-goog-if-generation-match:0r  r   Tr   r  zMHTTPError 412: At least one of the pre-conditions you specified did not hold.PreconditionExceptionNra  r   r   r   r   r   )r   r   fpath1gen_match_headerr   s        r   test_cp_generation_zero_matchz$TestCp.test_cp_generation_zero_match|  s     ""$J  ( 3F5 	NND*D&$z:JKL ^^	v	j			  F
 
mm"$ mm+V4r   c           
      b   | j                         }| j                  |d      }|j                  }| j                         }| j	                  |d      }d|z  }| j                  d|d|t        |      g       | j                  d|d|t        |      gdd	
      }| j                  r| j                  d|       n| j                  d|       | j                  d|dd|t        |      gdd	
      }| j                  r| j                  d|       y| j                  d|       | j                  d|       y)z?Tests that cp -v option handles the if-generation-match header.r  r_  r  r   r   zx-goog-if-generation-match:%sr  r   Tr   r  zpre-conditionr@  rG  z:Cannot specify both generation precondition and no-clobberArgumentExceptionzASpecifying x-goog-if-generation-match is not supported with cp -nN)	r  r   
generationrt  r   r   r   r   r   )r   r   r  r  r   rB  rC  r   s           r   test_cp_v_generation_matchz!TestCp.test_cp_v_generation_match  sJ    ++-J*xHF			B!F   BF6;NND*D&$v,GH ^^	v	f		  F 
mmOV,
mm+V4 ^^	tV	f		  F 
mm
FP mm'0
mm r   c                 0   | j                         }| j                  |d      }| j                         }| j                  |d      }| j	                  dd|t        |      g       | j	                  dd|t        |      gd      }| j                  d	|       y
)z4Tests that cp -nv works when skipping existing file.r  r_  r  rF  r   z-nvTrH  zSkipping existingN)r  r   rt  r   r   r   r   )r   r   r  r   rB  r   s         r   
test_cp_nvzTestCp.test_cp_nv  s     ++-J*xHF!F   BF 	NND%f67 ^^	ufd6l+4  AFMM%v.r   c                    | j                         }| j                  |d      }| j                  |d      }| j                         }| j                  |d      }| j	                  d||j
                         t        }ddt        |      f}t        |g      5  t        j                  |      }| j                         }| j                  ||      }| j	                  d||j
                         ddd       | j	                  dd	|j
                         | j                         }| j                  |
      }t        |      }	| j                  ddt        |      t        |	      gd      }
| j                  d|	j
                  z  |
       | j	                  d|j
                  |j
                         | j	                  d|j
                  |j
                         y# 1 sw Y   xY w)zD"Tests that cp -v returns the created object's version-specific URI.r  r_  r  rF  -vr  r  Nr  r   r   TrH  zCreated: %s
z-Dv)r  r   rt  r   _run_cp_minus_v_testr  r[   r   r4   r   urandomr
   r   r   r   )r   r   r  r  r   rB  size_thresholdr
  file_as_stringr  r   s              r   test_cp_v_optionzTestCp.test_cp_v_option  s    ++-J*xHF*xHF !F   BFdFFJJ7 N$&;/1	34	5zz.1n!!#f""&>"Jf
ffjj9	 
6 	dC4 !F   /F&!G^^	tT&\4=1  GF 	MM/GKK/8 	eVZZ< 	dFJJ

;1 
6	5s   AGGc                       j                  d||gd      }t        j                  d|      } j                  |       |j	                  d      t        t        dd       fd       } |        y	)
z6Runs cp -v with the options and validates the results.r   TrH  zCreated: (.*)\nr   rV  r  c                      j                  ddgd      } | j                  d      }j                  t        |      d       j	                  |d          y )Nr  -aTr  r/  rg   )r   r  assertGreaterr4  r   )r  r  created_uridst_strr   s     r   r  z,TestCp._run_cp_minus_v_test.<locals>._Check1  sT    ~~tT734~Hfll4 e UQ'
{E"I.r   N)r   r   r   assertIsNotNonegrouprU   r\  )r   optsrc_strrZ  r   matchr  rY  s   `  `   @r   rO  zTestCp._run_cp_minus_v_test   sn    ^^T39^NFII(&1E++a.K >3/ 4/ Ir   c                 @     j                         } j                  |d       j                  |d       j                          j                  ddt	              gdj                  f             t        t        dd	
       fd       } |        y)zTests cp with the -I option.r  rF  r  r   z-Ir/  )r  rV  r   r  c                  "   j                  dt              gd      } j                  t        j                  j                        |        j                  t        j                  j                        |        j                  | d       y )Nr  Tr  rg   )r   r   r   r   r   basenameassertNumLines)r  r   rB  fpath2r   s    r   r  z'TestCp.test_stdin_args.<locals>._Check1  si    ~~tT*%56d~Kf
mmBGG$$V,f5
mmBGG$$V,f5
&!$r   N)rt  r   ra  r   r   r   rU   r\  )r   r   r  r   rB  rd  s   `  @@@r   test_stdin_argszTestCp.test_stdin_args  s     !F   BF   BF""$JNND$Z 01FF#34  6 >3% 4% Ir   c                     | j                  d      }| j                  d      }| j                  |d      }| j                  dt        |      t        |      g       y )Nstandardstorage_classdurable_reduced_availabilityr   r_  r   ra  r   r   r   )r   re  rf  rP  s       r   !test_cross_storage_class_cloud_cpz(TestCp.test_cross_storage_class_cloud_cp'  s^    ##*#=K##4 $ 6K;HGNND$w-k):;<r   z(Test requires both S3 and GS credentialsc                 0   | j                  d      }| j                  d      }| j                  |d      }| j                  |d      }| j                  dt        |      t        |      g       | j                  dt        |      t        |      g       y )Ns3providergsr   r_  rF  r   rk  r   	s3_bucket	gs_buckets3_keygs_keys        r   test_cross_provider_cpzTestCp.test_cross_provider_cp/  s    !!4!0I!!4!0I)fEF)fEFNND$v,Y89NND$v,Y89r   zHThis test performs a large copy but remains here for debugging purposes.c           	         | j                  d      }| j                  d      }| j                  |ddz        }| j                  |ddz        }| j                  dt        |      t        |      g       | j                  dt        |      t        |      g       t	        d	d
t        t              fd	dt        t        dz        fg      5  | j                  dt        |      t        |      g       d d d        y # 1 sw Y   y xY w)Nrn  ro  rq  s   ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffi   r_  s   bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbr   r  r  json_resumable_chunk_size   )ra  r   r   r   r4   r   r[   rr  s        r   test_cross_provider_large_cpz#TestCp.test_cross_provider_large_cp8  s     !!4!0I!!4!0I)(3d(:  <F)(3d(:  <FNND$v,Y89NND$v,Y89	*?WN (*E #GcM 2 45 
6 nndDL$y/:;	
6 
6 
6s   'C==Dc                     | j                  d      }| j                  dd      }t        t        dz        }| j                  |d|z  d      }| j	                  d	d
d	ddt        |      t        |      g       y)zEEnsure daisy_chain works for an object that is downloaded in 2 parts.rn  ro  rq  T)rp  rl  g?   b)r   r   rl  z-ozs3:use-sigv4=Truezs3:host=s3.amazonaws.comr   N)ra  r   r   r   r   r   )r   rs  rt  	num_bytesrv  s        r   test_gs_to_s3_multipart_cpz!TestCp.test_gs_to_s3_multipart_cpJ  s     !!4!0I!!4!FI0367I)(,y(8/3  5F 	NN!4)CTVY r   zXThis test is slow due to creating many objects, but remains here for debugging purposes.c           	         | j                         }| j                         }d}t        |      D ]Y  }d|z  dz
  }d|z  }d|z  dz   }| j                  |d|z         | j                  |d|z         | j                  |d|z         [ | j                  ||dz         | j	                  d	d
dt        |d      t        |      g       | j                  ||dz         y)z6Ensure daisy chain cp works with a wide of file sizes.   rg   r      ar_  r}     crV  r#  r   r  z**N)ra  r_   r   r<  r   r   )r   r   rf  exponent_capione_byte_smallernormalone_byte_largers           r   test_daisy_chain_cp_file_sizesz%TestCp.test_daisy_chain_cp_file_sizesY  s     ""$J##%KL< A!tf1qo
:?O8OP
:vF
:8NO ! 	
L1,<=NN	tT	j$		k		
 	\A-=>r   c                      j                  d      } j                  d       j                  |d       j                  ddddd	dd
 j                  z  t	              g        j                  dddt	              g        j                  ddt	              gd       j                  ddt	              t	              gd      } j                  d|       t        t        dd       fd       } |        y)zTests cp with the -D option.rg  rh  rj  r   r_  setmetar  r  r  r  aclr   public-readr   Tr  r   z-DpnrH  zCopy-in-the-cloud disallowedrV  r   r  c                     t        j                        } j                  dd| gd      }j                  |d       j                  |d       j                  |d       j                  dd	| gd      }j	                  |       y )
Nr  r}  Tr  z"Cache-Control:\s+public,max-age=12r  zMetadata:\s+1:\s+abcdr  r   )r   r   r   rK  r   )r  r  new_acl_jsonacl_jsonrf  rP  r   s      r   rZ  z*TestCp.test_daisy_chain_cp.<locals>._Check  s    g112c~~tT3/t~Df
vDE
v;<
v78^^UE3$7t^Ll
x.r   N)ra  r   r   r  r   assertNotInrU   r\  )r   re  r   rZ  r  rf  rP  s   `   @@@r   test_daisy_chain_cpzTestCp.test_daisy_chain_cpp  s   ##*#=K##4 $ 6K;HG 	NN4:D $T666W	  	NNE5-g?@~~ueT'];4~PH
 ^^	vtG}	k		+/  1F 	3V<
>3/ 4/ Hr   zpgs_port is defined in config which can cause problems when uploading and downloading to the same local host portc                    | j                         }| j                         }| j                  |d| j                  z        }ddt        t              f}| j                  t        j                  t        dd                  }t        |g      5  | j                  dd	|d
t        |      t        |      gdd      }| j                  |j                  d      d       ddd       y# 1 sw Y   yxY w)z:Tests cp with the -D option when the download thread dies.r  r_  r  r  Fr  rE  r   --testcallbackfiler  r   Tr   z8ResumableDownloadException: Artifically halting downloadrV  N)ra  r   	halt_sizer   r[   r   pickledumpsr(   r4   r   r   r   rc  )r   re  rf  rP  r
  test_callback_filer   s          r   $test_daisy_chain_cp_download_failurez+TestCp.test_daisy_chain_cp_download_failure  s    
 ##%K##%K;)-)>   @G$&;S\J,,8BC - E	34	5~~
$&8$
w-
{

 /0,0  2f 
,,HJKLN 
6	5	5s   AC%%C.c           	          | j                         }| j                  dddt        |d      gddd      }| j                  r| j	                  d	|       y| j	                  d
|       y)zETests error when compression flag is requested on a streaming source.r   -Zr  r  Tr   zstreaming data)r   r   r  zGGzip content encoding is not currently supported for streaming uploads.z@gzip compression is not currently supported on streaming uploadsNr  r  s      r   test_streaming_gzip_uploadz!TestCp.test_streaming_gzip_upload  sr    ""$J^^	tS$z512	   F
 
mm mm
L
r   c           
         | j                  d      }| j                         }t        ddg      5  | j                  ddd|t	        |      gd	      }| j                  d
|       ddd       t        ddg      5  | j                  ddd|t	        |      gd	      }| j                  d|       ddd       y# 1 sw Y   SxY w# 1 sw Y   yxY w)z?Tests that the seek-ahead iterator estimates total upload work.rV  
test_filesr  task_estimation_threshold1r  task_estimation_forceTruer#  r   ro  TrH  ;Estimated work for this command: objects: 3, total size: 18Nr  r  r  Estimated work)rt  ra  r4   r   r   r   r  r   r   r   r   s       r   test_seek_ahead_upload_cpz TestCp.test_seek_ahead_upload_cp  s    1-F""$J	KJL 
M~~tVT*%5
6d  Lf
mm
GQ	
M 
KJL 
M~~tVT*%5
6d  Lf
'0	
M 
M
M 
M
M 
Ms   4B694C6B?Cc           	         | j                         }| j                  d      }| j                  |d       t        ddg      5  | j	                  dddt        |      |gd	      }| j                  d
|       d d d        t        ddg      5  | j	                  dddt        |      |gd	      }| j                  d|       d d d        y # 1 sw Y   SxY w# 1 sw Y   y xY w)NrV  test_objectsr  r  r#  r   ro  TrH  r  r  r  )rt  ra  r<  r4   r   r   r   r  r  s       r   test_seek_ahead_download_cpz"TestCp.test_seek_ahead_download_cp  s    !F"""2J
A.	KJL 
M~~tT*-v
6d  Lf
mm
GQ	
M 
KJL 
M~~tT*-v
6d  Lf
'0	
M 
M
M 
M
M 
Ms   4C4CCCc           	           j                         } j                          j                  |d       j                  dddt              t              g        j                  dddt              g        j                  ddt              gd	
      t	        t
        dd       fd       } |        y)z Tests copying with a canned ACL.r   r_  r   rV  r  r  r   r   Tr  rV  r   r  c                      t        j                        } j                  dd| gd      }j                  |       y )Nr  r   Tr  )r   r   r   r   )r  r  rf  rP  public_read_aclr   s     r   rZ  z)TestCp.test_canned_acl_cp.<locals>._Check  s>    g112c^^UE3$7t^Ll
5r   N)ra  r   r   r   rU   r\  )r   re  rZ  rf  rP  r  s   `  @@@r   test_canned_acl_cpzTestCp.test_canned_acl_cp  s    ##%K##%K;HGNN	t]	g	k		 	NNE5-g?@nneUDM%B37 % 9O >36 46
 Hr   c           
         | j                         }| j                  |d      }| j                  dddt        |      g       | j                  ddt        |      gd      }d	}| j	                  |d
      }| j                  ddd|t        |      g       | j                  ddt        ||      gd      }| j                  ||       t        }ddt        |      f}t        |g      5  d}	t        j                  |      }
| j	                  |	|

      }| j                  ddd|t        |      g       | j                  ddt        ||	      gd      }| j                  ||       ddd       y# 1 sw Y   yxY w)z)Tests uploading a file with a canned ACL.r   r_  r  r   r  r   Tr  r  r7  r   rV  r  r  resumable_barN)ra  r   r   r   r   r   r[   r   r4   r   rP  )r   re  rP  r  r8  r   r  resumable_sizer
  resumable_file_nameresumable_contentsresumable_fpathnew_resumable_acl_jsons                r   test_canned_acl_uploadzTestCp.test_canned_acl_upload  s    ##%K;HGNNE5-g?@nneUDM%B37 % 9O I)fEENND$ud;6GHI>>	tK34D " JL_l3N$&;/1	34	5+::n5++6I5G , Io
nn}o  $~~%k+>?
@  .   (>? 
6	5	5s   'A<E,,E5c                     | j                         }d}| j                  ||      }| j                  dt        |      dgd      }| j	                  ||j                  d             y )Nr   r_  r   r  Tr  r  )ra  r   r   r   r   r   )r   r   r   rP  r  s        r   test_cp_key_to_local_streamz"TestCp.test_cp_key_to_local_stream  s^    ""$JH:IG^^T4=#6d^KFMM(FMM'23r   c                     d}| j                  |      }| j                  d|dgd      }| j                  ||j                  t                     y )Ns   contentrE  r   r  Tr  )r   r   r   r   rD   )r   r   r   r  s       r   "test_cp_local_file_to_local_streamz)TestCp.test_cp_local_file_to_local_stream%  sJ    H2E^^T5#.d^CFMM(FMM$/0r   c                      j                          j                         }t        j                  j	                  |d      t        d      5 }	 d d d         j                  dt              g       t        t        dd       fd       } |        t        j                  j	                  |d      } j                  dt        d      |g        j                  t        j                  |             y # 1 sw Y   xY w)	N	zero_bytewr   rV  r   r  c                      j                  dt              gd      } j                  t        j                  j                        |        y )Nr  Tr  )r   r   r   r   r   rb  )r  dst_bucket_urir   r   s    r   r  z.TestCp.test_cp_zero_byte_file.<locals>._Check14  s<    ~~tT.%9:$~Of
mmBGG$$U+V4r   zero_byte_download)ra  rt  r   r   r   r   r   r   rU   r\  r   r   )r   src_dirunused_out_filer  download_pathr  r   s   `    @@r   test_cp_zero_byte_filezTestCp.test_cp_zero_byte_file+  s    &&(N  "GGGLL+.E	eS	_
 
NND%n!567
>35 45 IGGLL*>?MNND$~{;]KLOOBGGM*+ 
	s   C==Dc                       j                          j                          j                  dd        j                  dd       t        t        dd       fd	       } |        y
)zTests recursively copying from bucket to bucket.

    This should produce identically named objects (and not, in particular,
    destination objects named by the version-specific URI from source objects).
    obj0   abcrn  r      defrV  r   r  c                     j                  ddt              t              g       j                  ddj                  gd      } j                  j                  d|        j                  j                  d|        y Nr   r*  r  Tr  z/obj0
z/obj1
r   r   r  r   r   r  r  r   src_bucket_uris    r   _CopyAndCheckz8TestCp.test_copy_bucket_to_bucket.<locals>._CopyAndCheckO      
nndD$~"6^8LMN~~tT>+=+=>,0  2f
mm)>+E+E
FP
mm)>+E+E
FPr   N)r  r   rU   r\  )r   r  r  r  s   ` @@r   test_copy_bucket_to_bucketz!TestCp.test_copy_bucket_to_bucket?  s     //1N//1N"(%  ' 	"(%  '
 >3P 4P Or   c                       j                          j                  dd        j                  dd       t        t        dd       fd       } |        y	)
BTests copying from bucket to same bucket preserves file structure.dir1/file.txtrj  rn  dir2/file.txtrV  r   r  c                     j                  ddt              dz   t              dz   g       j                  ddj                  gd      } j                  t              dz   |        j                  t              d	z   |        y )
Nr   r*  /*/dstr  Tr  /dst/dir1/file.txt/dst/dir2/file.txtr   r   r  r   r  r   r   s    r   r  zVTestCp.test_copy_duplicate_nested_object_names_to_new_cloud_dir.<locals>._CopyAndCheckf  s    
nn
d"
f$&' ~~tT:>>:$~Of
mmD$';;VD
mmD$';;VDr   Nra  r   rU   r\  r   r  r   s   ` @r   8test_copy_duplicate_nested_object_names_to_new_cloud_dirz?TestCp.test_copy_duplicate_nested_object_names_to_new_cloud_dir[  so    ""$J"1&  ( 	"1&  (
 >3E 4E Or   c                       j                          j                  dd        j                  dd        j                  dd       t        t        dd       fd	       } |        y
)r  r  rj  rn  r  zdst/existing_file.txtrV  r   r  c                  T   j                  ddt              dz   t              dz   g       j                  ddj                  gd      } j                  t              dz   |        j                  t              d	z   |        j                  t              d
z   |        y )Nr   r*  r  r  r  Tr  r  r  z/dst/existing_file.txtr  r  s    r   r  z[TestCp.test_copy_duplicate_nested_object_names_to_existing_cloud_dir.<locals>._CopyAndCheck  s    
nn
d"
f$&' ~~tT:>>:$~Of
mmD$';;VD
mmD$';;VD
mmD$'??Hr   Nr  r  s   ` @r   =test_copy_duplicate_nested_object_names_to_existing_cloud_dirzDTestCp.test_copy_duplicate_nested_object_names_to_existing_cloud_dirr  s    ""$J"1&  ( 	"1&  ( 	"9&  (
 >3I 4I Or   z3Only s3 V4 signatures error on location mismatches.c                     d}d}d|z  }d|z  }d}t        dd|fg      5   j                  |       j                  dd	
        j                  dd
       d d d        t        dd|fg      5   j                  |      d d d        t        t        dd       fd       }t        dd|fg      5   |        d d d        y # 1 sw Y   oxY w# 1 sw Y   QxY w# 1 sw Y   y xY w)Nz	ap-east-1	us-east-2zs3.%s.amazonaws.comzs3.eu-west-1.amazonaws.comrn  host)locationr  r  rn  r   r  rV  r   r  c                     j                  ddt              t              g       j                  ddj                  gd      } j                  j                  d|        j                  j                  d|        y r  r  r  s    r   r  zOTestCp.test_copy_bucket_to_bucket_with_location_redirect.<locals>._CopyAndCheck  r  r   )r4   ra  r   rU   r\  )	r   src_bucket_regiondest_bucket_regionsrc_bucket_hostdest_bucket_hostclient_hostr  r  r  s	   `      @@r   1test_copy_bucket_to_bucket_with_location_redirectz8TestCp.test_copy_bucket_to_bucket_with_location_redirect  s    $$+.??O,/AA.K	fo>?	@((2C(Dn
>$*!'  ) >$*!'  ) 
A 
f.>?@	A((2D(En 
B >3P 4P 
fk:;	<o 
=	</ 
A	@ 
B	A 
=	<s#   ;C5C;C$CC!$C-c                       j                          j                          j                  dd        j                  dd       t        t        dd       fd	       } |        y
)zTests recursively copying from bucket to a directory.

    This should produce identically named objects (and not, in particular,
    destination objects named by the version- specific URI from source objects).
    r  r  rn  r   r  rV  r   r  c                  2   j                  ddt              g       g } t        j                        D ]<  \  }}}|D ]1  }| j	                  t        j
                  j                  ||             3 > t        |       } j                  t        |       d       j                  t        j
                  j                  j                  d      | d          j                  t        j
                  j                  j                  d      | d          y	z8Copies the bucket recursively and validates the results.r   r*  rg   r  r   r   r   Nr   r   r   walkr   r   r   sortedr   r4  r   dir_listdirnamer6  	filenamesfilenamedst_dirr   r  s        r   r  z5TestCp.test_copy_bucket_to_dir.<locals>._CopyAndCheck       nndD$~"6@Ah#%777#3
'1i!H
//"'',,w9
: " $4 !h
s8}a(

'',,w : :F
C
1+ 
'',,w : :F
C
1+r   Nra  rt  r   rU   r\  r   r  r  r  s   ` @@r   test_copy_bucket_to_dirzTestCp.test_copy_bucket_to_dir  s}     &&(N  "G"(%  ' 	"(%  '
 >3 4  Or   c                       j                  dd       j                          j                  dd        j                  dd       t        t        d	d
       fd       } |        y)zTests copying object from s3 to local dir with v4 signature.

    Regions like us-east2 accept only V4 signature, hence we will create
    the bucket in us-east2 region to enforce testing with V4 signature.
    rn  r  )rp  r  r  r  rn  r   r  rV  r   r  c                  2   j                  ddt              g       g } t        j                        D ]<  \  }}}|D ]1  }| j	                  t        j
                  j                  ||             3 > t        |       } j                  t        |       d       j                  t        j
                  j                  j                  d      | d          j                  t        j
                  j                  j                  d      | d          yr  r  r  s        r   r  z;TestCp.test_copy_object_to_dir_s3_v4.<locals>._CopyAndCheck  r  r   Nr  r  s   ` @@r   test_copy_object_to_dir_s3_v4z$TestCp.test_copy_object_to_dir_s3_v4  s     &&{&KN  "G"(%  ' 	"(%  '
 >3 4  Or   z]The boto lib used for S3 does not handle objects starting with slashes if we use V4 signaturec                 ,   | j                         }| j                         }| j                  |dd       | j                  |dd       | j                  |d      }| j	                  |d       | j                  |d       | j                  d	d
t        |      |g       g }t        j                  |      D ]<  \  }}}|D ]1  }|j                  t        j                  j                  ||             3 > t        |      }| j                  t        |      d       | j                  t        j                  j                  ||j                   d      |d          | j                  t        j                  j                  ||j                   d      |d          y)9Tests that we correctly handle leftover dir placeholders.r  r  rn  r   r  r   r   rV  r   r*  rg   r   r   Nra  rt  r   r  r  r<  r   r   r   r  r   r   r   r  r   r4  r   	r   r  r  rP  r  r  r6  r  r  s	            r   @test_recursive_download_with_leftover_slash_only_dir_placeholderzGTestCp.test_recursive_download_with_leftover_slash_only_dir_placeholder  sf    &&(N  "G"(%  ' 	"(%  '
 --ncBG(("52NND$^ 4g>?H!#!1I(Wh78   "2 hHS]A&RWW\\'>+E+EvNa["RWW\\'>+E+EvNa["r   c                 ,   | j                         }| j                         }| j                  |dd       | j                  |dd       | j                  |d      }| j	                  |d       | j                  |d       | j                  d	d
t        |      |g       g }t        j                  |      D ]<  \  }}}|D ]1  }|j                  t        j                  j                  ||             3 > t        |      }| j                  t        |      d       | j                  t        j                  j                  ||j                   d      |d          | j                  t        j                  j                  ||j                   d      |d          y)r  r  r  rn  r   r  zfoo/r   rV  r   r*  rg   r   r   Nr  r  s	            r   5test_recursive_download_with_leftover_dir_placeholderz<TestCp.test_recursive_download_with_leftover_dir_placeholder  sd   &&(N  "G"(%  ' 	"(%  '
 --nfEG(("52NND$^ 4g>?H!#!1I(Wh78   "2 hHS]A&RWW\\'>+E+EvNa["RWW\\'>+E+EvNa["r   c                     | j                         }| j                  |d      }| j                  ddt        |      t        | j	                  |d            gd      }| j                  |j                  d      d	       y )
Nr   r_  z-qr   o2TrH  zCopying r   )ra  r   r   r   r  r   rc  )r   r   rP  r   s       r   test_copy_quietzTestCp.test_copy_quiet4  s~    ""$J:GG^^dWT,,Z>?
 +/  0F 	V\\*-q1r   c                      j                          j                  d      }t        |d      5 }t        j                  t        |            }	 t        j                  |      }|j                  d      ddd        j                  d|t              g       t        t        dd	       fd
       } |        y# t        $ r t        j                  |      }Y zw xY w# 1 sw Y   nxY w)zTests that the uploaded object has the expected MD5.

    Note that while this does perform a file to object upload, MD5's are
    not supported for composite objects so we don't use the decorator in this
    case.
    rF  rE  r      
Nr   rV  r   r  c            	          j                  ddt              gd      } j                  | dt        j                  j                  d            z         y )Nr  r}  Tr  zHash\s+\(md5\):\s+%sr  )r   r   rK  r   rM  decode)r  r   file_md5r   s    r   r  z)TestCp.test_cp_md5_match.<locals>._Check1S  sS    ~~tT4
+;<,0  2f

)BIIhoog6N,OOQr   )ra  r   r   binascii	unhexlifyrJ   base64encodebytesAttributeErrorencodestringrstripr   r   rU   r\  )r   r   f_inmd5encoded_bytesr  r   r  s   `     @@r   test_cp_md5_matchzTestCp.test_cp_md5_match?  s     ""$J0E	eT	d7=>c1**3/ %%e,h 
 	NND%j!123 >3Q 4Q I  1++C01	 
	s/   C%C'C%C"C%!C""C%%C.z:Unicode handling on Windows requires mods to site-packagesc                     | j                  dj                  t              dj                  t              dj                  t                    S )Nu   foo-unicödeu   bar-unicödeu   manifest-unicöde)_ManifestUploadr   rD   r   s    r   test_cp_manifest_upload_unicodez&TestCp.test_cp_manifest_upload_unicode\  sB      5 5d ; . 5 5d ; 3 : :4 @B Br   c                 (    | j                  ddd      S )z$Tests uploading with a mnifest file.r  r  manifest)r  r   s    r   test_cp_manifest_uploadzTestCp.test_cp_manifest_uploadd  s     uj99r   c                    | j                         }t        ||      }| j                  |d      }| j                  |d      }t        |d      j	                          | j                  dd|||g       t        |d      5 }|j                         }	ddd       t        j                  r'	D 
cg c]  }
t        j                  |
t               }	}
| j                  t        	      d	       g d
}| j                  ||	d   j                         j                  d             |	d   j                         j                  d      }t        t!        ||            }| j                  |d   d|z          | j                  |d   |       d}t"        j"                  j%                  |d   |      }t"        j"                  j%                  |d   |      }| j                  ||kD  d       | j
                  t&        j(                  j
                  k(  r| j                  |d   d       | j                  t+        |d         d       | j                  t+        |d         d       | j                  |d   d       y# 1 sw Y   xY wc c}
w )z%Tests uploading with a manifest file.rF  r7  r   r  r   r}  r~  Nrg   
SourceDestinationStartEndMd5UploadIdSource SizeBytes TransferredResultDescriptionr   r  r   r'  file://r(  %Y-%m-%dT%H:%M:%S.%fZr)  r*  Tr+  zrL0Y20zC+Fzt72VPzMSk2A==r-  rV  r.  r/  OK)ra  r   r   r   closer   r  r   PY2	text_typerD   r   r4  r  r  dictzipdatetimestrptimetestcaseGsUtilIntegrationTestCaser   )r   r8  r   manifest_namer   r  r   logpathr   r  lineexpected_headersr  date_format
start_dateend_dates                   r   r  zTestCp._ManifestUploadi  s9   ""$J*k*F)fEE!!MC!HG#NND$78	gs	qkkme 

ww5:;UTs}}T4(Ue;SZ# 	%uQx~~'7'='=c'BCAhnn$$S)G3'12GE 	
 *K""++GG,<kJJ  ))'%.+FHX
*D1~~;;EEE
 wu~'ABS/0!4S!456:WX&-O 
	 <s   ?I/,!I</I9c                    | j                  d      }| j                  d      }| j                  d      }t        |d      j                          | j	                  dd|t        |      |gd       t        |d	      5 }|j                         }d
d
d
       t        j                  rg }D ]  }|j                  d      r|j                  d      }g }	|D ]\  }
|
j                  d      r8|	j                  t        j                  |
      j                  t                     L|	j                  |
       ^ |j                  dj!                  |	             |j                  |        |}| j#                  t%              d       g d}| j#                  ||d   j'                         j                  d             |d   j'                         j                  d      }| j#                  |d   d
d d| j(                  z         | j#                  |d   d
d d       d}t*        j*                  j-                  |d   |      }t*        j*                  j-                  |d   |      }| j#                  ||kD  d       | j#                  t/        |d         d       | j1                  t/        |d         d       | j#                  |d   d       y
# 1 sw Y   ;xY w)z'Tests downloading with a manifest file.r   rE  r   r  r   r}  Tr  r~  Nzb'r  rg   r&  r   r   r  z%s://   r1  r2  rV     r   r3  )r   r   r   r4  r   r   r  r   PY3
startswithr  r   astliteral_evalr  rD   r   r   r4  r  r   r9  r:  r   rb  )r   rP  r   r>  r   r  decode_linesr?  	some_strs
line_partssome_strr@  r  rA  rB  rC  s                   r   test_cp_manifest_downloadz TestCp.test_cp_manifest_download  s    0G-E!!3!/G#NN	tWd7mU34  I	gs	qkkme 

wwl$??4 jjo)*#h""4( 0 0 : A A$ GH)	 $
 

chhz2
3


d
#  eSZ# 	%uQx~~'7'='=c'BCAhnn$$S)GWQZ^Wt/D/D%DEWQZ^Y/)K""++GAJDJ  ))'!*kBHX
*D1S_a( 	C
OQ/WQZ&I 
	s   >KKc                    | j                         }dt        z  dz  }| j                  d|      }t        dg      5  | j	                  d|t        |      gd       | j	                  d	t        |      gdd
      }| j                  |j                  d      |       d d d        t        ddt        t        dz        fg      5  | j	                  d|t        |      gd       | j	                  d	t        |      gdd
      }| j                  |j                  d      |       d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)N   xrg   u   Аудиоархивr7  )r  r  r  r   TrH  cat)r  force_gsutilr  r  r  rV  )	r   rC   r   r4   r   r   r   r   r   )r   rP  file_contentsr   r  s        r   $test_copy_unicode_non_ascii_filenamez+TestCp.test_copy_unicode_non_ascii_filename  sK   !G 33a7M*@=YE	EF	G
nndE4=1nF~~ud7m4,0+/  1f v}}W-}= 
H 
*? #$<q$@ A C D 
E
nndE4=1nF~~ud7m4,0+/  1f v}}W-}=
E 
E 
H	G
E 
Es   A D,A D8,D58Ec           	         | j                         }ddz  }| j                         }| j                  d||       | j                  d||       | j                  d||       | j                  ddd	t        j
                  j                  |d
      t        |      g       | j                  |d       t        |d      }t        |d      }t        |d      }| j                  d|gd      }| j                  |d       | j                  d|gd      }| j                  |d       | j                  d|gd      }| j                  |d       | j                         }|||fD ]T  }	| j                  d|	t        |      g       t        |d      5 }
| j                  |
j                         |       d d d        V y # 1 sw Y   axY w)NrQ  '  	test.htmlr8  r   r   test.jstest.txtr   -zjs, htmlztest.*rV  r   Tr  Content-Encoding:\s+gzipr   )ra  rt  r   r   r   r   r   r   r<  rK  r  r   r   r   )r   r   r   r   uri1uri2uri3r  fpath4r  r   s              r   test_gzip_upload_and_downloadz$TestCp.test_gzip_upload_and_download  s   ""$Je|H!F+fxP)FXN*VhO 	NNdJ
VX&Z 
 	
A.
K(D
I&D
J'D^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?F ;<  "FdD!
nndCf./8,  "s   !F77G 	z0No compressed transport encoding support for S3.z9No compressed transport encoding support for the XML API.c                       fd} fd} fd}ddz  f fd	} |       \  }}} |||      } |||        ||       y)zTest gzip encoded files upload correctly.

    This checks that files are not tagged with a gzip content encoding and
    that the contents of the files are uncompressed in GCS. This test uses the
    -j flag to target specific extensions.
    c                      j                         } ddz  }j                         }g }dD ]%  }|j                  j                  |||             ' | ||fS )a  Setup the bucket and local data to test with.

      Returns:
        Triplet containing the following values:
          bucket_uri: String URI of cloud storage bucket to upload mock data
                      to.
          tmpdir: String, path of a temporary directory to write mock data to.
          local_uris: Tuple of three strings; each is the file path to a file
                      containing mock data.
      rQ  rW  )rX  rZ  r[  rY  )ra  rt  r   r   )r   r   r   
local_urisr  r   s        r   _create_test_datazQTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._create_test_data	  sq     $$&jh!!#fj:(('-)1   3	4 ; &*--r   c           
          j                   rd}nd}j                  ddd|t        j                  j	                  | d      t        |      gd      }j                  |d	       |S )
a<  Upload local test data.

      Args:
        tmpdir: String, path of a temporary directory to write mock data to.
        bucket_uri: String URI of cloud storage bucket to upload mock data to.

      Returns:
        stderr: String output from running the gsutil command to upload mock
                  data.
      zjs,htmlr]  r  r   -jtest*TrH  rV  )r   r   r   r   r   r   r<  )r   r   extension_list_stringr   r   s       r   _upload_test_datazQTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._upload_test_data,	  sr     
	!	! ) *~~
d1
'',,vw
'
z

 -1  2f !!*a0mr   c                     | \  }}}d}j                  |j                  |      |       j                  |j                  |      |       j                  |j                  |      |       y)a"  Ensure the correct files were marked for compression.

      Args:
        local_uris: Tuple of three strings; each is the file path to a file
                    containing mock data.
        stderr: String output from running the gsutil command to upload mock
                data.
      z2Using compressed transport encoding for file://{}.N)r   rL  r  )rf  r   local_uri_htmllocal_uri_jslocal_uri_txtassert_base_stringr   s         r   _assert_sent_compressedzWTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._assert_sent_compressedD	  sg     5?1nlMO
mm&--n=vF
mm&--l;VD
)00?Hr   rQ  rW  c                    t        | d      }t        | d      }t        | d      }	j                         }|||fD ]{  }	j                  d|gd      }	j                  |d       	j                  d|t        |      g       t	        |d	      5 }	j                  |j                         |       d
d
d
       } y
# 1 sw Y   xY w)zEnsure the files are not compressed when they are stored in the bucket.

      Args:
        bucket_uri: String with URI for bucket containing uploaded test data.
        contents: Byte string that are stored in each file in the bucket.
      rX  rZ  r[  r   Tr  r^  r   r   N)r   r   r   r  r   r   r   )
r   r   rn  ro  rp  rb  r  r  r   r   s
            r   _assert_stored_uncompressedz[TestCp.test_gzip_transport_encoded_upload_and_download.<locals>._assert_stored_uncompressedS	  s     J4n*i0l:z2m""$f ,>#TBF$?@c4<01&$1


1668X
.  	 ?  s   !B::C	Nr  )	r   rg  rl  rr  rt  r   r   rf  r   s	   `        r   /test_gzip_transport_encoded_upload_and_downloadz6TestCp.test_gzip_transport_encoded_upload_and_download		  sP    .00I :> /( &7%8"J
vz2FJ/
+r   c                 V   | j                         }d}| j                  d|      }t        ddt        t              fg      5  | j                  dddd	d
|t        |      gd      }| j                  |d       | j                  s| j                  d|       ddd       y# 1 sw Y   yxY w)a  Test non resumable, gzip encoded files upload correctly in parallel.

    This test generates a small amount of data (e.g. 100 chars) to upload.
    Due to the small size, it will be below the resumable threshold,
    and test the behavior of non-resumable uploads.
    sd   xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
   r  r   r  r  r  r#  r   -Jro  TrH  4send: Using gzip transport encoding for the request.N)
ra  rt  r4   r   r[   r   r   r<  r   r   r   r   r   r   r   s        r   9test_gzip_transport_encoded_parallel_upload_non_resumablez@TestCp.test_gzip_transport_encoded_parallel_upload_non_resumablel	  s     ""$JH2AF	*?WN   
!~~tT4
  f
 !!*b1%%L	
! 
! 
!s   ABB(c                 ~   | j                         }t        | j                        }| j                  d|      }t	        ddt        t              fg      5  | j                  dddd	d
|t        |      gd      }| j                  |d       | j                  s| j                  d|       ddd       y# 1 sw Y   yxY w)a  Test resumable, gzip encoded files upload correctly in parallel.

    This test generates a large amount of data (e.g. halt_size amount of chars)
    to upload. Due to the large size, it will be above the resumable threshold,
    and test the behavior of resumable uploads.
    sizerw  rx  r  r  r  r#  r   ry  ro  TrH  rz  N)ra  rW   r  rt  r4   r   r[   r   r   r<  r   r   r{  s        r   5test_gzip_transport_encoded_parallel_upload_resumablez<TestCp.test_gzip_transport_encoded_parallel_upload_resumable	  s     ""$J%4>>:H2AF	*?WN   
!~~tT4
  f
 !!*b1%%L	
! 
! 
!s   AB33B<c                    | j                         }ddz  }| j                         }| j                  d||       | j                  d||       | j                  d||       | j                  d||       | j                  dd	t        j
                  j                  |d
      t        |      g       | j                  |d       t        |d      }t        |d      }t        |d      }t        |d      }| j                  d|gd      }| j                  |d       | j                  d|gd      }| j                  |d       | j                  d|gd      }| j                  |d       | j                  d|gd      }| j                  |d       | j                         }	||||fD ]T  }
| j                  d|
t        |	      g       t        |	d      5 }| j                  |j                         |       d d d        V y # 1 sw Y   axY w)NrQ  rW  rX  rY  rZ  r[  testr   r  rj  r0  r   Tr  r^  r   )ra  rt  r   r   r   r   r   r   r<  rK  r   r   r   )r   r   r   r   r_  r`  ra  uri4r  rb  r  r   s               r   !test_gzip_all_upload_and_downloadz(TestCp.test_gzip_all_upload_and_download	  s   ""$Je|H!F+fxP)FXN*VhO&(KNN	t	fg	&	j		 	
A.
K(D
I&D
J'D
F#D^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?FV89  "FdD$'
nndCf./8,  (s   !G>>H	c           	      <   | j                         }ddz  }| j                         }| j                  d||      }| j                  d||      }| j                  dddt        j
                  j                  |d	      t        |      gd
      }| j                  |d       | j                  d|z  |       | j                  d|z  |       | j                  s| j                  d|       t        |d      }t        |d      }| j                         }	||fD ]{  }
| j                  d|
gd
      }| j                  |d       | j                  d|
t        |	      g       t        |	d      5 }| j                  |j                         |       ddd       } y# 1 sw Y   xY w)zTest gzip encoded files upload correctly.

    This checks that files are not tagged with a gzip content encoding and
    that the contents of the files are uncompressed in GCS. This test uses the
    -J flag to target all files.
    rQ  rW  r[  rY  r  r  r   ry  rj  TrH  rg   z2Using compressed transport encoding for file://%s.rz  r   r  r^  r   N)ra  rt  r   r   r   r   r   r   r<  r   r   r  r   r   r   )r   r   r   r   
local_uri1
local_uri2r   remote_uri1remote_uri2rb  r  r  r   s                r   3test_gzip_transport_encoded_all_upload_and_downloadz:TestCp.test_gzip_transport_encoded_all_upload_and_download	  s    ""$Je|H!F$$z,2.6 % 8J $$v,2.6 % 8J ^^	tT	fg	&	j		 	  F
 	
A.MM<
K 	MM<
K ##
mmJ z:.Kz6*K  "F[)~~vsm4~@f
&"=>
nndCf./8, 	 * s   &!FF	c                     g dg df}| j                   rt        \  }}}nd}d}d}|D ]:  }| j                  |d|      }| j                  ||       | j                  ||       < y)	z)Test that mixing compression flags error.)r   r  r\  html, jsa.jsb.js)r   r\  r  r  r  r  r   CommandExceptionz:Specifying both the -z and -Z options together is invalid.Tr  Nr   )_GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUTr   r   r   casesr   expected_error_prefixexpected_error_substringcaser   s          r   test_both_gzip_options_errorz#TestCp.test_both_gzip_options_error	  s     	76	8E 
3 Go,.F o0
F ~~d,0.=  ?f mm)62
mm,f5 r   c                     g dg df}| j                   rt        \  }}}nd}d}d}|D ]:  }| j                  |d|      }| j                  ||       | j                  ||       < y)	z0Test that mixing transport encoding flags error.)r   ry  ri  r  r  r  )r   ri  r  ry  r  r  r   r  z:Specifying both the -j and -J options together is invalid.Tr  Nr  r  s          r   /test_both_gzip_transport_encoding_options_errorz6TestCp.test_both_gzip_transport_encoding_options_error
  s     	76	8E 
3 Go,.F o0
F  ~~d,0.=  ?f mm)62
mm,f5 r   c                     g dg dg dg df}| j                   rt        \  }}}nd}d}d}|D ]:  }| j                  |d|	      }| j                  ||       | j                  ||       < y
)z@Test that mixing transport encoding and compression flags error.)r   r  ri  r  r  r  )r   ry  r\  r  r  r  )r   ri  r  r  r  r  )r   r\  r  ry  r  r  r   r  z@Specifying both the -j/-J and -z/-Z options together is invalid.Tr  Nr  r  s          r    test_combined_gzip_options_errorz'TestCp.test_combined_gzip_options_error&
  s     !O;=E
 
3 Go,.F o0
L  ~~d,0.=  ?f mm)62
mm,f5 r   c                     | j                  d      }| j                         }d|d d z  }| j                  dd|t        |      gd      }| j	                  d	|       | j                  |d
       y )N)tmpxyzr8  z%s*r   r*  TrH  Copying file:r   )r   ra  r   r   r   r<  )r   rB  r   wildcard_urir   s        r   /test_upload_with_subdir_and_unexpanded_wildcardz6TestCp.test_upload_with_subdir_and_unexpanded_wildcard=
  sy      +A BF""$J6#2;&L^^	t\4
#34D  JFMM/6*
A.r   c           
         | j                  d      }| j                         }t        dg      5  | j                  ddd|t	        |      gd      }| j                  d	|       d d d        | j                  |d
       y # 1 sw Y   xY w)Nr  r  r  check_hashesneverr  zContent-MD5: invalid-md5r   TrH  r  r   )r   ra  r4   r   r   r   r<  )r   rB  r   r   s       r   Btest_upload_does_not_raise_with_content_md5_and_check_hashes_neverzITestCp.test_upload_does_not_raise_with_content_md5_and_check_hashes_neverF
  s      E 3F""$J	BC	D~~+T6
  f mmOV, 
E 	
A. 
E	Ds   4A>>Bc                    | j                         }| j                         }| j                  |dd       | j                  |dd       | j                  |d       | j	                  ddt        |      |g       t        t        j                  j                  ||j                  d	d
            5 }| j                  d
dj                  |j                                      ddd       y# 1 sw Y   yxY w)z8Tests that cp works with object names ending with slash.zabc/s   dirrn  zabc/defr  rg   r   r*  abcdefr/  N)rt  ra  r   r<  r   r   r   r   r   r   r   r   r  )r   r   r   r   s       r    test_cp_object_ending_with_slashz'TestCp.test_cp_object_ending_with_slashQ
  s     !F""$J"(%  ' 	"+%  ' 	
A.NND$Z 0&9: 
bggll6:#9#95%H	IQ
udii67 
J	I	Is   .0C''C0c                 D   | j                         }| j                  |d      }| j                  |d       | j                  dk(  rd}nd}| j	                         5  | j                  dt        |      dgd	d
      }ddd       | j                  |       y# 1 sw Y   xY w)z6Tests that cp fails without read access to the object.r   r_  r   rn  AccessDeniedzAnonymous \S+ do(es)? not haver   r  Tr  N)ra  r   r<  r   SetAnonymousBotoCredsr   r   rK  )r   r   
object_uriexpected_error_regexr   s        r   test_cp_without_read_accessz"TestCp.test_cp_without_read_accessc
  s    
 ""$J""j6"JJ 	
A.$,>		#	#	%~~tT*%5u=,0./  1f 
& 	V12	 
&	%s   !BBc           	         | j                         }| j                         }| j                  |d       t        j                  j                  |d      }t        j                  |       t        j                  t        j                  j                  |d             t        j                  t        j                  j                  |d      t        j                  j                  |d             t        j                  t        j                  j                  |d             | j                  ddd|t        |      g       y)	z4Tests that cp -e -r ignores symlinks when recursing.r   rF  subdirmissingr   ro  -eN)ra  rt  r   r   r   r   r  r  rmdirr   r   )r   r   r   r  s       r   test_cp_minus_r_minus_ezTestCp.test_cp_minus_r_minus_ey
  s     ""$J!F 	v7WW\\&(+FHHVHHRWW\\&),- JJrww||FI.VY0OPHHRWW\\&),-NND$fd:.>?@r   c                    | j                         }| j                  |      }t        j                  j	                  |d      }| j                         }t        j                  ||       | j                  ddd|t        j                  j                  dt        |d      gd	      }| j                  d
|       | j                  r| j                  d|       n| j                  d|       | j                  ddd||t        |d      gdd      }| j                  d
|       | j                  r(| j                  d|       | j                  d|z  |       y | j                  d|       | j                  d|z  |       y )NrN  
cp_minus_er#  r   r  r+  filesTrH  zCopying filezSkipping symlinkzSkipping symbolic linkro  r   r  z#URL matched no objects or files: %sz%CommandException: No URLs matched: %s)rt  r   r   r   r   ra  r  r   sepr   r   r   )r   	fpath_dirrB  rd  r   r   s         r   test_cp_minus_ezTestCp.test_cp_minus_e
  s^   ""$I  	 2FWW\\)\2F""$JJJvv
 ^^dDbggkk*Z!
 +/  0F 	MM.&)
mm&/
mm,f5
 ^^	tT66	j'	"	$	  F
 	MM.&)
mm&/
mm9FBFK
mm,f5
mm;fDfMr   c                     d}| j                  |      }| j                         }|t        j                  d}| j	                  dd|t        |      g       | j                  ||       y)z'Tests that cp -m works with a wildcard.r  r  r+  r#  r   N)rt  ra  r   r  r   r   r<  )r   num_test_filestmp_dirr   r  s        r   test_cp_multithreaded_wildcardz%TestCp.test_cp_multithreaded_wildcard
  s`    N  N ;G""$J%rvv.LNND$d:.>?@
N;r   c                 `   d}| j                  d|      }| j                         }| j                  ddt        |      t        |      |g       t	        t
        j                  j                  |d      d      5 }|j                         }| j                  ||       ddd       y# 1 sw Y   yxY w)z@Tests that cp -m works when a source argument is provided twice.s   edger  )r   r   r#  r   r   N)
r   rt  r   r   r   r   r   r   r   r   )r   r  r  r  in_fpr   s         r   test_cp_duplicate_source_argsz$TestCp.test_cp_duplicate_source_args
  s     O""u"OJ  "GNND$Z 0$z2BGLM	bggll7E*D	1Uh
x1 
2	1	1s   8#B$$B-z<gsutil doesn't support S3 customer-supplied encryption keys.c                 @   | j                   t        j                  k(  rt        j                  d      S d}| j                  d|t              }| j                         }ddt        fg}t        |      5  | j                  dt        |      t        |      g       ddd       t        |d	      5 }| j                  |j                         |       ddd       | j                         }ddt        fdd
t        fddt        fg}t        |      5  | j                  dt        |      t        |      g       ddd       t        |d	      5 }| j                  |j                         |       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   WxY w# 1 sw Y   yxY w)z&Tests downloading an encrypted object.3gsutil does not support encryption with the XML APIrF  r  r   r   encryption_keyr  r  r   Nr   decryption_key1decryption_key2)test_apir   XMLr;   r  r   r7   r   r4   r   r   r   r   r   r:   r9   )r   r  r  r   r
  r   rd  boto_config_for_test2s           r   !test_cp_download_encrypted_objectz(TestCp.test_cp_download_encrypted_object
  sr    }}']]
?A AO""u,;2F # HJ !E%'79MNO	2	3
nndD,d5k:; 
4	eT	a
qvvx1 
   "F	#%9:	$&:;	$&:; 
3	4
nndD,d6l;< 
5	fd	q
qvvx1 
	 
4	3		 
5	4		s0   2'E0-!E<'F!F0E9<FFFc                 @   | j                   t        j                  k(  rt        j                  d      S d}| j                  d|t              }| j                         }| j                  dt        |      t        |      gdd      }| j                  d	t        z  |       y
)z@Tests downloading an encrypted object without the necessary key.r  rF  r  r  r   r   Tr   z*Missing decryption key with SHA256 hash %sN)r  r   r  r;   r  r   r7   r   r   r   r   r8   )r   r  r  r   r   s        r   -test_cp_download_encrypted_object_without_keyz4TestCp.test_cp_download_encrypted_object_without_key
  s     }}']]
?A AO""u,;2F # HJ !E^^	tJe-  F 	MM4'	()/1r   c                    | j                   t        j                  k(  rt        j                  d      S | j                         }t        |d      }d}| j                  |d      }ddt        fg}t        |      5  | j                  dt        |      t        |      g       ddd       | j                  |t               t        |      5  | j                         }| j                  dt        |d      t        |      g       t        |d	      5 }| j                  |j                         |       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)
z$Tests uploading an encrypted object.r  r  rF  r   r8  r  r  r   Nr   )r  r   r  r;   r  ra  r   r   r7   r4   r   AssertObjectUsesCSEKr   r   r   )r   r   r  rT  r   r
  rd  r   s           r   test_cp_upload_encrypted_objectz&TestCp.test_cp_upload_encrypted_object
  s.    }}']]
?A A""$Jj%(JM%HE%'79MNO 
2	3
nndDKj)9:; 
4 	j*>?	2	3""$f
nndDU3T&\BC=1 	 
4	3 
4	3 	 
4	3s1   9'D?	AE!E.E?EE	EE z1No resumable upload or encryption support for S3.c                 &   | j                   t        j                  k(  rt        j                  d      S | j                         }t        |d      }| j                  d| j                  z        }ddt        t              fddt        fg}| j                  t        j                  t        dd	                  }t        |      5  | j!                  d
d|||gdd      }| j#                  d|       | j!                  d
||gd      }| j#                  d|       | j!                  d|gd      }t%        |d      5 }| j#                  t'        |      |       ddd       ddd       | j)                  |t               y# 1 sw Y   (xY w# 1 sw Y   ,xY w)z@Tests that an encrypted upload resumes after a connection break.r  r  r  rE  r  r  r  Tr  r   r  r   r   Artifically halting uploadrH  Resuming uploadr   r  r   N)r  r   r  r;   r  ra  r   r   r  r   r[   r7   r  r  r(   r4   r   r   r   rI   r  )	r   r   object_uri_strr   r
  r  r   r  fps	            r   /test_cp_resumable_upload_encrypted_object_breakz6TestCp.test_cp_resumable_upload_encrypted_object_break  s    }}']]
?A A""$J*e,N)>?E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8% /0,0	  2f
 mm0&9~~tUN;4~Pf
mm%v.~~v~6d~Kft8<fE  
4 	n.BC  
4	3s%   A5F8E;F;F	 FFc                    | j                   t        j                  k(  rt        j                  d      S | j                         }t        |d      }d| j                  z  }| j                  |      }ddt        t              fddt        fg}| j                  t        j                  t        dd	                  }t        |      5  | j!                  d
d|||gdd      }| j#                  d|       ddd       ddt        t              fddt$        fddt        fg}t        |      5  | j!                  d
||gd      }| j#                  d|       ddd       | j'                  |t               y# 1 sw Y   xY w# 1 sw Y   ,xY w)zCTests that an encrypted upload resume uses original encryption key.r  r  r  rE  r  r  r  Tr  r   r  r   r   r  Nr  rH  r  )r  r   r  r;   r  ra  r   r  r   r   r[   r7   r  r  r(   r4   r   r   r9   r  	r   r   r  rT  r   r
  r  r   r  s	            r   7test_cp_resumable_upload_encrypted_object_different_keyz>TestCp.test_cp_resumable_upload_encrypted_object_different_key3  s    }}']]
?A A""$J*e,N4>>)M7E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8% /0,0	  2f
 mm0&9 
4 '(=s7|L&(924&(8:NOQ
 
3	4~~tUN;4~Pf
mm%v. 
5
 	n.BC' 
4	3 
5	4s   ,E/')E;/E8;Fc                 F   | j                   t        j                  k(  rt        j                  d      S | j                         }t        |d      }d| j                  z  }| j                  |      }ddt        t              fddt        fg}| j                  t        j                  t        dd	                  }t        |      5  | j!                  d
d|||gdd      }| j#                  d|       ddd       ddt        t              fddt$        fg}t        |      5  | j!                  d
||gd      }| j'                  d|       | j#                  d|       | j#                  d|       | j)                  |t$               ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zDTests that an encrypted upload does not resume without original key.r  r  r  rE  r  r  r  Tr  r   r  r   r   r  NrH  r  z%does not match current encryption keyzRestarting upload from scratch)r  r   r  r;   r  ra  r   r  r   r   r[   r7   r  r  r(   r4   r   r   r9   r  r  r  s	            r   5test_cp_resumable_upload_encrypted_object_missing_keyz<TestCp.test_cp_resumable_upload_encrypted_object_missing_keyW  s    }}']]
?A A""$J*e,N4>>)M7E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8% /0,0	  2f
 mm0&9 
4 '(=s7|L&(8:NOQ 
3	4~~tUN;4~Pf
(&1
mm;VD
mm4f= 0DE 
5	4 
4	3 
5	4s   ,FA#FFF c                 R    | j                  d|gd      }| j                  d|       y)z9Strongly consistent check that the object is unencrypted.r   Tr  zEncryption KeyN)r   r  )r   r  r  s      r   _ensure_object_unencryptedz!TestCp._ensure_object_unencrypted{  s*    ^^V^4D^IF%v.r   z#No resumable upload support for S3.c           
         | j                         }| j                  d| j                  z        }ddt        t              f}| j                  t        j                  t        dd                  }t        |g      5  | j                  dd||t        |      gd	d
      }| j                  d|       | j                  d|t        |      gd      }| j                  d|       ddd       y# 1 sw Y   yxY w)z=Tests that an upload can be resumed after a connection break.r  rE  r  r  Tr  r   r  r   r   r  rH  r  N)ra  r   r  r   r[   r  r  r(   r4   r   r   r   r   r   r   r
  r  r   s         r   test_cp_resumable_upload_breakz%TestCp.test_cp_resumable_upload_break  s     ""$J)>?E$&;S\J,,8qAB - D 
34	5~~
$&8%
z
 /0,0  2f mm0&9~~tUD,<=,0  2f
mm%v. 
6	5	5s   =A&C,,C5c                    | j                         }t        | j                        }| j                  d|      }ddt	        t
              f}| j                  t        j                  t        dd                  }t        |g      5  | j                  d	d
dd||t        |      gdd      }| j                  d|       | j                  d|       | j                  d	d
d|t        |      gd      }| j                  d|       | j                  d|       ddd       | j                         }t        |d      }| j                  d|gd      }	| j                  |	d       | j                  d
|t        |      g       t        |d      5 }
| j                  |
j!                         |       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z0Tests that a gzip encoded upload can be resumed.r~  r[  r7  r  r  Tr  rE  r  r   ry  r  r   r   rz  r  rH  r  Nr   r  r^  r   )ra  rW   r  r   r   r[   r  r  r(   r4   r   r   r   r  r   r   r   )r   r   r   	local_urir
  r  r   temp_uri
remote_urir  r   s              r   +test_cp_resumable_upload_gzip_encoded_breakz2TestCp.test_cp_resumable_upload_gzip_encoded_break  s    ""$J%4>>:H##j8#LI$&;S\J,,8qAB - D 
34	5~~
d02Di
z
 /0,0  2f mmJ
mm0&9~~tT4#J/1,0  2f mm%v.
mmJ! 
6( ""$Hj*-J^^VZ0^EF ;<NND*d8n56	h	
qvvx* 
	3 
6	52 
	s   BF3	!F?3F<?Gc                    | j                         }| j                  d| j                  z        }| j                  t        j
                  k(  r?| j                  t        j                  t        dt        j                  d                  }n>| j                  t        j                  t        dt        j                  d                  }ddt        t              f}t        |g      5  | j!                  dd	d
||t#        |      gd      }| j                  t        j
                  k(  r| j%                  d|       n| j%                  d|       ddd       y# 1 sw Y   yxY w)z7Tests that a resumable upload completes with one retry.r  rE  r  )unusedr  r  r  r  r  r  r   r  r   rH  zGot retryable failureRetryingN)ra  r   r  r  r   r  r  r  r&  r^   BadStatusLiner   BadStatusCodeErrorr   r[   r4   r   r   r   )r   r   r   r  r
  r   s         r   test_cp_resumable_upload_retryz%TestCp.test_cp_resumable_upload_retry  sB    ""$J)>?E }}'..
&q+*C*C F 9.   ..
&$77 :DE9F. G %&;S\J	34	5~~
*,>
z
 -.	  /f
 
+//	)-v6j&) 
6	5	5s   (A%EEc                    | j                   t        j                  k(  rt        j                  d      S | j                         }| j                  t        j                  t        dt        j                  d                  }ddt        dt        z        fdg}t        |      5  | j                  d	d
d|dt!        |d      gdt        z  d      }| j#                  d|       ddd       y# 1 sw Y   yxY w)zATests that a streaming resumable upload completes with one retry.z1XML does not support resumable streaming uploads.r  r  rE  r  ry  rz  )Botor0  2r  r   r  r  r     aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar   r  r  N)r  r   r  r;   r  ra  r   r  r  r&  r   r  r   r[   r4   r   r   r   )r   r   r  boto_configs_for_testr   s        r   (test_cp_resumable_streaming_upload_retryz/TestCp.test_cp_resumable_streaming_upload_retry  s     }}']]NOO""$J,,fll$Q(;(N(N%C	E7F, G
 '(C!#-023OQ	3	4~~
*,>
z5
! %.$7,-  /f mmJ' 
5	4	4s   #>C**C3z'preserve_acl flag not supported for S3.c           
         | j                         }| j                  |d      }| j                  ddddt        |      g       | j                  ddddt        |      g       | j	                         5  | j                  dd	t        |      t        |d
      gdd      }| j                  d|       d d d        y # 1 sw Y   y xY w)Nr   r_  r  chz-uz
AllUsers:Rz
AllUsers:Wr   z-pr  Tr   r  z0OWNER permission is required for preserving ACLs)ra  r   r   r   r  r   )r   r   r  r   s       r   test_cp_preserve_no_ownerz TestCp.test_cp_preserve_no_owner  s    ""$J""j6"JJ 	NNE4|T*5EFGNNE4|T*5EFG		#	#	%~~tJ'
E"$	  f
 mmFO 
&	%	%s   2>B99Cc                    | j                         }t        d      }|dz   |z   }| j                  dt        z  d      }ddt	        t
              f}t        |g      5  | j                  d|t        |      gd	
      }| j                  d|j                  |             d d d        ddt	        dt        z        f}t        |g      5  | j                  d|t        |      gd	
      }| j                  d|j                  |             d d d        | j                  dt        |d      |gd	
      }| j                  d|j                  |             y # 1 sw Y   xY w# 1 sw Y   WxY w)N   r   r  r  r  r  r  r   TrH  r   rg   )ra  r@   r   r\   r   r[   r4   r   r   r   rc  )r   r   final_size_stringfinal_progress_callbackr   r
  r   s          r   test_cp_progress_callbacksz!TestCp.test_cp_progress_callbacks  sb   ""$J/8/#58II5IE$&;S\J	34	5~~tUD,<=,0  2f
q&,,'>?@ 
6 %&;SW=MN	34	5~~tUD,<=,0  2f
q&,,'>?@ 
6 ^^T4
E#:EB*.  0FQ%<=> 
6	5
 
6	5s   AEAEEE&c                    | j                         }| j                  d| j                  z        }ddt        t              f}t        |g      5  | j                  d|t        |      g       ddd       y# 1 sw Y   yxY w)z;Tests that a basic resumable upload completes successfully.r  rE  r  r  r   N)ra  r   r  r   r[   r4   r   r   )r   r   r   r
  s       r   test_cp_resumable_uploadzTestCp.test_cp_resumable_upload  sp     ""$J)>?E$&;S\J	34	5
nndE4
#345 
6	5	5s   A55A>c                 r   | j                         }| j                  dd| j                  z        }ddt        t              f}t        |g      5  t        t        t        |d            t        j                  | j                        }| j                  t        j                  t        dd                  }	 | j                  d	d
||t        |d      gdd      }| j!                  d|       | j#                  t$        j&                  j)                  |      d|z         t$        j*                  dk(  rVt-        t/        j0                  t%        j.                  |      j2                              }| j5                  t-        d      |       t7        |       	 ddd       y# t7        |       w xY w# 1 sw Y   yxY w)z=Tests that a tracker file is created with a resumable upload.r  r  r7  r  r  Tr  rE  r   r  r   r   r  zTracker file %s not present.posixi  N)ra  r   r  r   r[   r4   rE   r   r   rH   UPLOADr  r  r  r(   r   r   r   r   r   ry  r   octr   S_IMODEst_moder   r=   )r   r   r   r
  tracker_filenamer  r   r   s           r   *test_resumable_upload_break_leaves_trackerz1TestCp.test_resumable_upload_break_leaves_tracker!  s    ""$J%$:OPE$&;S\J	34	5+
tJ6
79O9O
--  ..<< :4 CD / F,&(:EU#!
 12.2   4 	2F;'7869II	K 77gT\\"''*:";"C"CDE$


3u:t
,*+- 
6	5, 	*+- 
6	5s&   A(F-8CFF-F**F--F6c           
      ^   | j                         }| j                         }| j                  d|d| j                  z        }| j                  t	        j
                  t        dd                  }ddt        t              f}t        |g      5  | j                  d	d
||t        |      gdd      }| j                  d|       | j                  d|d| j                  z  dz        }| j                  d	|t        |      gdd      }| j                  d|       ddd       y# 1 sw Y   yxY w)zxTests a resumable upload where the uploaded file changes size.

    This should fail when we read the tracker data.
    r  r  rY  Tr  rE  r  r  r   r  r   r   r  rg   ResumableUploadAbortExceptionN)ra  rt  r   r  r  r  r(   r   r[   r4   r   r   r   )r   r   r  r   r  r
  r   s          r   /test_cp_resumable_upload_break_file_size_changez6TestCp.test_cp_resumable_upload_break_file_size_change?  sJ    ""$J  "G%'.)-)>   @E ,,8qAB - D %&;S\J	34	5~~
$&8%
z
 /0,0  2f mm0&9!!E)0+/$..+@1+D " Fe ~~tUD,<=./,0  2f mm3V< 
6	5	5s   BD##D,c           
         | j                   t        j                  k(  rt        j                  d      S | j                         }| j                         }| j                  d|dt        z  t        z        }| j                  t        j                  t        dt        t              dz                    }dd	t        t              f}dd
t        t        dz        f}t        ||g      5  | j                  dd||t!        |      gdd      }| j#                  d|       | j                  d|dt        z  t        z        }| j                  d|t!        |      gdd      }| j#                  d|       ddd       y# 1 sw Y   yxY w)zATests a resumable upload where the uploaded file changes content.zXML doesn't make separate HTTP calls at fixed-size boundaries for resumable uploads, so we can't guarantee that the server saves a specific part of the upload.r  r  rY  T   rE  r  r  ry  rz  r   r  r   r   r  r}  #doesn't match cloud-supplied digestN)r  r   r  r;   r  ra  rt  r   r[   r  r  r(   r   r   r4   r   r   r   r   r   r  r   r  resumable_threshold_for_testresumable_chunk_size_for_testr   s           r   2test_cp_resumable_upload_break_file_content_changez9TestCp.test_cp_resumable_upload_break_file_content_change^  s    }}']])* * ""$J  "G%'.)-')A   CE ,,fll"4#&w<##5	778, 9 %-.C$'L$2 %-/J%(3%7%9!		%'DE
G~~
$&8%
z
 /0,0  2f mm0&9!!E)0+/'>G+C " Ee ~~tUD,<=./,0  2f mm:FC
G 
G 
Gs   (B	E::Fc           
         | j                         }| j                         }| j                  d|dt        z  t        z        }| j                  t	        j
                  t        dt        t              dz                    }ddt        t              f}dd	t        t        d
z        f}t        ||g      5  | j                  dd||t        |      gdd      }| j                  d|       | j                  d|dt        z        }| j                  d|t        |      gdd      }| j                  d|       ddd       y# 1 sw Y   yxY w)zmTests a resumable upload where the uploaded file changes content.

    This should fail hash validation.
    r  r  rY  Tr  rE  r  r  ry  rz  r   r  r   r   r  r  N)ra  rt  r   r[   r  r  r(   r   r   r4   r   r   r   r  s           r   0test_cp_resumable_upload_break_file_smaller_sizez7TestCp.test_cp_resumable_upload_break_file_smaller_size  ss    ""$J  "G%'.)-')A   CE ,,fll"4#&w<##5	778, 9 %-.C$'L$2 %-/J%(3%7%9!		%'DE
G~~
$&8%
z
 /0,0  2f mm0&9!!E)0+/'> " ;e ~~tUD,<=./,0  2f mm3V<
G 
G 
Gs   6BEE
c           	         | j                   t        j                  k(  rt        j                  d      S | j                         }t        t        |d            }d}d}| j                  ||      }t        |      }t        |t        j                  | j                   |      }d}t        |z   j                  t              }	t               }
|
j!                  |	       |
j#                         }|t$        z   |z   dz   }d}| j'                  |||d	| t(        
      }t+        |t-        |j.                              }|g}t0        }t3        ||||       	 t5        dddt-        |      fddt(        fg      5  | j7                  d|t        |d      gd      }| j9                  d|       | j;                  t<        j>                  jA                  |      d|z         | j7                  dt        |d      gd      }| jC                  |j                  d      |       d	d	d	       tE        |       y	# 1 sw Y   xY w# tE        |       w xY w)z>Tests that an encrypted composite upload resumes successfully.r  r     foobarfoobarr  123_0rV  Nr   r   r   r  )encryption_key_sha256r  #parallel_composite_upload_thresholdr  r  (parallel_composite_upload_component_sizer  r   TrH  z/Found 1 existing temporary components to reuse.)Tracker file %s should have been deleted.rR  r  r  )#r  r   r  r;   r  ra  r   r   r   rE   rH   PARALLEL_UPLOADrF   r   rD   rK   update	hexdigestrG   r   r7   r   r   rH  r8   r   r4   r   r   rx  r   r   ry  r   r=   )r   r   dst_urlrT  r8  source_filesrc_urltracker_file_nametracker_prefixencoded_namecontent_md5digestcomponent_object_namecomponent_sizer  existing_componentexisting_componentsenc_key_sha256r   read_contentss                       r   )test_cp_composite_encrypted_upload_resumez0TestCp.test_cp_composite_encrypted_upload_resume  sd    }}']]
?A A""$J"4
E#:;GMI%%}09 & ;K";/G +7+:+J+J+/=='C N 0+=EEdKL(K|$""$F+.LL#$&*+ N""j/D,9/>,J2F # HJ ++@+.z/D/D+EG-.4N"#4#1#69GI
+
@?~ %';
<	! 	 ;Z 78   NGPGGNN,-7:KK	M tJ/F'G59 ' ;--g6F	" )*#	 	" )*s%   !H4 /B%H(H4 (H1-H4 4Ic           	         | j                   t        j                  k(  rt        j                  d      S | j                         }t        t        |d            }d}| j                  |d      }t        |      }t        |t        j                  | j                   |      }d}d}| j                  |ddt              }	t        |t        |	j                               }
|
g}t"        }t%        ||||j'                  d	             	 t)        d
dddt*        fg      5  | j-                  d|t        |d      gd      }| j/                  d|       | j1                  d|       | j3                  t4        j6                  j9                  |      d|z         | j-                  dt        |d      gd      }| j;                  |j=                  d	      |       ddd       t?        |       y# 1 sw Y   xY w# t?        |       w xY w)zETests that encrypted composite upload restarts given a different key.r  r  r  r  r  foo_1r   r  r  r  )r  r!  3r  r  r   TrH  zTdoes not match current encryption key. Deleting old components and restarting uploadz'existing temporary components to reuse.r"  rR  r  N) r  r   r  r;   r  ra  r   r   r   rE   rH   r#  r   r7   r   r   rH  r8   r   r  r4   r9   r   r   r  rx  r   r   ry  r   r   r=   )r   r   r&  rT  r'  r(  r)  r*  existing_component_namer  r0  r1  r2  r   r3  s                  r   *test_cp_composite_encrypted_upload_restartz1TestCp.test_cp_composite_encrypted_upload_restart  s    }}']]
?A A""$J"4
E#:;GM%%}%NK";/G +7+:+J+J+/=='C N%""j/6,22F # HJ ++B+.z/D/D+EG-.4N"#4n#6#1#8#8#AC+
@
E%';
<! 	
 ;Z 78   N<=C	E 	BFKGGNN,-7:KK	M tJ/F'G59 ' ;--g6F!	& )*'	 	& )*s%   G- B7G!G- !G*&G- -G:z$Test uses gs-specific KMS encryptionc                 N   | j                         }| j                  d      }t        |d      }t               }t	        dd|fddg      5  | j                  d||g       d d d        t	        d	g      5  | j                  ||       d d d        y # 1 sw Y   1xY w# 1 sw Y   y xY w)
N   abcdrE  composedr  r  r  r  r!  r  r   r  
prefer_apijsonra  r   r   r%   r4   r   AssertObjectUsesCMEK)r   r   r   obj_surikey_fqns        r   2test_kms_key_correctly_applied_to_composite_uploadz9TestCp.test_kms_key_correctly_applied_to_composite_upload  s    ""$J1EJ
+H02G		#W->C 

 nndE8,-
 
?@	A
'2 
B	A
 
 
B	As   
B3BBB$z#No composite upload support for S3.c                 D   | j                  d      }| j                  d      }t        |d      }t        ddg      5  | j	                  dd	d
||g       d d d        | j	                  dd|gd      }| j                  |dt        j                         y # 1 sw Y   BxY w)Nrg  rh  r;  rE  r<  r  r=  r   -snearliner  r}  Tr  z Storage class:          NEARLINEflags)ra  r   r   r4   r   assertRegexpMatchesWithFlagsr   
IGNORECASE)r   r   r   rC  r  s        r   2test_nearline_applied_to_parallel_composite_uploadz9TestCp.test_nearline_applied_to_parallel_composite_upload0  s    """<J1EJ
+H	>C 
 nndD*eX>?	

 ^^T42$^GF%%f(K.0mm & =
 
s   BBz$chmod on dir unsupported on Windows.c                 r   | j                         }t        t        t        |d            t        j
                  | j                        }t        j                  j                  |      }| j                  ddt        z        }ddt        t              f}t        j                  |      j                  }	 t        j                  |d       t!        |g      5  | j#                  d|t        |      gdd	
      }| j%                  d|       ddd       t        j                  ||       t        j                  j'                  |      rt        j(                  |       yy# 1 sw Y   UxY w# t        j                  ||       t        j                  j'                  |      rt        j(                  |       w w xY w)z9Tests a resumable upload with an unwritable tracker file.r  r  r7  r  r  r   r   r   Tr   Couldn't write tracker fileN)ra  rE   r   r   rH   r  r  r   r   r  r   r[   r   r   r	  chmodr4   r   r   ry  unlink)r   r   r
  tracker_dirr   r
  save_modr   s           r   test_cp_unwritable_tracker_filez&TestCp.test_cp_unwritable_tracker_fileB  s[    ""$J)T*e457M7M ''//"23K%$.IE$&;S\Jww{#++H
$hh{A!5 67ud:.> ?01.2   4 	4f=	 8 	hh{H%	(	)
		"# 
* 87 	hh{H%	(	)
		"# 
*s%   4"E) 3E	E) E&"E) )AF6c                    | j                  dt        z        }t        t        t	        |            t
        j                  | j                        }t        j                  j                  |      }| j                         }t        j                  |      j                  }	 t        j                  |d       ddt        t               f}t#        |g      5  | j%                  dt	        |      |g       ddd       ddt        t              f}t#        |g      5  | j%                  dt	        |      |gdd	
      }| j'                  d|       ddd       t        j                  ||       t        j                  j)                  |      rt        j*                  |       yy# 1 sw Y   xY w# 1 sw Y   axY w# t        j                  ||       t        j                  j)                  |      rt        j*                  |       w w xY w)z0Tests downloads with an unwritable tracker file.r   rE  r   r  r  r   Nr   Tr   rO  )r   r[   rE   r   r   rH   DOWNLOADr  r   r   r  r   r   r	  rP  r   rX   r4   r   r   ry  rQ  )r   r  r
  rR  r   rS  r
  r   s           r   (test_cp_unwritable_tracker_file_downloadz/TestCp.test_cp_unwritable_tracker_file_download_  s   
 ""FW,<"=J)T*-.0H0H ''//"23K!Eww{#++H$hh{A&(=s9~N!5 67d:.67 8 '(=s7|L!5 67tJ'7 ?01.2   4 	4f=	 8 	hh{H%	(	)
		"# 
* 87 87 	hh{H%	(	)
		"# 
*s=    4F7 F2&F7 3F+F7 F($F7 +F40F7 7AHNc           	      0   | j                         }d| j                  z  }| j                  |d||      }| j                         }| j                  t	        j
                  t        dd                  }t        |      5  | j                  dd|t        |      |gd	d
      }| j                  d|       t        t        |      t        j                  | j                        }	| j!                  t"        j$                  j'                  |	             | j                  dt        |      |gd
      }| j                  d|       ddd       t)        |d      5 }
| j+                  |
j-                         |d       ddd       y# 1 sw Y   @xY w# 1 sw Y   yxY w)zHelper function for different modes of resumable download break.

    Args:
      boto_config: List of boto configuration tuples for use with
          SetBotoConfigForTest.
      encryption_key: Base64 encryption key for object encryption (if any).
    r  r  r  Fr  rE  r   r  r   Tr   Artifically halting download.rH  Resuming downloadNr   File contents differ)ra  r  r   r   r  r  r(   r4   r   r   r   rE   r   rH   rV  r  r   r   r   isfiler   r   r   )r   boto_configr  r   rT  r  r   r  r   r
  r   s              r   (_test_cp_resumable_download_break_helperz/TestCp._test_cp_resumable_download_break_helper}  s    ""$J4>>)M""j/4,92@ # BJ !E,,8BC - E 
k	*~~
$&8
z
E /0,0  2f mm3V<+,@,G,;,D,D,0MM; oobggnn%567~~tT*%5u=,0  2f
mm'0 
+ 
eT	a
qvvx0FG 
	 
+	* 
	s   ?CF "F F	Fc                 H    | j                  ddt        t              fg       y)z>Tests that a download can be resumed after a connection break.r  r  N)r^  r   r[   r   s    r    test_cp_resumable_download_breakz'TestCp.test_cp_resumable_download_break  s$    11	(#g,73 r   c                     | j                   t        j                  k(  rt        j                  d      S | j                  ddt        t              fddt        fgt               y)zBTests that an encrypted download resumes after a connection break.r  r  r  r  )r  N)	r  r   r  r;   r  r^  r   r[   r7   r   s    r   *test_cp_resumable_encrypted_download_breakz1TestCp.test_cp_resumable_encrypted_download_break  s`     }}']]
?A A11
)3w<	8
$&:	;	=+ 2 -r   c           	         | j                   t        j                  k(  rt        j                  d      S | j                         }d| j                  z  }| j                  |d|t              }| j                         }| j                  t        j                  t        dd                  }dd	t        t              fdd
t        fg}t        |      5  | j!                  dd|t#        |      |gdd      }| j%                  d|       t'        t)        |      t*        j,                  | j                         }| j/                  t0        j2                  j5                  |             ddd       dd	t        t              fddt        fdd
t6        fg}	t        |	      5  | j!                  ddt#        |      g       ddd       dd	t        t              fdd
t6        fg}
t        |
      5  | j!                  dt#        |      |gd      }| j%                  d|       ddd       t9        |d      5 }| j;                  |j=                         |d       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   XxY w# 1 sw Y   yxY w)z=Tests that a download restarts with a rotated encryption key.r  r  r  r  Fr  rE  r  r  r  r   r  r   Tr   rY  Nr  rewritez-krH  zRestarting downloadr   r[  )r  r   r  r;   r  ra  r  r   r7   r   r  r  r(   r   r[   r4   r   r   r   rE   r   rH   rV  r   r   r   r\  r9   r   r   r   )r   r   rT  r  r   r  r
  r   r
  r  boto_config_for_test3r   s               r   1test_cp_resumable_encrypted_download_key_rotationz8TestCp.test_cp_resumable_encrypted_download_key_rotation  sq    }}']]
?A A""$J4>>)M""j/4,92F # HJ !E,,8BC - E &'<c'lK%'79MNP 
2	3~~
$&8
z
E /0,0  2f mm3V<+,@,G,;,D,D,0MM; oobggnn%567 
4 '(=s7|L&(924&(8:NOQ 
3	4
nnitJ'789 
5
 '(=s7|L&(8:NOQ	3	4~~tT*%5u=,0  2f
mm)62 
5 
eT	a
qvvx0FG 
	; 
4	3$ 
5	4 
5	4 
	s1   BII #2I,)"I8I I),I58Jc           	      l   | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        dd                  }ddt        t              f}t        |g      5  | j                  d	d
|t        |      |gdd      }| j                  d|       | j                  |dd| j                  z  |j                        }| j                  d	t        |      |gd      }| j                  d|       ddd       y# 1 sw Y   yxY w)zrTests that download restarts the file when the source object changes.

    This causes the etag not to match.
    r  r  rn  Fr  rE  r  r  r   r  r   Tr   rY  r}  r'  rH  rZ  N)ra  r   r  r   r  r  r(   r   r[   r4   r   r   r   rH  r  r   r   r  r   r  r
  r   s          r   'test_cp_resumable_download_etag_differsz.TestCp.test_cp_resumable_download_etag_differs  sK    ""$J""j/4,2T^^,C # EJ !E,,8BC - E$&;S\J	34	5~~
$&8
z
E /0,0  2f mm3V< $$$..(#-#8#8	 % :j
 ~~tT*%5u=,0  2f
*F3% 
6	5	5s   BD**D3z%Sliced download requires fast crcmod.z"No sliced download support for S3.c           
      l   | j                         }d| j                  z  }| j                  |d|      }| j                         }| j                  t	        j
                  t        dd                  }ddt        | j                        fdd	t        | j                        fd
g}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       | j                  |      }| j                  dt        |      t        |      g       | j                  dt        |      t        |      gd      }| j                  d|       t        |d      5 }	| j                  |	j                         |d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zCTests that a resumable download restarts if the generation differs.r;  r  rn  Fr  rE  r  r   sliced_object_download_thresholdr  %sliced_object_download_max_componentsr7  r   r  Tr   r  rY  rH   Restarting download from scratchr   r[  N)ra  r  r   r   r  r  r(   r   r4   r   r   r   r   r   r   )
r   r   rT  r  r   r  r
  r   identical_filer   s
             r   -test_cp_resumable_download_generation_differsz4TestCp.test_cp_resumable_download_generation_differs  s   
 ""$Jdnn,M""j/4,9 # ;J !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm3V< **M*Bn
nndD0$z2BCD~~j!4;
/t  Ef
mm6?t=2HI % 
4	3$ % 
4	3s%   /B<F*+"FF*F'	#F**F3c           	         | j                         }| j                         }| j                  |dd| j                  z        }| j                  t	        j
                  t        dd                  }ddt        t              f}t        |g      5  | j                  d	d
|t        |      |gdd      }| j                  d|       t        |dz   d      5 }t        | j                  dz        D ]  }|j                  d        	 ddd       | j                  d	t        |      |gdd      }| j!                  d|       | j                  d|       ddd       y# 1 sw Y   VxY w# 1 sw Y   yxY w)zETests download deletes the tracker file when existing file is larger.r  r  rn  Fr  rE  r  r  r   r  r   Tr   rY  _.gstmpr  rg   aNrZ  zDeleting tracker file)ra  r   r   r  r  r  r(   r   r[   r4   r   r   r   r   r_   r   r  )	r   r   r   r  r  r
  r   larger_filer6  s	            r   &test_cp_resumable_download_file_largerz-TestCp.test_cp_resumable_download_file_larger5  su   ""$J!E""j/4,04>>,A # CJ ,,8BC - E$&;S\J	34	5~~
$&8
z
E /0,0  2f mm3V<	!3';t~~)*A


C
  + ( ~~tT*%5u=./,0  2f *F3
mm+V4 
6	5 (' 
6	5s&   AE$/EAE$E!	E$$E-c                    | j                         }| j                         }| j                  |      }|dz   }t        |d      5 }|j	                  dt
        z         ddd       | j                  |ddt
        z        }| j                  d	d
t        |      gd      }t        j                  d|      }| j                  |d       | j                  t        |j                               dd       |j                  d      }	t!        t#        |      t$        j&                  | j(                        }
	 t        |
d      5 }|j	                  |	       ddd       ddt+        t
              f}t-        |g      5  | j                  dt        |      |gdd      }| j/                  d|       | j/                  d|       | j1                  t2        j4                  j7                  |             | j1                  t2        j4                  j7                  |
             | j1                  t2        j4                  j7                  |             ddd       t2        j4                  j9                  |
      rt3        j:                  |
       yy# 1 sw Y   'xY w# 1 sw Y   BxY w# 1 sw Y   YxY w# t2        j4                  j9                  |
      rt3        j:                  |
       w w xY w)aK  Tests that we do not re-download when tracker file matches existing file.

    We only compare size, not contents, so re-download should not occur even
    though the contents are technically different. However, hash validation on
    the file should still occur and we will delete the file then because
    the hashes differ.
    rN  rr  r  abcdNr     efghrn  r  r}  Tr  \s*ETag:\s*(.*)Could not get object ETagr   "Did not match expected single ETagr  r  r   r  Download already completer  )ra  rt  r   r   r   r[   r   r   r   r   r   r[  r   r4  groupsr\  rE   r   rH   rV  r  r   r4   r   rx  r   r   r\  ry  rQ  )r   r   r  r   temp_download_filer  r  r  
etag_matchetagr
  
tracker_fpr
  r   s                 r   *test_cp_resumable_download_content_differsz1TestCp.test_cp_resumable_download_content_differsP  sf    ""$J  "Gw/E*	 #	&"hhv  
' ""j/4,3g,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-q9;AD)*>u*E*9*B*B*.--9$ #&* '&(=s7|L!5 67tJ'7 ?.201   3 	16:<fE(:;<(89:./ 8 
(	)
		"# 
*A 
'	&" '& 87 
(	)
		"# 
*sI   J1J( =J&J( 5CJJ( JJJ( J%!J( (7Kc                    | j                         }| j                         }| j                  |      }dt        z  }|dz   }t	        |d      5 }|j                  |       ddd       | j                  |d|      }| j                  dd	t        |      gd
      }t        j                  d|      }	| j                  |	d       | j                  t        |	j                               dd       |	j                  d      }
t!        t#        |      t$        j&                  | j(                        }t	        |d      5 }|j                  |
       ddd       	 ddt+        t              f}t-        |g      5  | j                  dt        |      |gd
      }| j/                  d|       | j1                  t2        j4                  j7                  |             ddd       t2        j4                  j9                  |      rt3        j:                  |       yy# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   XxY w# t2        j4                  j9                  |      rt3        j:                  |       w w xY w)z>Tests download no-ops when tracker file matches existing file.rN  r;  rr  r   Nr  rn  r  r}  Tr  ry  rz  r   r{  r  r  r  r   rH  r|  )ra  rt  r   r[   r   r   r   r   r   r   r   r[  r   r4  r}  r\  rE   r   rH   rV  r  r   r4   r   rx  r   r   r\  ry  rQ  )r   r   r  r   matching_contentsr~  r  r  r  r  r  r
  r  r
  r   s                  r   *test_cp_resumable_download_content_matchesz1TestCp.test_cp_resumable_download_content_matches  s   ""$J  "Gw/E')*	 $	'2hh ! 
( ""j/4,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-q9;AD)*>u*E*9*B*B*.--9 
	$
t 
%
$&(=s7|L!5 67tJ'7 ?.2   416:(89: 8 
(	)
		"# 
*5 
(	' 
%	$ 87 
(	)
		"# 
*s=   H7HH3 0A H'H3 HH$'H0,H3 37I*c                    | j                         }| j                         }| j                  |dt        z        }| j	                  |ddt        z        }| j                  ddt        |      gd	      }t        j                  d
|      }| j                  |d       | j                  t        |j                               dd       |j                  d      }|dz  }t        t        |      t         j"                  | j$                        }t'        |d      5 }	|	j)                  |       ddd       	 ddt+        t              f}
t-        |
g      5  | j                  dt        |      |gd      }| j/                  d|       t'        |d      5 }|j1                         }| j                  |dt        z  d       ddd       | j3                  t4        j6                  j9                  |             ddd       t4        j6                  j;                  |      rt5        j<                  |       yy# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   XxY w# t4        j6                  j;                  |      rt5        j<                  |       w w xY w)zETests that download overwrites when tracker file etag does not match.r;  rF  r  rx  rn  r  r}  Tr  ry  rz  r   z/Did not match regex for exactly one object ETagnonmatchingr  Nr  r  r   rH  rZ  r~  efghzQFile not overwritten when it should have been due to a non-matching tracker file.)ra  rt  r   r[   r   r   r   r   r   r[  r   r4  r}  r\  rE   r   rH   rV  r  r   r   r   r4   r  r   rx  r   r   r\  ry  rQ  )r   r   r  r   r  r  r  r  r
  r  r
  r   r  r   s                 r   3test_cp_resumable_download_tracker_file_not_matchesz:TestCp.test_cp_resumable_download_tracker_file_not_matches  s9   ""$J  "Gw79JKE""j/4,3g,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-qFHADMD)*>u*E*9*B*B*.--9 
	$
t 
%$&(=s7|L!5 67tJ'7 ?.2   4,f5%ZZ\(


(45  	(89: 8 
(	)
		"# 
*# 
%	$  87 
(	)
		"# 
*sH   H-2I >I+H:96I/I -H7:I	?III 7J	c                     | j                         }| j                  dd      }| j                  ddddt        |      t        |d      g       | j                  dt        |d      |g       y	)
ATests that upload and download of a doubly-gzipped file succeeds.looks-zipped.gzr   r7  r  content-type:application/gzipr   r  r  N)ra  r   r   r   )r   r   r   s      r   test_cp_double_gzipzTestCp.test_cp_double_gzip  sn    ""$J*;fMENN-tTUZ 
 	NND$z5159:r   c                    | j                         }| j                  dd      }| j                  dddddt        |      t        |d	      gd
      }| j                  r%| j                  d|       | j                  d|       n$| j                  d|       | j                  d|       | j                  dt        |d	      |g       y)r  r  r   r7  r   r  r  r   ry  r  TrH  zb'Content-Encoding': b'gzip'z!"contentType": "application/gzip"z'Content-Encoding': 'gzip'zcontentType: 'application/gzip'NrA  )r   r   r   r   s       r   %test_cp_double_gzip_transport_encodedz,TestCp.test_cp_double_gzip_transport_encoded  s    
 ""$J*;fME^^t4dDUZ
 +/  0F 
mm6?
mm7@
mm4f=
mm7@NND$z5159:r   z%TODO(b/293885158) Timeout on Windows.c           
      J     j                         t        j                  d       t        t	         j
                        D cg c]%  }t        j                  t        j                        ' c}      j                  d      }t        j                           j                  d|      } j                  dddt        |      t              g       t        t        dd	
       fd       } |       } j                  | j
                  d        j                         } j                  t!        j"                  t%        dd                  }ddt        t&              f}t)        |g      5   j                  dd|t              t        |      gdd	      }	 j+                  d|	        j+                  d|	       t-        t.        j0                  j3                  ddt4                    }
t7        |      |
kD  xr |
dkD  xr
 t9               }|rt:        j<                  }nt:        j>                  }tA        tC        |      | jD                        } jG                  tH        jJ                  jM                  |              jG                  tH        jJ                  jM                  d|z                j                  dt              t        |      gd      }	 j+                  d|	       tO        |d      5 } jQ                  |jS                         |d       ddd        jU                  tH        jJ                  jM                  |              jU                  tH        jJ                  jM                  d|z               ddd       yc c}w # 1 sw Y   vxY w# 1 sw Y   yxY w)zDTests that download can be resumed successfully with a gzipped file.r   r  zunzipped.txtr7  r   r\  txtrV  r   r  c                     j                  dt              gd      } t        j                  d|       }j	                  |d       j                  t        |j                               dd       t        |j                  d            S )NduTr  z
(\d+)\s+.*zCould not get object sizer   z0Did not match regex for exactly one object size.)
r   r   r   r   r[  r   r4  r}  longr\  )r  
size_matchr  r   s     r   _GetObjectSizez>TestCp.test_cp_resumable_download_gzip.<locals>._GetObjectSize  sz    ~~tT*%56d~Kf99]F3j
:'BC
s:,,./IK*""1%&&r   zCompresed object size was not large enough to allow for a halted download, so the test results would be invalid. Please increase the compressed object size in the test.Fr  rE  r  r  r  Tr  rY  z!Downloading to temp gzip filenamerk  z	%s_.gztmprH  rZ  r   File contents did not match.N)+r   randomseedr   r_   r  choicestringascii_lettersr   r   r   r   rU   r\  rb  r  r  r(   r[   r4   r   rY   r  r  r   r   r4  rB   rH   SLICED_DOWNLOADrV  rE   r   r  r   r   r   r\  r   r   r   rx  )r   r6  r   rB  r  object_sizerd  r  r
  r   sliced_download_thresholdsliced_downloadtrackerfile_typer
  r   r  s   `              @r   test_cp_resumable_download_gzipz&TestCp.test_cp_resumable_download_gzip  s!    ""$J
KKN5:4>>5J5Jf**+5J vg  KKM  >H MFNND$tF|T*5EFG >3' 4' !"KT^^	#$   "F,,8BC - E %&;S\J	34	5~~
$&8
z

v,
 -1./  1f mm3V<
mm7@ #7
++//($FBD#E X)BB 02Q60-/  
*::*33+,@,H,<dmmM
 oobggnn%567
oobggnn[6%9:;~~j!4<
0  Ff
mm'08-KL 
rww~~&678
rww~~kF&:;<I 
6	5;| C 
6	5s,   *N/FN6"NA'NN	NN"c                     t        | d      s-t        j                  dd      }| j                  |      | _        | j                  S )Ntest_data_favicon_filer6  ztests/test_data/favicon.ico.gzrE  )hasattrr9  r:  r   r  )r   r   s     r   _GetFaviconFilezTestCp._GetFaviconFile6  sD     412!!'+KLh$($7$7$7$Jd!&&&r   c                    | j                         }| j                  |d      }| j                         }| j                  dddddt	        |      t	        |      g       t        j                  |      5 }dt               i}t        j                  ||       |d   j                         }ddd       | j                         }| j                  dt	        |      t	        |      g       t        |d	      5 }dt               i}t        j                  ||       |d   j                         }ddd       | j                         y# 1 sw Y   xY w# 1 sw Y   (xY w)
a5  Tests chunked transfer encoded download handling.

    Tests that download works correctly with a gzipped chunked transfer-encoded
    object (which therefore lacks Content-Length) of a size that gets fetched
    in a single chunk (exercising downloading of objects lacking a length
    response header).
    r  )r   r   r  zContent-Encoding:gzipzContent-Type:image/x-iconr   r  Nr   )ra  r   r  r   r   gzipr   rK   rA   CalculateHashesFromContentsr-  r   r   )	r   r   r  input_filenamer  	hash_dictin_file_md5rd  out_file_md5s	            r   !test_cp_download_transfer_encodedz(TestCp.test_cp_download_transfer_encoded?  s7    ""$J""je"LJ))+NNN%t-H$^Z  
>	"b&(#i00Y?e$++-k 
#   "FNND$z*DL9:	fd	r&(#i00Y?u%,,.l 
 	[,/ 
#	" 
	s   36E36EEEc           	         | j                         }d| j                  z  }| j                  |d|      }| j                         }| j                  t	        j
                  t        dd                  }ddt        t              fd	g}t        |      5  | j                  d
d|t        |      |gdd      }| j                  d|       | j                  d
t        |      |gd      }| j                  d|       | j                  d|       t        |d      5 }| j                  |j                         |d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z=Tests that resumble downloads work with check_hashes = never.r;  r  rn  Fr  rE  r  r  r  r   r  r   Tr   rY  rH  rZ  z-Found no hashes to validate object downloadedr   r  N)ra  r  r   r   r  r  r(   r   r[   r4   r   r   r   r   r   r   )	r   r   r   r  r   r  r
  r   r   s	            r   -test_cp_resumable_download_check_hashes_neverz4TestCp.test_cp_resumable_download_check_hashes_neverc  sh    ""$J'H""j/4,4 # 6J !E,,8BC - E &'<c'lK?A	2	3~~
$&8
z
E /0,0  2f mm3V<~~tT*%5u=,0  2f
mm'0
mmCVLt8-KL  
4	3  
4	3s%   BE"E	8E	E	EEc           
         | j                         }| j                  dt        z        }ddt        t              f}| j                  t	        j
                  t        d|                  }t        |g      5  | j                  dd||t        |      gdd	
      }ddd       | j                  d       | j                  d|       y# 1 sw Y   .xY w)zFTests that a not found exception is raised if bucket no longer exists.s   aarE  r  r  r  r   r  Tr   r  NzDeleting bucketzbucket does not exist)ra  r   r[   r   r  r  r  r4   r   r   r   r  s         r   'test_cp_resumable_upload_bucket_deletedz.TestCp.test_cp_resumable_upload_bucket_deleted  s     ""$JG);<E$&;S\J,,fll5aD7F, G 
34	5~~
$&8%
z
 -1./  1f 
6 	MM#V,MM)62 
6	5s   7#CCc                 z   | j                         }| j                  |ddt        z        }| j                         }ddt	        t              fdddt	        t              fdg}t        |      5  | j                  d	t        |      |g       t        t        |      | j                        }|D ]0  }| j                  t        j                  j                  |             2 t        |d
      5 }| j!                  |j#                         dt        z  d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z<Tests that sliced object download works in the general case.r  r  rn  r  r  )r  test_assume_fast_crcmodr  rk  rl  r   r   r[  N)ra  r   r[   r   r   r4   r   r   r?   r   r  rx  r   r   r\  r   r   r   )r   r   r  r   r
  tracker_filenamesr
  r   s           r   test_cp_sliced_downloadzTestCp.test_cp_sliced_download  s"    ""$J""j/4,2W,< # >J !E
 
(#g,75	5s7|D@	 
2	3
nndD,e45 <
u
%t}}6/
(89: 0 t6G#35KL  
4	3  
4	3s%   -A>D1+)D%D1%D.	*D11D:c           
         | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        dd                  }ddt        | j                  dz        fdd	t        | j                        fd
g}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       | j                  t        j                  j                  |dz                t!        t#        |      | j$                        }|D ]0  }| j'                  t        j                  j                  |             2 	 ddd       t        |      5  | j                  dt        |      t        |      gd      }| j)                  d|       | j'                  t        j                  j                  |dz                t+        |d      5 }	| j-                  |	j/                         d| j                  z  d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)z:Tests sliced download works when resumability is disabled.r  r;  rn  Fr  rE  r  r  rk  r  rm  4r   r  Tr   r  not downloaded successfullyrr  NrH  rZ  r   r[  )ra  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r\  r?   r   r  rx  r  r   r   r   
r   r   r  r   r  r
  r   r  r
  r   s
             r   #test_cp_unresumable_sliced_downloadz*TestCp.test_cp_unresumable_sliced_download  s$   
 ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnnq.@*AB	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16:
oobggnnUY%678 <
u
%t}}6/
(89: 0 
4& 
2	3~~j!4;
/t  Ef
*F3
rww~~ei&789t7T^^#;/	1  
4	3' 
4	32  
4	3s2   0CH?A8I?/I.I?II	II c           
      P   | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        dd                  }ddt        | j                        fdd	t        | j                        fd
g}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       t        t        |      | j                        }|D ]0  }| j                  t         j"                  j%                  |             2 | j                  dt        |      |gd      }| j                  d|       t        t        |      | j                        }|D ]0  }| j'                  t         j"                  j%                  |             2 t)        |d      5 }	| j+                  |	j-                         d| j                  z  d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z/Tests that sliced object download is resumable.r  r  rn  Fr  rE  r  r  rk  rl  r   r  Tr   r  r  rH  rZ  r   r[  N)ra  r   r  r   r  r  r(   r   r4   r   r   r   r?   r   r  r   r   r   r\  rx  r   r   r   r  s
             r   test_cp_sliced_download_resumez%TestCp.test_cp_sliced_download_resume  s   
 ""$J""j/4,2T^^,C # EJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: <
u
%t}}6/
'789 0 ~~tT*%5u=,0  2f
mm'0 <
u
%t}}6/
(89: 0 t6DNN#:/	1 5 
4	34 5 
4	3s%   -D#H/H?HH	HH%c           
      r   | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        d                  }ddt        | j                        fddt        | j                        fd	g}t        |      5  | j                  d
d|t        |      t        |      gdd      }| j                  d|       t        t        |      | j                        }|D ]0  }| j                  t         j"                  j%                  |             2 | j                  d
t        |      |gd      }| j                  d|       | j                  d|       t        t        |      | j                        }|D ]0  }| j'                  t         j"                  j%                  |             2 t)        |d      5 }	| j+                  |	j-                         d| j                  z  d       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zDTest sliced download resumability when some components are finished.r  r  rn  r  rE  r  r  rk  rl  r   r  Tr   r  r  rH  rZ  r|  r   r[  N)ra  r   r  r   r  r  r)   r   r4   r   r   r   r?   r   r  r   r   r   r\  rx  r   r   r   r  s
             r   &test_cp_sliced_download_partial_resumez-TestCp.test_cp_sliced_download_partial_resume  s   
 ""$J""j/4,2T^^,C # EJ !E,,A!DE - G 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: <
u
%t}}6/
'789 0 ~~tT*%5u=,0  2f
mm'0
mm/8 <
u
%t}}6/
(89: 0 t6DNN#:/	1 7 
4	36 7 
4	3s%   ,D5H-!/H!H-!H*	&H--H6c           
         | j                         }| j                  |dd| j                  z        }| j                  d      }| j                  t	        j
                  t        dd                  }dd	t        | j                        fdd
t        | j                        fdg}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       | j                  t        j                  j                  |dz                t!        t#        |      | j$                        }|D ]0  }| j                  t        j                  j                  |             2 t'        |dz   d      5 }	|	j)                  d       ddd       | j                  dt        |      |gdd      }| j                  d|       | j                  d|       | j                  d|       t!        t#        |      | j$                        }|D ]0  }| j+                  t        j                  j                  |             2 | j+                  t        j                  j                  |dz                | j+                  t        j                  j                  |             ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z?Tests differing file contents are detected by sliced downloads.r  r  rn  r   rE  Fr  r  r  rk  rl  r   r  Tr   r  r  rr  zr+bs   altered file contentsNrZ  r  zHashMismatchException: crc32c)ra  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r\  r?   r   r  r   r   rx  r  s
             r   .test_cp_sliced_download_resume_content_differsz5TestCp.test_cp_sliced_download_resume_content_differsD  sy   
 ""$J""j/4,2T^^,C # EJ -E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: oobggnnUY%678 <
u
%t}}6/
'789 0 	!5)Q	() * ~~tT*%5u=,0./  1f mm'0
mm:FC
mm3V< <
u
%t}}6/
(89: 0 rww~~ei&789
rww~~e,-M 
4	3& *)' 
4	3s&   /CJ:J-DJ:-J7	2J::Kc           
         | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        dd                  }ddt        | j                        fdd	t        | j                        fdd
t        | j                  dz        fdg}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       ddd       ddt        | j                        fdd	t        | j                        fdd
t        | j                  dz        fdg}t        |      5  | j                  dt        |      |gd      }| j                  d|       | j                  d|       | j                  d|       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zTests sliced download doesn't break when the boto config changes.

    If the number of components used changes cross-process, the download should
    be restarted.
    r  r;  rn  Fr  rE  r  r  rk  %sliced_object_download_component_sizer0  r  r   r  Tr   r  r  Nrg   )r  rm  r  rH  z+Sliced download tracker file doesn't match rn  rZ  )ra  r   r  r   r  r  r(   r   r4   r   r   r   r  rh  s          r   .test_cp_sliced_download_component_size_changedz5TestCp.test_cp_sliced_download_component_size_changed  s    ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK	:	T^^q 	!	#@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: 
4 
(#dnn*=>	5s4>>7JK	:	T^^q 	!	#@ 
2	3~~tT*%5u=,0  2f
mmBFK
mm6?
*F3 
4	3% 
4	3$ 
4	3s   >G#AGGGc           
         | j                         }| j                  |dd| j                  z        }| j                         }| j                  t	        j
                  t        dd                  }ddt        | j                        fdd	t        | j                        fd
g}t        |      5  | j                  dd|t        |      t        |      gdd      }| j                  d|       | j                  t        j                  j                  |dz                t!        t#        |      | j$                        }|D ]0  }| j                  t        j                  j                  |             2 	 ddd       ddt        | j                        fdd	t        | j                  dz        fd
g}t        |      5  | j                  dt        |      |gd      }| j'                  d|       | j)                  t        j                  j                  |dz                D ]0  }| j)                  t        j                  j                  |             2 t+        |d      5 }	| j-                  |	j/                         d| j                  z         ddd       ddd       y# 1 sw Y   ,xY w# 1 sw Y   xY w# 1 sw Y   yxY w)zTests temporary files are not orphaned if sliced download is disabled.

    Specifically, temporary files should be deleted when the corresponding
    non-sliced download is completed.
    r  r;  rn  Fr  rE  r  r  rk  r  r   r  Tr   r  r  rr  NrH  rZ  r   )ra  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r\  r?   r   r  r  rx  r   r   r   r  s
             r   .test_cp_sliced_download_disabled_cross_processz5TestCp.test_cp_sliced_download_disabled_cross_process  sw    ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16:
oobggnnUY%678 <
u
%t}}6/
'789 0 
4( 
(#dnn*=>	5s4>>A;M7NO@ 
2	3~~tT*%5u=,0  2f
*F3
rww~~ei&789 0
(89: 0t7T^^#;<  
4	33 
4	3F  
4	3s2   -CJ8B$J4.J(
J4J%(J1	-J44J=c                 4    dD ]  }| j                  |        y )N)i    i  )start_over_error_test_helper)r   start_over_errors     r   .test_cp_resumable_upload_start_over_http_errorz5TestCp.test_cp_resumable_upload_start_over_http_error  s      ''(89	r   c           
      j   | j                         }t        t        dz        }| j                  |      }ddt	        t
              f}| j                  t        j                  k(  r0| j                  t        j                  t        dd                  }nK| j                  t        j                  k(  r.| j                  t        j                  t        d                  }t        |g      5  | j                  dd	|t!        |      gd
      }| j#                  d|       d d d        y # 1 sw Y   y xY w)Nr0  r~  rE  r  r  r  r  r   r  TrH  zRestarting upload of)ra  rW   r\   r   r   r[   r  r   JSONr  r  r   r  r	  r4   r   r   r   )r   r   r   
rand_charsr   r
  r  r   s           r   r  z#TestCp.start_over_error_test_helper  s   ""$J (gk;J4E$&;S\J}}(((..<< Fq# NO / Q	+//	)..
9!
<9>. ? 
34	5~~
$&8%
z
 -1	  2f
 mm*F3 
6	5	5s   ,4D))D2c                     | j                         }| j                  |dd      }ddt        |      dz   t        |      t        |      dz   g}| j                  |d	       | j                  d
dt        |      z  g       y )Nr  r   rn  r   z-cz/foo2z/dir/r   )r   r   z
%s/dir/foo)ra  r   r   r   )r   r   r  
cp_commands       r   test_cp_minus_czTestCp.test_cp_minus_c  s    ""$J""j/4,2 # 4J
 	Z7"ZZ7"J 	NN:qN1NNFL4
+;;<=r   c           	      D   | j                   t        j                  k(  rt        j                  d      S | j                         }| j                  |dd      }t        t        t        j                         t               | j                        }|j                         }t        j                  |j                   |j"                  j                   |j$                        }t        j                  |j"                  | j'                  d      |j(                        }|j+                  ||       | j-                  |j/                  |j"                  |j                   dd	g
      j0                  |j/                  |j"                  |j                   dd	g
      j0                  d       y)zTests the JSON Rewrite API.&Rewrite API is only supported in JSON.r  rF  rn  )r   bucketcontentTypeobjectr  r   r  customerEncryptionmd5Hashfields;Error: Rewritten object's hash doesn't match source object.N)r  r   r  r;   r  ra  r   r   r   logging	getLoggerr   r   r  apitools_messagesObjectr   r  r  MakeTempNamer  
CopyObjectr   GetObjectMetadatar  )r   r   r  
gsutil_apikeysrc_obj_metadatadst_obj_metadatas          r   test_rewrite_cpzTestCp.test_rewrite_cp  sy   }}']]CDD""$J""j/4,2 # 4J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<LN )//&&x($002 *,<=$$%5%<%<%5%:%:-A-6-8 	% 	9 :A$$%5%<%<%5%:%:-A-6-8 	% 	9 :AG	Ir   c           	         | j                   t        j                  k(  rt        j                  d      S | j                         }| j                  d      }| j                  |ddt        z  dz   d      }t        t        t        j                         t               | j                        }|j                         }t        j                   |j"                  |j$                  j"                  |j&                  |j(                  j+                  d	      
      }| j-                  d      }t        j                   |j.                  ||j0                        }t3        |j$                  |j"                  |j$                  |j"                  | j                         }		 	 |j5                  ||t7        t        dz        j8                  t               | j;                  d       | j?                  t@        jB                  jE                  |	             |j5                  ||tG        t        dz        j8                  t               | jI                  t@        jB                  jE                  |	             | jK                  |jM                  |j$                  |j"                  ddg      jN                  |jM                  |j$                  |j"                  ddg      jN                  d       tQ        |	       y# t<        $ r Y w xY w# tQ        |	       w xY w)zETests the JSON Rewrite API, breaking and resuming via a tracker file.r  rj  rh  r     12rF  Trk  "'r   r  r  r  r  r  rg   progress_callbackmax_bytes_per_callExpected RewriteHaltException.r  r  r  r  N))r  r   r  r;   r  ra  r   r\   r   r   r  r  r   r   r  r  r  r   r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   ry  r   rx  r   r  r  r=   )
r   r   bucket_uri2r  r  r  r  dst_obj_namer  r)  s
             r   test_rewrite_cp_resumezTestCp.test_rewrite_cp_resume:  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
%+..0M")A+1//3t18	 	 	:
 			23
 oobggnn%678 

>w?@@A BBF$$  & rww~~&789


&
&'7'>'>'7'<'</C/8/: ' ; <C7

&
&'7'>'>'7'<'</C/8/: ' ; <C7
I	K )*; " : )*s,   AK* DK: *	K73K: 6K77K: :Lc           	         | j                   t        j                  k(  rt        j                  d      S | j                         }| j                  d      }| j                  |ddt        z  dz   d      }t        t        t        j                         t               | j                        }|j                         }t        j                   |j"                  |j$                  j"                  |j&                  |j(                  j+                  d	      
      }| j-                  d      }t        j                   |j.                  ||j0                        }t3        |j$                  |j"                  |j$                  |j"                  | j                         }		 	 |j5                  ||t7        t        dz        j8                  t               | j;                  d       | j                  |ddd      }
|
j                         }t        j                   |j"                  |j$                  j"                  |j&                  |j(                  j+                  d	      
      }| j?                  t@        jB                  jE                  |	             |j5                  ||t               | jG                  t@        jB                  jE                  |	             | jI                  |jK                  |j$                  |j"                  ddg      jL                  |jK                  |j$                  |j"                  ddg      jL                  d       tO        |	       y# t<        $ r Y w xY w# tO        |	       w xY w)zBTests that Rewrite starts over when the source object has changed.r  rj  rh  r  r  rF  Trk  r  r  r  r  rg   r  r  )r  r  r  r  r  N)(r  r   r  r;   r  ra  r   r\   r   r   r  r  r   r   r  r  r  r   r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   ry  rx  r   r  r  r=   )r   r   r  r  r  r  r  r  r  r)  object_uri2key2src_obj_metadata2s                r   %test_rewrite_cp_resume_source_changedz,TestCp.test_rewrite_cp_resume_source_changed  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
-+..0M")A+1//3t18	 	 	:
 			23 %%27/56: & <k   "d+22yy!!''yyu%	' oobggnn%678 -,/6  8
 rww~~&789


&
&'8'?'?'8'='=/C/8/: ' ; <C7

&
&'7'>'>'7'<'</C/8/: ' ; <C7
I	K )*K " J )*s,   AM E*M 	MM MM M+c           	      T   | j                   t        j                  k(  rt        j                  d      S | j                         }| j                  d      }| j                  |ddt        z  dz   d      }t        t        t        j                         t               | j                        }|j                         }t        j                   |j"                  |j$                  j"                  |j&                  |j(                  j+                  d	      
      }| j-                  d      }t        j                   |j.                  ||j0                        }t3        |j$                  |j"                  |j$                  |j"                  | j                         }		 	 |j5                  ||dt7        t        dz        j8                  t               | j;                  d       | j?                  t@        jB                  jE                  |	             |j5                  ||dt               | jG                  t@        jB                  jE                  |	             |jI                  |j$                  |j"                  g d      }
| jK                  |jI                  |j$                  |j"                  ddg      jL                  |
jL                  d       d}|
jN                  D ]  }|jP                  dk(  sd} | j?                  |d       tS        |	       y# t<        $ r Y =w xY w# tS        |	       w xY w)z:Tests that Rewrite starts over when the arguments changed.r  rj  rh  r  r  rF  Trk  r  r  r  r  privaterg   )
canned_aclr  r  r  r  )r  r  )r  r  r  r  r  r  r  FallUsersz-New object was not written with a public ACL.N)*r  r   r  r;   r  ra  r   r\   r   r   r  r  r   r   r  r  r  r   r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   ry  rx  r  r   r  r  entityr=   )r   r   r  r  r  r  r  r  r  r)  new_obj_metadatafound_public_acl	acl_entrys                r   &test_rewrite_cp_resume_command_changedz-TestCp.test_rewrite_cp_resume_command_changed  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
,+	..)20M")A+1//3t18 	 	: 			23
 oobggnn%678 ,,'4/6  8 rww~~&789#55

!
!


9 6 ; 

&
&'7'>'>'7'<'</C/8/: ' ; <C7

"
"
IK '++)z)!
 , oo&EG )*G " F )*s2   AL
 DL (L 
	LL LL L'*POSIX attributes not available on Windows.zTest requires fast crcmod.c                 b    | j                         }| j                         }t        | ||d       y)zTests use of the -P flag with cp from a bucket to a local dir.

    Specifically tests combinations of POSIX attributes in metadata that will
    pass validation.
    Tr   N)ra  rt  r   )r   r   r   s      r   .test_cp_preserve_posix_bucket_to_dir_no_errorsz5TestCp.test_cp_preserve_posix_bucket_to_dir_no_errors!  s.     ""$J!F&tZtLr   c                     | j                         }| j                         }| j                  |dd      }t        | |||d       y)zTests use of the -P flag with cp from a bucket to a local dir.

    Specifically, combinations of POSIX attributes in metadata that will fail
    validation.
    r   s   objrn  Tr  N)ra  rt  r   r   )r   r   r   r   s       r   +test_cp_preserve_posix_bucket_to_dir_errorsz2TestCp.test_cp_preserve_posix_bucket_to_dir_errors-  sO     ""$J!F


z(-%+  -C %T:sF$Or   c                 @    | j                         }t        | |d       y)z>Tests use of the -P flag with cp from a local dir to a bucket.Tr  N)ra  r   )r   r   s     r   -test_cp_preseve_posix_dir_to_bucket_no_errorsz4TestCp.test_cp_preseve_posix_dir_to_bucket_no_errors<  s     ""$J&tZtDr   c                     | j                  d      }| j                         }| j                  ddd||gdd      }| j                  d	|       y
)zDTest that cp -s operations to a non-cloud destination are prevented.r   rE  r   rG  rg  r   Tr   z9Cannot specify storage class for a non-cloud destination:N)r   rt  r   r   )r   
local_filedest_dirr   s       r   'test_cp_minus_s_to_non_cloud_dest_failsz.TestCp.test_cp_minus_s_to_non_cloud_dest_failsB  s]    $$f$5J!!#H^^T4ZJ,-*.  0F 	MMMr   zNeed Boto version > 2.46.1c                 B   | j                         }| j                  |dd      }t        |      dz   }ddd}|| j                     }| j	                  dd	|t        |      |g       | j	                  d
|gd      }| j                  |d|z  t        j                         y )Nr  r   rn  r  Standard_iAdurable_REDUCED_availability)rn  rq  r   rG  r   Tr  zStorage class:\s+%srI  )ra  r   r   r   r   rK  r   rL  )r   r   r  object2_surinondefault_storage_classri  r  s          r   (test_cp_specify_nondefault_storage_classz/TestCp.test_cp_specify_nondefault_storage_classN  s    ""$J""j/4,2 # 4J 
#e+L
 ,  -T-B-BCMNND$tJ/?NO^^V\2$^GF%%f&<}&L,.MM & ;r   z&Test uses gs-specific storage classes.c                 >   | j                  d      }d}| j                  d|      }t        |      dz   |z   }| j                  ddd	||g       t	        d
g      5  | j                  d|gd      }ddd       | j                  dt        j                         t        |      dz   }| j                  d||g       t	        d
g      5  | j                  d|gd      }ddd       | j                  |dt        j                         t        |      dz   }| j                  ddd	||g       t	        d
g      5  | j                  d|gd      }ddd       | j                  |dt        j                         y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   ExY w)zETests that object storage class is set correctly with and without -s.rH  rh  zfoo-origr   r  r   r   rG  rg  r>  r   Tr  NzStorage class:\s+STANDARDrI  z/foo-nlzStorage class:\s+NEARLINEz/foo-std)ra  r   r   r   r4   rK  r   rL  )r   r   local_fnamelocal_fpathfoo_cloud_surir  foo_nl_surifoo_std_suris           r   'test_cp_sets_correct_dest_storage_classz.TestCp.test_cp_sets_correct_dest_storage_classc  s    """<JK%%v%MK*%+k9NNND$
KHI	?@	A~~v~6d~Kf 
B%%f&B,.MM & ; z"Y.KNND.+67	?@	A~~v{34~Hf 
B%%f&B,.MM & ; 
#j0LNND$
KFG	?@	A~~v|4D~If 
B%%f&B,.MM & ;1 
B	A 
B	A 
B	As$   E:	F9F:FFFc                    | j                         }d}d}t               }| j                  ||d      }t        dd|fg      5  | j	                  dt        |      t        |      d|g       d d d        t        d	g      5  | j                  t        |      d||       d d d        y # 1 sw Y   ?xY w# 1 sw Y   y xY w)
Nr  r  r   rn  r  r  r   r   r>  )ra  r%   r   r4   r   r   rB  )r   r   	obj1_name	obj2_namerD  r  s         r   >test_kms_key_correctly_applied_to_dst_obj_from_src_with_no_keyzETestCp.test_kms_key_correctly_applied_to_dst_obj_from_src_with_no_key  s    ""$JII02G :,5)/   1G 
*:GDE	F
nng:&	245 
G 
?@	A
4
+;Y G ') 
B	A 
G	F 
B	As   ,B,!B8,B58Cc                 X   | j                         }| j                  d      }d}t        |      dz   |z   }t               }t	        dd|fg      5  | j                  d||g       d d d        t	        dg      5  | j                  ||       d d d        y # 1 sw Y   1xY w# 1 sw Y   y xY w)	Nr;  rE  r  r   r  r  r   r>  rA  )r   r   r   r   rC  rD  s         r   9test_kms_key_correctly_applied_to_dst_obj_from_local_filez@TestCp.test_kms_key_correctly_applied_to_dst_obj_from_local_file  s    ""$J1EHJ#%0H02G	*:GDE	F
nndE8,- 
G 
?@	A
'2 
B	A 
G	F 
B	As   B8B B B)c                 |   d}| j                         }| j                  d|z        }d}t        |      dz   |z   }t               }t	        dd|fddt        |      fg      5  | j                  d	||g       d d d        t	        d
g      5  | j                  ||       d d d        y # 1 sw Y   1xY w# 1 sw Y   y xY w)Nr  r  rE  r  r   r  r  r  r   r>  )ra  r   r   r%   r4   r   r   rB  )r   r  r   r   r   rC  rD  s          r   (test_kms_key_works_with_resumable_uploadz/TestCp.test_kms_key_works_with_resumable_upload  s    %""$J0C)CDEHJ#%0H02G	*:GD (*? #$7 8 :; 
< nndE8,-
<
 
?@	A
'2 
B	A
< 
<
 
B	As   !B&
B2&B/2B;c                    | j                         }d}d}t               }t        t        j                        }t	        | j                  ||d|            }t	        |      d|}t        dd|fg      5  | j                  d	||g       d d d        t        d
g      5  | j                  ||       d d d        y # 1 sw Y   1xY w# 1 sw Y   y xY w)Nr  r  )key_namer   r   r   r   kms_key_namer   r  r  r   r>  )	ra  r%   r,   CONSTANT_KEY_NAME2r   r   r4   r   rB  )r   r   r  r  key1_fqnkey2_fqn	obj1_suri	obj2_suris           r   @test_kms_key_correctly_applied_to_dst_obj_from_src_with_diff_keyzGTestCp.test_kms_key_correctly_applied_to_dst_obj_from_src_with_diff_key  s    ""$JII13H1$779HZ&/#)'/ 	 	12I  
+Y7I	*:HEF	G
nndIy12 
H 
?@	A
	84 
B	A	 
H	G 
B	As   6B;C;CCz5Copying KMS-encrypted objects prohibited with XML APIc                 (   | j                         }d}d}t               }t        | j                  ||d|            }t        |      d|}| j	                  d||g       t        dg      5  | j                  |       d d d        y # 1 sw Y   y xY w)Nr  r  r   r'  r   r   r>  )ra  r%   r   r   r   r4   AssertObjectUnencrypted)r   r   r  r  r*  r,  r-  s          r   @test_kms_key_not_applied_to_nonkms_dst_obj_from_src_with_kms_keyzGTestCp.test_kms_key_not_applied_to_nonkms_dst_obj_from_src_with_kms_key  s     ""$JII13HZ&/#)'/ 	 	12I  
+Y7INND)Y/0 
?@	A
""9- 
B	A	As   -BBzWOnly Windows paths need to be normalized to use backslashes instead of forward slashes.c                     | j                         }| j                  |dd       | j                         }| j                  dd|z  t	        |      g       | j                  ddt	        |      z  g       y )Nr   r   )r   r8  r   r   z	%s\./obj1r   z%s/obj1)rt  r   ra  r   r   )r   r  r   s      r   ;test_windows_path_with_back_and_forward_slash_is_normalizedzBTestCp.test_windows_path_with_back_and_forward_slash_is_normalized  so       "Gw&6J""$JNND,0$z2BCD 	NNFIZ(889:r   c                     | j                  g ddd      }| j                  r| j                  d|       y| j                  d|       y)z*Tests that cp -m - anything is disallowed.)r#  r   r  rs  Tr   r  zSWARNING: Using sequential instead of parallel task execution to transfer from stdinzBCommandException: Cannot upload from a stream when using gsutil -mNrX  )r   r   s     r    test_cp_minus_m_streaming_uploadz'TestCp.test_cp_minus_m_streaming_upload  sQ    ^^5*.,-  /F 
mm!"(* mm
N
r   c                    | j                  d      }| j                  d      }| j                  dt        |      |gd      }t	        |d      5 }| j                  |j                         d       d d d        y # 1 sw Y   y xY w)Nr   rE  rF  r   TrH  r   )r   r   r   r   r   r   r   rO  s        r   'test_cp_overwrites_existing_destinationz.TestCp.test_cp_overwrites_existing_destination	  ss    0G0E^^T4=%8^MF	eT	a
qvvx( 
		s   !A::Bc                    d}| j                         }| j                  |      }dt        |d      |g}g }t        d      D ]E  }t	        j
                  | j                  |g      }|j                          |j                  |       G |D cg c]  }|j                          c} | j                  t        t        j                  |            |       y c c}w )Nrw  r  r   r+  rg   r  )rt  ra  r   r_   r  r  r   r  r   r   r   r4  r   listdir)	r   test_file_counttemporary_directoryr   cp_argsthreadsr6  threadts	            r   >test_downloads_are_reliable_with_more_than_one_gsutil_instancezETestCp.test_downloads_are_reliable_with_more_than_one_gsutil_instance  s    O,,."""@JT*c*,?@GG1Xt~~WIFfllnnnV  w!QVVXwS$789?K  s   C)N)r  r  r  r  rC   r  r;  rC  r3   rQ  rT  r]  rg  r#   r{  r  r;   skipIfrV   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$   r  r   r%  r"   r>  rD  rI  rK  rS  rO  re  rl  
skipUnlessr+   rw  r  r{  r  r  r  r*   r  r  r  r  r  r  r  r  r  r  r  r  r!   r  r   r  r  r
  r  r  r!  r$  r  rO  rU  rc  ru  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r4  r9  rE  rM  r    rT  rW  r^  r`  rb  rf  ri  rB   rp  ru  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r  r   r"  r$  r.  r1  r3  r5  r7  r@  r  r   r   r4  r4  c  s   ) '*)BE !) !) !* !*"E  !EF/ G !/: !EF+ G !+> !EF8??:JK"O L G !"OH 8??:DE ; ! F;& 8??:DE ; ! F;( 8??:DE 1 ! F1. !? !? 8??:DE %H ! F%HN 8??:DE K ! FK& 8??:DE ; ! F;.  ! !<:> 8??:GH  ! I$ !( !(T ! !> !7 !7. !+ !+& !:;M < !M* !<=G > !G$ !/G !/Gb& EFN GN` !235 4 !54 !23(  4 !( T !/ !/  !EF'< G !'<R& ! !&= 8|%OP: Q: 8|%OP8== ' (<( Q< 8|%OP Q 8== < =?=?*$L 8
o LMNMN0"1$1$. !@ !@@41 !, !,&8.6 BC# D#J!F 8|%OP! Q!F  < ="="8"8	2: 8??:OQ B !QB
 !: !:2.h !-' !-'^ !> !>8 !- !-< ?@IJ ^, ! K A^,@ ?@IJ  ! K A2 ?@IJ  ! K A2 !- !-@ ?@IJ -- ! K A--^6.606./	/ !8 !8"3, 8??:JKA LA  8??:JK$N L$NL< !	2 !	2 LM 2 ! N2< LM 1 ! N1& LM 2 ! N22 @AD BD8 @A!D B!DF @A!F B!FF/
 23/ 4/( ?@IJ &+ ! K A&+P 23* 4*8 23( 4(, 67P 8P 23? 4?& 236 46 23, 4,: 23= 4=< 23"D 4"DH 23 = 4 =D 23@+ 4@+D 233+ 43+j 343 53  23= 4=" 238??:EF $ ! G 4 $2 8??:EF $ ! G $: ?C$HL LM- N- LM0H N0Hd !4 !4F 8+->@12%J 3@%JN56-$^"$H$$L	; ?@IJ ; ! K A;$ 8??:FG I= ! HI=V'"0H !M !M8 233 43$ 12M 3M: 8+->@12+1 3@+1Z 8+->@12,1 3@,1\ 8+->@12-1 3@-1^ 8+->@126. 3@6.p 8+->@12-4 3@-4^ 8+->@128= 3@8=t 23: 4:4,> I<F+PN+`M+^ 8??:KL8+-/KLM M MM 8??:KLP MP 8??:KLE ME
 *+; ,;( 56#; 7#;J 34) 5)* 343 53 343 53  345 55, 34EF. G 5.& 8;	; !) !) !L !Lr   r4  c                      e Zd ZdZd Zd Zd Zd Z ej                  e
d       ej                  d ej                  d	      
      ej                  j                  edd      d                      Z ej                  e
d       ej                  d ej                  d	      
      ej                  j                  edd      d                      Z ej                  d       ej                  d       ej                  d ej                  ej&                  j(                        
      d                      Zd Zy)TestCpUnitTestszUnit tests for gsutil cp.c                 ~   | j                  d      }| j                  |d      }d|j                         _        | j	                         }| j                  dt        |      |gd      }|j                  d	   }| j                  d
t        |             | j                  |d   d       | j                  d|d          y)4Tests a download with no valid server-supplied hash.rn  ro  r   r_  12345r   Treturn_log_handlerwarningrg   r   zQNon-MD5 etag \(12345\) present for key .*, data integrity checks are not possiblezIntegrity cannot be assuredr   N)ra  r   r  r  rt  
RunCommandr   messagesr   r4  rK  r   )r   r   r  r  log_handlerwarning_messagess         r   testDownloadWithNoHashAvailablez/TestCpUnitTests.testDownloadWithNoHashAvailable%  s     ""D"1J""j6"JJ 'J  "G//$j)97(C59 " ;K"++I6Q,-. 23 	MM/1A!1DEr   c                 L   | j                  d      }| j                  |d      }dt        j                  j                  z   }| j                  t              5 }| j                  dt        |      |g       | j                  t        |      d       ddd       y# 1 sw Y   yxY w)	rF  rn  ro  r   r_  
random_dirr   z%Invalid destination path: random_dir/N)ra  r   r   r   r  assertRaisesr   rK  r   r   r   )r   r   r  destination_patherrors        r   9testDownloadWithDestinationEndingWithDelimiterRaisesErrorzITestCpUnitTests.testDownloadWithDestinationEndingWithDelimiterRaisesError6  s     ""D"1J""j6"JJ#bggkk1			?	+u
oodT*-/?@A
s5z#JK 
,	+	+s   9BB#c                    | j                         }| j                  |dd      }| j                  |dd       | j                         }t        dg      5  | j	                  dt        |      |g       d d d        t        |d      5 }| j                  |j                         d       d d d        y # 1 sw Y   ?xY w# 1 sw Y   y xY w)	Nr  r   rn  zfoo/barrF  r  r   r   )	ra  r   r   r4   rK  r   r   r   r   )r   r   r  r   r   s        r    test_object_and_prefix_same_namez0TestCpUnitTests.test_object_and_prefix_same_nameA  s    ""$J""j/4,2 # 4J 	"+%  ' !E	BC	D
oodT*-u56 
E	eT	a
qvvx( 
	 
E	D		s   B1!B=1B:=Cc                 F   | j                         }| j                  d      }t        dg      5  | j                  d|t	        |      gd      }d d d        j
                  d   }| j                  dt        |             | j                  d	|d
          y # 1 sw Y   IxY w)Nr;  rE  r  r   TrH  rJ  r   z)Found no hashes to validate object uploadr   )	ra  r   r4   rK  r   rL  r   r4  r   )r   r   r   rM  rN  s        r   !test_cp_upload_respects_no_hashesz1TestCpUnitTests.test_cp_upload_respects_no_hashesP  s    ""$J1E	BC	DOOD5$z2B*C7; $ =k 
E #++I6Q,-.MM="1%' 
E	Ds    BB r  z
os.geteuidr   )return_value)newchownT)autospecc                     | j                  d      }t        j                         }t        t        t
        i      |_        t        ||dd       |j                  |t
        d       y )Nr;  rE  entriesFT)	r   r  r  rL   rQ   rf   metadatarR   assert_called_once_withr   
mock_chownr   r   s       r   #test_posix_runs_chown_as_super_userz3TestCpUnitTests.test_posix_runs_chown_as_super_user[  sV     1E

"
"
$C'70CDCLuc5$7&&ugr:r   r   c                     | j                  d      }t        j                         }t        t        t
        i      |_        t        ||dd       |j                          y )Nr;  rE  r_  FT)	r   r  r  rL   rQ   rf   rb  rR   assert_not_calledrd  s       r   *test_posix_skips_chown_when_not_super_userz:TestCpUnitTests.test_posix_skips_chown_when_not_super_usere  sP     1E

"
"
$C'70CDCLuc5$7  "r   zFgslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessaryz/gslib.command.Command._GetProcessAndThreadCountzgslib.command.Command.Apply)specc                     | j                  d      }| j                         }d|_        | j                  d|t	        |      g       |j                  t        t	        |            t        j                  d       |j                  d d d d       y )Nr  r  )rg   rV  r   rF  F)process_countthread_countparallel_operations_overrideprint_macos_warning)	r   ra  rZ  rK  r   rc  r   r   ANY)r   !mock_get_process_and_thread_countmock_trigger_reauthr   r   s        r   test_cp_triggers_reauthz'TestCpUnitTests.test_cp_triggers_reautho  s     %1D""$J59%2OOD4j!123//T*-.	 &==%)!	 > r   c                 J    g d}t        |       | j                  |g d       y )N)z
--flag-keyz
flag-value)rV  r  rV  zdoes-not-exist)ru  )rV  
publicReadrv  )r   r   )r   sub_optss     r   'test_translates_predefined_acl_sub_optsz7TestCpUnitTests.test_translates_predefined_acl_sub_opts  s(    *H+H5X  P Qr   N)r  r  r  r  rO  rU  rW  rY  r;   rA  rV   r   patchMockr  r   rf  ri  r   CommandApplyrs  ry  r  r   r   rD  rD  "  sJ   !F"	L)	' 8??:KL4::l			q 9:::R40; 1 ; M; 8??:KL4::l			q 9:::R40# 1 ; M# 4::NP4::?@4::+$))!6!6799 AP
*Qr   rD  c                       e Zd ZdZd Zy)TestCpShimUnitTestsz Unit tests for shimming cp flagsc                 >   | j                         }| j                  d      }t        ddg      5  t        ddd      5  | j	                  dd	d
dddddddd|t        |      gd      }dj                  |j                  d         }| j                  dj                  t        j                  d      |t        |            |       dj                  |j                  d         }| j                  d|       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr;  rE  )r  use_gcloud_storager  )r  hidden_shim_modedry_runr  fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr   r  rG  ro  r*  rG  z
some-classrM  z-UrV  r  TrH  r/  infozGcloud Storage Command: {} storage cp --ignore-symlinks --no-clobber -r -r --storage-class some-class --print-created-message --skip-unsupported --predefined-acl publicRead {} {}rJ  z%Use the -m flag to enable parallelism)ra  r   r4   r5   rK  r   r   rL  r   rL  r]   _get_gcloud_binary_path)r   r   r   mock_log_handler
info_lines
warn_liness         r   test_shim_translates_flagsz.TestCpShimUnitTests.test_shim_translates_flags  s8   ""$J1E	GHJ 
K 6<)" 	  ??4$dD,dD52

 ?C + D YY/88@A
\\b\b11*=uZ ]"#-	/ YY/88CD
=zJ%	
K 
K	 	
K 
Ks#   DB7D6DD	DDN)r  r  r  r  r  r  r   r   r  r    s    (Kr   r  )T)r  
__future__r   r   r   r   rI  r  r  r9  r  r  r   r  r9  r  r   r   r  r   r  r;   r   apitools.base.pyr	   r   r  r
   boto.exceptionr   r   boto.storage_urir   r6  r   r   r   gslib.cloud_apir   gslib.commands.configr   gslib.commands.cpr   gslib.cs_api_mapr   gslib.daisy_chain_wrapperr   gslib.discard_messages_queuer   gslib.exceptionr   gslib.gcs_json_apir   gslib.parallel_tracker_filer   r   gslib.project_idr   gslib.storage_urlr   gslib.tests.rewrite_helperr   r   r   gslib.tests.testcasetestsr;  gslib.tests.testcase.baser    )gslib.tests.testcase.integration_testcaser!   r"   r#   r$   gslib.tests.utilr%   r&   r'   r  r(   r)   r*   r+   r,   r-   r   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   "gslib.third_party.storage_apitoolsr<   r  gslib.tracker_filer=   r>   r?   gslib.ui_controllerr@   gslib.utilsrA   gslib.utils.boto_utilrB   gslib.utils.constantsrC   rD   gslib.utils.copy_helperrE   rF   rG   rH   gslib.utils.hashing_helperrI   rJ   rK   gslib.utils.metadata_utilrL   gslib.utils.posix_utilrM   rN   rO   rP   rQ   rR   rS   rT   gslib.utils.retry_utilrU   gslib.utils.system_utilrV   gslib.utils.text_utilrW   gslib.utils.unit_utilrX   rY   rZ   r[   r\   r]   r   	six.movesr^   r_   rG  r   r  gslib.testsr`   ra   rb   rc   rd   re   rf   r  r   r   r   r   r   r  r   r	  r  r&  r<  r4  GsUtilUnitTestCaserD  ShimUnitTestBaser  r  r   r   <module>r     s!   ( &  % ' 
      	    	   
   >   7 / -     = J E ( B = + ) 9 F . 2 I D ; ' ' 7 ? ? @ A ? , 8 7 @ ( ) 0 0 * , < - , : 1 2 $ 1 < 1 1 % W 0 8 @ 7 & 6 : & 6 ? B 3 I ? - : + , ( * + = ? 4 ( . 8 + 6 3 ) ) ! 
 !  77	$ +,/,*&-* )Y)xm.`I<X%

6V 6.Pf P,QF QB<6 <0|BLX// |BL~ElQh11 lQ^K(33 Kr   