
                            d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl'm)Z) ddl'm*Z* ddl+m,Z- ddl.m/Z/ ddl.m0Z0 ddl.m1Z1 ddl.m2Z2 ddl.m3Z3 dd l4m5Z5 dd!l4m6Z6 dd"l4m7Z7 dd#l4m8Z8 dd$l4m9Z9 dd%l4m:Z: dd&l4m;Z; dd'l4m<Z< dd(l4m=Z= dd)l4m>Z> dd*l?m@Z@mAZA  e@ eAd+d+d,             dd-lBmCZC e3j                  ZE G d. d/e&      ZF G d0 d1e&      ZGy)2z8Unit tests for parallel upload functions in copy_helper.    )absolute_import)print_function)division)unicode_literalsN)
exceptions)BucketListingObject)BucketListingPrefix)CloudApi)ResumableUploadAbortException)ResumableUploadException)!ResumableUploadStartOverException)ServiceException)CreateOrGetGsutilLogger)DiscardMessagesQueue)CommandException)
GcsJsonApi)ObjectFromTracker)StorageUrlFromString)MockCloudApi)GsUtilUnitTestCase)GSMockBucketStorageUri)SetBotoConfigForTest)unittest)storage_v1_messages)copy_helper)parallelism_framework_util)
posix_util)system_util)hashing_helper)_CheckCloudHashes)_DelegateUploadFileToObject)_GetPartitionInfo) _SelectUploadCompressionStrategy)_SetContentTypeFromFile)ExpandUrlToSingleBlr)FilterExistingComponents)GZIP_ALL_FILES)%PerformParallelUploadFileToObjectArgs)"WarnIfMvEarlyDeletionChargeApplies)add_moveMovedModulemockzunittest.mock)r,   c                      e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Z ej$                  dddddd      Z ej*                  d ej,                   ej0                  e                  d        Zd Zd Zd Zd Z ej*                  d      d        Z ej*                  d       d!        Z ej*                  d       d"        Z d# Z!y$)%TestCpFuncsz7Unit tests for parallel upload functions in cp command.c                    t        ddd      \  }}| j                  d|       | j                  d|       t        ddd      \  }}| j                  d|       | j                  d|       t        ddd      \  }}| j                  d|       | j                  d|       t        ddd      \  }}| j                  d|       | j                  d	|       t        dd
z  dz   ddz  d      \  }}| j                  ddz  dz   |       | j                  d|       t        dd
z  dz   dd      \  }}| j                  d|       | j                  ddz  dz   |       t        ddd      \  }}| j                  d|       | j                  d|       y)z%Tests the _GetPartitionInfo function.i,     
      i-     i+                 d   i  3   2   N)r"   assertEqual)selfnum_componentscomponent_sizes      5platform/gsutil/gslib/tests/test_copy_helper_funcs.pytestGetPartitionInfoz TestCpFuncs.testGetPartitionInfoL   s    (9c2'F$^^R(R( (9c2'F$^^R(R( (9c2'F$^^R(R( (9a'D$^^Q'S.) (9"c'QC9;(=$^^b#g]N3R( (9"c'QB'O$^^R(b#g]N3 (9c2'F$^^Q'R(    c                    t               }| j                  d      }| j                  dd      }t        j                         }d}d}| j                  dd      }t        t        |            }t        | j                  d	|d
|      }	t        |d      5 }
t        |
      }ddd       |j                  t        j                  ||      d       t        |dd||	d||||dd      }| j                  dd      }t        t        |            }t        | j                  d	|d
|      }t        |dd||d||||dd      }| j                  dd      }t        t        |            }t        | j                  d	|d
|      }t        | j                  d      d      5 }
t        |
      }ddd       |j                  t        j                  ||      d       t        |dd||d||||dd      }| j                  dd      }t        t        |            }t        |dd|dd||||dd      }| j                  dd      }t        |d      5 }
t        |
      }ddd       |j                  t        j                  |d      d       ||||||||i}t        |d      t        |d      t        |d      t        |d      g}t        | j                  d	|      }t        ||||      \  }}} |D !cg c]  }!|!d   	 }}!|||fD ]  }"| j!                  |"|v         | j#                  dt%        |             | j#                  |j&                  j(                  |d   j(                         | j#                  dt%        |              t        | j                  d	|d
|      }#| j#                  |#j(                  | d   j(                         y# 1 sw Y   2xY w# 1 sw Y   2xY w# 1 sw Y   xY wc c}!w )z0Tests upload with a variety of component states.bucketfoo   asdf	file_namecontentsContentTypeStorageClassfoo1   1:///rbNrD   namemd5HashrI   r   r7    Ffoo2   2foo4   4   _foo5   5foo6   6)r   MakeTempNameCreateTempFiler   
CreateLockr   strdefault_provideropen#_CalculateB64EncodedMd5FromContentsMockCreateObjectWithMetadataapitools_messagesObjectr(   r   r&   
assertTruer<   lendst_url
url_string)$r=   mock_apibucket_nametracker_filetracker_file_lockcontent_typestorage_classfpath_uploaded_correctlyfpath_uploaded_correctly_urlobject_uploaded_correctly_urlf_infpath_uploaded_correctly_md5args_uploaded_correctlyfpath_not_uploadedfpath_not_uploaded_urlobject_not_uploaded_urlargs_not_uploadedfpath_wrong_contentsfpath_wrong_contents_urlobject_wrong_contents_urlfpath_wrong_contents_md5args_wrong_contentsfpath_remote_deletedfpath_remote_deleted_urlargs_remote_deletedfpath_no_longer_usedfile_md5dst_argsexisting_components
bucket_urlcomponents_to_uploaduploaded_componentsexisting_objects_to_deleteiargno_longer_used_urls$                                       r@   (testFilterExistingComponentsNonVersionedz4TestCpFuncs.testFilterExistingComponentsNonVersionedr   s   ~H##H-K&&&IL2==? !L"M  $22V<@  3  B#7$%$' $8			-E	G%H! 
&	-%H%N" 
.))*;*B*B%,+. 48	 * 9 D !Q(D%r<'u6 ,,v,M1#6H2IJ2--{<NOQ=Aq"8\=,4(  ..$.O3C8L4MN 4			-A	C!D 
d!!4!0$	74!DT!J 
8))*;*B*B!(+* 48	 * 9 @a$<!2|]'u6  ..$.O3C8L4MN?a$<b"m\3Dd  ..$.O	"D	)T4T:h 
*))*;*B*B+;37 * 9
 	!"9-11	H 	2B7.3.3.3	 &'+'<'<k'K LJ %=	&
H%>!.)<=)<A1Q4)<=!#68KL
ooc112 MQ/01,44??(+668Q678-			-A	CD 	'22/2==?u 
.	-> 
8	7. 
*	)4 >s*   O$O O-8O:O O*-O7c                    t               }| j                  d      }|j                  |       d}d}| j                  dd      }t	        j
                         }| j                  dd      }t        t        |            }t        |d	      5 }	t        |	      }
d
d
d
       |j                  t        j                  ||
      d      }t        | j                  d|d|d|j                        }t        |dd|||j                  ||||d
d      }|}t        t        |            }|j                  t        j                  |||
      d      }t        | j                  d|d|d|j                        }t        |dd|||j                  ||||d
d      }| j                  dd      }t        t        |            }t        | j                  d      d	      5 }	t        |	      }d
d
d
       |j                  t        j                  ||      d      }t        | j                  d|d|d|j                        }t        |dd||d||||d
d      }||||i}t!        ||j                        t!        ||j                        t!        ||j                        g}t        | j                  d|      }t#        ||||      \  }}}|D cg c]  }|d   	 }}| j%                  |g|       | j%                  |j&                  j(                  |d   j(                         |j&                  j*                  |j&                  j                  f|j&                  j*                  |j&                  j                  fg} |D ]+  }!| j-                  |!j*                  |!j                  f| v        - | j%                  t/        |       t/        |             y
# 1 sw Y   >xY w# 1 sw Y   xY wc c}w )z2Tests upload with versionined parallel components.rD   rJ   rK   rE   rF   rG   rL   rM   rP   NrQ   rT   rN   rO   #r   r7   FrX   rY   rZ   rU   )r   r_   MockCreateVersionedBucketr`   r   ra   r   rb   rd   re   rf   rg   rh   rc   
generationr(   r   r&   r<   rk   rl   object_nameri   rj   )"r=   rm   rn   rq   rr   ro   rp   rs   rt   rv   rw   object_uploaded_correctlyru   rx   fpath_duplicatefpath_duplicate_urlduplicate_uploaded_correctly duplicate_uploaded_correctly_urlargs_duplicater}   r~   r   object_wrong_contentswrong_contents_urlr   r   r   r   r   r   r   r   expected_to_deleteuris"                                     r@   %testFilterExistingComponentsVersionedz1TestCpFuncs.testFilterExistingComponentsVersioned   sW    ~H##H-K&&{3 !L"M&&&IL2==?  $22V<@  3  B#7$%$' 	&	-%H%N" 
. ( E E  &>)E	G 	 !F !
 %9			-E	"	-	-	/%0! D !Q(D%'@'K'Km\3Dd	 /O.s?/CD#+#H#H  &5)E	G 	 $I $ 
 (<			-E	%	0	0	2(3$ ;A2($//}'u	6N  ..$.O3C8L4MN	d!!4!0$	74!DT!J 
8$AA  &:)A	C 	 B 
 .			-A		)	)	+, @a$<Bm\4( 	!"91H 	27BB	D/:EE	G.0B0M0MN &'+'<'<k'K LJ %=	&
H%>!.)<=)<A1Q4)<=)*,@A,44??(+668.66BB.66AAC)11==)11<<>? *
oos7;MMN *S+,c2L.MNg 
.	-L 
8	7F >s   OO O-O O*c                     t        j                  t              }t        d      }t	        dg      5  t        j                  ||d       d d d        |j                  j                  ddgd	       y # 1 sw Y   )xY w
Nspecgs://bucketGSUtil0trigger_reauth_challenge_for_parallel_operationsTruer4   worker_countrD   locationgs)fieldsprovider)	r,   Mockr
   r   r   r   .TriggerReauthForDestinationProviderIfNecessary	GetBucketassert_called_once_withr=   rm   destination_urls      r@   testReauthChallengeIsPerformedz*TestCpFuncs.testReauthChallengeIsPerformedJ  sv    yyh'H*=9O	N 
 @@
8!5
 ..x7Al8< / >
 
s   A33A<c                     t        j                  t              }t        d      }t	        j
                  ||d       |j                  j                          y )Nr   r   r4   r   )r,   r   r
   r   r   r   r   assert_not_calledr   s      r@   *testReauthChallengeIsNotPerformedByDefaultz6TestCpFuncs.testReauthChallengeIsNotPerformedByDefaultX  sE    yyh'H*=9O>>?GLMO ((*rB   c                     t        j                  t              }t        d      }t	        dg      5  t        j                  ||d       d d d        |j                  j                          y # 1 sw Y   $xY w)Nr   zdir/filer   r4   r   	r,   r   r
   r   r   r   r   r   r   r   s      r@   2testReauthChallengeNotPerformedWithFileDestinationz>TestCpFuncs.testReauthChallengeNotPerformedWithFileDestinationb  se    yyh'H*:6O	N 
 @@
8!5
 ((*
 
   A..A7c                     t        j                  t              }t        d      }t	        dg      5  t        j                  ||d       d d d        |j                  j                          y # 1 sw Y   $xY w)Nr   zgs://bucket*r   r4   r   r   r   s      r@   >testReauthChallengeNotPerformedWhenDestinationContainsWildcardzJTestCpFuncs.testReauthChallengeNotPerformedWhenDestinationContainsWildcardn  se    yyh'H*>:O	N 
 @@
8!5
 ((*
 
r   c                     t        j                  t              }t        d      }t	        dg      5  t        j                  ||d       d d d        |j                  j                          y # 1 sw Y   $xY w)Nr   r   r   r7   r   r   r   s      r@   6testReauthChallengeNotPerformedWithSequentialExecutionzBTestCpFuncs.testReauthChallengeNotPerformedWithSequentialExecutionz  se    yyh'H*=9O	N 
 @@
8!5

 ((*
 
r   c                    t        j                  t              }t        j                  j                  d      |j                  _        t        d      }t        dg      5  | j                  t        j                  j
                  d      5  t        j                  ||d       d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nr   zReauth errorr   r   r4   r   )r,   r   r
   pyu2ferrorsPluginErrorr   side_effectr   r   assertRaisesRegexr   r   r   s      r@   $testReauthChallengeRaisesReauthErrorz0TestCpFuncs.testReauthChallengeRaisesReauthError  s    yyh'H%*\\%=%=n%MH"*=9O	N 
 !!%,,":":NKBBXA	7 L
 
 LK
 
s$    +CB5$C5B>	:CC
c                 (   t        j                  t              }t        |j                  _        t        d      }t        dg      5  t        j                  ||d       d d d        |j                  j                  ddgd	       y # 1 sw Y   )xY wr   )r,   r   r
   	Exceptionr   r   r   r   r   r   r   r   s      r@   &testReauthChallengeSilencesOtherErrorsz2TestCpFuncs.testReauthChallengeSilencesOtherErrors  s    yyh'H%.H"*=9O	N 
 @@
8!5
 ..x7Al8< / >
 
s   BBc                 f   t        t        t        d      t                     }d|j                  _        t        j                  ddidd      }|j                  |      }| j                  t        |t                     d|j                  _        t        j                  ddidd      }|j                  |      }| j                  t        |t                     d|j                  _        t        j                  ddidd      }|j                  |      }| j                  t        |t                     t        j                  ddidd      }|j                  |      }| j                  t        |t                     t        j                  dd	idd      }|j                  |      }| j                  t        |t                     t        j                  dd
idd      }|j                  |      }| j                  t        |t                     t        j                  d      }|j                  |      }| j                  t        |t                     t        j                  d      }|j                  |      }| j                  t        |t                     | j!                  d|j"                         y)zFTests that _TranslateApitoolsResumableUploadException works correctly.	copy_testTstatusi  NFi  i  i  i  zAborting transferzadditional bytes left in streamz&this can happen if a file changes size)r   r   r   r   http"disable_ssl_certificate_validationapitools_exceptions	HttpError*_TranslateApitoolsResumableUploadExceptionri   
isinstancer   r   r   r   TransferErrorassertInreasonr=   
gsutil_apiexctranslated_excs       r@   -testTranslateApitoolsResumableUploadExceptionz9TestCpFuncs.testTranslateApitoolsResumableUploadException  s9   23K@024J :>JOO6

'
'3t
DCJJ3ONOOJ~/?@A9>JOO6

'
'3t
DCJJ3ONOOJ~/GHI9>JOO6

'
'3t
DCJJ3ONOOJ~/GHI

'
'3t
DCJJ3ONOO>#DEG 
'
'3t
DCJJ3ONOO>#DEG 
'
'3t
DCJJ3ONOOJ~/LMN

+
+,?
@CJJ3ONOOJ~/LMN

+
+,M
NCJJ3ONOOJ~/LMNMM: '')rB   c                 
   t        t        t        d      t                     }t	        j
                  d      }|j                  |      }| j                  t        |t                     | j                  d|j                         y)z-Test that StreamExhausted error gets handled.r   zNot enough bytesz]if this issue persists, try deleting the tracker files present under ~/.gsutil/tracker-files/N)r   r   r   r   r   StreamExhaustedr   ri   r   r   r   r   r   s       r@   <testTranslateApitoolsResumableUploadExceptionStreamExhaustedzHTestCpFuncs.testTranslateApitoolsResumableUploadExceptionStreamExhausted  so    23K@024J 
-
-.@
ACJJ3ONOOJ~/LMNMM	23A3H3HJrB   c                 P   t         j                  rt        j                  d      S d}| j	                         }| j                  ||d      }d}t        j                  ||t        j                  j                  z   |z          t        j                  d      }t        j                  |t        j                  j                  z   |z         }d	|j                  _        d
|j                  _        d
|j                  _        t!        dg      5  t#        ||       ddd       | j%                  d|j&                         t        j                  d      }t!        dg      5  t#        ||       ddd       | j%                  d|j&                         y# 1 sw Y   yxY w# 1 sw Y   2xY w)z=Tests that content type is correctly determined for symlinks.z/use_magicfile features not available on Windowss:   <html><body>And you thought I was just text!</body></html>zhtml_in_disguise.txt)tmpdirrI   rH   link_to_realfileNcontentTyper   TF)r   use_magicfiler   ztext/html; charset=us-ascii)r   r   Falsez
text/plain)r   
IS_WINDOWSr   skipCreateTempDirr`   ossymlinkpathsepr,   	MagicMock	IsFileUrlreturn_valueIsStreamIsFifor   r$   r<   r   )r=   surprise_htmltemp_dir_pathtxt_file_path	link_namedst_obj_metadata_mocksrc_url_stubs          r@   testSetContentTypeFromFilez&TestCpFuncs.testSetContentTypeFromFile  sc   ]]LMMQM&&(M''}1>2H ( JM #IJJ}mbggkk9IEF NNt<>>mbggkk.I"+/, -L*.L').L&',L$ 
BC	Dl,AB 
E2*668 !NNt<	CD	El,AB 
F\#8#D#DE 
E	D 
F	Es   FFFF%c                 h   t         j                  j                         }|D ]  \  }}t        j                  d       }t        j                  d|z         }d|j
                  _        d|j                  _        d|j                  _        t        ||       | j                  ||j                          y )Nr   zfile.r   TF)r   COMMON_EXTENSION_RULESitemsr,   r   r   r   r   r   r$   r<   r   )r=   extension_rules	extensionexpected_content_typer   r   s         r@   4testSetsContentTypesForCommonFileExtensionsCorrectlyz@TestCpFuncs.testSetsContentTypesForCommonFileExtensionsCorrectly  s    !88>>@O,;(	("nn>^^)0CDl,0l)+0l().l&l,AB
,.C.O.OP -<rB   i           	      z	time.time)r   )newc           	      ~   t        j                  d      }t        d      }| j                  | j                  t	        j
                  dd      z
  fD ]m  }t        j                  d|      }t        j                  j                  |d      5 }t        |||       |j                  d	d
|j                  d       ddd       o | j                  | j                  t	        j
                  dd      z
  fD ]m  }t        j                  d|      }t        j                  j                  |d      5 }t        |||       |j                  d	d|j                  d       ddd       o | j                  | j                  t	        j
                  dd      z
  fD ]m  }t        j                  d|      }t        j                  j                  |d      5 }t        |||       |j                  d	d|j                  d       ddd       o t        j                  j                  |d      5 }t        j                  d| j                  t	        j
                  dd      z
        }t        |||       |j                          ddd       t        j                  j                  |d      5 }t        j                  d| j                  t	        j
                  dd      z
        }t        |||       |j                          ddd       t        j                  j                  |d      5 }t        j                  d| j                  t	        j
                  dd      z
        }	t        ||	|       |j                          ddd       t        j                  j                  |d      5 }t        j                  d| j                        }
t        ||
|       |j                          ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w# 1 sw Y   xY w# 1 sw Y   %xY w# 1 sw Y   xY w# 1 sw Y   yxY w)zCTests that WarnIfEarlyDeletionChargeApplies warns when appropriate.testzgs://bucket/object      )dayshoursNEARLINE)storageClasstimeCreatedwarnzWarning: moving %s object %s may incur an early deletion charge, because the original object is less than %s days old according to the local system time.nearliner2   NY   COLDLINEcoldlineZ   il  ARCHIVEarchiveim  r7   )r  secondsSTANDARD)loggingLoggerr   _PI_DAYdatetime	timedeltarg   rh   r,   patchobjectr)   assert_called_withrl   r   )r=   test_loggersrc_urlobject_time_createdrecent_nearline_objmocked_warnrecent_archive_objold_nearline_objold_coldline_objold_archive_objnot_old_enough_nearline_objs              r@   &testWarnIfMvEarlyDeletionChargeAppliesz2TestCpFuncs.testWarnIfMvEarlyDeletionChargeApplies  s   
 ..(K"#78G !%dll ( 2 2" E/F  G-44!/BD ::[&1[*74G+6	8&&23=		$ 21 G !%dll ( 2 2" E/F  G-44!/BD ::[&1[*74G+6	8&&23=		$ 21 G !%dll ( 2 22 F/G  H,33 .AC ::[&1[*74F+6	8&&23<		% 21 H 
		;	/;*11!llX%7%7R%KKM )2BKP##% 
0 
		;	/;*11!llX%7%7R%KKM )2BKP##% 
0 
		;	/;)00 llX%7%7S!%LLNo )/;O##% 
0 
		;	/;$5$<$<!t||%=!(2M)46##% 
0	/y 21 21 21 
0	/ 
0	/ 
0	/ 
0	/sW   ,O&+,O3,P !AP APAP'?P3&O0	3O=	 P
	PP$'P03P<c                     d}t         }|D ]h  }t        |d|d      \  }}| j                  |       | j                  |       t        |d|d      \  }}| j                  |       | j                  |       j y )N)zfile://testtest.xmlztest.pyFT)r'   r#   ri   assertFalse)r=   pathsextsr   zippedgzip_encodeds         r@   &testSelectUploadCompressionStrategyAllz2TestCpFuncs.testSelectUploadCompressionStrategyAllY  sz    2ED=
tU$fl
oof
|$=
tT#fl
v
ool# rB   c                     t        dddgd      \  }}| j                  |       | j                  |       t        dddgd      \  }}| j                  |       | j                  |       y )Nr/  Fxmlyaml)r#   ri   r0  r=   r3  r4  s      r@   )testSelectUploadCompressionStrategyFilterz5TestCpFuncs.testSelectUploadCompressionStrategyFilterf  sl    ;EE7E+FLOOF\";EF8U,FLV\"rB   c                 l    t        dddgd      \  }}| j                  |       | j                  |       y )Nr/  Tnot_matching)r#   r0  ri   r9  s      r@   ,testSelectUploadCompressionStrategyComponentz8TestCpFuncs.testSelectUploadCompressionStrategyComponentp  s7    ;D>*D2FLVOOL!rB   c           	         t        j                         }t        j                         |_        d }t        |d|dddd       \  }}| j	                  |d       | j	                  |d       | j                  |j                  j                         y )Nc                       yN)ab rC  rB   r@   DelegateUploadzHTestCpFuncs.testDelegateUploadFileToObjectNormal.<locals>.DelegateUploadz      rB   urlFrA  rB  )r,   r   closer!   r<   ri   called)r=   mock_streamrD  elapsed_timeuploaded_objects        r@   $testDelegateUploadFileToObjectNormalz0TestCpFuncs.testDelegateUploadFileToObjectNormalv  s|    ))+K		K %@{E5%%G!L/ 	\3'_c*OOK%%,,-rB   z	os.unlinkc           	         t        j                         }t        j                         |_        t        j                         }d|_        d }t	        |||dddd       \  }}| j                  |d       | j                  |d       | j                  |j                         | j                  |j                  j                         y )NSamplec                       yr@  rC  rC  rB   r@   rD  zHTestCpFuncs.testDelegateUploadFileToObjectZipped.<locals>.DelegateUpload  rE  rB   TFrA  rB  )r,   r   rG  r   r!   r<   ri   rH  )r=   mock_unlinkrI  mock_upload_urlrD  rJ  rK  s          r@   $testDelegateUploadFileToObjectZippedz0TestCpFuncs.testDelegateUploadFileToObjectZipped  s    ))+K		KiikO"*O %@dE5$%P!L/ 	\3'_c*OOK&&'OOK%%,,-rB   z/gslib.command.concurrent_compressed_upload_lockc           	      r    t        j                         }t        j                         |_         fd}t        |d|dddd       \  }} j	                  |d        j	                  |d        j                  |j                  j                          j                  j                  j                         y )Nc                  P    j                   j                  j                         yr@  )ri   	__enter__rH  	mock_lockr=   s   r@   rD  zMTestCpFuncs.testDelegateUploadFileToObjectGzipEncoded.<locals>.DelegateUpload  s    
ooi))001rB   rF  FTrA  rB  )r,   r   rG  r!   r<   ri   rH  __exit__r=   rW  rI  rD  rJ  rK  s   ``    r@   )testDelegateUploadFileToObjectGzipEncodedz5TestCpFuncs.testDelegateUploadFileToObjectGzipEncoded  s    ))+K		K
 %@{E4%F!L/ 	\3'_c*OOK%%,,-OOI&&--.rB   c           	      r    t        j                         }t        j                         |_         fd}t        |d|dddd       \  }} j	                  |d        j	                  |d        j                  |j                  j                          j                  j                  j                         y )Nc                  P    j                   j                  j                         yr@  )r0  rU  rH  rV  s   r@   rD  zVTestCpFuncs.testDelegateUploadFileToObjectGzipEncodedComposite.<locals>.DelegateUpload  s     
y**112rB   rF  FTrA  rB  )	r,   r   rG  r!   r<   ri   rH  r0  rX  rY  s   ``    r@   2testDelegateUploadFileToObjectGzipEncodedCompositez>TestCpFuncs.testDelegateUploadFileToObjectGzipEncodedComposite  s    ))+K		K
 %@{E4t%E!L/ 	\3'_c*OOK%%,,-Y''../rB   c                 Z    t        j                  dddd      }| j                  d|       y )NTF)file_size_will_changesize)r   GetSourceFieldsNeededForCopyassertNotIn)r=   r   s     r@   1testDoesNotGetSizeSourceFieldIfFileSizeWillChangez=TestCpFuncs.testDoesNotGetSizeSourceFieldIfFileSizeWillChange  s+    55dE7FVV$rB   N)"__name__
__module____qualname____doc__rA   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r,   r   r   r   ConvertDatetimeToPOSIXr-  r5  r:  r=  rL  rR  rZ  r]  rc  rC  rB   r@   r.   r.   I   s1   ?$)Ln?`fOP>+
+
+	+
7> *)X
JF@Q HdAr2q"5'4::k $..@z@@IKLL&LL&\$#". 4::k. .& 4::?@/ A/& 4::?@0 A0&%rB   r.   c                   N   e Zd Z ej                  d       ej                  d      d               Z ej                  d       ej                  d      d               Z ej                  d       ej                  d      d               Z ej                  d       ej                  d      d               Z ej                  d       ej                  d      d               Z	 ej                  d       ej                  d      d	               Z
 ej                  d      d
        Z ej                  d      d        Z ej                  d      d        Z ej                  d      d        Z ej                  d      d        Z ej                  d      d        Zd Zy)TestExpandUrlToSingleBlrzgslib.cloud_api.CloudApiz.gslib.utils.copy_helper.CreateWildcardIteratorc           	          t        d      }t        t        |      g      |_        t	        d|ddt        d            \  }}| j                  |       | j                  ||       y Ngs://test/helloworldgs://test/hello*/
project_idFr   )r   iterr	   r   r%   r   ri   r<   r=   mock_CreateWildcardIteratormock_gsutil_apistorage_urlexp_urlhave_existing_dst_containers         r@   $testContainsWildcardMatchesNotObjectz=TestExpandUrlToSingleBlr.testContainsWildcardMatchesNotObject  sh     ''=>K/3	[	)*0,,-A_lE,..*W) 	OO/0Wk*rB   c           	          t        d      }t        t        |      g      |_        t	        d|ddt        d            \  }}| j                  |       | j                  ||       y rl  )r   rp  r   r   r%   r   r0  r<   rq  s         r@   !testContainsWildcardMatchesObjectz:TestExpandUrlToSingleBlr.testContainsWildcardMatchesObject  sj     ''=>K/3	[	)*0,,-A_lE,..*W) 	01Wk*rB   c           
          t        t        t        d            t        t        d            g      |_        | j	                  t
              5  t        d|ddt        d             d d d        y # 1 sw Y   y xY w)Nrm  zgs://test/helloworld2rn  ro  Fr   )rp  r   r   r   assertRaisesr   r%   r   r=   rr  rs  s      r@   #testContainsWildcardMultipleMatchesz<TestExpandUrlToSingleBlr.testContainsWildcardMultipleMatches  sm     0401GHI01HIJ5 0, 
		+	,. "9+"FH 
-	,	,s   A..A7c           
          t        g       |_        | j                  t              5  t	        d|ddt        d             d d d        y # 1 sw Y   y xY w)Nrn  ro  Fr   )rp  r   r{  r   r%   r   r|  s      r@   testContainsWildcardNoMatchesz6TestExpandUrlToSingleBlr.testContainsWildcardNoMatches  sG     04Bx,			+	,. "9+"FH 
-	,	,s   AAz,gslib.utils.copy_helper.StorageUrlFromStringc           	          t        j                         }d|j                  _        d|j                  _        ||_        t        d|ddt        d            \  }}| j                  |       | j                  ||       y )NT
/home/testro  Fr   )	r,   r   	isFileUrlr   IsDirectoryr%   r   ri   r<   r=   mock_StorageUrlFromStringrs  mock_storage_urlru  rv  s         r@   testLocalFileDirectoryz/TestExpandUrlToSingleBlr.testLocalFileDirectory  sx     yy{.2+04  --=*-Ao|U,..*W) 	OO/0W./rB   c           	          t        j                         }d|j                  _        d|j                  _        ||_        t        d|ddt        d            \  }}| j                  |       | j                  ||       y )NTFr  ro  r   )	r,   r   r  r   r  r%   r   r0  r<   r  s         r@   testLocalFileNotDirectoryz2TestExpandUrlToSingleBlr.testLocalFileNotDirectory  sz     yy{.2+05  --=*-Ao|U,..*W) 	01W./rB   c           	          t        t        j                  dt        j                  j                        g      |j
                  _        t        d|ddt        d            \  }}| j                  |       | j                  |t        d             y )Nfolder/gs://test/folderro  Fr   rp  r
   CsObjectOrPrefixCsObjectOrPrefixTypePREFIXListObjectsr   r%   r   ri   r<   r   r=   rs  ru  rv  s       r@   testNoSlashPrefixExactMatchz4TestExpandUrlToSingleBlr.testNoSlashPrefixExactMatch  s    /3!!)"*"?"?"F"F	H5 0O, .BO\5,..*W) 	OO/0W23EFGrB   c           	          t        t        j                  dt        j                  j                        g      |j
                  _        t        d|ddt        d            \  }}| j                  |       | j                  |t        d             y )Nz
folderone/r  ro  Fr   )rp  r
   r  r  r  r  r   r%   r   r0  r<   r   r  s       r@   testNoSlashPrefixSubstringMatchz8TestExpandUrlToSingleBlr.testNoSlashPrefixSubstringMatch$  s    /3!!,"*"?"?"F"F	H5 0O, .BO\5,..*W) 	01W23EFGrB   c           	      H   t        t        j                  t        j                  d      t        j
                  j                        g      |j                  _        t        d|ddt        d            \  }}| j                  |       | j                  |t        d             y )Nzfolder_$folder$)rR   r  ro  Fr   )rp  r
   r  rg   rh   r  OBJECTr  r   r%   r   ri   r<   r   r  s       r@   testNoSlashFolderPlaceholderz5TestExpandUrlToSingleBlr.testNoSlashFolderPlaceholder1  s    /3!!$$*;<))00	25 0O,
 .BO\5,..*W) 	OO/0W23EFGrB   c           	          t        g       |j                  _        t        d|ddt	        d            \  }}| j                  |       | j                  |t        d             y )Nr  ro  Fr   )rp  r  r   r%   r   r0  r<   r   r  s       r@   testNoSlashNoMatchz+TestExpandUrlToSingleBlr.testNoSlashNoMatch?  s]    /3BxO,-AO\5,..*W) 	01W23EFGrB   c           	          t        t        j                  dt        j                  j                        g      |j
                  _        t        d|ddt        d            \  }}| j                  |       | j                  |t        d             y )Nr  gs://test/folder/ro  Fr   r  r  s       r@   testWithSlashPrefixExactMatchz6TestExpandUrlToSingleBlr.testWithSlashPrefixExactMatchI  s    /3!!)"*"?"?"F"F	H5 0O, .B_lE,..*W) 	OO/0W23FGHrB   c           	          t        g       |j                  _        t        d|ddt	        d            \  }}| j                  |       | j                  |t        d             y )Nr  ro  Fr   )rp  r  r   r%   r   ri   r<   r   r  s       r@   testWithSlashNoMatchz-TestExpandUrlToSingleBlr.testWithSlashNoMatchV  s[    /3BxO,-A_lE,..*W) 	OO/0W23FGHrB   c           
          t        j                  ddg      }t        dg      5  t        d d d  |d       |d             d d d        y # 1 sw Y   y xY w)N
FakeObjectrS   )r   check_hashesneverrA  )rS   rB  )loggerr$  rk   src_obj_metadatadst_obj_metadata)collections
namedtupler   r    )r=   r  s     r@   &testCheckCloudHashesIsSkippedCorrectlyz?TestExpandUrlToSingleBlr.testCheckCloudHashesIsSkippedCorrectly`  sP    ''ykBJ	BC	Dt $ $)3C)@)3C)@	B 
E	D	Ds   AAN)rd  re  rf  r,   r   rw  ry  r}  r  r  r  r  r  r  r  r  r  r  rC  rB   r@   rj  rj    s   4::()4::>?
+ @ *
+ 4::()4::>?
+ @ *
+ 4::()4::>?H @ *H 4::()4::>?H @ *H 4::()4::<=
0 > *
0 4::()4::<=0 > *0 4::()
H *
H 4::()
H *
H 4::()H *H 4::()H *H 4::()
I *
I 4::()I *I	BrB   rj  )Hrg  
__future__r   r   r   r   r  r  r  r   r   apitools.base.pyr   r   gslib.bucket_listing_refr   r	   gslib.cloud_apir
   r   r   r   r   gslib.commandr   gslib.discard_messages_queuer   gslib.exceptionr   gslib.gcs_json_apir   gslib.parallel_tracker_filer   gslib.storage_urlr   gslib.tests.mock_cloud_apir   "gslib.tests.testcase.unit_testcaser   gslib.tests.utilr   r   r   "gslib.third_party.storage_apitoolsr   rg   gslib.utilsr   r   r   r   r   gslib.utils.copy_helperr    r!   r"   r#   r$   r%   r&   r'   r(   r)   sixr*   r+   	six.movesr,   "CalculateB64EncodedMd5FromContentsre   r.   rj  rC  rB   r@   <module>r     s    ? & %  '    	  > 8 8 $ 9 4 = , 1 = , ) 9 2 3 A 3 1 % W # 2 " # & 5 ? 5 D ; 8 < 2 I F % VV_	5 6  55 ${	%$ {	%|bB1 bBrB   