
    	t                        d dl mZ d dl mZ d dl mZ d dl mZ d dlmZ d dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlZd dlZd dlZd d	lmZ d d
lmZ d dl Z d dl!m"Z" d dl!m#Z# d dl$m%Z% d dl&m'Z( d dl)m*Z* d dl+m,Z,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 e3smd dl6Z6d dl7Z7d Z8d Z9d Z:d Z; e*jx                           e=e*j|                  d      Z? e
j                         ZA e%d       ZB e%d       ZC e%d       ZD e%d       ZEdodZFdpdZGdpd ZHd!ZI e1eI      ZJd"ZK e1eK      ZLd#ZM e1eM      ZNd$ZO e1eO      ZPd%ZQd&ZRd'ZSd(ZTd)ZUd*ZVd+ZWd,ZXd-ZYd.ZZd/Z[d0Z\d1Z]d2Z^d3Z_d4Z`d5Zad6Zbd6ZcdZddZed7Zfd8Zgd9Zhd:Zid;Zjd<Zk G d= d>el      Zmemj                  fd?Zod@ ZpdA Zqej                  j                  dBdCd      xr ej                  j                  dBdDd      Ztej                  j                  dBdEdF      j                  dG      Zv e-       Zwej                  j                  dBdHd      duZxej                  j                  dBdId      duZyej                  j                  dJdKdL      j                         dMk7  Z{dN Z|dO Z} e}       Z~dP ZdQ ZdR Z G dS dTe j                        Z e       Z G dU dVe j
                        ZdWZdX ZdY ZdZ Zd[ Zd\ Zed]        Zedqd^       Zed_        Zed`        Zda Zdb Zdc Zedd        Zde Zdf Z G dg dhel      Z G di djel      Z G dk dlel      Z G dm dne      Zy)r    )absolute_import)print_function)division)unicode_literals)contextmanagerN)urllib)	cStringIO)KmsApi)PopulateProjectId)ResumableDownloadException)ResumableUploadException)LazyWrapper)
posix_util)UsingCrcmodExtensionHasUserSpecifiedGsHost)UTF8)#Base64Sha256FromBase64EncryptionKey)
IS_WINDOWS)MakeHumanReadablec                  :   t        t        j                         D  cg c]  } | j                   c}       }d}|dk  r+||v r|dz  }	 t        j                  |       |dz  }|dk  r+t        d      c c} w # t
        $ r |cY S t        $ r Y t        d      w xY w)Ni  l    rT    zUnable to generate GID for )sortedgrpgetgrallgr_gidgetgrgidKeyErrorOverflowError	Exception)groupall_gidgids      #platform/gsutil/gslib/tests/util.pyGetInvalidGidr$   @   s    ?uell?@G C 

	q 	Sq 

$ 1
221 @"   
 
1
22s   A3A8 8BBBc                      t               } t               }t        |      dk(  r| S t        |      D cg c]
  }|| k7  s	| c}d   S c c}w )Nr   r   )GetPrimaryGidGetUserGroupslenlist)primary_giduser_groupsgs      r#   GetNonPrimaryGidr-   \   sP      /K/K
;1K(=(!A,<A(=a@@=s
   
AAc                  *    t        j                         S N)osgetgid     r#   r&   r&   i   s    99;r3   c                      t        t               gt        j                         D  cg c]%  } t	               | j
                  v s| j                  ' c} z         S c c} w r/   )setr&   r   r   	USER_NAMEgr_memr   )r,   s    r#   r'   r'   l   sJ     "%,,.L.QIK1884K.LM N NLs   A
A
   c                  H    t        j                  t              j                  S r/   )pwdgetpwuidUSER_IDpw_namer2   r3   r#   <lambda>r>   s   s    #,,w"7"?"?r3   c                      t        t        j                         D  cg c]  } | j                   c}       d   dz   S c c} w )Nr   )r   r:   getpwallpw_uid)users    r#   r>   r>   x   s/    fclln=nddkkn=>rBQF=s   <c                      t               S r/   )r$   r2   r3   r#   r>   r>   }       MOr3   c                      t               S r/   )r'   r2   r3   r#   r>   r>      rE   r3   Fc                 f   |rd}n!t         j                  j                  ddd      }ddj                  |      ddg}t        j
                  dddt               z   g|z   | z   }t               st        t        j                        g|z   }n|}|D cg c]  }t        j                  |       c}S c c}w )	zFAdds config options to a list of strings defining a gsutil subcommand.FGSUtiluse_gcloud_storagez-ozGSUtil:use_gcloud_storage={}z#GSUtil:hidden_shim_mode=no_fallbackz--testexceptiontraceszGSUtil:default_project_id=)botoconfiggetboolformatgslibGSUTIL_PATHr   InvokedFromParFilestrsys
executablesix
ensure_str)raw_commandforce_gsutilrI   gcloud_storage_settinggsutil_command#gsutil_command_with_executable_pathparts          r#   GetGsutilCommandr\      s     ,,X7K-24 $++,>?
+	 0$"%6%88   ++. 
	+.s~~+>*?.*P'*8'+N	O+N4#..
+N	OO	Os   B.c                    t         j                  j                         }|r|j                  |       t	               }t        j                  |      D ]0  \  }}t        j                  |      |t        j                  |      <   2 t        t         d      rt         j                  nd}t        j                  | t        j                  t        j                  t        j                  ||      S )zCReturns a subprocess.Popen object for for running a gsutil command.setsidN)stdoutstderrstdinenv
preexec_fn)r0   environcopyupdatedictrT   	iteritemsrU   hasattrr^   
subprocessPopenPIPE)cmdenv_varsrb   envstrkvrc   s          r#   GetGsutilSubprocessrr      s    


#JJx6&mmC da #q 1F3>>! ! $B1ryyt*			#!+!+ *$%/
1 1r3   c           
      x    |Kt         j                  r&t        |t              s+|j	                  t
              }n|j	                  t
              }d|i} fd}t         j                  rd|d<   n&t        j                  d|      }|j                            j                  di |}t         j                  sj                          	 |D cg c]  }t        j                  |       }}|S c c}w # t        $ r@ |D cg c]+  }t        j                  |t        j                  d            - nc c}w }}Y |S w xY w)Ninputc                      t        j                  t        j                   j                        t        j
                         y r/   )r0   killpggetpgidpidsignalSIGKILL)processs   r#   Killz$CommunicateWithTimeout.<locals>.Kill   s"    IIbjj%v~~6r3   ih  timeoutFr2   )rT   PY3
isinstancebytesencoder   	threadingTimerstartcommunicatecancelensure_textUnicodeDecodeErrorlocalegetpreferredencoding)r{   ra   comm_kwargsr|   timerc_outoutputs   `      r#   CommunicateWithTimeoutr      s   

wwu%T"ll4 e% +7 	WW K	OOC&E	KKM
'


,
,%		LLN3895S__V$5E9 
, :	  F 	 ; ;E BCE 
 
,s0   C0 C+'C0 +C0 0D9=0D.-D98D9s,   iMSM9eeXliDZHSBJZO71R98tfeW/+87VXTpk5chGd6Y=s,   4TSaQ3S4U+5oxAbByA7HgIigD51zfzGed/c03Ts2TXc=s,   HO4Q2X28N/6SmuAJ1v1CTuJjf5emQcXf7YriKzT1gj0=s,   U6zIErjZCK/IpIeDS0pJrDayqlZurY8M9dvPJU0SXI8=s,   MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=s   dnF5x6K/8ZZRzpfSlMMM+w==s   barzN7UdGUp1E+RbVvZSTy1R8g==zCrcTMQ==s   bar2zIk4lOfUiA+szcorNIotEMg==zQScXtg==s   bar3z9iW6smjfu9hm0A//VQTQfw==zs0yUtQ==s   bar4zkPCx6uZiUOU7W6E+cDCZFg==zZ4bwXg==s   bar5z758XbXQOVkp8fTKMm83NXA==zle1zXQ==Tz&/tmp/.boto.parallel_upload_test_configzPThis sync will orphan file(s), please fix their permissions before trying again.z$Mode for %s won't allow read access.z+GID for %s doesn't exist on current system.z+UID for %s doesn't exist on current system.z,Insufficient access with uid/gid/mode for %sc                   (    e Zd ZdZdZdZdZdZdZdZ	y)	KmsTestingResourceszCConstants for KMS resource names to be used in integration testing.zus-central1z$keyring-for-gsutil-integration-testsz key-for-gsutil-integration-testsz!key-for-gsutil-integration-tests2zkey-for-gsutil-no-authz-cryptokey-for-gsutil-integration-tests-%d%d%dN)
__name__
__module____qualname____doc__KEYRING_LOCATIONKEYRING_NAMECONSTANT_KEY_NAMECONSTANT_KEY_NAME2"CONSTANT_KEY_NAME_DO_NOT_AUTHORIZEMUTABLE_KEY_NAME_TEMPLATEr2   r3   r#   r   r     s2    K" 8, 9: (@$ Nr3   r   c                 $   t        t        j                               }|j                  t	        d      t
        j                  t
        j                        }|j                  ||       }t        ddd|gd      }t        |      }t        |       |S )zAEnsures test keys exist and that the service agent is authorized.N)locationkms	authorizez-kT)rW   )r
   logging	getLoggerCreateKeyRingr   r   r   r   CreateCryptoKeyr\   rr   r   )key_namekms_apikeyring_fully_qualified_namekey_fully_qualified_namerm   r{   s         r#   "AuthorizeProjectToUseTestingKmsKeyr   )  s     7$$&''!(!6!6&&"33 "7 "5
 %44"H.%d4LM&*	,#$'!	!!r3   c                 D    t        j                  |t        |       z        S )a	  Builds a regex to match a file name for a file that would be orphaned.

  Args:
    obj: Object uri.
    err_str: The error string to search for.

  Returns:
    A regex that will match the file name and with the error text for a file
    that would be orphaned.
  )recompileObjectToURI)objerr_strs     r#   BuildErrorRegexr   <  s     
Gk#..	//r3   c                 f     t         fd|j                         j                  d      D              S )a  Returns set of object name tails.

  Tails can be compared between source and dest, past the point at which the
  command was done. For example if test ran {cp,mv,rsync}
  gs://bucket1/dir gs://bucket2/dir2, the tails for listings from bucket1
  would start after "dir", while the tails for listings from bucket2 would
  start after "dir2".

  Args:
    start_point: The target of the cp command, e.g., for the above command it
                 would be gs://bucket1/dir for the bucket1 listing results and
                 gs://bucket2/dir2 for the bucket2 listing results.
    listing: The listing over which to compute tail.

  Returns:
    Object name tails.
  c              3   :   K   | ]  }|t              d    y wr/   )r(   ).0lstart_points     r#   	<genexpr>zTailSet.<locals>.<genexpr>\  s!     G+FaQs; !+Fs   
)r5   stripsplit)r   listings   ` r#   TailSetr   J  s'    $ 
G7==?+@+@+FG	GGr3   Credentialsaws_access_key_idaws_secret_access_keygs_service_key_file z.p12gs_hostgs_portrH   
prefer_apijsonXMLc                 >    t         rt        j                  |       S d S )Nc                     | S r/   r2   funcs    r#   r>   z!SkipForP12Creds.<locals>.<lambda>r      r3   )HAS_P12_CREDSunittestskipreasons    r#   SkipForP12Credsr   n  s    ==  r3   c                  P    d } t         s		 dd l} | d uS | d uS # t        $ r Y | d uS w xY w)Nr   )r   argcompleteImportError)r   s    r#   _ArgcompleteAvailabler   t  sH    +	 
D	  D	    
	D	  s    	%%c                 P   | j                  dd      } t        t        j                  j	                  |             }t        j                  |d         |d<   |d   j                  d      r|d   dd |d<   t        j                  j                  |      }|j                  dd      }|S )zNormalizes the path component of a URI.

  Args:
    uri: URI to normalize.

  Returns:
    Normalized URI.

  Examples:
    gs://foo//bar -> gs://foo/bar
    gs://foo/./bar -> gs://foo/bar
  zgs://zfile://   z//r   N)	replacer)   r   parseurlparse	posixpathnormpath
startswith
urlunparse)uriparsedunparseds      r#   _NormalizeURIr     s    & 	GY'#%%c*+&  +&)AY$q	!"F1I\\$$V,(i1(	/r3   c                     | j                   s| j                  s| j                  dk(  ry| j                   xs | j                  S )zReturns a the generation for a StorageUri.

  Args:
    uri: boto.storage_uri.StorageURI object to get the URI from.

  Returns:
    Generation string for the URI.
  s3null)
generation
version_idscheme)r   s    r#   GenerationFromURIr     s3     ..CNN
zzT		)3>>)r3   c                 R   t        |       rVdj                  t        j                  j	                  t        j                  j
                  | j                  g|             S t        | t        j                        r/dj                  t        j                  j
                  | g|       S t        j                  | j                        }|r@|D cg c]  }t        j                  |       }}t        dj                  |g|z               }|j                  d      r|dd }|S c c}w )a  Returns the storage URI string for a given StorageUri or file object.

  Args:
    obj: The object to get the URI from. Can be a file object, a subclass of
         boto.storage_uri.StorageURI, or a string. If a string, it is assumed to
         be a local on-disk path.
    *suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo')
               would return the URI for a key name 'foo' inside the given
               bucket.

  Returns:
    Storage URI string.
  z	file://{}/Nr@   )is_filerM   r0   pathabspathjoinnamer   rT   string_typesr   r   r   endswith)r   suffixesr   suffixsuffixes_lists        r#   r   r     s     S\bggoobggll388 /F<D/F G H HS%%&bggll3::;; #;CD8S__V,8MD
#!67
8C 	\\#
cr(C	* Es   D$c                        e Zd Z fdZ xZS )GSMockConnectionc                 D    d|d<   d| _         t        t        |   |i | y )Ngsproviderr   )debugsuperr   __init__)selfargskwargs	__class__s      r#   r   zGSMockConnection.__init__  s)    F:DJ	
D*D;F;r3   )r   r   r   r   __classcell__)r   s   @r#   r   r     s    < <r3   r   c                   L    e Zd Zd
dZddZddZddZddZddZddZ	dd	Z
y)GSMockBucketStorageUriNc                     t         S r/   )mock_connection)r   access_key_idsecret_access_keys      r#   connectzGSMockBucketStorageUri.connect  s    r3   c                 "    | j                         S )z:Dummy implementation to allow parallel uploads with tests.)new_key)r   
componentsheaderss      r#   composezGSMockBucketStorageUri.compose  s    <<>r3   c                      y)NUSr2   r   r  s     r#   get_locationz#GSMockBucketStorageUri.get_location      r3   c                 R    t         j                  j                  j                         S r/   )rJ   r   corsCorsr  s     r#   get_corszGSMockBucketStorageUri.get_cors  s    77<<r3   c                 R    t         j                  j                  j                         S r/   )rJ   r   encryptionconfigEncryptionConfigr  s     r#   get_encryption_configz,GSMockBucketStorageUri.get_encryption_config  s    77##4466r3   c                      y r/   r2   r  s     r#   get_lifecycle_configz+GSMockBucketStorageUri.get_lifecycle_config  r  r3   c                      y r/   r2   r  s     r#   get_website_configz)GSMockBucketStorageUri.get_website_config  r  r3   c                      y r/   r2   r  s     r#   get_versioning_configz,GSMockBucketStorageUri.get_versioning_config  r  r3   )NNr/   )r   r   r   r  r  r  r  r  r  r  r  r2   r3   r#   r  r    s*    7r3   r  TestRemoveSectionc                    t         j                  j                  | |d      }t         j                  j                  |       s7|j	                  | t
        df       t         j                  j                  |        |j	                  | ||f       |!t         j                  j                  | |       yt         j                  j                  | ||       y)a  Sets boto configuration temporarily for testing.

  SetBotoConfigForTest should be called by tests instead of this function.
  This will ensure that the configuration is reverted to its original setting
  using _RevertBotoConfig.

  Args:
    section: Boto config section to set
    name: Boto config name to set
    value: Value to set
    revert_list: List for tracking configs to revert.
  N)	rJ   rK   gethas_sectionappendTEST_BOTO_REMOVE_SECTIONadd_sectionremove_optionr5   )sectionr   valuerevert_list
prev_values        r#   _SetBotoConfigr,    s     {{wd3*		 	 	)!94@AKKG$gtZ01
]KKgt,KKOOGT5)r3   c                 (   g }| D ]e  \  }}}|<|t         k(  r|j                  |       $t        j                  j	                  ||       Et        j                  j                  |||       g |D ]!  }t        j                  j                  |       # y)zReverts boto config modifications made by _SetBotoConfig.

  Args:
    revert_list: List of boto config modifications created by calls to
                 _SetBotoConfig.
  N)r%  r$  rJ   rK   r'  r5   remove_section)r*  sections_to_remover(  r   r)  s        r#   _RevertBotoConfigr0    s}     )gtU}	)	)!!'*!!'40
kkoogtU+ * $gKKw' $r3   c                 B     t        j                          fd       }|S )a`  Decorator for tests that perform file to object transfers, or vice versa.

  This forces the test to run once normally, and again with special boto
  config settings that will ensure that the test follows the parallel composite
  upload and/or sliced object download code paths.

  Args:
    func: Function to wrap.

  Returns:
    Wrapped function.
  c                       | i | t         s*t               rt        g d      5   | i | d d d        y y y # 1 sw Y   y xY w)N))rH   #parallel_composite_upload_threshold1)rH    sliced_object_download_thresholdr4  )rH   %sliced_object_download_max_components3)rH   check_hashesalways)RUN_S3_TESTSr   SetBotoConfigForTest)r   r   r   s     r#   Wrapperz.SequentialAndParallelTransfer.<locals>.Wrapper1  sS     	$&02 ! 	 	df	 	 3<	 	s	   	;A)	functoolswraps)r   r<  s   ` r#   SequentialAndParallelTransferr?  #  s'     ??4  
.r3   c                 ^    i }| D ]%  }|d   |d   |d   }}}||vri ||<   ||||   |<   ' |S )a  Converts the input config list to a dict that is easy to write to a file.

  This is used to reset the boto config contents for a test instead of
  preserving the existing values.

  Args:
    boto_config_list: list of tuples of:
        (boto config section to set, boto config name to set, value to set)
        If value to set is None, no entry is created.

  Returns:
    Dictionary of {section: {keys: values}} for writing to the file.
  r   r   r   r2   )boto_config_listsectionsconfig_entryr(  keyr)  s         r#   _SectionDictFromConfigListrE  C  sZ     (&l'?LO\!_%SGhhw$hw ' 
/r3   c           	         t        |d      5 }t        j                  |       D ]M  \  }}|j                  d|z         t        j                  |      D ]  \  }}|j                  |d|d        O 	 ddd       y# 1 sw Y   yxY w)zFWrites a section dict from _SectionDictFromConfigList to tmp_filename.wz[%s]
z = r   N)openrT   rh   write)section_dicttmp_filenametmp_filer(  key_value_pairsrD  r)  s          r#   _WriteSectionDictToFilerN  \  sl    L#($'MM,$? nnX'(o6*#uc512 7 %@ s   A&A==Bc               #   V   K   t        dg      5  d ddd       y# 1 sw Y   yxY ww)zHSets a dummy project in boto config for the duration of a 'with' clause.)rH   default_project_id
dummy_projN)r;  r2   r3   r#   SetDummyProjectForUnitTestrR  e  s#     
 KLM	 NMMs   )	)&)c              #     K   g }d}	 t        j                  d      \  }}t        j                  |       |r| D ]P  }|d   }t        j
                  r%t        |t              r|j                  t              }t        |d   |d   ||       R t        |d      5 }t        j                  j                  |       ddd       nt        t!        |       |       t#        |      5  d ddd       t%        |       |r	 t        j&                  |       yy# 1 sw Y   FxY w# 1 sw Y   :xY w# t(        $ r Y yw xY w# t%        |       |r&	 t        j&                  |       w # t(        $ r Y w w xY ww xY ww)aA  Sets the input list of boto configs for the duration of a 'with' clause.

  This preserves any existing boto configuration unless it is overwritten in
  the provided boto_config_list.

  Args:
    boto_config_list: list of tuples of:
        (boto config section to set, boto config name to set, value to set)
    use_existing_config: If True, apply boto_config_list to the existing
        configuration, preserving any original values unless they are
        overwritten. Otherwise, apply boto_config_list to a blank configuration.

  Yields:
    Once after config is set.
  Nzgsutil-temp-cfg)prefixr   r   r   rG  )tempfilemkstempr0   closerT   r~   r   r   decoder   r,  rH  rJ   rK   rI  rN  rE  _SetBotoConfigFileForTestr0  removeOSError)rA  use_existing_configrevert_configsrK  tmp_fdboto_config
boto_valuerL  s           r#   r;  r;  n  sV    " .,#++3DEFLHHV)+ ^
77
E*#**40J{1~{1~z%	' * c"h(# #" 89IJ*, 
#<	0 
1 n%
		,  #" 
1	0  	 n%
		,  s   E3BD;  D9)D; "D 'D; /E3=D, E3DD;  D)%D; ,	D85E37D88E3;E0
E E0 	E,)E0+E,,E00E3c              #      K   fd}i | D ]$  }t         j                  j                  |      |<   & 	  ||        d  |       y#  |       w xY ww)z0Sets OS environment variables for a single test.c                     t        j                  |       D ]a  \  }}t        j                  j	                  |      |<   ||t        j                  |<   >|t        j                  v sQt        j                  |= c y r/   )rT   rh   r0   rd   r"  )dict_to_applyrp   rq   
old_valuess      r#   _ApplyDictToEnvironmentz6SetEnvironmentForTest.<locals>._ApplyDictToEnvironment  sX    m,1jjnnQ'jm	


1

?JJqM -r3   N)r0   rd   r"  )env_variable_dictre  rp   rd  s      @r#   SetEnvironmentForTestrg    sU      *aJJNN1%JqM (-.	J'J's   1AA
 	A

AAc              #   l  K   	 t         j                  d   }d}| t         j                  d<   	 d |rt         j                  d<   yt         j                  j                  dd       y# t        $ r d}Y \w xY w# |rt         j                  d<   w t         j                  j                  dd       w xY ww)a  Sets a given file as the boto config file for a single test.

  This function applies only the configuration in boto_config_path and will
  ignore existing configuration. It should not be called directly by tests;
  instead, use SetBotoConfigForTest.

  Args:
    boto_config_path: Path to config file to use.

  Yields:
    When configuration has been applied, and again when reverted.
  BOTO_CONFIGTFN)r0   rd   r   pop)boto_config_pathold_boto_config_env_variableboto_config_was_sets      r#   rY  rY    s      #%::m#<  /"**]*	 ">bjjjjnn]D) 
    ">bjjjjnn]D)s=   B4A( B4A9 7B4(A63B45A66B498B11B4c                      t        j                  d      } g }t        j                  t        j
                        D ]:  \  }}}| j                  |      }|s|j                  |j                  d             < |S )z?Returns a list of the names of the test modules in gslib.tests.z^test_(?P<name>.*)$r   )	r   r   pkgutiliter_modulesgslib_tests__path__matchr$  r    )matchernames_modnamems        r#   GetTestNamesry    sd    JJ-.'
%++K,@,@Ama!gAll1776?# B 
,r3   c                 v    t         j                  rt        | t              S t        | t        j
                        S r/   )rT   PY2r   fileioIOBase)r   s    r#   r   r     s'    WWc4  	C	##r3   c                     t        | t        j                  t        j                  f      r | j	                  dd      j                         S t        dj                  t        |                   )a  Returns a copy of the given name with any invalid characters replaced.

  Args:
    name Union[str, unicode, bytes]: The bucket name to transform into a valid name.

  Returns:
    Union[str, unicode, bytes] The version of the bucket name containing only
      valid characters.
  rv  -z*Unable to format name. Incorrect Type: {0})	r   rT   	text_typebinary_typer   lower	TypeErrorrM   type)r   s    r#   MakeBucketNameValidr    sW     s}}coo67<<S!''))
@GGT
  r3   c              #     K   d}	 t        j                         }| rt        j                  |        	 d | r|rt        j                  |       yyy# t        $ r Y Cw xY w# | r|rt        j                  |       w w w xY ww)a  Changes the working directory for the duration of a 'with' call.

  Args:
    new_working_directory: The directory to switch to before executing wrapped
      code. A None value indicates that no switching is necessary.

  Yields:
    Once after working directory has been changed.
  N)r0   getcwdr[  chdir)new_working_directoryprev_working_directorys     r#   WorkingDirectoryr    s       	YY[
 HH"#'	!7hh%& "8 
 		 !7hh%& "8s=   BA BA! B	ABAB!A>>Bc                  l    t               j                  dd       } | syd| j                  j                  v S )N
__loader__F	zipimport)globalsr"  r   r   )loaders    r#   rP   rP     s1    9==t,&		((33	33r3   c                 F    t               rt        j                  |       S d S )Nc                     | S r/   r2   r   s    r#   r>   z SkipForParFile.<locals>.<lambda>  r   r3   )rP   r   r   r   s    r#   SkipForParFiler    s    ==  r3   c                       e Zd ZdZd Zd Zy)HaltingCopyCallbackHandlerzFTest callback handler for intentionally stopping a resumable transfer.c                      || _         || _        y r/   )
_is_upload_halt_at_byte)r   	is_uploadhalt_at_bytes      r#   r   z#HaltingCopyCallbackHandler.__init__&  s    DO%Dr3   c           
          || j                   k\  rgt        j                  j                  d| j                   dt	        |      dt	        |      d       | j
                  rt        d      t        d      y)aD  Forcibly exits if the transfer has passed the halting point.

    Note that this function is only called when the conditions in
    gslib.progress_callback.ProgressCallbackWithTimeout.Progress are met, so
    self._halt_at_byte is only precise if it's divisible by
    gslib.progress_callback._START_BYTES_PER_CALLBACK.
    zHalting transfer after byte z. r   z transferred.
zArtifically halting upload.Artifically halting download.N)r  rR   r`   rI  r   r  r   r   )r   total_bytes_transferred
total_sizes      r#   callzHaltingCopyCallbackHandler.call+  si     $"4"44	jj01HIZ(*+ 
&'DEE()HII 5r3   Nr   r   r   r   r   r  r2   r3   r#   r  r  #  s    N&
Jr3   r  c                       e Zd ZdZd Zd Zy)#HaltOneComponentCopyCallbackHandlerz=Test callback handler for stopping part of a sliced download.c                      d | _         || _        y r/   )_last_progress_byter  )r   r  s     r#   r   z,HaltOneComponentCopyCallbackHandler.__init__A  s    #D%Dr3   c                     | j                   S| j                   | j                  cxk  r|k  r4n	 || _         yt        j                  j	                  d       t        d      || _         y)zCForcibly exits if the passed the halting point since the last call.NzHalting transfer.
r  )r  r  rR   r`   rI  r   )r   current_progress_bytetotal_size_unuseds      r#   r  z(HaltOneComponentCopyCallbackHandler.callG  s[      ,  4#5#5M8MM  5D 
jj./&'FGG4Dr3   Nr  r2   r3   r#   r  r  >  s    E&5r3   r  c                       e Zd ZdZddZy)
TestParamsak  Allows easier organization of test parameters.

  This class allows grouping of test parameters, which include args and kwargs
  to be used, as well as the expected result based on those arguments.

  For example, to test an Add function, one might do:

  params = TestParams(args=(1, 2, 3), expected=6)
  self.assertEqual(Add(*(params.args)), params.expected)
  Nc                     |
t               n|| _        |
t               n|| _        || _        t        |t         t        f      st        d      t        | j                  t              st        d      y )Nz'TestParam args must be a tuple or list.z TestParam kwargs must be a dict.)tupler   rg   r   expectedr   r)   r  )r   r   r   r  s       r#   r   zTestParams.__init__\  s`    <TDI"N$&DKDMdUDM*?@@dkk4(899 )r3   )NNN)r   r   r   r   r   r2   r3   r#   r  r  P  s    	:r3   r  c                       e Zd ZdZd Zd Zy)CaptureStdoutzContext manager.

  For example, this function has the lines printed by the function call
  stored as a list in output:

  with CaptureStdout() as output:
    function(input_to_function)
  c                 d    t         j                  | _        t               xt         _        | _        | S r/   )rR   r_   _stdoutr	   	_stringio)r   s    r#   	__enter__zCaptureStdout.__enter__q  s"    ::DL"++-CJKr3   c                     | j                  | j                  j                         j                                | `| j                  t
        _        y r/   )extendr  getvalue
splitlinesr  rR   r_   )r   r   s     r#   __exit__zCaptureStdout.__exit__v  s4    KK'')4467CJr3   N)r   r   r   r   r  r  r2   r3   r#   r  r  g  s    
r3   r  )Fr/   )T)
__future__r   r   r   r   
contextlibr   r=  r   r   r0   ro  r   r   r}  ry   rj   rR   rU  r   r   rT   	six.movesr   r	   rJ   crcmodrN   gslib.kms_apir
   gslib.project_idr   mock_storage_servicegslib.cloud_apir   r   gslib.lazy_wrapperr   gslib.teststestsrq  gslib.utilsr   gslib.utils.boto_utilr   r   gslib.utils.constantsr   gslib.utils.encryption_helperr   gslib.utils.system_utilr   gslib.utils.unit_utilr   r   r:   r$   r-   r&   r'   InitializeDefaultModeintSYSTEM_POSIX_MODEDEFAULT_MODEgetuidr<   r6   INVALID_UIDINVALID_GIDUSER_GROUPSr\   rr   r   TEST_ENCRYPTION_KEY1TEST_ENCRYPTION_KEY1_SHA256_B64TEST_ENCRYPTION_KEY2TEST_ENCRYPTION_KEY2_SHA256_B64TEST_ENCRYPTION_KEY3TEST_ENCRYPTION_KEY3_SHA256_B64TEST_ENCRYPTION_KEY4TEST_ENCRYPTION_KEY4_SHA256_B64TEST_ENCRYPTION_KEY_S3TEST_ENCRYPTION_KEY_S3_MD5TEST_ENCRYPTION_CONTENT1TEST_ENCRYPTION_CONTENT1_MD5TEST_ENCRYPTION_CONTENT1_CRC32CTEST_ENCRYPTION_CONTENT2TEST_ENCRYPTION_CONTENT2_MD5TEST_ENCRYPTION_CONTENT2_CRC32CTEST_ENCRYPTION_CONTENT3TEST_ENCRYPTION_CONTENT3_MD5TEST_ENCRYPTION_CONTENT3_CRC32CTEST_ENCRYPTION_CONTENT4TEST_ENCRYPTION_CONTENT4_MD5TEST_ENCRYPTION_CONTENT4_CRC32CTEST_ENCRYPTION_CONTENT5TEST_ENCRYPTION_CONTENT5_MD5TEST_ENCRYPTION_CONTENT5_CRC32CRUN_INTEGRATION_TESTSRUN_UNIT_TESTSr:  USE_MULTIREGIONAL_BUCKETS%PARALLEL_COMPOSITE_UPLOAD_TEST_CONFIGORPHANED_FILEPOSIX_MODE_ERRORPOSIX_GID_ERRORPOSIX_UID_ERRORPOSIX_INSUFFICIENT_ACCESS_ERRORobjectr   r   r   r   r   rK   r"  HAS_S3_CREDSr   r   HAS_NON_DEFAULT_GS_HOSTHAS_GS_HOSTHAS_GS_PORTupperUSING_JSON_APIr   r   ARGCOMPLETE_AVAILABLEr   r   r   MockConnectionr   r  MockBucketStorageUrir  r%  r,  r0  r?  rE  rN  rR  r;  rg  rY  ry  r   r  r  rP   r  r  r  r  r)   r  r2   r3   r#   <module>r     s"    ' %  ' %    	   	 	   
    
        .  6 4 * ! " N & M . 3 38AN #*""$Z1115,BIIK'?@) FH+ 34+ 34+P>1, L G "E#  G "E#  G "E#  G "E#  I 8 ! 9 ", " 9 ", " 9 ", " 9 ", " 9 ",   ! (P %( ; @@"P N& N. !22"&0H* /BDI N/FM  /DbIRRSYZ02 kkoomY=TIkkoomY=TI<!')).E:! ./ <*<<+:: < #$1FF 8 / *0((@23 
 
 + +\ ( (, * *>$& ' '64J J65& 5$: :.D r3   