dZddlmZddlmZddlmZddlmZddlZddlmZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZdd lm Z dd lm Z dd l!m"Z"dd l#m$Z$ddl%Z%ddl&Z&dd l'm(Z(dd l'm)Z)ddl'm*Z*ddl'm+Z+ddl'm,Z,ddl'm-Z-ddl'm.Z.ddl'm/Z/ddl'm0Z0ddl'm1Z1ddl'm2Z2ddl'm3Z3ddl4m5Z5ddl6m7Z7ddl6m8Z8ddl6m9Z9ddl6m:Z:ddl6m;Z;ddl6mZ>dd l?m@Z@dd!lAmBZBdd"lAmCZCdd#lAmDZDdd$lEmFZFdd%lGmHZHdd&lGmIZIdd'lGmJZJdd(lGmKZKdd)lGmLZLdd*lMmNZNdd+lMmOZOdd,lPmQZQdd-l&mRZRdd.lSmTZTdd/lSmUZUdd0lSmVZVdd1lSmWZWdd2lXmYZZdd3l[m\Z\dd4l[m]Z]dd5l^m_Z_dd6l^m`Z`dd7l^maZadd8l^mbZbdd9l^mcZcdd:l^mdZddd;l^meZeddl^mhZhdd?l^miZidd@l^mjZjddAlkmlZlddBlkmmZmddClkmnZnddDlkmoZoddElpmqZqddFlpmrZrddGlpmsZsddHlpmtZtddIlpmuZuddJlvmwZwddKlvmxZxddLlymzZzddMlym{Z{ddNlym|Z|ddOl}m~Z~ddPl}mZddQl}mZddRl}mZddSlmZddTlmZddUlmZddVlmZddWlmZddXlmZddYlmZddZlmZdd[lmZdd\lmZdd]lmZdd^lmZdd_lmZdd`lmZddalmZddblmZddclmZdddlmZddelmZddflmZddglmZddhlmZddilmZddjlmZddklmZddllmZddmlmZddnlmZddolmZddplmZddqlmZddrlmZddslmZddtlmZddulmZddvlmZddwlmZej^reZerddlZeejfr eljhndxaeljladyZdzZd{Zd|Zgd}Zed~dZeddZeddZdZdZeZdZdZeejfr eljhndxaeljladddZGddeȫZdZdZddZedgdZ ddZdZdZdZdZdZdZ ddZdZdZdZ ddZdZdZ ddZdZ ddZ ddZ ddZdZdZdZdZ ddZdZ ddZ ddZ ddZdZdZ ddZdZdZddZdZGddeZ ddZ ddZ ddZ ddZ ddZ ddZddZdZ ddZ ddZdezdezdezdĜZdńZdƄZdDŽZ ddȄZGdɄdeZGd˄deȫZGd̈́deȫZGdτdАeZdфZd҄ZdӄZddԄZ dՄZ y)z(Helper functions for copy functionality.)absolute_import)print_function)division)unicode_literalsN) namedtuple) attrgetter)range) protojson)config)AccessDeniedException)ArgumentException)CloudApi)EncryptionException)NotFoundException)PreconditionException) Preconditions)ResumableDownloadException)ResumableUploadAbortException)ResumableUploadException)!ResumableUploadStartOverException)ServiceException)MAX_COMPOSE_ARITY)0DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE)+DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD)-DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE)-DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS)(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)DEFAULT_GZIP_COMPRESSION_LEVEL) ApiSelector)DaisyChainWrapper)CommandException)HashMismatchException)InvalidUrlError)FilePart)GenerateComponentObjectPrefix)ReadParallelUploadTrackerFile)$ValidateParallelCompositeTrackerData))WriteComponentToParallelUploadTrackerFile)WriteParallelUploadTrackerFile)FileProgressCallbackHandler)ProgressCallbackWithTimeout)#ResumableStreamingJsonUploadWrapper) storage_url)ContainsWildcard)GenerationFromUrlAndString)IsCloudSubdirPlaceholder)StorageUrlFromString)storage_v1_messages) FileMessage)RetryableErrorMessage)DeleteDownloadTrackerFiles)DeleteTrackerFile)ENCRYPTION_UPLOAD_TRACKER_ENTRY)GetDownloadStartByte)GetTrackerFilePath)GetUploadTrackerData)#RaiseUnwritableTrackerFileException)ReadOrCreateDownloadTrackerFile)"SERIALIZATION_UPLOAD_TRACKER_ENTRY)TrackerFileType)!WriteDownloadComponentTrackerFile)WriteJsonDataToTrackerFile)parallelism_framework_util) stet_util)temporary_file_util) text_util)GetJsonResumableChunkSize)GetMaxRetryDelay) GetNumRetries)ResumableThreshold)UsingCrcmodExtension)GetCloudApiInstance)GetDownloadSerializationData)DEFAULT_FILE_BUFFER_SIZE)MIN_SIZE_COMPUTE_LOGGING)UTF8) CryptoKeyType)CryptoKeyWrapperFromKey)FindMatchingCSEKInBotoConfig)GetEncryptionKeyWrapper)Base64EncodeHash)"CalculateB64EncodedMd5FromContents)CalculateHashesFromContents)CHECK_HASH_IF_FAST_ELSE_FAIL)CHECK_HASH_NEVER) ConcatCrc32c)GetDownloadHashAlgs)GetMd5)GetUploadHashAlgs)HashingFileUploadWrapper)ObjectIsGzipEncoded) AtomicDict)$CheckMultiprocessingAvailableAndInit)PutToQueueWithTimeout) ATIME_ATTR)ConvertDatetimeToPOSIX)GID_ATTR) MODE_ATTR) MTIME_ATTR)ParseAndSetPOSIXAttributes)UID_ATTR)CheckFreeSpace) GetFileSize)GetStreamFromFileUrl) IS_WINDOWS)AddS3MarkerAclToObjectMetadata)CopyObjectMetadata)DEFAULT_CONTENT_TYPE)ObjectMetadataFromHeaders)PreconditionsFromHeaders)S3MarkerAclFromObjectMetadata) DivideAndCeil)HumanReadableToBytes)MakeHumanReadable)SECONDS_PER_DAY)TEN_MIB)CreateWildcardIterator)managerF_corruptzF/gsutil/tmp/parallel_composite_uploads/for_details_see/gsutil_help_cp/ad PARALLEL_UPLOAD_SALT_TO_PREVENT_COLLISIONS. The theory is that no user will have prepended this to the front of one of their object names and then done an MD5 hash of the name, and then prepended PARALLEL_UPLOAD_TEMP_NAMESPACE to the front of their object name. Note that there will be no problems with object name length since we hash the original name. )crc32ccustomerEncryptionetag generationmd5Hashsize%PerformParallelUploadFileToObjectArgszfilename file_start file_length src_url dst_url canned_acl content_type storage_class tracker_file tracker_file_lock encryption_key_sha256 gzip_encoded%PerformSlicedDownloadObjectToFileArgszicomponent_num src_url src_obj_metadata_json dst_url download_file_name start_byte end_byte decryption_key!PerformSlicedDownloadReturnValueszKcomponent_num crc32c bytes_transferred component_total_size server_encodingi GZIP_ALL_FILESi` lz text/markdownzapplication/gzip)mdtgzceZdZdZy)FileConcurrencySkipErrorz@Raised when skipping a file due to a concurrent, duplicate copy.N__name__ __module__ __qualname____doc__*platform/gsutil/gslib/utils/copy_helper.pyrr+sHrrcL|jjt|y)9Simple exception handler to allow post-completion status.N)loggererrorstrclses r_RmExceptionHandlerr/s**3q6rc|jjt||xjdz c_|jj dt j y)rz* Encountered exception while copying: %s N)rrrop_failure_countdebug traceback format_excrs r_ParallelCopyExceptionHandlerr4sJ**3q6!**C'')+rc8t|j|j|j}t ||}|5d}t j |jj|jj|j|j}|j} tjrtj |_ t#|j$||j|j||||j&|t(ddd|j*}tjr||_ dddd} t-|j.|j0| |j&|j2|S#tjr||_ wwxYw#1swYexYw) aOFunction argument to Apply for performing parallel composite uploads. Args: cls: Calling Command class. args: PerformParallelUploadFileToObjectArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: StorageUrl representing a successfully uploaded component.  thread_stateN)namebucket contentType storageClassFT) gzip_extsallow_splitting is_component gzip_encodedencryption_key_sha256)r$filename file_start file_lengthrJapitools_messagesObjectdst_url object_name bucket_name content_type storage_class prefer_apiglobal_copy_helper_opts canned_aclrXML_UploadFileToObjectsrc_urlrrrr( tracker_filetracker_file_lockr) rargsrfp gsutil_api preconditionsdst_object_metadataorig_prefer_apiret components r"_PerformParallelUploadFileToObjectr<sf t0@0@A""3\B* M,22 \\ % %||''%%'' ) !++O0 + +!,   " $ 0 0 $ 3 - * # # =*.05-1-1->-> @c ! + + / G J!f)+   jj 66 8 * ! + + /  ,E rs%A$F A4E3F3F  FFCopyHelperOpts perform_mv no_clobber daisy_chainread_args_from_stdin print_ver use_manifest preserve_aclrskip_unsupported_objectstest_callback_filedest_storage_classc :t|||||||||| |  atS)z9Creates CopyHelperOpts for passing options to CopyHelper.r)rrrs rCreateCopyHelperOptsrs8+/7++ - ! rctS)z.Returns namedtuple holding CopyHelper options.)rrrrGetCopyHelperOptsrs  ! rcpd}|jr`|jtjk(rd} tj|jj }t j |rd}|rtjjStjjS#t$rYAwxYw)zGet download strategy based on the destination object. Args: dst_url: Destination StorageUrl. Returns: gsutil Cloud API DownloadStrategy. FT) IsFileUrlrosdevnullstatst_modeS_ISCHROSErrorrDownloadStrategyONE_SHOT RESUMABLE)rdst_is_specialmodes r_SelectDownloadStrategyrs. bjj(n WWW(( ) 1 1d d   $ $ - --  $ $ . ..    sAB)) B54B5c|jr|jr"|jr"|jr|st d|zyyy)aEnsures the destination URL names a container. Acceptable containers include directory, bucket, bucket subdir, and non-existent bucket subdir. Args: exp_dst_url: Wildcard-expanded destination StorageUrl. have_existing_dst_container: bool indicator of whether exp_dst_url names a container (directory, bucket, or existing bucket subdir). command_name: Name of command making call. May not be the same as the calling class's self.command_name in the case of commands implemented atop other commands (like mv command). Raises: CommandException: if the URL being checked does not name a container. zuDestination URL must name a directory, bucket, or bucket subdirectory for the multiple source form of the %s command.N)r IsDirectory IsCloudUrlIsBucketr!) exp_dst_urlhave_existing_dst_container command_names rInsistDstUrlNamesContainerrs_${'>'>'@K$8$8$: & <>JK LL '%;rc>|ry|jr |xs|xr|Sy)a"Checks whether dst_url should be treated as a bucket "sub-directory". The decision about whether something constitutes a bucket "sub-directory" depends on whether there are multiple sources in this request and whether there is an existing bucket subdirectory. For example, when running the command: gsutil cp file gs://bucket/abc if there's no existing gs://bucket/abc bucket subdirectory we should copy file to the object gs://bucket/abc. In contrast, if there's an existing gs://bucket/abc bucket subdirectory we should copy file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc exists, when running the command: gsutil cp file1 file2 gs://bucket/abc we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). Finally, for recursive copies, if the source is a container then we should copy to a container as the target. For example, when running the command: gsutil cp -r dir1 gs://bucket/dir2 we should copy the subtree of dir1 to gs://bucket/dir2. Note that we don't disallow naming a bucket "sub-directory" where there's already an object at that URL. For example it's legitimate (albeit confusing) to have an object called gs://bucket/dir and then run the command gsutil cp file1 file2 gs://bucket/dir Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: have_multiple_srcs: Bool indicator of whether this is a multi-source operation. dst_url: StorageUrl to check. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. src_url_names_container: bool indicator of whether the source URL is a container. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. TN)r)have_multiple_srcsrhave_existing_dest_subdirsrc_url_names_containerrecursion_requesteds r _ShouldTreatDstUrlAsBucketSubDirrs2X    > $ <)<?rc|r|ry|jr|j S| xr| xr|jS)avChecks that dst_url names a single file/object after wildcard expansion. It is possible that an object path might name a bucket sub-directory. Args: src_url_names_container: Bool indicator of whether the source for the operation is a container (bucket, bucket subdir, or directory). have_multiple_srcs: Bool indicator of whether this is a multi-source operation. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. dst_url: StorageUrl to check. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. F)rrIsObject)rrrrrs r_ShouldTreatDstUrlAsSingletonrsR(4  ""$ $$" " +D'D      rc>|jsy|jj|j}|j |j\}}}|t j vxr0|t j Dcgc]}|jdz|zc}vScc}w)acReturns True if not FileUrl ending in relative path symbols. A URL is invalid if it is a FileUrl and the parent directory of the file is a relative path symbol. Unix will not allow a file itself to be named with a relative path symbol, but one can be the parent. Notably, "../obj" can lead to unexpected behavior at the copy destination. We examine the pre-recursion "url", which might point to "..", to see if the parent is valid. If the user does a recursive copy from a URL, it may not end up the final parent of the copied object. For example, see: "dir/nested_dir/obj". If you ran "cp -r dir gs://bucket" from the parent of "dir", then the "url" arg would be "dir", but "nested_dir" would be the parent of "obj". This actually doesn't matter since recursion won't add relative path symbols to the path. However, we still return if "url" is valid because there are cases where we need to copy every parent directory up to "dir" to prevent file name conflicts. Args: url: StorageUrl before recursion. Returns: Boolean indicating if the "url" is valid as a parent directory. Tz://)rversionless_url_stringrstripdelim rpartitionr-RELATIVE_PATH_SYMBOLSscheme)url#url_string_minus_trailing_delimiter_after_last_delimitersymbols r_IsUrlValidParentDirr3s2  (+(B(B(I(I ii)% C N N ii!!Q [%F%F F #99#9f **u v %9# #s=Bc |jr|jr |r tdt|||||r|S|jrF|js|j r&|r"|jrdnd} td| z|S|sh|sf|j j |jd} t|jj|j|j| S|s"|s |jrX|jrH|jj|js#t|j|j}t|} | s|rtdj|d|j vxr | xr|xs|} | s"|r~|j#s|jr^t%||} |j t'| dj)|j}| sG|j+|jd}n(|j j |jd}|jst-|||||r|j r[|j j|jr6|j j|j|j|}n0|j r |jnd }|j xsd ||}|j/}|j1|j|j|_|S) aQConstructs the destination URL for a given exp_src_url/exp_dst_url pair. Uses context-dependent naming rules that mimic Linux cp and mv behavior. Args: src_url: Source StorageUrl to be copied. exp_src_url: Single StorageUrl from wildcard expansion of src_url. src_url_names_container: True if src_url names a container (including the case of a wildcard-named bucket subdir (like gs://bucket/abc, where gs://bucket/abc/* matched some objects). have_multiple_srcs: True if this is a multi-source request. This can be true if src_url wildcard-expanded to multiple URLs or if there were multiple source URLs in the request. has_multiple_top_level_srcs: Same as have_multiple_srcs but measured before recursion. exp_dst_url: the expanded StorageUrl requested for the cp destination. Final written path is constructed from this plus a context-dependent variant of src_url. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. recursion_requested: True if a recursive operation has been requested. preserve_posix: True if preservation of posix attributes has been requested. Returns: StorageUrl to use for copy. Raises: CommandException if destination object name not specified for source and source is a stream. z/Cannot preserve POSIX attributes with a stream.stream named pipez2Destination object name needed when source is a %szmPresence of multiple top-level sources and invalid expanded URL make file name conflicts possible for URL: {}z**N)rIsStreamr!rIsFiforrrr1 url_stringrrendswithrr#formatrrGetPathBeforeFinalDirlenlstrip partitionrClonereplace)r exp_src_urlrrhas_multiple_top_level_srcsrrrpreserve_posix type_textsrc_final_compsrc_url_is_valid_parentpreserve_src_top_level_dirssrc_url_path_sans_final_dir dst_key_namernew_exp_dst_urls rConstructDstUrlr[s>N+"6"6"8^ L MM"#:K19 %@  99? JJR"&W-K-K!K"=!8"="=#<";!%<&1&<&<&>&1&=&=&? #8"M55 '()++16'--+@ '!++GMM:2>l**55gmmDRHL!A+'@2"4;#:#:#C#C$!,!8!8!?!?   ")//?l$/#:#:ke!,!8!8"$!#"$%*L:l %%'/ , 4 4W]]5@5F5F!H/ rci}|rN|D]I}tj||jjdj d||<K|S)N ascii)base64 b64encodedigestrdecode) digestersdigestsalgs r_CreateDigestsFromDigestersr*sS '%% C.   !##)6%=cl .rc i}d|vr t|d<d|vr"tjjd|d<t |d5}t ||t |jt||djdddi}tj|D]!\}} t| j||<#|S#1swYFxYw) aCreates a base64 CRC32C and/or MD5 digest from file_name. Args: status_queue: Queue for posting progress messages for UI/Analytics. algs: List of algorithms to compute. file_name: File to digest. src_url: StorageUrl for local object. Used to track progress. src_obj_metadata: Metadata of source object. Returns: Dict of algorithm name : base 64 encoded digest md5rzzcrc-32crbHashing)roperation_name)callback_processorN)rZcrcmod predefinedCrcopenrUr+rr*callsix iteritemsrS hexdigest) status_queuealgs file_namersrc_obj_metadata hash_dictrr(alg_namer%s r_CreateDigestsFromLocalFiler? s) d]xIe  ++// :Ih It )3N$4$9$9$?(4077@%BCG$ 4HI '-- 2h()9)9);@++ !  "25C)5*6?7Y+N P QQ K  kk 89@B  kk 89BD rcft|txs t|txrd|jvS)aRChecks to see if the server attempted to clobber a file. In this case we specified via a precondition that we didn't want the file clobbered. Args: e: The Exception that was generated by a failed copy operation Returns: bool indicator - True indicates that the server did attempt to clobber an existing file. 412)rGrrmessage)rs rIsNoClobberServerExceptionrds4 a. / K a1 2 Iu 7ILrch|jry|j}tjj |}tjj |rt d|jd|dtjj|rt d|jd|dy)aChecks whether copying exp_src_url into dst_url is not possible. This happens if a directory exists in local file system where a file needs to go or vice versa. In that case we print an error message and exits. Example: if the file "./x" exists and you try to do: gsutil cp gs://mybucket/x/y . the request can't succeed because it requires a directory where the file x exists. Note that we don't enforce any corresponding restrictions for buckets, because the flat namespace semantics for buckets doesn't prohibit such cases the way hierarchical file systems do. For example, if a bucket contains an object called gs://bucket/dir and then you run the command: gsutil cp file1 file2 gs://bucket/dir you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: exp_src_url: Expanded source StorageUrl. dst_url: Destination StorageUrl. Raises: CommandException: if errors encountered. NzCannot retrieve z> because a file exists where a directory needs to be created (z).z because a directory exists (z%) where the file needs to be created.) rrrpathdirnameisfiler!r isdir)rrdst_path final_dirs rCheckForDirFileConflictrls2   (ggooh')WW^^I '22I? @@WW]]8 '22H> ??rc Fttjddt} t |t | \} }i}g}t | D]}tjt|jz}t}|j||j}|tz|zdzt|z}|j!}||_|| dz kr|}n || dz |zz }||z}t%|j||||||||| | | }|j'||||<|S)aPartitions a file into FilePart objects to be uploaded and later composed. These objects, when composed, will match the original file. This entails splitting the file into parts, naming and forming a destination URL for each part, and also providing the PerformParallelUploadFileToObjectArgs corresponding to each part. Args: canned_acl: The user-provided canned_acl, if applicable. content_type: content type for the component and final objects. dst_bucket_url: CloudUrl for the destination bucket. file_size: The size of fp, in bytes. fp: The file object to be partitioned. random_prefix: The randomly-generated prefix used to prevent collisions among the temporary component names. src_url: Source FileUrl from the original command. storage_class: storage class for the component and final objects. tracker_file: The path to the parallel composite upload tracker file. tracker_file_lock: The lock protecting access to the tracker file. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: dst_args: The destination URIs for the temporary component objects. rA(parallel_composite_upload_component_sizerr)rsr rDr_GetPartitionInforr r6rEPARALLEL_UPLOAD_STATIC_SALTrrZupdater8PARALLEL_UPLOAD_TEMP_NAMESPACErrrrappend)rrdst_bucket_url file_sizer random_prefixrrrrrrrnnum_componentscomponent_sizedst_args file_namesi encoded_name content_md5r%temp_file_name tmp_dst_urlfile_part_lengthoffset func_argss r_PartitionFilersVJ.B jjEAC.D*&7"$L&N">>(*  a $$%@277%JKL(K|$  " " $F#&DDvMF#N &&(K,KNQ '$(:n'LM  F5 )7Km\3D|-In%(H^7 !: /rcjt|j|jjddzdS)zGets component number from component CloudUrl. Used during parallel composite upload. Args: component: CloudUrl representing component. Returns: component number rrN)intrrfind)rs r_GetComponentNumberr7s3 Y " "9#8#8#>#>s#Ca#G#H I JJrc  tj} t|j} |j|j}t t }|r |jnd}t|tj||}t|| \}}}t|||||| || tt \}}||jdnd}||n t!|}t#||||t%j&}i}t)||j*| |||||j,||||  }t/||| |\}}}|D]}t0j2|j4f||j6j8<t;|j<t1|j>|j6tj|j4dtA|j6t0j2|D]}|djB}t0jD|d f||<t;|j<t1||dtjd|d tA|dt0jD |D]F}t;|j<t1||tjdt0jF H|jItJ|| d tLjNjP|jRjTd }g}|D]}|jW|d||D cgc]} | d c} z}!tY|!tY|k(rt[|!t@}!g}"|!D]g}#t\j^ja|#jb}$|#jertg|#jh|$_4|"jW|$i|jk|"|||jgd|}% |!|z}&|jIt|&ttLjNjP|jRjT|!D]\}|jB} t;|j<t1||tjd tA|||d ||d^|D]F}t;|j<t1||tjd t0jF H |5tu|dddn twdtj| z }'|'|%fScc} w#t;|j<t1||tjd Y'xYw#tl$r4| joddjq|jszYwxYw#1swYxYw#|5tu|dddw#1swYwxYwxYw)aUploads a local file to a cloud object using parallel composite upload. The file is partitioned into parts, and then the parts are uploaded in parallel, composed to form the original destination object, and deleted. Args: fp: The file object to be uploaded. src_url: FileUrl representing the local file. dst_url: CloudUrl representing the destination file. dst_obj_metadata: apitools Object describing the destination object. canned_acl: The canned acl to apply to the object, if any. file_size: The size of the source file in bytes. preconditions: Cloud API Preconditions for the final object. gsutil_api: gsutil Cloud API instance to use. command_obj: Command object (for calling Apply). copy_exception_handler: Copy exception handler (for use in Apply). logger: logging.Logger for outputting log messages. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, crc32c, and size fields populated. providerNr"r)rrFrfinished component_num message_typerr)rrrr)rr)rtotal_bytes_transferredT arg_checkerparallel_operations_overrideshould_return_resultsrkey)r)rzr}r)rrfieldsencryption_tuple)rr)rrrr)rz:Failed to delete some of the following temporary objects:  zSSome temporary components were not uploaded successfully. Please retry this upload.)<timer1bucket_url_stringGetApiSelectorrrRr crypto_key_sha256r9r>PARALLEL_UPLOADr&r'_DeleteTempComponentObjectFnrr&r%r)rA CreateLockrrrFilterExistingComponentsr3COMPONENT_TO_UPLOADrrr r`r9rrrEXISTING_COMPONENTEXISTING_OBJECT_TO_DELETEApplyrgslibcommandDummyArgCheckerParallelOverrideReasonSLICErsrsortedrComposeRequestSourceObjectsValueListEntryr HasGenerationlongr} ComposeObject ExceptionrIjoinkeysr6r!)(rrrrJrrurr command_objcopy_exception_handlerrr start_timert api_selectorencryption_keywrapperrtracker_file_nameexisting_enc_key_sha256existing_prefixexisting_componentsrvrcomponents_inforycomponents_to_uploadexisting_objects_to_deleter component_str cp_resultsuploaded_components cp_resultr{ componentsrequest_components component_urlr<composed_objectobjects_to_delete elapsed_times( r_DoParallelCompositeUploadrEsFyy{*'(A(AB.**GNN*C,1&931BB9= ))8)H)H)5w@ 9&"O ,P0/0.+ *,?,A(?' 4?177@EI&5&A?0-BD !!2!.!47LN 1;;=/ J,88*%)#,99--2G)5 7(#;$nj#B,(i'')>)>5@OI%%001I%%%%IIK"..""5i6G6G"H!,!@!@  BC('iaL77M&1&D&D&/l&4OM"GaLIIK""1"5il"C!,!?!?  AB '.iGIIK"!,!F!F  HI.  (M--//#.#E#E#K#K !"*iy|,"4G%H4Gqad4G%HH*_H % (; NOTE: You are uploading one or more large file(s), which would run significantly faster if you enable parallel composite uploads. This feature can be enabled by editing the "parallel_composite_upload_threshold" value in your .boto configuration file. However, note that if you do this large files will be uploaded as `composite objects `_,which means that any user who downloads such objects will need to have a compiled crcmod installed (see "gsutil help crcmod"). This is because without a compiled crcmod, computing checksums on composite objects is so slow that gsutil disables downloads of composite objects.TN)rsr rDrr r r'PARALLEL_COMPOSITE_SUGGESTION_THRESHOLDsuggested_sliced_transfers_locksuggested_sliced_transfersinfortextwrapwrap) rrrrrurrrall_factors_but_sizes r _ShouldDoParallelCompositeUploadr(s 2)= jj@<>)?%  nn  ..D . BaG:: ( ' + +K 8 DII MM % & '*. . /37";/! )$  ;#F#J ; : :<% ) (s 9AC&&C/c|xstj}t|r_tt ||||}t |dk7rt d|z|d}t|j|j fSt|}|jr||jfS|jr|dfSt|r|dfS|jjd}|j!|j"|d|j$ddg } | D]} | j&t(j*j,k(r| j.|dzk(r|dfcS| j&t(j*j0k(r,| j.j2|jd zk(r|dfcS| j&t(j*j0k(s| j.j2|jk(s|d fcS||fS) a-Expands wildcard if present in url_str. Args: url_str: String representation of requested url. gsutil_api: gsutil Cloud API instance to use. project_id: project ID to use (for iterators). treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. logger: logging.Logger instance to use for output. If None, the root Logger will be used. Returns: (exp_url, have_existing_dst_container) where exp_url is a StorageUrl and have_existing_dst_container is a bool indicating whether exp_url names an existing directory, bucket, or bucket subdirectory. In the case where we match a subdirectory AND an object, the object is returned. Raises: CommandException: if url_str matched more than 1 URL. ) project_idrrz)Destination (%s) must match exactly 1 URLrT/prefixesz items/name)prefix delimiterrrz _$folder$F)loggingLoggerr.listrwrr!r1r rrrrr0rr ListObjectsrrdatatyperCsObjectOrPrefixTypePREFIXdataOBJECTr) url_strrr"treat_nonexistent_object_as_subdirr blr_expansionblrr-r list_iterator obj_or_prefixs rExpandUrlToSingleBlrrhs6  %W^^%&gw)*4&, ./M  =Q H$% &&  C ! 0clln2D EE$W-+[ 0002 33[  k*    " " ) )# .&(()@)@06362=2D2D1;\0J )L- %m ("?"?"F"FFfsl*4   H$A$A$H$H H    ! ![%<%<{%J J4   H$A$A$H$H H    ! ![%<%< <5 !!5%< 9 ::rcXtjdddsy|jsyt|jry|dk(ry |j |j dg|jy#t$r/}t|tjjrYd}~yd}~wwxYw)alMakes a request to the destination API provider to trigger reauth. Addresses https://github.com/GoogleCloudPlatform/gsutil/issues/1639. If an API call occurs in a child process, the library that handles reauth will fail. We need to make at least one API call in the main process to allow a user to reauthorize. For cloud source URLs this already happens because the plurality of the source name expansion iterator is checked in the main thread. For cloud destination URLs, only some situations result in a similar API call. In these situations, this function exits without performing an API call. In others, this function performs an API call to trigger reauth. Args: destination_url (StorageUrl): The destination of the transfer. gsutil_api (CloudApiDelegator): API to use for the GetBucket call. worker_count (int): If greater than 1, assume that parallel execution is used. Technically, reauth challenges can be answered in the main process, but they may be triggered multiple times if multithreading is used. Returns: None, but performs an API call if necessary. rA0trigger_reauth_challenge_for_parallel_operationsFNrlocation)rr) r getboolrr.r  GetBucketrrrrGpyu2ferrors PluginError)destination_urlr worker_countrs r.TriggerReauthForDestinationProviderIfNecessaryrs: BE K   # # % o001 Q  ##| '')  !U\\--. / s)A11 B):%B$$B)c|jrK|jdk(r<|jr,tjdd|j }t |}|S)aTranslates Windows pathnames to cloud pathnames. Rewrites the destination URL built by ConstructDstUrl(). Args: src_url: Source StorageUrl to be copied. dst_url: The destination StorageUrl built by ConstructDstUrl(). Returns: StorageUrl to use for copy. \z\\r)rrrresubr r1)rr trans_url_strs rFixWindowsNamingr sMgmmt38J8J8LFF5#w'9'9:M"=1G .rcX|jrg|jrWtjj|j}tjj|j}||k(S|j |j k(xr|j |j k(S)zChecks if src_url and dst_url represent the same object or file. We don't handle anything about hard or symbolic links. Args: src_url: Source StorageUrl. dst_url: Destination StorageUrl. Returns: Bool indicator. )rrrfnormpathrr r})rr new_src_path new_dst_paths r SrcDstSamers W..077##G$7$78L77##G$7$78L < ''   '"4"4 4 5   '"4"4 46rcN|jr|r|jrd|jz}nd}|jrH|js|j r(|jrdnd}|j d||y|j d|j |y)a*Logs copy operation, including Content-Type if appropriate. Args: logger: logger instance to use for output. src_url: Source StorageUrl. dst_url: Destination StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. z [Content-Type=%s]r zrzCopying from %s%s...zCopying %s%s...N)rrrr r rr )rrrrJcontent_type_msgsrc_texts r_LogCopyOperationr5s/""+.>.J.JJ g..0GNN4D#,,.yLH KK&2BC KK!7#5#57GHrc 0t|}tt}tj} t |j ||dj } tjrPttjd5} tj| jj } ddd|j|||jtj|| |j t"|| } tj} |j%}t'|| j|_t)|j t+||| t*j,|j.d| | z |j.|| j0fS#1swYxYw)aPerforms copy-in-the cloud from specified src to dest object. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata for source object; must include etag and size. dst_url: Destination CloudUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API instance to use for the copy. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. Copyingrrr/r-N)src_generationrrprogress_callbackrrdecryption_tuplerTrrr)rPrRr rr*r9r5rrr4pickleloadsread CopyObjectr}rrUPLOAD_RETURN_FIELDSrr/r`r3FILE_CLOUD_COPYrr~)rr<rrJrrdecryption_keydecryption_keywrapperrrr test_fpdst_objend_time result_urls r_CopyObjToObjInTheCloudrLsi42.A1&9yy{*1*2I2I:A:AAJLMQD// %88$ ?7 ,,w||~6;; @  ! !"2"2181C1C-D-O-O0=4E+2>>)=3H3H " J'YY[(}}*4Z5<5G5GI*'*::',, !" Z !1!6!6 // 9 @ ?s <.F  Fc|j|jr|js|jsq|j}d}|dk7rMt j j|}tjdddrts tjddd|gtjtj }|j\}}|jj!|j"j!|j$d k7s|rt'd ||j$|fz|j)}t+j,|}n?|j1d\} } } | t2vr t2| }nt5j6|\}} |st8}||_yyyyy#t.$r}t'd |d |d}~wwxYw)zDetects and sets Content-Type if src_url names a local file. Args: src_url: Source StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. N-rA use_magicfileFfilez-bz--mime)stdoutstderrrzAEncountered error running "file -b --mime %s" (returncode=%d). %sz,Encountered OSError running "file -b --mime z" .)rrr r rrrfrealpathr rrk subprocessPopenPIPE communicatercloser returncoder!rr6 ensure_strrrCOMMON_EXTENSION_RULES mimetypes guess_typern) rrJrrreal_file_pathpoutputrrr extensions r_SetContentTypeFromFiler0s""*w/@/@/B    W^^%5%%KLcww'' 4n /5 9* #hG&0oo&0oo7!--/-&% ((..  ((..  \\Q %"'*8!,,)NOP P , 5, )33C81i . ./ :,%00@/, )l#/ K&6 0C*4 # q"# # #sCF66 G?GGc t|j||dj}tjrPt tjd5} t j| jj}dddtj} tt} |js|jr7|j||tj||| |j t"| } n7|j%||tj|||| |j t"| } tj} | | z }|| fS#1swYxYw)aUploads the file using a non-resumable strategy. This function does not support component transfers. Args: src_url: Source StorageUrl to upload. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. Uploadingrr-N)object_metadatarrr rrrr) r3rrrr rrrr)r*r9r5rrr4r rrrrRr r r UploadObjectStreamingrrr UploadObject)rsrc_obj_filestream src_obj_sizerrJrrrr rrruploaded_objectrrs r_UploadFileToObjectNonResumabler9s:42 "#'$  // %88$ ?7 ,,w||~6;; @yy{*1&9 7>>+ 66(*55#+.#!7 #O!--(*55 #+.#!. #OYY[(J&,  &&C @ ?s .EEc t|tj|j|jt t } | r'| jr| jjdndfd} t|} | r|jd|jd} |r t|nd}t|j|||d j}t j"rPt%t j"d 5}t'j(|j+j}dddt-j,}d }| rM |j/||t j0||j|| | t2| || }d } | s tA | rMt-j,}||z }|fS#1swYxYw#t4$r}|jd|jz|dz }|t7k} | s |jd|j8z|j;|j8|jn#t<$rYn t>$rwxYwtA|jdd|jdn2#tA|jdd|jdwxYw|jd|j||fzd} |jCd |r t|nd}t|j|||d j}tE|jtG|t-j,|t-jHtKtMjLd|zztOYd}~d}~wtP$rd } wxYw#| s tAwwxYw)aUploads the file using a resumable strategy. Args: src_url: Source FileUrl to upload. Must not be a stream. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. logger: for outputting log messages. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. rr"NcLttt|i}t|y)aCreates a new tracker file for starting an upload from scratch. This function is called by the gsutil Cloud API implementation and the the serialization data is implementation-specific. Args: serialization_data: Serialization data used in resuming the upload. N)r7r=rr@)serialization_datarrrs r_UploadTrackerCallbackz<_UploadFileToObjectResumable.._UploadTrackerCallback/s* ()>*C0B,C D0$7rrzResuming upload for %sTr2)rrrr/r-r) r3rrrrr<rrtracker_callbackr rFz:Caught ResumableUploadStartOverException for upload of %s.rz8Checking that bucket %s exists before retrying upload...zDeleted tracker file z for resumable upload of z before retrying.zRestarting upload of %s from scratch (retry #%d) after exception indicating we need to start over with a new resumable upload ID: %s) num_retriesr))r9r>UPLOADrrrRr rr&r:rr rr*r9r5rrr4r rrrUploadObjectResumablerrrrGrrr rr6seekr`r4sleepminrandomrFr)rr6r7rrJrrrrrrr= tracker_data retryablerr rrnum_startover_attemptsr8rrrrrs @@r_UploadFileToObjectResumablerI s8) %%8:2&9 #8#J#J--44W=  8"&7LN, KK('*<*<=)2>%g.D-1! " #'$ // %88$ ?7 ,,w||~6;; @yy{* H-"88 *,77%>>)0%1-#9 %oir+,S VYY[(J&,  ((u @ ?8 -3# kkN$$%& !)MO;i   K N$++, --44w~~N "    +, #4g6H6HJ K +, #4g6H6HJ K kk P   !7 ; <=la 6B)'2m5  ! !%$ & '+d   ! !  $ ,B DE  jj fmmo$:!:;  "## ) i  +,sn.F;;G;G N33N#AI  J I J I  J#.N#/KCN#N6#N33N66Ocd}d}|jd}|r|rd}||fS|tk(s|rt|dkDr |d|vr| }|}||fS)aSelects how an upload should be compressed. This is a helper function for _UploadFileToObject. Args: object_name: The object name of the source FileUrl. is_component: indicates whether this is a single component or whole file. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: A tuple: (If the file should be gzipped locally, if the file should be gzip transport encoded). Fr Trr)splitrr)rrrr zipped_filegzip_encoded_file fname_partss r _SelectUploadCompressionStrategyrOsx,+!!#&+ l ' '' ^# s;'!+ B90L""K$ ' ''rcX||tk\r|jd|tj\}}d} |j s|j r t d|(t|dt|zkrt d|ztjddt}tj|d| }|jt}|r)|j!||jt}|r)|r|j#t%j"||j#t$j&j)|} t|d } t+|| | fS#|r|j#t%j"||j#wxYw) aCompresses a to-be-uploaded local file to save bandwidth. This is a helper function for _UploadFileToObject. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file - will be consumed and closed. src_obj_size (int or None): Size of the source file. logger: for outputting log messages. Returns: StorageUrl path to compressed file, read stream of the compressed file, compressed file size. NzCompressing %s (to tmp)...zgzip compression is not currently supported on streaming uploads. Remove the compression flag or save the streamed output temporarily to a file before uploading.rz|Inadequate temp space available to compress %s. See the CHANGING TEMP DIRECTORIES section of "gsutil help cp" for more info.rAgzip_compression_levelwb) compresslevelr-)rMrtempfilemkstempr r r!rhrr getintrgzipr4rGZIP_CHUNK_SIZEwriter&rrfgetsizer1) rr6r7rgzip_fh gzip_pathgzip_fpcompression_levelr gzip_sizecompressed_filestreams r_ApplyZippedUploadCompressionras$,2J"J KK,g6!))+7I 'W^^-  4 55^I%>'(3|+<'<&= BDKL MM h0H&DFii 47HIG  " "? 3D  mmD  $ $_ 5d  mmoHHWggooi()y$/ i (*? JJ  mmoHHWs CE009F)c2d}d} |r0|s.tjj5|\}}dddn |\}}|j|r$ t j |j ||fS||fS#1swYCxYw#t$r"|jd|j Y||fSwxYw#|j|rL t j |j w#t$r|jd|j YwwxYwwxYw)aHandles setup and tear down logic for uploads. This is a helper function for _UploadFileToObject. Args: upload_delegate: Function that handles uploading the file. upload_url: StorageURL path to the file. upload_stream: Read stream of the file being uploaded. This will be closed after the upload. zipped_file: Flag for if the file is locally compressed prior to calling this function. If true, the local temporary file is deleted after the upload. gzip_encoded_file: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. parallel_composite_upload: Set to true if this upload represents a top-level parallel composite upload (not an upload of a component). If true, resource locking is skipped. logger: For outputting log messages. Returns: The elapsed upload time, the uploaded object. NzZCould not delete %s. This can occur in Windows because the temporary file is still locked.) rr!concurrent_compressed_upload_lockr&runlinkrrwarning) upload_delegate upload_url upload_streamrLrMparallel_composite_uploadrrr8s r_DelegateUploadFileToObjectrj s-2,/G!: == : :(7(9% o ; :'6&7#lOG *(()  && &&) ; : G ./9/E/E G  && G G *(()G ./9/E/E GG sWB6 A<B6B<BB6%B32B36D C*)D*%DDDDc   !"r jstjdd}|r|_|!| |t|j | | \}r sj d|n`|r^t |||\! d_jsd_n1djjvrxjdz c_ sEtjt!tjtjdd }d }tt!fd xsiD}t#| !|t$j& }d nt)kxs"|j+xs|j-}|j+s|j-rCj/j0 t2j4k(r }t7|t9 |st;rt= |!"n "  !f d}!"fd} !"f d}|r|}n|r|}n|}t?|! ||\}}|s' tA|}tC||j|dj]}|jZ|_-t_||jZ|_- sEtjt!tjtjd||j`||jbfS#tD$rtFrltIjJjLjN|jP}jtRz_&jU|j0 jWjXj|jZj0wxYw)a|Uploads a local file to an object. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file to be read and closed. src_obj_size (int or None): Size of the source file. dst_url: Destination CloudUrl. dst_obj_metadata: Metadata to be applied to the destination object. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: for outputting log messages. command_obj: command object for use in Apply in parallel composite uploads. copy_exception_handler: For handling copy exceptions during Apply. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. rAcontent_languagez+Using compressed transport encoding for %s.rWz no-transformz ,no-transformFr Nc36K|]}||fywNr.0r) hash_algss r z&_UploadFileToObject.. FosC3)*o)rrrcN t tj SN)r)rrr) rrrJrrrMrr upload_sizerhrgs rCallParallelCompositeUploadz8_UploadFileToObject..CallParallelCompositeUploads; %m&0&-&6&=&H&H&1&3&0&1&<&,3D F Frc *tSrv)r9)rJrrrMrrwrgwrapped_filestreamsrCallNonResumableUploadz3_UploadFileToObject..CallNonResumableUploads) *:+=+6+2+;+8+58I KKrc . t  S)N)rr)rI) rJrrrMrrrrwrgrzs rCallResumableUploadz0_UploadFileToObject..CallResumableUploads/ ' (:(3(/(8(5(2(.5A5F H HrT)rYrrr|r}r)2contentLanguager get_valuerOrrracontentEncoding cacheControllowerr`r9r3r FILE_UPLOADr[dictrrrrHr r rrrJSONr,rErr\rjr*r`r"_RENAME_ON_HASH_MISMATCHrrrrr|_RENAME_ON_HASH_MISMATCH_SUFFIXr DeleteObjectrr}rr/rr~)#rr6r7rrJrrrrrrrrrrlrLrr8r'rinon_resumable_upload orig_streamrxr{r}delegater(corrupted_obj_metadatarrMrqrwrhrgrzs# ``````` ` @@@@@@rrrEsX !1!A!A''2DE)9&*$-+#C <L$B + | LL>H-J#\6.;*J {'-$  ( (&4# /<<BBD D##6# JIIK!,!8!8$"  $%,/!)Fio2oFF)>  (335!, 31.01+4;4D4D4F+!..*W^^-8K NOTE: You are downloading one or more large file(s), which would run significantly faster if you enabled sliced object downloads. This feature is enabled by default but requires that compiled crcmod be installed (see "gsutil help crcmod").TN)rsr rDrrVrrVrzrIrWrrrrrrrrrrr) download_strategyr<rrrmax_componentsrKparallel_hashing hashing_okay use_slices r_ShouldDoSlicedDownloadrR sq&: jj=9;&<"==!H!NP.  8^#?A%,,G1E1G!L%8. rsrt)r start_byteend_byterNrzr)rJr decode_messagerrsrc_obj_metadata_jsonrYrrzr_DownloadObjectToFileResumablerrrrrrrcrcValuer) rrrrr<r'bytes_transferredserver_encoding crc32c_valrqs @r"_PerformSlicedDownloadObjectToFiler s $#3\B*--.?.F.F.2.H.HJ!#**2B2I2IK)Fio2oFF)*H ll ll  jj&&}}(( +*&o* 8$--J *4+=+=z+<+/==4??+JQ+N+: <SLICED_DOWNLOADr4rijsonloadr}r&rIrIOError ValueErrorrGrENOENTrr5rYdumpsr;strerror) r<rrrrrwrrrexisting_file_sizetracker_file_datars r#_MaintainSlicedDownloadTrackerFilesr s:,    ,/11 ()8)H)H)57 "  $ 'B$R-222+S1l))L1 F #'7'<'< < L )-=-H-H H , - ? hhj  H''( ) hhjWl3 G  %"''(33* $567 & %% : /!Z AGGu||$; kk$c!f./ / hhj & % G'(91::FFGshA3F!.F H5 ?H) H5G<$AG72G?7G<<G??'H&)H2.H52H55 I>IIc\eZdZdZdZedZdZejfdZ dZ dZ dZ y ) SlicedDownloadFileWrapperaWraps a file object to be used in GetObjectMedia for sliced downloads. In order to allow resumability, the file object used by each thread in a sliced object download should be wrapped using SlicedDownloadFileWrapper. Passing a SlicedDownloadFileWrapper object to GetObjectMedia will allow the download component tracker file for this component to be updated periodically, while the downloaded bytes are normally written to file. cX||_||_||_d|_||_||_y)a;Initializes the SlicedDownloadFileWrapper. Args: fp: The already-open file object to be used for writing in GetObjectMedia. Data will be written to file starting at the current seek position. tracker_file_name: The name of the tracker file for this component. src_obj_metadata: Metadata from the source object. Must include etag and generation. start_byte: The first byte to be downloaded for this parallel component. end_byte: The last byte to be downloaded for this parallel component. N)_orig_fp_tracker_file_name_src_obj_metadata_last_tracker_file_byte _start_byte _end_byte)selfrrr<rrs r__init__z"SlicedDownloadFileWrapper.__init__ s2DM/D-D#'D !DDNrc0t|jddS)z >@ ?$--.}}))+,I $$,4777)CDNNQ..'(?(?(,(>(>(8:&6d" /rc8|tjk(r,|jj||jzdzn|jj|||j |jj cxkr|jdzksJJyr)rSEEK_ENDrrBrrr)rrwhences rrBzSlicedDownloadFileWrapper.seek= sw  mm$..0145 mm(   t}}113 It~~7I II II Irc6|jjSrn)rrrs rrzSlicedDownloadFileWrapper.tellD s ==   rc8|jjyrn)rflushrs rrzSlicedDownloadFileWrapper.flushG sMMrcR|jr|jjyyrn)rr&rs rr&zSlicedDownloadFileWrapper.closeJ s }} mmrN)rrrrrpropertyrrYrSEEK_SETrBrrr&rrrrr sC* 0 06"!# J rrc ttjddt}tjddt }t |j||\}}g} g} t|D]v} | |z} t| dz|zdz |jdz } | j| | z dztj|}| jt| ||||| | |x| | fS)aPartitions an object into components to be downloaded. Each component is a byte range of the object. The byte ranges of the returned components are mutually exclusive and collectively exhaustive. The byte ranges are inclusive at both end points. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object with non-pickleable fields removed. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: components_to_download: A list of PerformSlicedDownloadObjectToFileArgs to be used in Apply for the sliced download. rA%sliced_object_download_component_sizerr)rsr rDrrVrrorr rDrsr encode_messager)rr<rrrsliced_download_component_sizerrwrxcomponents_to_downloadcomponent_lengthsr{rrrs r_PartitionObjectrO s.$8 jjB>@$A ==!H!NP.$5^-K$M ..  a^#JAEn-13C3H3H13LMHX 2Q67&445EF!!-a:O.57I.8(.< >? ! !2 22rc (d|_t|||||\} } t| } t|||||| t |d5} | j |j dddt| D]\}}|j|jz dz}t||||j||}||jz }t| t||tj|d|tj||jt | |t"j$j&|j(j*d}t|| krt-d|j.zt1|t3d  }|d j4}|.t7d| D]}t9|||j4| |}!d }t;|}|D]}t| t||tj|j<d|j>tj ||j@z }|jBxr)|jBjEjGd }|s|rt-d|j.z||fS#1swY,xYw)aDownloads a cloud object to a local file using sliced download. Byte ranges are decided for each thread/process, and then the parts are downloaded in parallel. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. command_obj: command object for use in Apply in parallel composite uploads. logger: for outputting log messages. copy_exception_handler: For handling copy exceptions during Apply. api_selector: The Cloud API implementation used. decryption_key: Base64-encoded decryption key for the source object, if any. status_queue: Queue for posting file messages for UI/Analytics. Returns: (bytes_transferred, crc32c) bytes_transferred: Number of bytes transferred from server this call. crc32c: a crc32c hash value (integer) for the downloaded bytes, or None if crc32c hashing wasn't performed. Nr+brF)rrrrbytes_already_downloadedTrzSSome components of %s were not downloaded successfully. Please retry this download.rrrrrWzTDownload of %s failed because the server sent back data with an unexpected encoding.)$r{rrrr4truncater enumeraterrr8r`r3rCOMPONENT_TO_DOWNLOADrrrrrrrr!rrrrzr rXr]component_total_sizerrrrr )rr<rrrrrrrr9rrrwrr{rrdownload_start_byterrrzr expect_gzipr server_gzips r_DoSlicedDownloadr sH)-%.> *r rrrrs" rrr sD$$q(HZ'!+-4')**GNN*C,/%%-_}44M "s  % (BGGJ,,gnn,EL$R-L.**Z'+>A+M)*  $&3 4 55 - ]0JJ#z1L;L7LH kk,m<  kk '(576$,,. $n1J>1  %j&=&=4A.5.55>  @AE Fm 2 241NBD ww}%-'!H H  $ $S%6%6t%< ="s4y) 2 2kk!n3)#$&'+d11 '::D AW"LL8== B%**.@.BB $R):? gg!""11       (1''&++$55??>>/-2>B2Do* hhj!44q8 O ++U B AB $  NNO88],   hhj sEFMA&M.M/CMMM N#NNN Ncd} t|d}t|d|j}t|j||dj } t jrPtt jd5} tj| jj } ddd|j|j|j||j|jt j"j$|j&||| t)| } |r|j+ |j| fS#1swYxYw#|r|j+wwxYw) a^Downloads an object to a local file using the non-resumable strategy. This function does not support component transfers. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. digesters: Digesters corresponding to the hash algorithms that will be used for validation. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. Nrr)rrrr-)r}rrrr<r'r r )r4rKrr*r9r5rrr rrrrrr}rrrrrrPr&) rr<rrrr'rrr<r rrs r!_DownloadObjectToFileNonResumabler s@4 "   # &B6!**A*AC4$ &'+d  11 '::D AW"LL8== B!// %%$))"33<<-+0@0 BO hhj    //' B A  hhj s%A1E5.D;#A7E;EEEc |jj|jrX|jdj t j d|jztd|jz|j|j} t|} t| |||}t|||\}}t5tj|drt dt|<ddd|j"xr| }t%|||j&t)fd xsiD}d}|j*d k(}d }t-j,}|s|r1t/|||||||| ||j0 \}}d |vr||d _nv| t4j6j8urt;||||||| \}}nD| t4j6j<urt?|||||||| \}}ntAd| d|t-j,}|xr|jCjd}tE||||||||| ||| | | }t5tjG|dddtI|j0tK|||tJjL|j*d||z |||fS#1swYxYw#1swY\xYw)aDownloads an object to a local file. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. command_obj: command object for use in Apply in sliced downloads. copy_exception_handler: For handling copy exceptions during Apply. allow_splitting: Whether or not to allow sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve POSIX attributes. use_stet: Decrypt downloaded file with STET binary if available on system. Returns: (elapsed_time, bytes_transferred, dst_url, md5), where time elapsed excludes initial GET. Raises: FileConcurrencySkipError: if this download is already in progress. CommandException: if other errors encountered. rzSkipping attempt to download to filename ending with slash (%s). This typically happens when using gsutil to download from a subdirectory created by the Cloud Console (https://cloud.google.com/console)zInvalid destination path: %srFTN) consider_md5rc36K|]}||fywrnrros rrrz(_DownloadObjectToFile..# rsrtr)rr9rzrzInvalid download strategy z chosen forfile rW)is_rsyncruse_stet) message_timerrr)'rr rrIrrrr#rrrrropen_files_lockopen_files_maprDrr~rYrzrrrrr9rrrrrrrr!r_ValidateAndCompleteDownloaddeleter`r3 FILE_DOWNLOAD)rr<rrrrrrrrrrrrsliced_downloadrrrr'rrrrrzrr local_md5rqs @r_DownloadObjectToFiler s J !!'--0 KK   N     !"# 87;N;NN OO**GNN*C,-g6+,=?O,;VE/'7w7G'Q#m,e4 $$)-N%& "))A/.A,!&/;2B2I2IK)Fio2oFF)/',,1yy{* #G$4$+$6$/$*$:$03A1;1H1H J  Y '- ($ h77@@ @+L      ',( h77AA A+I       ' ,( /1CE FFYY[(LO$9$9$;$D$DV$L+*6+2+;+2+8+6+4+4+=+7+<+54<:H4<>) ,-''*88',, !" Z !2GY GGsZs &K"KKKc|j}d}|D] }||r d}n|r t|}nt| j||||}d}d} t |||||t || |s|s|}ntj |}| t"kDr|j%d|d} t'j(|d}t)|d5}|j+t,}|r)|j/||j+t,}|r)ddd|r|j5 tj||s5 t| j||||}t |||||t || |rt7j8||||tj||t;||| | d |vr|d Sy#t $r}|r|jdd}no| tjk(r|jd|}d}nFt || trtj||tztj|Yd}~d}~wwxYw#1swYNxYw#t0$r}d t3|vr|r|Yd}~qd}~wwxYw#|r|j5wwxYw#t $rGt || trtj||tztj|wxYw) aValidates and performs necessary operations on a downloaded file. Validates the integrity of the downloaded file using hash_algs. If the file was compressed (temporarily), the file will be decompressed. Then, if the integrity of the file was successfully validated, the file will be moved from its temporary download location to its permanent location on disk. Args: logger: For outputting log messages. src_url: StorageUrl for the source object. src_obj_metadata: Metadata for the source object, potentially containing hash values. dst_url: StorageUrl describing the destination file. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. server_gzip: If true, the server gzipped the bytes (regardless of whether the object metadata claimed it was gzipped). digesters: dict of {string, hash digester} that contains up-to-date digests computed during the download. If a digester for a particular algorithm is None, an up-to-date digest is not available and the hash must be recomputed from the local file. hash_algs: dict of {string, hash algorithm} that can be used if digesters don't have up-to-date digests. temporary_file_name: Temporary file name that was used for download. api_selector: The Cloud API implementation used (used tracker file naming). bytes_transferred: Number of bytes downloaded (used for logging). gsutil_api: Cloud API to use for service and status. is_rsync: Whether or not the caller is the rsync function. Used to determine if timeCreated should be used. preserve_posix: Whether or not to preserve the posix attributes. use_stet: If True, attempt to decrypt downloaded files with the STET binary if it's present on the system. Returns: An MD5 of the local file, if one was calculated as part of the integrity check. TFNzDHash did not match but server gzipped the content, will recalculate.zMHash did not match but server may have gzipped the content, will recalculate.z/Uncompressing temporarily gzipped file to %s...r-rRzNot a gzipped filerrr,)rr*r?r9r`r5r"rrrrrrenamerrdrCrrvrrWr4rrXrYrrr&rBdecrypt_downloadrf)rrr<rrrr'rqtemporary_file_namerrrrrrfinal_file_namedigesters_succeededr)rZdigest_verifiedhash_invalid_exceptionrunzipped_temporary_file_namer]f_outrs rr r u sh''/ c S>! .y9L.z/F/F/8:M/68HJL/ "2Ow 52 ;#6 $7#F#Fw#O 7" kkC!#G -t4g ,d 3u||O, ++d o.$ 4  II!"  01H1H1:1M18:JLl67$4o! ,7 w1M%')) (/:_-&.,:<  l  u   ll"#o  ( ll  !o ,7 ! %!$CC E  %& - R 4 3 % Q ',B%$ %    !  ,7 ! ( (+J J L   ./  si F3 "IAII64J3 I  }|j@tBz|_tE|}|jG|||j|| |jI|jJ|j@|jL|j wxYw)a Copies from src_url to dst_url in "daisy chain" mode. See -D OPTION documentation about what daisy chain mode is. Args: src_url: Source CloudUrl src_obj_metadata: Metadata from source object dst_url: Destination CloudUrl dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: For outputting log messages. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. z"Cross-provider cp -p not supportedNr-)rr rr)r3rrrrrrr2r) r3rrrrrr r>rr~)rr rrTr )-rrrNotImplementedErroraclrr4r rrr5r]rRr rr rr5rrrAr*r9r%rUr"rrrrrr|rrrPrrrr}rr/r`r3FILE_DAISY_COPYr~)rr<rrJrrrrr rrrr upload_fpr8rrrrs r_CopyObjToObjDaisyChainModer+P s@** nn& B CC - -// %88$ ?7 ,,w||~6;; @,,<=1&9yy{* 0 5 5 *4G2C/= ?) /a --(*55##  " "..0O!66(*55##  " "5  # #& ()-..70OYY[( fgw0@%'.}}*4Z5D5O5OQ*'*::',, !" Z !1!6!6  ! ! ##i @ ?d  077$$!((## % '22> ?5nE2,%,^^-B-B D G//#//'6'A'A%,^^5 ' s8.II$I!$CL+ct}|r<|jgd|r|jdg|s|jdgnl|jgd|r|jdtzdg|r;dtzdtzdtzdt zdt zg}|j||r|jddg|r|jdgt|S)aDetermines the metadata fields needed for a copy operation. This function returns the fields we will need to successfully copy any cloud objects that might be iterated. By determining this prior to iteration, the cp command can request this metadata directly from the iterator's get/list calls, avoiding the need for a separate get metadata HTTP call for each iterated result. As a trade-off, filtering objects at the leaf nodes of the iteration (based on a remaining wildcard) is more expensive. This is because more metadata will be requested when object name is all that is required for filtering. The rsync command favors fast listing and comparison, and makes the opposite trade-off, optimizing for the low-delta case by making per-object get metadata HTTP call so that listing can return minimal metadata. It uses this function to determine what is needed for get metadata HTTP calls. Args: dst_is_cloud: if true, destination is a Cloud URL. skip_unsupported_objects: if true, get metadata for skipping unsupported object types. preserve_acl: if true, get object ACL. is_rsync: if true, the calling function is rsync. Determines if metadata is needed to verify download. preserve_posix: if true, retrieves POSIX attributes into user metadata. delete_source: if true, source object will be deleted after the copy (mv command). file_size_will_change: if true, do not try to record file size. Returns: List of necessary field metadata field names. )rcomponentCountcontentDispositionrrrrzr{r|r}r~ mediaLinkmetadatametagenerationr timeCreatedr(r) rzrrr{r|r/r~rr}z metadata/%sr2r)setrqrerarcrdrgr) dst_is_cloudrrrr delete_sourcefile_size_will_changesrc_obj_fields_set posix_fieldss rGetSourceFieldsNeededForCopyr9 sNu&( )  !;] KL * $ * $ ( " ) # ( " l - ~./  !!rZim)nearlinecoldlinearchivecn|jdk(r|r|jr|jr|jj}tj |d}|rV|t |jz}tj|kr&|jd||j|tzyyyyyyy)aWarns when deleting a gs:// object could incur an early deletion charge. This function inspects metadata for Google Cloud Storage objects that are subject to early deletion charges (such as Nearline), and warns when performing operations like mv that would delete them. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with necessary fields (per GetSourceFieldsNeededForCopy). logger: logging.Logger for outputting warning. rNzWarning: moving %s object %s may incur an early deletion charge, because the original object is less than %s days old according to the local system time.) rr2rrEARLY_DELETION_MINIMUM_LIFETIMErDrbrrIr ru)rr<robject_storage_classearly_deletion_cutoff_secondsminimum_delete_ages r"WarnIfMvEarlyDeletionChargeAppliesrDIsnn!1""'7'D'D+88>>@$C$G$Gd%$!$ ' !1!=!= > ? ) )  ; !'"4"4 )_ <  > * % (E""2rcx|jdk(r+tjr|jdk(r t yyy)aSkips unsupported object types if requested. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with storageClass field (per GetSourceFieldsNeededForCopy). Raises: SkipGlacierError: if skipping a s3 Glacier object. s3GLACIERN)rrrrSkipGlacierError)rr<s rMaybeSkipUnsupportedObjectrIhs?nn66##y0  17rc|jrMt|jjt}|s%t d|jjd||Sy)aEnsures a matching decryption key is available for the source object. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with optional customerEncryption field. Raises: EncryptionException if the object is encrypted and no matching key is found. Returns: Base64-encoded decryption key string if the object is encrypted and a matching key is found, or None if object is not encrypted. z(Missing decryption key with SHA256 hash z#. No decryption key matches object N)r{rQ keySha256r r)rr<rs rGetDecryptionCSEKrLys`((1++55v?N    . . 8 8' C DD )rc|r|j}ni}t|}|jr|jdk(r t |}n t }d}d}d}|jr|jr+|j|jk(rt jsd}t jr t|||t||t||}|j}|j|_ t jr|jr|jdk(r|jst!d|z|j|_|jdk(rt#|}|rt%||n|rt'j(|||}n|} t+|}|j7s|j9rd}nF|r|jr|s |j}n)t:j<j?|j@}t jBr| jE|jFd ||jdk(r/|dk7r*|(|tHkDrt!d |d tKtHdtLr[|jOrK|j7r;tQjRt+|jUt:jVt jXr|jZr t]dd|_-|jOr3t:j<j_|j@r ta|jr? |jc|jd|j@|j}|r ta|jr|j@|_4|jd|_5|jr#|j@|_4|jd|_5n tm||totp}|r=|jrttjvk(r |jdk(r|jx|_=|rt}||dt j~rt j~|_@tqjddtk(rd|_Ct|||||jr|jOr\t|jt||tjtj|dt|||||||||| | | S|rWt|jt||tjtj|dt|||||||St|jt||tjtj|dt||||||||S|jr;t||||||||||| ||  }|rt;j|jh|St|jt||tjtj|dt|||j|}|j7s)|j7st|j@|| | |S#t,$rY}d|dt/|d }|j0r+|xj2d z c_|j5|Yd}~yt!|d}~wwxYw#tf$rd}YwxYw)aPerforms copy from src_url to dst_url, handling various special cases. Args: logger: for outputting log messages. src_url: Source StorageUrl. dst_url: Destination StorageUrl. gsutil_api: gsutil Cloud API instance to use for the copy. command_obj: command object for use in Apply in parallel composite uploads and sliced object downloads. copy_exception_handler: for handling copy exceptions during Apply. src_obj_metadata: If source URL is a cloud object, source object metadata with all necessary fields (per GetSourceFieldsNeededForCopy). Required for cloud source URLs. If source URL is a file, an apitools Object that may contain file size, or None. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload or download. headers: optional headers to use for the copy operation. manifest: optional manifest for tracking copy operations. gzip_exts: List of file extensions to gzip, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve posix attributes. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts. Streaming files compressed is only supported on the JSON GCS API. use_stet: If True, will perform STET encryption or decryption using the binary specified in the boto config or PATH. Returns: (elapsed_time, bytes_transferred, version-specific dst_url) excluding overhead like initial GET. Raises: ItemExistsError: if no clobber flag is specified and the destination object already exists. SkipUnsupportedObjectError: if skip_unsupported_objects flag is specified and the source is an unsupported type. CommandException: if other errors encountered. rNFTzZNo OWNER permission found for object %s. OWNER permission is required for preserving ACLs.rFzError opening file "z": r rr"zV" exceeds the maximum gsutil-supported size for an S3 upload. S3 objects greater than zB in size require multipart uploads, which gsutil does not support.zASpecifying x-goog-if-generation-match is not supported with cp -nrr)overriderArBr )rrrrrr)rrr)r9r<r)TcopyrorrrprrrrrDrIrLrrrr(r!rqrlrBencrypt_uploadrjrrcontinue_on_errorrrr r rrfrZrrSetr S3_MAX_UPLOAD_SIZErtrkrmsvcrtsetmodefilenoO_BINARYr gen_matchr rItemExistsErrorGetObjectMetadatarrrrr0rRr crypto_typerOCMEK crypto_key kmsKeyNamermrrrDrWr~rr`r9r3rrrrrr)r+rrdr r#rf)rrrrrrr<rheadersmanifestrrrrrdst_obj_headersrJrr6rcopy_in_the_cloudr7acl_textsource_stream_urlrrc dst_objectruploaded_metadataresults r PerformCopyrisn llnOO /? gnn4,_=M!OM. 7>>!A # / /))(2BFKw(89&w0@AN#((L#3#?#? ++0B0B0D 4 (8(<(< ,.5 67 7.11 4 01AB  ()98 D#227GVL! (/0ABW^^-l .33H&**lWW__%6%B%BCl)) LL##V\: nnD''  #$67 9 ::G%%'G,<,<,> NN'07792;;G'' 9 ::!"mrww~~g.A.AB      11'2E2E292E2E;B>>2K  #//%11&11 ' 3 3g'783F;))]-?-??$$9$D$D!')9EJ//$;$N$N! ZZ.)-== $FGW.>?   ! ! giik#.#<#<'$  &'#7#3#*#-#)#.#93B2@,42@,4 6 6   ! ! giik#.#>#>'$  &'%W%5%,%5%2%/4B DD  ! ! giik#.#>#>'$  &'))9)0)9)6)3)/8FHH.g.@.:.5.>.;.8.4.9.D8A>M;G I  $))*   ! ! giik#.#>#>'$  &'w&,6,C,C0@Bf    (8(8(:"7#6#6#3,42@ BmI (293q6Bg  & &$$)$ Ww''(z s1 [2\8 \5A\0% \00\58 ]]cHeZdZdZdZdZdZdZdZdZ d dZ d Z d Z y ) Manifestz2Stores the manifest items for the CpCommand class.ci|_i|_tj|_t j j||_|j|jyrn) itemsmanifest_filterrArlockrrf expanduser manifest_path_ParseManifest_CreateManifestFile)rrfs rrzManifest.__init__sPDJD*557DI++D1Drc  tjj|jrt |jd5}d}t j |}|D]G}|r# |jd}|jd}d}|}|}|dvs9||j|<I dddyy#t$rtd|jzwxYw#1swYyxYw#t$rtd |jzwxYw) z|Load and parse a manifest file. This information will be used to skip any files that have a skip or OK status. rTSourceResultz$Missing headers in manifest file: %sF)OKskipNzCould not parse %s) rrfrrqr4csvreaderindexrr!rnr) rf first_rowrzrow source_index result_indexsourcerhs rrrzManifest._ParseManifests H ** +$$$c *a)::a=&c;"yy2 "yy2 I&F&F'.4d""6*+ * , ;&'M'+'9'9(:;;;+ *& H 1D4F4FF GGHsG?C!C!"B0CC&C!0"CCCC!C!!"Dc||jvS)z?Returns whether the specified src url was marked as successful.)rn)rsrcs r WasSuccessfulzManifest.WasSuccessfuls $&& &&rcV tjj|jr,tj|jj dk(rt jrJt|jdd5}tj|}|jgddddyt|jdd5}tj|}|jgddddyy#1swYyxYw#1swYyxYw#t$r td wxYw) z;Opens the manifest file and assigns it to the file pointer.rrr newline) ru DestinationStartEndMd5UploadIdz Source SizezBytes Transferredrv DescriptionNrRrzCould not create manifest file.)rrfrrqrst_sizer6rFr4rywriterwriterowrr!)rr|rs rrszManifest._CreateManifestFiles#@ww~~d001 774%% & . .! 3 77D&&R8AZZ]F OO  98D&&a0AZZ]F OO  10% 49810 @ > ??@sNA=D?)C;(D1D)D1D;DDDD DDD(cn|y||jvr||j||<y||i|j|<yrnrm)rrrvalues rrSz Manifest.Sets; }  djj"djjoce djjorc|j|d||j|d||j|dtjjtjjj dy)N source_uridestination_urirtztzinfo)rSdatetimenowtimezoneutcr)r source_urlrs r InitializezManifest.Initialize)sbHHZz2HHZ*O<HHZx'8'8'<'<@Q@Q@U@U'<'V'^'^fj'^'klrcl|j|d||j|d||j|d||j|dtjjtjjj d|j ||j|y)Nbytesrh descriptionrrr)rSrrrrr_WriteRowToManifestFile_RemoveItemFromManifest)rrrrhrs r SetResultzManifest.SetResult/sHHZ"34HHZ6*HHZ 4HHZX%6%6%:%:h>O>O>S>S%:%T%\%\dh%\%ij  ,  ,rc |j|}|d|dd|djzd|djzd|vr|dndd|vr|dndd |vrt|d ndd |vrt|d nd|d |d g }|Dcgc]}tj|}}|j 5t r*tjrt|jd dd}nt|jd d}tj|}|j||jdddycc}w#1swYyxYw)zBWrites a manifest entry to the manifest file for the url argument.rrz%sZrrr,r  upload_idrrrhrarrN)rm isoformatrr6r(rorkrFr4rqryrrr&)rrrow_itemrrr|rs rrz Manifest._WriteRowToManifestFile7sAzz#H"# &0022 $..00 H,"!,!8b!'8!3HV ")X"5HW 2 D04 4teCNN5 !tD 4   ##S!R 8 ##S! ,zz!}f oodggi  5 s D;4A>EE c|j|=yrnr)rrs rrz Manifest._RemoveItemFromManifestTs 3rN)r ) rrrrrrrrrsrSrrrrrrrrkrks7:H@'%@N %m -:rrkceZdZdZy)rZzHException class for objects that are skipped because they already exist.NrrrrrZrZ[sPrrZc"eZdZdZfdZxZS)SkipUnsupportedObjectErrorCException for objects skipped because they are an unsupported type.c8tt| d|_y)NUnknown)superrrunsupported_typer __class__s rrz#SkipUnsupportedObjectError.__init__cs $d46%Drrrrrr __classcell__rs@rrr`K&&rrc"eZdZdZfdZxZS)rHrc8tt| d|_y)NrG)rrHrrrs rrzSkipGlacierError.__init__ks D*,%Drrrs@rrHrHhrrrHc|j}|jrS|jtdd}|j |dk(ryd|j |j |dzS|jrd|jzS|jj |j |d}t||S)a?Returns the path section before the final directory component of the URL. This handles cases for file system directories, bucket, and bucket subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', and for file://dir we'll return file:// Args: url: StorageUrl representing a filesystem directory, cloud bucket or bucket subdir. exp_src_url: StorageUrl representing the fully expanded object to-be-copied; used for resolving cloud wildcards. Returns: String name of above-described path, sans final path separator. zfile://Nrz file://%srz%s://) rrr rfindrrrr$ResolveWildcardsInPathBeforeFinalDir)rrsep past_schemepath_sans_final_dirs rrrps #]]_..Y1K"  ;--c2==cB1E EE\\^ SZZ --c2==cB1E -.A; OOrcbt|s|St|jjd}|jjd}t t |D]}t||s||||<dj |}|j}||_|jS)afReturns the path section for a bucket subdir with wildcards resolved. This handles cases for bucket subdirectories where the initial source URL contains a wildcard. In this case, src_url must be wildcard-expanded before calculating the final directory. Example: A bucket containing: gs://bucket/dir1/subdir/foo gs://bucket/dir2/subdir/foo and source URL gs://bucket/*/subdir and src_url_path_sans_final dir gs://bucket/* should yield final path gs://bucket/dir1 or gs://bucket/dir2 according to the expanded source URL. Args: src_url_path_sans_final_dir: URL string with wildcards representing a bucket subdir as computed from GetPathBeforeFinalDir. exp_src_url: CloudUrl representing the fully expanded object to-be-copied. Returns: String name of above-described path, sans final path separator. r) r.r1rrKr rrrr )rrwildcarded_src_obj_pathexpanded_src_obj_pathpath_segment_indexresolved_src_pathfinal_path_urls rrrs6 5 6 &&1!##.;uuSz%1177<!#&=">?/0BCD 2 301 @ hh67$$&.0.  " ""rcft||}tt||d}t||}||fS)aGets info about a file partition for parallel file/object transfers. Args: file_size: The number of bytes in the file to be partitioned. max_components: The maximum number of components that can be composed. default_component_size: The size of a component, assuming that max_components is infinite. Returns: The number of components in the partitioned file, and the size of each component (except the last, which will have a different size iff file_size != 0 (mod num_components)). r)rrmaxrD)rurdefault_component_sizerwrxs rroros?!,BC.s>>:A>.!N;. . ))rct||} |j|j|j|j|j y#t $rYywxYw)zGWrapper func to be used with command.Apply to delete temporary objects.rN)rJrrrr}rr)r url_to_deleterrs rrrs_"3 5*  M55)55'4'?'?%2%9%9;    s=A AAc g}|Dcgc]}|j}}|D]}||vs|j||g}g} g} |D]} | j|jvs| j|vrD|j} | j| _| j| _| j| r|| j} t | j | j| j}t|} | j}|j|j|j|j|jgd}|j}||k7rx|j| |j| j| jsO| jj}| j|_| j|| jj} | j| _| j| | jf|j| j| rt!j"dt%| || | fScc}w#t$rd}YwxYw)a]Determines course of action for component objects. Given the list of all target objects based on partitioning the file and the list of objects that have already been uploaded successfully, this function determines which objects should be uploaded, which existing components are still valid, and which existing components should be deleted. Args: dst_args: The map of file_name -> PerformParallelUploadFileToObjectArgs calculated by partitioning the file. existing_components: A list of ObjectFromTracker objects that have been uploaded in the past. bucket_url: CloudUrl of the bucket in which the components exist. gsutil_api: gsutil Cloud API instance to use for retrieving object metadata. Returns: components_to_upload: List of components that need to be uploaded. uploaded_components: List of components that have already been uploaded and are still valid. Each element of the list contains the dst_url for the uploaded component and its size. existing_objects_to_delete: List of components that have already been uploaded, but are no longer valid and are in a versioned bucket, and therefore should be deleted. )r{r|r~)r}rrNz0Found %d existing temporary components to reuse.)rrsrrr}r$rrrrTrr[rrr~rrrr)ryr bucket_urlrrrexisting_component_namescomponent_nameobjects_already_chosenrrtracker_objectrdst_arg file_partr}r dst_metadata cloud_md5!invalid_component_with_generations rrrsj:-@-@ i-@!n55!!(>":;! "+n""(--/9""&<<    c&22co%00cn '',~112G))7+=+= ,,.I5Y?K g11      ''>>: 2$>?  " "-4OO,A,A,C)7E7P7P)4"))*KL OO ! ! #c%00cn  #w':':!;<##N$>$>?a,d LLC()+  35O PPIX isIr?r@ gslib.utilsrArBrCrDgslib.utils.boto_utilrErFrGrHrIgslib.utils.cloud_api_helperrJrKgslib.utils.constantsrLrMrNgslib.utils.encryption_helperrOrPrQrRgslib.utils.hashing_helperrSrTrUrVrWrXrYrZr[r\gslib.utils.metadata_utilr]&gslib.utils.parallelism_framework_utilr^r_r`gslib.utils.posix_utilrarbrcrdrerfrggslib.utils.system_utilrhrirjrkgslib.utils.translation_helperrlrmrnrorprqgslib.utils.unit_utilrrrsrtrurvgslib.wildcard_iteratorrwrFrrrU is_availabletop_level_managerr rr rrrrrprrrrrXrrrrTrrr)rrrrrrrrrrrrrrr*r?rUr`rdrlrrrrrrrrrrr0r9rIrOrarjrrrrrobjectrrrrrrr r#r%r+r9r@rDrIrLrirkrZrrHrrrorrrrrrs /&%' "     0 1-$/-1)694=,4RMOOJ@(7,1+$EELQF??P.862W,690>313B>A.@92!+!;2/46<E::&7AFA7IBC73:-8?9=WH-9+,-=+2/8.I=?DCH/631):77 $46CC( 9 9IMP 8,779!",M4)3+)% )3+))*% %/'%!" '*;', (46CC( 9 9IMP#H"<"G"G"I   IyI +9 x, /  %*$)%*.3#(&+&+$(27,0,0!@! /6L40?f :% `$)ePD=J! @DFL"%?d*. %JZ K2-2`'R15 =>"D"& $$"nb RvRji &&&1&P<,#^*0  bQr