=jdZddlmZddlmZddlmZddlZddlZddlZddlZddl m Z ddl m Z ddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlZdZGddej6ZGddeZGddeZGddeZGddeZ dZ!dZ"dZ#dZ$dZ%d Z&d!Z'd"Z(Gd#d$e)Z*Gd%d&e)Z+d'Z,d(Z-d)Z.dd*ej^j`ej^j`fd+Z1Gd,d-e)Z2Gd.d/e)Z3Gd0d1ejhZ5y)2z4Utilities for interacting with Google Cloud Storage.)absolute_import)division)unicode_literalsN)apis) gcloudignore)config) exceptions)execution_utils)log) resources)files) iso_duration) platformszgs://ceZdZdZy)Errorz)Base class for exceptions in this module.N__name__ __module__ __qualname____doc__2lib/googlecloudsdk/api_lib/storage/storage_util.pyrr+s1rrceZdZdZy) GsutilErrorz-Exception raised when gsutil cannot be found.Nrrrrrr/s5rrc"eZdZdZfdZxZS)InvalidNameErrorz.Error indicating that a given name is invalid.cRtt| dj||||y)Nz>Invalid {type} name [{name}]: {reason} See {url} for details.)namereasontypeurl)superr__init__format)selfrr type_namer" __class__s rr$zInvalidNameError.__init__6s2 D* "#)6tF/8c$*$CDr)rrrrr$ __classcell__r(s@rrr3s6DDrrc*eZdZdZdZdZfdZxZS)InvalidBucketNameErrorz5Error indicating that a given bucket name is invalid.bucketz9https://cloud.google.com/storage/docs/naming#requirementscZtt| |||j|jyN)r#r,r$TYPEURLr&rr r(s rr$zInvalidBucketNameError.__init__B$ $0 fdii+rrrrrr0r1r$r)r*s@rr,r,=s= $C#++rr,c*eZdZdZdZdZfdZxZS)InvalidObjectNameErrorz5Error indicating that a given object name is invalid.objectz8https://cloud.google.com/storage/docs/naming#objectnamescZtt| |||j|jyr/)r#r6r$r0r1r2s rr$zInvalidObjectNameError.__init__Lr3rr4r*s@rr6r6Gs= $B#++rr6zeBucket names must contain only lowercase letters, numbers, dashes (-), underscores (_), and dots (.).z8Bucket names must start and end with a number or letter.zBucket names must contain 3 to 63 characters. Names containing dots can contain up to 222 characters, but each dot-separated component can be no longer than 63 characters.zjBucket names cannot be represented as an IP address in dotted-decimal notation (for example, 192.168.5.4).zmObject names can contain any sequence of valid Unicode characters, of length 1-1024 bytes when UTF-8 encoded.zFObject names must not contain Carriage Return or Line Feed characters.cD|jd}dt|cxkrdkrnntd|Drt|tt |t t jt jzdzz rt|tt |d|dzt t jt jzz rt|tt|dk(r0d j|jrt|ty y ) aValidate the given bucket name according to the naming requirements. See https://cloud.google.com/storage/docs/naming#requirements Args: name: the name of the bucket, not including 'gs://' Raises: InvalidBucketNameError: if the given bucket name is invalid .c38K|]}t|dkDyw)?N)len).0cs r z&_ValidateBucketName..rs'HZA Zsz-_.rN)splitr?anyr,VALID_BUCKET_LENGTH_MESSAGEsetstringascii_lowercasedigitsVALID_BUCKET_CHARS_MESSAGEVALID_BUCKET_START_END_MESSAGEjoinisdigit#VALID_BUCKET_DOTTED_DECIMAL_MESSAGE)r componentss r_ValidateBucketNamerSfszz#* s4y C C'HZ'H$H 'B CCYV++fmm;eCDD 'A BBa48 s6#9#9FMM#IJJ 'E FF_bggj199; 'J KK<rc|jtr|ttd}n|}t|j dy)N/) startswithGSUTIL_BUCKET_PREFIXr?rSrstrip)r"rs rValidateBucketUrlrYs: ^^() s'() *D Ddkk#&'rcfeZdZdZdZedZedZed dZdZ dZ dZ d Z d Z y ) BucketReferencez=A wrapper class to make working with GCS bucket names easier.c||_y)zMCreates a BucketReference. Args: bucket: str, The bucket name Nr-)r&r-s rr$zBucketReference.__init__s DKrc&||jS)z=Create a bucket reference from a bucket message from the API.r)clsr-s r FromMessagezBucketReference.FromMessages v{{ rcd|tjj|djS)z=Parse a bucket URL ('gs://' optional) into a BucketReference.zstorage.buckets) collection)r REGISTRYParser-)r`r"s rFromUrlzBucketReference.FromUrls/ y!!''8I'J rc|r*|jtstjd t ||j|S#t $r-}tjt j|d}~wwxYw)zEValidates that the argument is a reference to a Cloud Storage bucket.zIMust be a valid Google Cloud Storage bucket of the form [gs://somebucket]N) rVrWargparseArgumentTypeErrorrYr,six text_typerf)r`valuerequire_prefixerrs r FromArgumentzBucketReference.FromArgumentswe../CD  & &  ; ;;u  ";  & &s}}S'9 ::;s A B(A;;Bc8dj|jS)Nzgs://{}r%r-r&s rToUrlzBucketReference.ToUrls   DKK ((rc8dj|jS)Nz"https://storage.googleapis.com/{0}rqrrs r GetPublicUrlzBucketReference.GetPublicUrls / 6 6t{{ CCrc4|j|jk(Sr/r]r&others r__eq__zBucketReference.__eq__s ;;%,, &&rc&|j| Sr/ryrws r__ne__zBucketReference.__ne__{{5! !!rc,t|jSr/)hashr-rrs r__hash__zBucketReference.__hash__s  rN)T)rrrrr$ classmethodrarfrorsruryr|rrrrr[r[s`E   )D'"rr[ceZdZdZdZdZdZedZedZ e dZ e dZ e d Z e dd Ze dd Ze d Zd ZdZdZdZdZy)ObjectReferencezGWrapper class to make working with Cloud Storage bucket/objects easier.z&^gs://(?P[^/]+)/(?P.+)z^gs://(?P[^/]+)/?c ||_||_yr/r-r)r&r-rs rr$zObjectReference.__init__sDKDIrc|jS)z:Emulates the object field on the object core/resource ref.r_rrs rr7zObjectReference.objects 99rc,t|jS)z9Gets a bucket reference for the bucket this object is in.)r[r-rrs r bucket_refzObjectReference.bucket_refs 4;; ''rc<||j|jS)z?Create an object reference from an object message from the API.r)r`objs rrazObjectReference.FromMessages szz388 $$rct|dt|jdcxkrdksnt|td|vsd|vrt|t |||S)zCreate an object reference from a bucket reference and a name.)rr-)r`rrs r FromBucketRefzObjectReference.FromBucketRefs << ))4 00rctj|j|tj}|r0|j |j d|j dStj|j |tj}|r&|r||j ddStddtd)z?Parse an object URL ('gs://' required) into an ObjectReference.r-r7rEz Empty object name is not allowedz"Must be of form gs://bucket/object) rematchGSUTIL_OBJECT_REGEXDOTALLrgroupGSUTIL_BUCKET_REGEXr6 ValueError)r`r"allow_empty_objectrs rrfzObjectReference.FromUrls HHS,,c299 =E \\%++h/X1F GG HHS,,c299 =E 5;;x("--$R)KLL 9 ::rc |j||S#ttf$r-}tjt j |d}~wwxYw)N)r)rfr6rrhrirjrk)r`r"rrns rrozObjectReference.FromArgumentsJ; [[1C[ DD "J /;  & &s}}S'9 ::;sA(A  AcF |j|y#t$rYywxYw)NFT)rfr)r`paths r IsStorageUrlzObjectReference.IsStorageUrls, kk$   s   cNdj|j|jS)Nz gs://{}/{}r%r-rrrs rrszObjectReference.ToUrls   t{{DII 66rcNdj|j|jS)Nz$https://storage.googleapis.com/{}/{}rrrs rruzObjectReference.GetPublicUrls 1 8 8dii PPrcD|j|jk(Sr/)rsrws rryzObjectReference.__eq__s ::<5;;= ((rc&|j| Sr/r{rws rr|zObjectReference.__ne__r}rc4t|jSr/)rrsrrs rrzObjectReference.__hash__s  rN)F)rrrrrrr$propertyr7rrrarrrfrorrsruryr|rrrrrrsOA4   ( (%%  11 ; ;;; 7Q)"rrc.tjddS)z:Import and return the appropriate storage messages module.storagev1) core_apisGetMessagesModulerrr GetMessagesr   $ $Y 55rc.tjddS)z1Import and return the appropriate storage client.rr)rGetClientInstancerrr GetClientr%rrctjj}|sItjd}|r't j dj||Stdtjj|dS)z)Determines the path to the gsutil binary.gsutilzUsing gsutil found at [{path}])rz]A path to the storage client `gsutil` could not be found. Please check your SDK installation.) rPaths sdk_bin_path file_utilsFindExecutableOnPathr debugr%rosrrO)r gsutil_paths r_GetGsutilPathr*st,,, 11(;K ii077[7IJ  E FF lH --rFct}|rd|gn|g}|||z }tjjtjjk(rt j |dzg|}nt j|g|}tjdjdj|t j|d||S)acRuns the specified gsutil command and returns the command's exit code. WARNING: This is not compatible with python 3 and should no longer be used. Args: command_name: The gsutil command to run. command_args: List of arguments to pass to the command. run_concurrent: Whether concurrent uploads should be enabled while running the command. out_func: str->None, a function to call with the stdout of the gsutil command. err_func: str->None, a function to call with the stderr of the gsutil command. Returns: The exit code of the call to the gsutil command. z-mz.cmdzRunning command: [{args}]] )argsT)no_exitout_funcerr_func) rrOperatingSystemCurrentWINDOWSr ArgsForCMDToolArgsForExecutableToolr rr%rOExec) command_name command_argsrun_concurrentrr command_pathr gsutil_argss rRunGsutilCommandr9s, !,!/$ l^$LD&&(I,E,E,M,MM!001FNNK!77 LtLK)) ( / /SXXk5J / KL   k4'/'/ 11rceZdZdZdZy) FileMetadataa FileMetadata contains information about a file destined for GCP upload. Attributes: root: str, The root directory for considering file metadata. path: str, The path of this file, relative to the root. size: int, The size of this file, in bytes. c||_||_tjjtjj |||_y)zCollect file metadata. Args: root: str, The root directory for considering file metadata. path: str, The path of this file, relative to the root. N)rootrrgetsizerOsize)r&rrs rr$zFileMetadata.__init__hs5DIDI T4 89DIrNrrrrr$rrrrr_s  :rrceZdZdZddZy)SnapshotaSnapshot is a manifest of the source in a directory. Attributes: src_dir: str, The root of the snapshot source on the local disk. ignore_file: optional str, an override for .gcloudignore. include_gitignore: bool, whether the local .gitignore shouild be included in the ignored files. files: {str: FileMetadata}, A mapping from file path (relative to the snapshot root) to file metadata. dirs: [str], The list of dirs (possibly empty) in the snapshot. uncompressed_size: int, The number of bytes needed to store all of the files in this snapshot, uncompressed. any_files_ignored: bool, whether any files were ignored. Nc`||_i|_g|_d|_t j |jd||}d|_tjtj|jD]\}}}tjj||j}|D] } tjj|| } tjj| rDtjj| s%t!j"dj%| |dk7r tjj|| n| } |j'| sd|_t)|j| } | |j| <|xj| j*z c_ |ddD]i} |dk7r tjj|| n| }|j'|ds|j-| O|jj/|ky)NrF) write_on_disk ignore_fileinclude_gitignorez5Ignoring [{}] which is a symlink to non-existent pathr:T)is_dir)src_dirr dirsuncompressed_sizerGetFileChooserForDirany_files_ignoredrwalkrjrkrrelpathrOislinkexistsr infor% IsIncludedrrremoveappend)r&rrr file_chooserdirpathdirnames filenamesrfnamerfpathfmdnamedpaths rr$zSnapshot.__init__sDLDJDID44 + L #D+-''#-- 2M*N&(I6g%ww||GU+ 77>>$ t(< ((ELL  073 We,E&&u-#'$  $,, . 5 "'')$A;%073 We,E&&uT&: //%   )+Or)NTrrrrrrts  ' rrcPeZdZdZdZdZdZdZdZdZ eezZ e ezZ e ezZ de zZ y) ObjectLockRetentionDurationaSpecialized object lock duration class derived from iso_duration.Duration. Retention period for object lock feature follows non iso standard values for years and months. A year is considered to be 365.25 days and every month is expected to have 31 days. This class makes the required changes so that calculations are adjusted accordingly when parsing the duration string. gv@i@B< N)rrrr_DAYS_PER_YEAR_MICROSECONDS_PER_SECOND_SECONDS_PER_MINUTE_MINUTES_PER_HOUR_HOURS_PER_DAY_MONTHS_PER_YEAR_SECONDS_PER_HOUR_SECONDS_PER_DAY_SECONDS_PER_YEAR_SECONDS_PER_MONTHrrrrrsY.$.),==&7&7,,rr)6r __future__rrrrhrrrJgooglecloudsdk.api_lib.utilrrgooglecloudsdk.command_lib.utilrgooglecloudsdk.corerr r r r googlecloudsdk.core.utilr rrrrjrWrrrrr,r6rMrNrHrQrrrSrYr7r[rrrrfile_only_loggerrrrrDurationrrrrr sN ;&' 98&*/#)81. 2J  26%6DzD+-++-+""<@'(# .JLH(2f2jTfTn6 6 . #'$)!2288!2288 #1L:6:*7 v7 t-,"7"7-r