-dZddlmZddlmZddlmZddlZddlZddlZddlm Z ddl m Z ddl mZdd l mZdd lmZdd lmZddlm Zdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZdZ ejBdZ"GddejFZ$GddejFZ%dZ&dZ'dZ(dZ)Gdde*Z+dZ, d"d Z-d!Z.y)#z/Utility methods used by the deploy_app command.)absolute_import)division)unicode_literalsN) exceptions) metric_names) storage_api) storage_util) context_util)storage_parallel)log)metrics) propertiesencoding)files)times)mapceZdZfdZxZS)LargeFileErrorcPtt| dj|||y)NzCannot upload file [{path}], which has size [{size}] (greater than maximum allowed size of [{max_size}]). Please delete the file or add to the skip_files entry in your application .yaml file and try again.)pathsizemax_size)superr__init__format)selfrrr __class__s 9lib/googlecloudsdk/api_lib/app/deploy_app_command_util.pyrzLargeFileError.__init__3s* .$( &d&A D__name__ __module__ __qualname__r __classcell__r s@r!rr1sDDr"rceZdZfdZxZS) MultiErrorct|dkDrdj|}ndj|}djtt|}t t j|#||z||_ y)NrzMultiple errors occurred {0} zAn error occurred {0} z ) lenrjoinrstrrcore_exceptionsErrorrerrors)roperation_descriptionr1msg errors_stringr s r!rzMultiError.__init__=sf 6{Q , 3 34I Jc % , ,-B CcKKC 01M /  /m0CDDKr"r#r(s@r!r*r*;s r"r*ci}dj|j}|D]u}tjj ||}t j j|tj}dj ||g} | |d|t|<wtj|d|} | D]} tjj| }||vrtjd nnU +/  CI &8D8  / /   sA%DD1*A D1D.-D.c|ytjtj}|tz }||jz |kS)aDetermines whether a GCS object is close to end-of-life. In order to reduce false negative rate (objects that are close to deletion but aren't marked as such) the returned filter is forward-adjusted with _TTL_MARGIN. Args: ttl: datetime.timedelta, TTL of objects, or None if no TTL. obj: storage object to check. Returns: True if the ojbect is safe or False if it is approaching end of life. T)rNowUTC _TTL_MARGIN timeCreated)ttlobjnowdeltas r! _IsTTLSafervs< [  %))# %  E ))r"c i}tj}t||tfd|j |D}d\}} |D]9} t j j|| } t j jtj| ds t j j|| } t j jtj| d} |r| |kDr t| | ||| d} | | z } | |vr+tjdj| || z }n| || <| stj djt#d |z| z d  <|S) aBuilds a map of files to upload, indexed by their hash. This skips already-uploaded files. Args: manifest: A dict containing the deployment manifest for a single service. source_dir: The relative source directory of the service. bucket_ref: The GCS bucket reference to upload files into. tmp_dir: The path to a temporary directory where generated files may be stored. If a file in the manifest is not found in the source directory, it will be retrieved from this directory instead. max_file_size: int, File size limit per individual file or None if no limit. Raises: LargeFileError: if one of the files to upload exceeds the maximum App Engine file size. Returns: A dict mapping hashes to file paths that should be uploaded. c3NK|]}t|r|jywN)rvname).0orrs r! z&_BuildFileUploadMap..s'.'L!$S!,vv'Ls"%)rrzutf-8rr9zSkipping upload of [{f}])fz)Incremental upload skipped {pct}% of datagY@)pct)r StorageClientrlset ListBucketr<rr-existsrEncodegetsizerr rErinforound)rJr:rHrI max_file_sizefiles_to_uploadriexisting_items skipped_size total_sizerLrMrrNrrs @r!_BuildFileUploadMaprse,/,,..!.*=#.~'@'@'L...!, h Z2I 77>>(//)gF G'',,w1i 77??8??9wG HD - 9dM ::"9-I$JN" ii*11H1=>dl#,oi  hh:AAEL(:5q9B;<+. r"ceZdZdZy)FileUploadTaskc.||_||_||_yry)rNrrK)rrNrrKs r!rzFileUploadTask.__init__sDNDI DOr"N)r$r%r&rr"r!rrs!r"rctjjjj xst j }g}t|jD]L\}}tjj||}t j||}|j|Nt j||dy)aEUploads files to App Engine Cloud Storage bucket using threads. Args: files_to_upload: dict {str: str}, map of checksum to local path bucket_ref: storage_api.BucketReference, the reference to the bucket files will be placed in. Raises: MultiError: if one or more errors occurred during file upload. T) num_threadsshow_progress_barN)rVALUESappnum_file_upload_threadsGetIntr DEFAULT_NUM_THREADSsorteditemsr ObjectReference FromBucketRefrappend UploadFiles)rrHrtasksrNr dest_obj_reftasks r!_UploadFilesThreadsrs""&&>>EEG6!55 %  5 5 78oi//==j>GIL  * *4 >D LL 9 u+157r"ctjtjt j 5}t ||||}t|||||}t||dddtjjdtjdjtjtj|S#1swYqxYw)aCopies application files to the Google Cloud Storage code bucket. Use the Cloud Storage API using threads. Consider the following original structure: app/ main.py tools/ foo.py Assume main.py has SHA1 hash 123 and foo.py has SHA1 hash 456. The resultant GCS bucket will look like this: gs://$BUCKET/ 123 456 The resulting App Engine API manifest will be: { "app/main.py": { "sourceUrl": "https://storage.googleapis.com/staging-bucket/123", "sha1Sum": "123" }, "app/tools/foo.py": { "sourceUrl": "https://storage.googleapis.com/staging-bucket/456", "sha1Sum": "456" } } A 'list' call of the bucket is made at the start, and files that hash to values already present in the bucket will not be uploaded again. Args: upload_dir: str, path to the service's upload directory source_files: [str], relative paths to upload. bucket_ref: The reference to the bucket files will be placed in. max_file_size: int, File size limit per individual file or None if no limit. Returns: A dictionary representing the manifest. NzFile upload done.zManifest: [{0}])r CustomTimedEventrCOPY_APP_FILES_STARTr=TemporaryDirectoryrRrrr statusPrintrrCOPY_APP_FILES)rFrGrHrrIrJrs r!CopyFilesToCodeBucketrsT <<<=$$&''L*g7H)*j'=BO4 ' **&'((  # #H -. <667 /'&s *CCcdtjjdk(r|jddS|S)zHReformat a filename for the deployment manifest if it is Windows format.\r7)r<rsepreplace)filenames r!rBrB:s*WW[[D   D# && /r"ry)/__doc__ __future__rrrrfr@r<apitools.base.pyrr_googlecloudsdk.api_lib.apprgooglecloudsdk.api_lib.storagerr googlecloudsdk.appengine.toolsr "googlecloudsdk.command_lib.storager googlecloudsdk.corer/r r rgooglecloudsdk.core.utilrrr=r six.movesr_DEFAULT_NUM_THREADSrgrpr0rr*rRrlrvrobjectrrrrBrr"r!rs 6&' >3677?=#'*-8*!h  # D_**D && ,^9>**3l!V!74596rr"