
    1                        d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlm	Z	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZ G d d ej>                  ej@                  e!            Z" G d de"      Z# G d de"      Z$ G d de$      Z% G d de$      Z&y)zGClasses that represent and execute different upload strategies for GCS.    )absolute_import)division)unicode_literalsN)encoding_helper)transfer)errors)
retry_util)metadata_util)apis)gcs_resource_reference)resource_reference)s3_resource_reference)	copy_util)log)
properties)retry)scaled_integerc                   R    e Zd ZdZ	 	 ddZd Zd Zd Zej                  d        Z
y)	_Uploadz1Base class shared by different upload strategies.Nc	                     || _         || _        || _        || _        || _        || _        || _        || _        t        j                  dd      | _
        y)a$  Initializes an _Upload instance.

    Args:
      gcs_api (gcs_api.GcsApi): The API used to execute the upload request.
      http_client: An httplib2.Http-like object.
      source_stream (io.IOBase): Yields bytes to upload.
      destination_resource (resource_reference.ObjectResource|UnknownResource):
        Metadata for the destination object.
      should_gzip_in_flight (bool): Should gzip encode upload in flight.
      request_config (gcs_api.GcsRequestConfig): Tracks additional request
        preferences.
      posix_to_set (PosixAttributes|None): Set as custom metadata on target.
      source_resource (FileObjectResource|ObjectResource|None): Contains the
        source StorageUrl and source object metadata for daisy chain transfers.
        Can be None if source is pure stream.
    storagev1N)_gcs_api_http_client_source_stream_destination_resource_should_gzip_in_flight_request_config_posix_to_set_source_resourcer   GetMessagesModule	_messages)	selfgcs_apihttp_clientsource_streamdestination_resourceshould_gzip_in_flightrequest_configposix_to_setsource_resources	            5lib/googlecloudsdk/api_lib/storage/gcs_json/upload.py__init__z_Upload.__init__-   sW    6 DM#D'D!5D"7D)D%D+D++It<DN    c                 B   t        | j                  t        j                        r{| j                  j
                  j                  rZt        | j                  j                  d      r9t        j                  | j                  j                  j                        |_        y y y y )Nacl)
isinstancer    r   ObjectResourcer   resource_argspreserve_aclhasattrmetadatacopydeepcopyr0   )r#   destination_metadatas     r,   J_copy_acl_from_source_if_source_is_a_cloud_object_and_preserve_acl_is_truezR_Upload._copy_acl_from_source_if_source_is_a_cloud_object_and_preserve_acl_is_trueT   s     	4((*<*K*KL  ..;;D))22E: "&



(
(
,
," ; < 	Mr.   c           
      X   d}t        | j                  t        j                        r*t	        j
                  | j                  j                        }nt        | j                  t        j                        rt| j                  j                  r^t	        j                  | j                  j                  j                         D ci c]  \  }}|t	        j                  |       c}}      }|r:t        j                  || j                  j                   j"                        |_        yyc c}}w )z>Update object metadata with contexts from the source resource.N)r1   r    r   GcsObjectResourcer
   6parse_custom_contexts_dict_from_resource_contexts_dictcontextsr   S3ObjectResourcetags+get_contexts_dict_from_custom_contexts_dictitems!get_context_value_dict_from_valuer   DictToMessager"   ObjectContextsValue)r#   object_metadatacontexts_to_setkeyvalues        r,   %_update_object_metadata_with_contextsz-_Upload._update_object_metadata_with_contextsa   s   O5GG 
N
N##,, 
 
4EE
 
			#	#EE #'"7"7"<"<"B"B"DG #EJC !CC
 #EG  	 !0!>!>

..


-
-"o Gs    D&
c                 
   | j                   j                  r?t        | j                  j                  j
                  | j                   j                        }nd}| j                  j                  | j                  j                  j                  | j                  j                  j                        }t        | j                  t        j                        rc| j                  j                  rMt!        j"                  | j                  j                  | j                  j                  j$                        |_        | j)                  |       | j+                  |       t-        j.                  || j                   | j                  | j0                  t,        j2                  j4                         | j                  j	                  |j6                  |t9        j:                  | j                         | j                   j<                  |      S )z>Get an insert request that includes validated object metadata.N)namebucket)attributes_resourcer*   method_type)rN   objectifGenerationMatchifMetagenerationMatchpredefinedAcl)r   predefined_acl_stringgetattrr"   StorageObjectsInsertRequestPredefinedAclValueValuesEnumrE   r   storage_urlresource_namebucket_namer1   r    r   r2   custom_fieldsr   DictToAdditionalPropertyMessageMetadataValuer6   rK   r:   r
   *update_object_metadata_from_request_configr   
MethodTypeOBJECT_INSERTrN   r   get_generation_match_value!precondition_metageneration_match)r#   predefined_aclrG   s      r,   _get_validated_insert_requestz%_Upload._get_validated_insert_request   s   11
..
4
4''



4
46n
 nnn++''33AA))55AA , CO 	4((*<*K*KL++

9
9##11nn##11  	..?SS << 11''!,,:: >>55%%#>>  "   BB$ 6 & &r.   c                      y)z1Performs an upload and returns an Object message.N r#   s    r,   runz_Upload.run        	r.   )NN)__name__
__module____qualname____doc__r-   r:   rK   re   abcabstractmethodri   rg   r.   r,   r   r   *   s>    9 %=NB+&Z 	 	r.   r   c                       e Zd ZdZd Zy)SimpleUploadz&Uploads objects with a single request.c                 z   | j                   j                  }t        j                  | j                  |j
                  | j                  |j                        }| j                  |_	        t        j                  |_        | j                  j                  j                  j                  | j!                         |      S )N)gzip_encoded
total_sizeupload)r   r3   r   Uploadr   content_typer   sizer   
bytes_httpSIMPLE_UPLOADstrategyr   clientobjectsInsertre   )r#   r3   apitools_uploads      r,   ri   zSimpleUpload.run   s    ((66Moo""00 %%	'O
 "&!2!2O'55O==''..**,_ / F Fr.   N)rk   rl   rm   rn   ri   rg   r.   r,   rr   rr      s    .Fr.   rr   c                   L    e Zd ZdZd Zd Zej                  d        Zd Z	d Z
y)_BaseRecoverableUploadz7Common logic for strategies allowing retries in-flight.c           	         | j                   j                  }t        |dd      }t        j                  j
                  j                  j                         }t        j                  | j                  |j                  dt        j                  t        j                  j
                  j                  j                               | j                   ||      }t        j"                  |_        |S )z9Returns an apitools upload class used for a new transfer.rz   NF)auto_transfer	chunksizert   ru   num_retries)r   r3   rV   r   VALUESr   max_retriesGetIntr   rx   r   ry   r   ParseIntegerupload_chunk_sizeGetr   RESUMABLE_UPLOADr}   )r#   r3   rz   r   r   s        r,   _get_uploadz"_BaseRecoverableUpload._get_upload   s    ((66M=&$/D##++77>>@Koo"" --%%77;;=?00!O  (88Or.   c                     | j                   j                  sJ| j                  j                  j                  j                  | j                         | j                          yy)z1Inserts a a new object at the upload destination.rv   N)_apitools_uploadinitializedr   r~   r   r   re   rh   s    r,   _initialize_uploadz)_BaseRecoverableUpload._initialize_upload   sN      ,,
mm""))

,
,
.t7L7L * N -r.   c                      y)zBResponsible for pushing bytes to GCS with an appropriate strategy.Nrg   rh   s    r,   *_call_appropriate_apitools_upload_strategyzA_BaseRecoverableUpload._call_appropriate_apitools_upload_strategy   rj   r.   c                 B   t        |t        j                        sy| j                  j	                          | j                  j
                  | j                  kD  r"| j                  j
                  | _        d|_        t        j                  dj                  ||             y)z.Returns True if the failure should be retried.Fr   z.Retrying upload after exception: {}. Trace: {}T)r1   r   RetryableApiErrorr   RefreshResumableUploadStateprogress_last_progress_byteretrialr   debugformat)r#   exc_type	exc_valueexc_tracebackstates        r,   _should_retry_resumable_uploadz5_BaseRecoverableUpload._should_retry_resumable_upload   s~     i!9!9:557%%(@(@@!%!6!6!?!?demII !6(M:<r.   c                    | j                         | _        | j                  | j                  _        t	        j
                  | j                         | j                          | j                  j                  | _        	 t	        j                  | j                  | j                        }| j"                  j$                  j&                  j)                  | j"                  j$                  j&                  j+                  d      |      S # t        j                  $ r)}t        j                  dj!                  |            d}~ww xY w)zAUploads with in-flight retry logic and returns an Object message.)targetshould_retry_ifz7Max retrial attempts reached. Aborting upload.Error: {}Nr   )r   r   r   r{   r	   set_retry_funcr   r   r   retryerr   r   r   MaxRetrialsExceptionr   ResumableUploadAbortErrorr   r   r~   r   ProcessHttpResponseGetMethodConfig)r#   http_responsees      r,   ri   z_BaseRecoverableUpload.run   s    ,,.D'+'8'8D$d334#44==D! ((@@==?m =='';;$$44X>O O %% !,,fQi! !!s   <+D E $D;;E N)rk   rl   rm   rn   r   r   ro   rp   r   r   ri   rg   r.   r,   r   r      s5    ?"N 	 	 Or.   r   c                       e Zd ZdZd Zy)StreamingUploadzHUploads objects from a stream with support for error recovery in-flight.c                 6    | j                   j                         S )z5Calls StreamInChunks since the final size is unknown.)r   StreamInChunksrh   s    r,   r   z:StreamingUpload._call_appropriate_apitools_upload_strategy  s      //11r.   N)rk   rl   rm   rn   r   rg   r.   r,   r   r     s
    P2r.   r   c                   F     e Zd ZdZ	 	 	 	 d fd	Z fdZ fdZd Z xZS )ResumableUploadzDUploads objects with support for resuming between runs of a command.c           
      X    t         t        |   ||||||||	       || _        |
| _        y)a4  Initializes a ResumableUpload instance.

    See super class for arguments not described below.

    New Args:
      serialization_data (dict): JSON used by apitools to resume an upload.
      tracker_callback (Callable[[dict]|None]): Function that writes a tracker
        file with serialization data.
    )r*   r+   N)superr   r-   _serialization_data_tracker_callback)r#   r$   r%   r&   r'   r(   r)   r*   serialization_datar+   tracker_callback	__class__s              r,   r-   zResumableUpload.__init__  sC    . 
/4)!' * 	  2D-Dr.   c                    t         j                  j                  j                  j	                         }| j
                  tt        j                  j                  | j                  t        j                  | j
                        | j                  j                  j                  d| j                  |      S t         | G         S )zCCreates a new transfer object, or gets one from serialization data.F)r   rt   r   )r   r   r   r   r   r   r   rx   FromDatar   jsondumpsr   r~   httpr   r   r   )r#   r   r   s     r,   r   zResumableUpload._get_upload;  s    ##++77>>@K+__%%



**T--
.
--


#
#22! & # # 9d/11r.   c                     | j                   t        |           | j                  &| j                  | j                  j
                         yy)zEInserts an object if not already inserted, and writes a tracker file.N)r   r   r   r   r   r   )r#   r   s    r,   r   z"ResumableUpload._initialize_uploadJ  sG    'It/1)
T22EEF *r.   c                     | j                   r| j                  j                         S | j                  j                         S )zDCalls StreamMedia, or StreamInChunks when the final size is unknown.)r   r   r   StreamMediarh   s    r,   r   z:ResumableUpload._call_appropriate_apitools_upload_strategyR  s7    """"1133 ""..00r.   )NNNN)	rk   rl   rm   rn   r-   r   r   r   __classcell__)r   s   @r,   r   r     s*    L ".H2G1r.   r   )'rn   
__future__r   r   r   ro   r7   r   apitools.base.pyr   r   googlecloudsdk.api_lib.storager   r	   'googlecloudsdk.api_lib.storage.gcs_jsonr
   googlecloudsdk.api_lib.utilr   ,googlecloudsdk.command_lib.storage.resourcesr   r   r   +googlecloudsdk.command_lib.storage.tasks.cpr   googlecloudsdk.corer   r   googlecloudsdk.core.utilr   r   sixwith_metaclassABCMetarQ   r   rr   r   r   r   rg   r.   r,   <module>r      s    N &  ' 
   , % 1 5 A , O K N A # * * 3 
H	 c  f5 H	VF7 F"BOW BOJ2, 2G1, G1r.   