
                           d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl	m
Z
 ddlZddlZddlZddlZddlZddlZddlZddlZdd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddl Z ddl!m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl#m'Z' ddl#m(Z( ddl#m)Z) ddl#m*Z* ddl#m+Z+ ddl#m,Z, ddl#m-Z- ddl#m.Z. ddl#m/Z/ ddl#m0Z0 ddl#m1Z1 ddl#m2Z2 ddl3m4Z4 dd l5m6Z6 dd!l5m7Z7 dd"l8m9Z9 dd#l8m:Z: dd$l8m;Z; dd%l8m<Z< dd&l8m=Z= dd'l8m>Z> dd(l8m?Z? dd)l@mAZA dd*lBmCZC dd+lDmEZE dd,lFmGZG dd-lHmIZJ dd.lHmKZL dd/lMmNZN dd0lMmOZO dd1lMmPZP dd2lMmQZQ dd3lMmRZR dd4lSmTZT dd5lSmUZU dd6lSmVZV dd7lSmWZW dd8lSmXZX dd9lSmYZY dd:lSmZZZ dd;l[m\Z\ dd<l[m]Z] dd=l^m_Z_ dd>l^m`Z` dd?l^maZa dd@l^mbZb ddAl^mcZc ddBldmeZe ddCldmfZf ddDldmgZg ddEldmhZh ddFlimjZj ddGlkmlZl ddHlmmnZnmoZo ddIlpmqZq ddJlrmsZs ddKlrmtZt ddLlrmuZu ddMlrmvZv ddNlrmwZw ddOlrmxZx ddPlymzZz ej                  re|Z}eLj                  Z~e~j                  Zej                  ZdQZej                  ej
                  ej                  ej                  fZej                  ej                  ej                  ej                  e
j                  e
j                  e
j                  ej                   e j"                  j$                  ej&                  ej(                  ej*                  ej,                  efZ eg dR      ZddSdTdUdVdWdXZddSdYdZdTdUdVd[Zej;                         Zej?                  e        ed\d]g      Zd^Zd_Zd`ZdaZ G db dce'      Zy)dz>JSON gsutil Cloud API implementation for Google Cloud Storage.    )absolute_import)print_function)division)unicode_literals)contextmanagerN)http_client)encoding)
exceptions)http_wrapper)transfer)CalculateWaitForRetry)config)context_config)AccessDeniedException)ArgumentException)BadRequestException)CloudApi)EncryptionException)NotEmptyException)NotFoundException)PreconditionException)Preconditions) PublishPermissionDeniedException)ResumableDownloadException)ResumableUploadAbortException)ResumableUploadException)!ResumableUploadStartOverException)ServiceException)CommandException)SetUpJsonCredentialsAndCache)isP12Credentials)BytesTransferredContainer)&DownloadCallbackConnectionClassFactory)HttpWithDownloadStream)HttpWithNoRetries)$UploadCallbackConnectionClassFactory)WrapDownloadHttpRequest)WrapUploadHttpRequest)ImpersonationCredentials)NoOpCredentials)ProgressCallbackWithTimeout)PopulateProjectId)storage_v1_client)storage_v1_messages)DeleteTrackerFile)GetRewriteTrackerFilePath)HashRewriteParameters)ReadRewriteTrackerFile)WriteRewriteTrackerFile)GetCertsFile)GetGcsJsonApiVersion)GetJsonResumableChunkSize)GetMaxRetryDelay)
GetNewHttp)GetNumRetries)JsonResumableChunkSizeDefined)ListToGetFields)ValidateDstObjectMetadata)IAM_POLICY_VERSION)NUM_OBJECTS_PER_LIST_PAGE)REQUEST_REASON_ENV_VAR)REQUEST_REASON_HEADER_KEY)UTF8)#Base64Sha256FromBase64EncryptionKey)CryptoKeyType)CryptoKeyWrapperFromKey)FindMatchingCSEKInBotoConfig)AddAcceptEncodingGzipIfNeeded)LogAndHandleRetries)CreatePayloadGetFinalUrl)GetPrintableExceptionString)CreateBucketNotFoundException)%CreateNotFoundExceptionForObjectWrite)CreateObjectNotFoundException)DEFAULT_CONTENT_TYPE)PRIVATE_DEFAULT_OBJ_ACL)REMOVE_CORS_CONFIG)ServiceAccountCredentialsi  )acldefaultObjectAclz	items/aclzitems/defaultObjectAclzitems/ownerownerauthenticatedReadprivateprojectPrivate
publicReadpublicReadWrite)Nauthenticated-readrV   project-privatepublic-readzpublic-read-writebucketOwnerReadbucketOwnerFullControl)NrZ   zbucket-owner-readzbucket-owner-full-controlrV   r[   r\   crc32cmd5Hashskipz4Insufficient OAuth2 scope to perform this operation.zstorage.googleapis.comzstorage.mtls.googleapis.comc                   Z    e Zd ZdZ	 	 	 	 	 	 	 dI fd	Zd Zd Zd Zd Zd	 Z	dJd
Z
dJdZdKdZ	 	 	 dLdZ	 	 dKdZdJdZd ZdKdZd Zd Z	 	 	 	 	 dMdZdJdZ	 	 	 	 dNdZdKdZdLdZd Z	 	 	 	 	 dMdZdJdZd Zed        ZdJdZ	 	 dKdZ 	 	 	 dLd Z!	 	 dKd!Z"dJd"Z#	 	 	 dLd#Z$d$ Z%	 dJd%Z&dJd&Z'dJd'Z(dddd(e)jT                  jV                  ddddddfd)Z,	 	 	 	 	 	 dOd*Z-	 	 	 	 	 	 dOd+Z.	 	 	 	 	 	 dPd,Z/ddddddddde0jb                  dd(fd-Z2d. Z3	 	 	 	 	 	 	 	 dQd/Z4	 	 	 	 	 	 	 dRd0Z5	 	 	 	 	 	 	 	 	 	 dSd1Z6	 	 	 	 	 	 	 	 	 dTd2Z7	 	 	 dLd3Z8	 	 	 	 dNd4Z9	 	 	 dLd5Z:dJd6Z;dKd7Z<d8 Z=	 	 	 dLd9Z>d: Z?d; Z@d< ZAd= ZBd> ZC	 dUd?ZDd@ ZEdA ZFdB ZGdC ZH	 	 	 	 dNdDZIdE ZJdF ZKdG ZL	 	 	 	 dNdHZM xZNS )V
GcsJsonApiz=Google Cloud Storage JSON implementation of gsutil Cloud API.Nr   c                    t         t        |   |||d||||	|
	       t               | _        t               | _        t        | ||       | j                         | _	        | j                         | _        | j                  rU| j                  j                  | j                        | _        | j                  j                  | j                        | _        n"| j                  | _        | j                  | _        t!        | j                        r?t#        | j                  j                         t%        | j                  j                         n*t#        | j                         t%        | j                         d| _        t)        j*                  ddd      }t-        j.                         rKt-        j.                         j0                  r-|rt3        dj5                  t6        |            t6        | _        n|xs t:        | _        |s't)        j*                  dd	d      }|rt3        d
|z        t)        j*                  ddd      }|s/t)        j*                  ddd      }|rt3        d|z        d| _        ndt)        j*                  dd      z   | _        t?               | _         | j&                  | j8                  z   | j<                  z   dz   dz   | j@                  z   dz   | _!        |rtE        jF                  d|z        nd| _$        i }t)        j*                  ddd      }|r|r||d<   | jK                  |       |dk\  }|dk\  }tM        jN                  | jB                  | j                  ||| j                  | j@                  | jH                  |      | _(        tS               | _*        | jT                  | jP                  _*        tW               | _,        | jX                  | jP                  _,        t[        | j\                        | jP                  _/        d| jP                  _0        tc        | j                  td              r| jP                  jg                  dd       yy)a  Performs necessary setup for interacting with Google Cloud Storage.

    Args:
      bucket_storage_uri_class: Unused.
      logger: logging.logger for outputting log messages.
      status_queue: Queue for relaying status to UI.
      provider: Unused.  This implementation supports only Google Cloud Storage.
      credentials: Credentials to be used for interacting with Google Cloud
                   Storage.
      debug: Debug level for the API implementation (0..3).
      http_headers (dict|None): Arbitrary headers to be included in every request.
      trace_token: Trace token to pass to the API implementation.
      perf_trace_token: Performance trace token to use when making API calls.
      user_project: Project to be billed for this request.
    gs)providerdebughttp_headerstrace_tokenperf_trace_tokenuser_project)credentialszhttps://Credentialsgs_json_hostNz"use_client_certificate" is enabled, which sets gsutil to use the host {}. However, a custom host was set using "gs_json_host": {}. Please set "use_client_certificate" to "False" or comment out the "gs_json_host" line in the Boto config.gs_hostzJSON API is selected but gs_json_host is not configured, while gs_host is configured to %s. Please also configure gs_json_host and gs_json_port to match your desired endpoint.gs_json_portgs_portzJSON API is selected but gs_json_port is not configured, while gs_port is configured to %s. Please also configure gs_json_host and gs_json_port to match your desired endpoint. :/zstorage/ztoken:%s)tracegs_json_host_headerHost   )urlhttplog_requestlog_responserl   versiondefault_global_paramsadditional_http_headers)status_queueTkey'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM)4superrc   __init__r4   
certs_filer8   rz   r    _GetNewDownloadHttpdownload_http_GetNewUploadHttpupload_httprl   	authorizeauthorized_download_httpauthorized_upload_httpr!   r'   r(   	http_baser   getr   get_context_configuse_client_certificater   format	MTLS_HOST	host_baseDEFAULT_HOST	host_portr5   api_versionurl_baseapitools_messagesStandardQueryParametersglobal_params_UpdateHeadersapitools_client	StorageV1
api_clientr7   max_retry_waitr9   num_retriesrG   r   
retry_func(overwrite_transfer_urls_with_client_base
isinstancer*   AddGlobalParam)selfbucket_storage_uri_classloggerr   rf   rl   rg   rh   ri   rj   rk   rn   ro   rp   rq   r   rv   r{   r|   	__class__s                      %platform/gsutil/gslib/gcs_json_api.pyr   zGcsJsonApi.__init__   s   8 
*d$%=%+%1.2+02>1<6F2> % @ #nDODI v;G 113D--/D&*&6&6&@&@


'd#$($4$4$>$>t?O?O$Pd!&*&8&8d#$($4$4d!(()d;;@@AD77<<=d;;<D778DN::m^TBL))+))+BB	 vi68 	8 !dn#3|dn

=)T:g	L  	 ::m^TBL

=)T:g	L  	
 dnVZZ~FFdn+-D^^dnn4t~~EK "&"2"23589DM ,7 +BB;&(<@ 	 ! **]4I4P|(;f%/0A:KQJL%//MMYY!$$  "00 79DO +,D%)%8%8DOO"$D"&"2"2DOO!4&&"(DOO?CDOO<$""O4 oo$$U%NP 5    c                     t        | j                  t              r| j                  j                  S t        | j                  t              r| j                  j
                  S t        d      )z%Returns the service account email id.zBCannot get service account email id for the given credential type.)r   rl   r)   service_account_idrQ   service_account_emailr   r   s    r   GetServiceAccountIdzGcsJsonApi.GetServiceAccountIdY  sZ    $""$<=000	D$$&?	@333 r   c                     | j                   r|j                  | j                          | j                  r| j                  |d<   t        j                  j                  t              }|r
||t        <   y y )Ncookie)rh   updaterj   osenvironr   r?   r@   )r   headersrequest_reasons      r   r   zGcsJsonApi._UpdateHeadersd  s^    nnT&&' //ghZZ^^$:;N+9g'( r   c                 "    t        t              S )N
http_class)r8   r$   r   s    r   r   zGcsJsonApi._GetNewDownloadHttpq  s    !788r   c                 "    t        t              S )zCReturns an upload-safe Http object (by disabling httplib2 retries).r   )r8   r%   r   s    r   r   zGcsJsonApi._GetNewUploadHttpt  s    !233r   c                 b   t        | j                  t              rV| j                  j                  }| j                  j                  }|j                  ||      }|j                  |j                  fS t        | j                  t              r| j                  j                  |      S t        d      )zReturns the Signed Content.zKAuthentication using a service account is required for signing the content.)r   rl   r)   apir   SignBlobkeyId
signedBlobrQ   	sign_blobr   )r   string_to_signiam_cred_apir   responses        r   _GetSignedContentzGcsJsonApi._GetSignedContentx  s    $""$<=%%))l++>>&&'9>Jh^^X0000	D$$&?	@''77 r   c                 J    |du xs t         j                  t        |            S )a  Checks Returns true if ACL related values are in fields set.

    Args:
      fields: list or set of fields. May be in GET ['acl'] or List
          ['items/acl'] call format.

    Returns:
      True if an ACL value is requested in the input fields, False otherwise.
    N)_ACL_FIELDS_SETintersectionsetr   fieldss     r   _FieldsContainsAclFieldz"GcsJsonApi._FieldsContainsAclField  s"     T>F_99#f+FFr   c                 t    |r6t         j                  t        |            rd|vrt        dt         z        yyy)zEEnsures customerEncryption field is included if hashes are requested.customerEncryptionzhgsutil client error: customerEncryption must be included when requesting potentially encrypted fields %sN)_ENCRYPTED_HASHES_SETr   r   r   r   s     r   ._ValidateEncryptionFieldsForMetadataGetRequestz9GcsJsonApi._ValidateEncryptionFieldsForMetadataGetRequest  sK     	(55c&kBF*79NOP P 	+ Cr   c                 Z   t        j                  || j                  t              }t        j                         }|rdj                  t        |            |_        	 | j                  j                  j                  ||      S # t        $ r}| j                  ||       Y d }~y d }~ww xY w)N)bucketuserProjectoptionsRequestedPolicyVersion,r   bucket_name)r   !StorageBucketsGetIamPolicyRequestrk   r=   r   joinr   r   r   bucketsGetIamPolicy TRANSLATABLE_APITOOLS_EXCEPTIONS_TranslateExceptionAndRaise)r   r   rf   r   apitools_requestr   es          r   GetBucketIamPolicyzGcsJsonApi.GetBucketIamPolicy  s    (JJ%%&8: &==?M XXc&k2mC__$$112B@M 2 O O+ C
&&qk&BBCs   &B 	B*B%%B*c                 r   |t        |      }t        j                  |||| j                        }t        j                         }|rdj                  t        |            |_        	 | j                  j                  j                  ||      S # t        $ r}| j                  ||||       Y d }~y d }~ww xY w)Nr   object
generationr   r   r   r   object_namer   )longr   !StorageObjectsGetIamPolicyRequestrk   r   r   r   r   r   objectsr   r   r   )	r   r   r   r   rf   r   r   r   r   s	            r   GetObjectIamPolicyzGcsJsonApi.GetObjectIamPolicy  s     
#j(JJ%%	'
 &==?M XXc&k2m>__$$112B@M 2 O O+ >
&&q3>3>2< ' > >>s   '&B 	B6B11B6c                 0   |t        |      }t        j                  ||||| j                        }t        j                         }	 | j
                  j                  j                  ||      S # t        $ r}| j                  |||       Y d }~y d }~ww xY w)N)r   r   r   policyr   r   r   r   )
r   r   !StorageObjectsSetIamPolicyRequestrk   r   r   r   SetIamPolicyr   r   )	r   r   r   r   r   rf   api_requestr   r   s	            r   SetObjectIamPolicyzGcsJsonApi.SetObjectIamPolicy  s     
#j#EE%%'K &==?M@__$$11+@M 2 O O+ @
&&q3>3> ' @ @@s   &A. .	B7BBc                    t        j                  ||| j                        }t        j                         }	 | j                  j
                  j                  ||      S # t        $ r}| j                  ||       Y d }~y d }~ww xY w)N)r   r   r   r   r   )	r   !StorageBucketsSetIamPolicyRequestrk   r   r   r   r   r   r   )r   r   r   rf   r   r   r   s          r   SetBucketIamPolicyzGcsJsonApi.SetBucketIamPolicy  s    (JJ6t7H7HJ%==?MC__$$112B@M 2 O O+ C
&&qk&BBCs   &A 	B(B  Bc	                    | j                         }	t        |	||||||||	      \  }
}t        j                  r|
j	                  t
              }
| j                  |
      \  }}|j                  d|	d|d       t        ||d   ||      S ),See CloudApi class for function doc strings.)		client_idmethoddurationpathr   r   regionsigned_headersstring_to_sign_debugz.Key ID used to sign blob for service account "z": ""host)	r   rH   sixPY3encoderA   r   rg   rI   )r   r   r   r   r   r   r   r   r   r   r   canonical_query_stringkey_idraw_signatures                 r   SignUrlzGcsJsonApi.SignUrl  s     113-:$%1	.3*N* ww%,,T2n 22>BFM
LL$f. /}nV&<d-/ /r   c                    t         j                  j                  j                  }| j	                  |      r$t         j                  j                  j
                  }t        j                  ||| j                        }t        j                         }|rdj                  t        |            |_
        	 | j                  j                  j                  ||      S # t        $ r}| j                  ||       Y d}~yd}~ww xY w)r   )r   
projectionr   r   r   r   N)r   StorageBucketsGetRequestProjectionValueValuesEnumnoAclr   fullrk   r   r   r   r   r   r   Getr   r   )r   r   rf   r   r  r   r   r   s           r   	GetBucketzGcsJsonApi.GetBucket  s    #<<++EE ##F+%>>--dd (AA%%' &==?M XXc&k2m
C__$$(()97D ) F F+ C
&&qk&BBCs   2&C 	C?"C::C?c                     	 t        j                  |      }| j                  j                  j	                  |      S # t
        $ r}| j                  ||       Y d}~yd}~ww xY w)r   )r   r   N)r   &StorageBucketAccessControlsListRequestr   bucketAccessControlsListr   r   )r   r   requestr   s       r   ListBucketAccessControlsz#GcsJsonApi.ListBucketAccessControls  s^    C!HHg__1166w??+ C
&&qk&BBCs   := 	A#AA#c                     	 t        j                  ||      }| j                  j                  j	                  |      S # t
        $ r}| j                  |||       Y d}~yd}~ww xY w)r   )r   r   r   N)r   &StorageObjectAccessControlsListRequestr   objectAccessControlsr  r   r   )r   r   r   r  r   s        r   ListObjectAccessControlsz#GcsJsonApi.ListObjectAccessControls   sl    @!HH[2g__1166w??+ @
&&q3>3> ' @ @@s   ;> 	A%A  A%c           	      6   t         j                  j                  j                  }| j	                  |      r$t         j                  j                  j
                  }|}	|s
t               }g }
dD ]F  }t        |	|d      }|st        j                  |      r)t        |	|d       |
j                  |       H |	j                  r+|	j                  t        k(  rg |	_        |
j                  d       |	j                  r.|	j                  d   t        k(  rg |	_        |
j                  d       d}|r?|
j                  d       t         j                  j!                  | j#                  |            }d}|r?|
j                  d       t         j                  j%                  | j'                  |            }t        j                  ||	||j(                  ||| j*                        }t        j,                         }|rdj/                  t1        |            |_        | j4                  j7                  |
      5  	 | j4                  j8                  j;                  ||	      cddd       S # t<        $ r}| j?                  |       Y d}~nd}~ww xY w	 ddd       y# 1 sw Y   yxY w)
r   )billing
encryption	lifecycleloggingmetadataretentionPolicy
versioningwebsiteNcorsr   rS   rR   )r   bucketResourcer  ifMetagenerationMatchpredefinedAclpredefinedDefaultObjectAclr   r   r   ) r   StorageBucketsPatchRequestr
  r  r   r  r   getattrr	   MessageToDictsetattrappendr"  rP   rS   rO   PredefinedAclValueValuesEnum_BucketCannedAclToPredefinedAcl)PredefinedDefaultObjectAclValueValuesEnum_ObjectCannedAclToPredefinedAclmeta_gen_matchrk   r   r   r   r   r   IncludeFieldsr   Patchr   r   )r   r   r  
canned_aclcanned_def_aclpreconditionsrf   r   r  bucket_metadataapitools_include_fieldsmetadata_fieldattrpredefined_aclpredefined_def_aclr   r   r   s                     r   PatchBucketzGcsJsonApi.PatchBucket+  sV    $>>++EE ##F+%@@--dd O#om !	 _nd;d	h,,T26&&~6	  4 48J Jo$$V,((((+/FF)+o&$$%78N$$U+)DD44 @@LN  $$%78

6
6
3
322>BD 
 )CC&+::$#5%%' &==?M XXc&k2m		&	&'>	?,&&,,-=;H - J 
@	? . ,((++,+ 
@	?	?s0   ,J.&I	J'I=8J=JJJc                 X   	 t        |      }t        j                  ||| j
                        }t        j                         }	 | j                  j                  j                  ||      S # t        $ r t        d      w xY w# t        $ r}| j                  ||       Y d }~y d }~ww xY w)Nz6LockRetentionPolicy Metageneration must be an integer.r   r$  r   r   r   )r   
ValueErrorr   r   (StorageBucketsLockRetentionPolicyRequestrk   r   r   r   LockRetentionPolicyr   r   )r   r   metagenerationrf   r   r   r   s          r   rA  zGcsJsonApi.LockRetentionPolicy|  s    DN+n 	BB"0))	+ 
 &==?MC__$$88
- 9 9 9  D
BD DD , C
&&qk&BBCs#   A+ &B +B 	B)B$$B)c                    t         j                  j                  j                  }| j	                  |      r$t         j                  j                  j
                  }|st        j                         }||_        |j                  r|j                  j                         |_        |j                  r|j                  j                         |_
        t        |      }t        j                  |||| j                        }t        j                         }|rdj                  t        |            |_        	 | j"                  j$                  j'                  ||      S # t(        $ r}	| j+                  |	|       Y d}	~	yd}	~	ww xY w)r   )r   projectr  r   r   r   r   N)r   StorageBucketsInsertRequestr
  r  r   r  BucketnamelocationlowerstorageClassr,   rk   r   r   r   r   r   r   Insertr   r   )
r   r   
project_idr  rf   r   r  r   r   r   s
             r   CreateBucketzGcsJsonApi.CreateBucket  sG    $??++EE ##F+%AA--dd "))+hHM"++113h&3399;h":.J(DD%%	'
 &==?M XXc&k2mC__$$++,<:G , I I+ C
&&qk&BBCs   1&E 	E>!E99E>c                    |s
t               }t        j                  ||j                  | j                        }	 | j
                  j                  j                  |       y# t        $ r}t        | j                  ||      t              rM| j                  |dg      }|j                  r-|j                  j                  rt        d|j                        | j!                  ||       Y d}~yd}~ww xY w)r   r>  r   r   r   VersionedBucketNotEmptystatusN)r   r   StorageBucketsDeleteRequestr0  rk   r   r   Deleter   r   _TranslateApitoolsExceptionr   r  r   enabledstatus_coder   )r   r   r5  rf   r   r   r6  s          r   DeleteBucketzGcsJsonApi.DeleteBucket  s    #om(DD+::%%'
C
oo$$%56+ 
C	

*
*1+
*
F


 ..l^.L%%/*D*D*L*L!";)*8 8
&&qk&BB
Cs   %A   	C4)BC//C4c              #     K   t         j                  j                  j                  }| j	                  |      r$t         j                  j                  j
                  }t        |      }t        j                  |t        || j                        }t        j                         }|r4d|vr|j                  d       dj                  t        |            |_        	 | j                  j                  j!                  ||      }| j'                        D ]  }	|	  |j(                  rt        j                  ||j(                  t        || j                        }	 | j                  j                  j!                  ||      }| j'                  |      D ]  }	|	  |j(                  ryy# t"        $ r}| j%                  |       Y d}~d}~ww xY w# t"        $ r}| j%                  |       Y d}~od}~ww xY ww)r   )rD  
maxResultsr  r   nextPageTokenr   r   N)rD  	pageTokenrZ  r  r   )r   StorageBucketsListRequestr
  r  r   r  r,   NUM_BUCKETS_PER_LIST_PAGErk   r   addr   r   r   r   r   r  r   r   _YieldBucketsr[  )
r   rL  rf   r   r  r   r   bucket_listr   r   s
             r   ListBucketszGcsJsonApi.ListBuckets  s    #==++EE ##F+%??--dd ":.J(BB,%%	'
 &==?M		&

?# XXc&k2m*OO++001A?L 1 Nk
 $$[1l 2 
#
#*DD--.''),oo--223CAN 3 P
 &&{3& 4 
#
# , *
&&q))*  . ,((++,sa   CG7'F)  AG7'G  &G7'G7)	G2GG7GG7	G4G/*G7/G44G7c              #   P   K   |j                   r|j                   D ]  }|  yyw)z0Yields buckets from a list returned by apitools.N)items)r   ra  r   s      r   r`  zGcsJsonApi._YieldBuckets  s)     %%& & s   $&c           
   #     K   t         j                  j                  j                  }| j	                  |      r$t         j                  j                  j
                  }t        j                  |||||t        | j                        }t        j                         }	|rt        |      }d|vr|j                  d       t        |      j                  t              rD|j                  d       |j                  d       |j                  d       |j                  d       dj                  |      |	_        d	}
d
}|
rd}
t        j                  ||||||t        | j                        }	 | j                   j"                  j%                  ||	      }j*                  rd	}
|j*                  }| j-                  |      D ]i  }|j.                  t0        j2                  j4                  k(  r<| j7                  ||j8                  t        |            }|t:        k(  r[|r||_        n	 | k |
ry
y
# t&        $ r}| j)                  ||       Y d
}~d
}~ww xY ww)r   )r   prefix	delimiterversionsr  rZ  r   r[  list_fieldszitems/customerEncryptionzitems/kmsKeyNamezitems/generationz
items/namer   TNF)r   rf  rg  rh  r  r\  rZ  r   r   r   rO  )r   StorageObjectsListRequestr
  r  r   r  r>   rk   r   r   r_  r;   r   r   r   r   r   r   r  r   r   r[  _YieldObjectsAndPrefixesdatatyper   CsObjectOrPrefixTypeOBJECT_DecryptHashesIfPossibledata_SKIP_LISTING_OBJECT)r   r   rf  rg  all_versionsrf   r   r  r   r   	list_pagenext_page_tokenobject_listr   object_or_prefixdecrypted_metadatas                   r   ListObjectszGcsJsonApi.ListObjects  s:     $==++EE ##F+%??--dd (BB,%%' &==?M6{f		&

?#	
*l+@A
B 	

-.

%&

%&

<  XXf-mIO
i*DD#.'')Eoo--223CAN 3 P
 
	"	"	%33";;KH
$$(E(E(L(LL#<<##$V,  =  .
  #77!$6!  I+  . E(((DDEs7   E&I)'H, BI*I,	I5IIIIc           
         t        |      }| j                  |      s| j                  ||       yd}| j                  ||      r9t	        j
                  | j                  ||j                  |j                  |      }n| j                  ||      r|j                  j                  }t        j                  r!t        |t              s|j!                  d      }t#        |t$              }|rBt	        j
                  | j                  ||j                  |j                  |t'        |            }|y	  |       S # t(        $ r	 t*        cY S w xY w)a.  Attempts to decrypt object metadata.

    Helps fetch encrypted object metadata fields (if requested) for API method
    calls that requested them but did not receive them; this occurs for
    ListObjects(), along with GetObjectMetadata() calls for which the object is
    CSEK-encrypted but no decryption information was supplied.

    This method issues a StorageObjectsGetRequest (if object hash fields were
    requested but not retrieved), formatted appropriately depending on the
    encryption method.

    Args:
      bucket_name: String of bucket containing the object.
      object_metadata: apitools Object describing the object. Must include
          name, generation, and customerEncryption fields.
      fields: ListObjects-format fields to return.

    Returns:
      - apitools_messages.Object with decrypted hash fields if decryption was
      performed.
      - None if the object was not encrypted, or CSEK-encrypted but we could not
      find a matching decryption key.
      - gcs_json_api._SKIP_LISTING_OBJECT if the object no longer exists.
    ri  rO  Nr   r   asciir   r   decryption_tuple)r;   _ObjectIsEncrypted2_RaiseIfNoDesiredHashesPresentForUnencryptedObject _ObjectKMSEncryptedAndNeedHashes	functoolspartial_GetObjectMetadataHelperrG  r   !_ObjectCSEKEncryptedAndNeedHashesr   	keySha256r   r  r   bytesr  rE   r   rD   r   rr  )r   r   object_metadatar   
get_fieldsget_metadata_func
key_sha256decryption_keys           r   rp  z#GcsJsonApi._DecryptHashesIfPossibleL  sV   2 !V4J""?3
==oEO > Q ,,_4> - @ $++

'
'



$// 
	/	/7A 
0 
C
 #55??j	*e,!((1*3JGn	%--))  &114^DF  "   " "!"s   2D9 9E
Ec              #   B  K   |j                   rA|j                   D ]2  }t        j                  |t        j                  j                         4 |j
                  rB|j
                  D ]2  }t        j                  |t        j                  j                         4 y y wN)rd  r   CsObjectOrPrefixrn  ro  prefixesPREFIX)r   rv  	cloud_objrf  s       r   rl  z#GcsJsonApi._YieldObjectsAndPrefixes  s     "(()''	(0(E(E(L(LN 	N ) ((&''(0(E(E(L(LN 	N ) s   BBc              #      K   |rI| j                   j                  j                         }| j                   j                  j                  |       d |r| j                   _        yyw)z@Wrapped code makes apitools requests with the specified headers.N)r   r   copyr   )r   r   old_headerss      r   _ApitoolsRequestHeadersz"GcsJsonApi._ApitoolsRequestHeaders  sR      OO;;@@Bk
oo--44W=	0;doo- s   A%A'c                     i }|rS|j                   t        j                  k(  r6|j                  |d<   |j                  |d<   t        |j                        |d<   |S )z:Returns headers dict matching crypto_tuple, or empty dict.x-goog-encryption-algorithmx-goog-encryption-keyx-goog-encryption-key-sha256)crypto_typerC   CSEK
crypto_alg
crypto_keyrB   )r   crypto_tupler   s      r   _EncryptionHeadersFromTuplez&GcsJsonApi._EncryptionHeadersFromTuple  s]    G00M4F4FF/;/F/Fg+,)5)@)@g%&
-l.E.E
F ,-Nr   c                 :   i }|rJ|j                   t        j                  k(  r-|j                  |d<   |j                  |d<   |j
                  |d<   |rJ|j                   t        j                  k(  r-|j                  |d<   |j                  |d<   |j
                  |d<   |S )zAReturns headers dict matching the provided tuples, or empty dict.z'x-goog-copy-source-encryption-algorithmz!x-goog-copy-source-encryption-keyz(x-goog-copy-source-encryption-key-sha256r  r  r  )r  rC   r  r  r  crypto_key_sha256)r   r~  encryption_tupler   s       r   _RewriteCryptoHeadersFromTuplesz*GcsJsonApi._RewriteCryptoHeadersFromTuples  s     G,88M<N<NN

%
% 78 
%
% 12 
,
, 89 ,88M<N<NN/?/J/Jg+,)9)D)Dg%&

,
, ,- Nr   c                    | j                  ||||      }| j                  |      }	 |rR| j                  | j                  |            5  | j                  j
                  j                  ||      cddd       S | j                  j
                  j                  ||      S # 1 sw Y   yxY w# t        $ r}| j                  ||||       Y d}~yd}~ww xY w)a  Get metadata, adding necessary headers for CSEK decryption if necessary.

    Args:
      bucket_name: (str) Bucket containing the object.
      object_name: (str) Object name.
      generation: (int) Generation of the object to retrieve.
      fields: (list<str>) If present, return only these Object metadata fields,
          for example: ['acl', 'updated'].
      decryption_tuple: (gslib.utils.encryption_helper.CryptoKeyWrapper) The
          CryptoKeyWrapper for the desired decryption key.

    Raises:
      ArgumentException for errors during input validation.
      ServiceException for errors interacting with cloud storage providers.

    Returns:
      (apitools_message.Object) The retrieved object metadata.
    r{  rO  r  r   Nr   )	'_CreateApitoolsObjectMetadataGetRequest&_GetApitoolsObjectMetadataGlobalParamsr  r  r   r   r  r   r   )	r   r   r   r   r   r~  r   r   r   s	            r   r  z#GcsJsonApi._GetObjectMetadataHelper  s    0 CC[Z D H??v?NM>	)),,:J,KM((,,-=;H - JM M
 &&**+;9F + H 	HM M , >
&&q3>3>2< ' > >>s;   #B0 'B$3	B0 =&B0 $B-)B0 -B0 0	C9CCc                 :   | j                  |       t        j                  j                  j                  }| j                  |      r$t        j                  j                  j                  }|rt        |      }t        j                  ||||| j                        S )NrO  )r   r   r  r   r   )	r   r   StorageObjectsGetRequestr
  r  r   r  r   rk   )r   r   r   r   r   r  s         r   r  z2GcsJsonApi._CreateApitoolsObjectMetadataGetRequest  s    
 	77v7F#<<++EE ##F+%>>--dd 
#j55%%' 'r   c                 p    t        j                         }|rdj                  t        |            |_        |S )Nr   )r   r   r   r   r   )r   r   r   s      r   r  z1GcsJsonApi._GetApitoolsObjectMetadataGlobalParams	  s.    %==?M XXc&k2mr   c           
      J   	 | j                  ||||      }| j                  ||      s|S |j                  j                  }t        j
                  r!t        |t              s|j                  d      }t        |t              }|s!t        d|d| j                  d|d|      t        j
                  r%t        |t              s|j                  t              }| j                  ||||t        |            S # t        $ r}	| j!                  |	|||	       Y d
}	~	y
d
}	~	ww xY w)r   r{  rO  r|  z(Missing decryption key with SHA256 hash z#. No decryption key matches object z://rt   r}  r   N)r  r  r   r  r   r  r   r  r  rE   r   r   rf   rA   rD   r   r   )
r   r   r   r   rf   r   r  r  r  r   s
             r   GetObjectMetadatazGcsJsonApi.GetObjectMetadata  s6   #>55k6AAK=C 6 Eo 33O;A 4 C
 #55??j	*e,!((1*3JGn! [BC 	C 
.%0)006.**

2>B + D D , >
&&q3>3>2< ' > >>s   )C: CC: :	D"DD"c                 H    t        |j                  xs |j                        S r  )boolr   
kmsKeyName)r   r  s     r   r  zGcsJsonApi._ObjectIsEncrypted;  s%    22 +**, ,r   c                     |yd|v xr |j                    }d|v xr |j                   }|r(|j                  s|rt        d|j                  z        yyy)zBRaise an exception if none of the desired hash fields are present.Nr`   r_   zNService did not provide requested hashes in metadata for unencrypted object %s)r`   r_   componentCountr   rG  )r   r  r   md5_requested_but_missingcrc32c_requested_but_missings        r   r  z=GcsJsonApi._RaiseIfNoDesiredHashesPresentForUnencryptedObject?  s     ~!*f!4 "=%4%<%<!< $,$6 %?(7(>(>$> !
 	"/*H*H$"$3$8$89: : 	% +I!r   c                     |t         j                  |      sy|j                  xr |j                   xr |j                   S )a  Checks if an object is KMS-encrypted and lacking requested hash fields.

    Args:
      object_metadata: (apitools_messages.Object) Object metadata returned from
          an API call; may or may not contain hash-related metadata fields.
      fields: (list<str>) Requested object metadata fields.

    Returns:
      True if a GetObjectMetadata request should be issued on the KMS-encrypted
      object in order to retrieve encrypted hash fields, False otherwise.
    F)r   r   r  r_   r`   r   r  r   s      r   r  z+GcsJsonApi._ObjectKMSEncryptedAndNeedHashesR  sJ     "7"D"DV"L&& (/E/E+E (''')r   c                     |t         j                  |      sy|j                  xr |j                   xr |j                   S )a  Checks if an object is CSEK-encrypted and lacking requested hash fields.

    Args:
      object_metadata: (apitools_messages.Object) Object metadata returned from
          an API call; may or may not contain hash-related metadata fields.
      fields: (list<str>) Requested object metadata fields.

    Returns:
      True if a GetObjectMetadata request including a decryption key should be
      issued on the CSEK-encrypted object in order to retrieve encrypted hash
      fields, False otherwise.
    F)r   r   r   r_   r`   r  s      r   r  z,GcsJsonApi._ObjectCSEKEncryptedAndNeedHashese  sM     ~2??G.. G&&&G/>/F/F+FHr   Fc                    |rt        |      }|}|
r|
dz   }n-|r+t        j                  t        j                  |            d   }|r|t        d       ||	|       t               }|	|_        t        ||||      }|j                         }|| j                  _        d|i| j                  _        |r| j                  rWt        j                  t        j                  |            }|dxx   d| j                  z  z  cc<   t        j                  |      }t        j                   j#                  ||| j$                  j&                  | j(                  | j$                  	      }n-t        j                   j+                  |d
|| j(                        }| j,                  |_        t1        j2                  |||| j4                        }t7        d| j8                        |_        	 |t<        j>                  j@                  k(  r| jC                  |||||||||	|
||      S | jE                  ||||||||	|
||      S # tF        $ r}| jI                  ||||       Y d}~yd}~ww xY w)r      
total_sizeNz`Download size is required when callbacks are requested for a download, but no size was provided.)r  progress_callback	digestershttpsry   z&trace=token%%3A%s)r   clientF)auto_transferr  r   r   Tis_data_transferr   )compressed_encodingr   
start_byteend_byteserialization_datar~  r   r  r  r  r  r~  r   )%r   jsonloadsr   
ensure_strr   r"   bytes_transferredr#   GetConnectionClassr   streamconnectionsri   dumpsapitools_transferDownloadFromDatar   rz   r   
FromStreamr   
bytes_httpr   r  rk   rG   r   r   r   DownloadStrategy	RESUMABLE_PerformResumableDownload_PerformDownloadr   r   )r   r   r   download_streamrf   r   object_sizer  download_strategyr  r  r  r  r  r~  outer_total_sizebytes_downloaded_containercallback_class_factorydownload_http_classserialization_dictapitools_downloadr   r   s                          r   GetObjectMediazGcsJsonApi.GetObjectMediay  s   $ 
#j #!A	
..+
,..:< 		! !, - 	- 
$45!:!<3=0C"#+	
 1CCE !0D&-/B%CD"
 
		!ZZ7I(JK5!%9D<L<L%LL!!ZZ(:;+44==


//

&& > " ,44??
 &&	 @ ( $(#@#@ (AA%%	' $7D,=,=$? %>	h77AA	A--& 3!!1- . / 	/ $$[%0%4%5%60:9L0:.68J6F % 
H 
	H" , >
&&q3>3>2< ' > >>s   !9H7 H7 7	I IIc                    d}|	}|| j                   k  r	 | j                  ||||||||	|
||      S y y # t        $ r}| j                  |       |j	                         }	|	|_        |	|kD  r|	}d}|dz  }|| j                   kD  r#t        d| j                   t        |      fz        t        j                  t        || j                               | j                  j                  t        j                        r9| j                  j!                  d|	t        |      t#        j$                                t'        j(                  |j*                         Y d }~nd }~ww xY w|| j                   k  rZ?)Nr   r  r  5Transfer failed after %d retries. Final exception: %smax_waitz=Retrying download from byte %s after exception: %s. Trace: %s)r   r  HTTP_TRANSFER_EXCEPTIONS$_ValidateHttpAccessTokenRefreshErrortellr  r   rJ   timesleepr   r   r   isEnabledForr  DEBUGrg   	traceback
format_excapitools_http_wrapperRebuildHttpConnectionsr  )r   r   r   r  r   r  r  r   r  r  r  r  r~  retrieslast_progress_byter   s                   r   r  z$GcsJsonApi._PerformResumableDownload  s    G#
T%%
%2*$$[%0%4%5%60:9L0:.68J6F % 
H 
	H & & &*11!4$))+
7A"4**)
'1T%%%& +E!<Q!?@AB B 	

(4;N;NOP;;##GMM2
++

M5a8""$& 	44((	* 	*K&* T%%
%s   3 
EDEEc                    |
s(	 | j                   j                  j                  ||       d }d| j                   j
                  i}t        ||       | j                  |       |j                  | j                  |             |s|	!|j                  |||	d       |j                  S |j                  |||d       |j                  S # t        $ r}| j	                  ||||       Y d }~d }~ww xY w)	N)downloadr   c                      y r   )unused_responseunused_download_objects     r   _NoOpCallbackz2GcsJsonApi._PerformDownload.<locals>._NoOpCallbackT  s    
r   
user-agent)r  F)additional_headersstartend
use_chunks)callbackfinish_callbackr  r  )r   r   r  r   r   
user_agentrF   r   r   r  GetRangeStreamMediar	   )r   r   r   r  r   r  r   r  r  r  r  r~  r   r  r  s                  r   r  zGcsJsonApi._PerformDownload=  s-    @##$4-> 	$ 	@ 	doo00 ""46IK 	*+(()9:< X)  4F'1%-,1 ! 3 %%%	 ##]4A7I/4 $ 6 %%%S . @((5@5@4> 	) 	@ 	@@s   'C
 
	C2C--C2c
                 Z   t         j                  j                  j                  }
| j	                  |      r$t         j                  j                  j
                  }
|s
t               }|rt        |      }d}g }|r?|j                  d       t         j                  j                  | j                  |            }|j                  dk(  r|j                  d       |j                  dk(  r|j                  d       |j                  dk(  r|j                  d       |j                  dk(  r|j                  d       t        j                  |||||
|j                  |j                   || j"                  	      }t        j$                         }|rd	j'                  t)        |            |_        	 | j,                  j/                  |      5  | j,                  j0                  j3                  ||
      cddd       S # 1 sw Y   yxY w# t4        $ r}| j7                  ||||       Y d}~yd}~ww xY w)r   NrR   rr   cacheControlcontentDispositioncontentEncodingcontentLanguage)	r   r   objectResourcer   r  ifGenerationMatchr$  r%  r   r   r   r   )r   StorageObjectsPatchRequestr
  r  r   r  r   r   r+  r,  r/  r  r  r  r  	gen_matchr0  rk   r   r   r   r   r   r1  r   r2  r   r   )r   r   r   r  r3  r   r5  rf   r   rk   r  r:  r7  r   r   r   s                   r   PatchObjectMetadatazGcsJsonApi.PatchObjectMetadatax  s    $>>++EE ##F+%@@--dd  #om
#jN $$U+)DD44 @@LN 
 "$$^4""b($$%9:2%$$%672%$$%67(CC'11+::$%%	' &==?M XXc&k2m>??(()@A&&,,-=;H - J BAA , >
&&q3>3>2< ' > >>s6   *H 'G6,	H 6G?;H ?H 	H*H%%H*c                    t        |       d}|r.t        j                  j                  | j	                  |            }t               }|r|r|} |d|       t        |||| j                  | j                        }|j                         }||d| j                  _        d| j                  j                  i}| j                  |       	 d}d}d}|	s|j                  }|st         }|s
t#               }t        j                  |j$                  ||j&                  |j(                  || j*                        }t        j,                         }|rdj/                  t1        |            |_        | j5                  |      }|t6        j8                  k(  rt7        j:                  |||d	| j<                  |
      }||_        | j@                  |_!        | jE                  |      5  | j                  jF                  jI                  |||      cddd       S d| j                  j                  i}|jK                  |       | j                  |       | jM                  || j@                  |||	|||||
|||      S # 1 sw Y   yxY w# tN        $ rS}tQ        | jR                  |j$                        }| jU                  ||j$                  |jV                  |       Y d}~yd}~ww xY w)a  Upload implementation. Cloud API arguments, plus two more.

    Additional args:
      apitools_strategy: SIMPLE_UPLOAD or RESUMABLE_UPLOAD.
      total_size: Total size of the upload; None if it is unknown (streaming).

    Returns:
      Uploaded object metadata.
    Nr   )r  r  r   rg   )rz   r  r  )r   r   r
  r$  r%  r   r   r  T)r  r  r   gzip_encodeduploadr   r   r   not_found_exception),r<   r   StorageObjectsInsertRequestr,  r/  r"   r&   r   rg   r  r   r  r   r  r   contentTyperN   r   r   r  r0  rk   r   r   r   r   r  r  SIMPLE_UPLOADUploadr   strategyr   r  r  r   rK  r   _PerformResumableUploadr   rL   rf   r   rG  )r   upload_streamr  r3  sizer5  r  rf   r   r  tracker_callbackr  apitools_strategyr  r  r:  bytes_uploaded_containerr  upload_http_classr  content_typer   r   encryption_headersapitools_uploadr   r  s                              r   _UploadObjectzGcsJsonApi._UploadObject  s   4 o.N)EE44 @@LN   9:Tj4 A +{{jj /AAC!"$D  	doo00 	*+?Plm&22-,'/-,HH"))"+55"/">">())+ *AAC!$#f+!6-
;;' < ) 
/==	=+22=3?>BAE?C?O?O@LN $5 %)%@%@"))*<=((//0@7F>K 0 M >= $//44
 	!!"45./++466d 13C35E 1<	A 	A >=$ , PA
--//1
&&q3B3I3I3B3G3G;N ' P PPs?   	DI; (I/	I; A I; /I84I; 8I; ;	KA	KKc           	      z   	 |rdt         j                  j                  ||| j                  j                  | j
                  || j                        }t               |_        ||_        n~t        j                  |||t               d| j
                  |      }||_	        ||_        | j                  |      5  | j                  j                  j                  |||       d d d        t        d| j                        |_        d }|j                   |	_        |
r% |
t%        j&                  |j(                               d}|j                   }|| j
                  k  r	 |s!|rt+               s|j-                  |||	      }n|j/                  |||	      }| j                  j                  j1                  | j                  j                  j3                  d
      |      }||r ||j4                  |j4                         |S y y # 1 sw Y   $xY w# t6        $ r}| j9                  |       t;        j<                  |j                         || j
                  k  r	 |j?                          |j                   }||	_        n# t6        $ r}| j9                  |       t;        j<                  |j                         |dz  }|| j
                  kD  rtA        d| j
                  |fz        tC        jD                  tG        || jH                               Y d }~nd }~ww xY w|| j
                  k  r|kD  r|}d}na|dz  }|| j
                  kD  r#tA        d| j
                  tK        |      fz        tC        jD                  tG        || jH                               | jL                  jO                  tP        jR                        r9| jL                  jU                  d|tK        |      tW        jX                                Y d }~nd }~ww xY w|| j
                  k  r# tZ        $ r}| j]                  |      }|r| d }~ww xY w)N)r   r  r  F)r  	chunksizer  r   r  r  Tr  c                      y r  r  )r  unused_upload_objects     r   r  z9GcsJsonApi._PerformResumableUpload.<locals>._NoOpCallback]  s    r   r   )r  r   r  rK  r  r  r  z;Retrying upload from byte %s after exception: %s. Trace: %s)/r  r  r  r   rz   r   r6   r%  r  r  r  r   rK  rG   r   r   progressr  r  r  r  r:   r  StreamInChunksProcessHttpResponseGetMethodConfigr  r  r  r  r  RefreshResumableUploadStater   r  r  r   r   rJ   r   r  r  r  rg   r  r  r   *_TranslateApitoolsResumableUploadException)r   r  r   r   r  r  r  r   r   r  r  addl_headersr  r  r"  r  r  r  http_responseprocessed_responser   r  e2resumable_exs                           r   r  z"GcsJsonApi._PerformResumableUpload6  s   ~	+22;;OO  ((%?? < $ %>$?!%;" ,22/1((%' $5 %;")),7
//
!
!
(
()90?7D ) F 8
 $7d.?.?$Ao  4C3K3K0	O$F$FGHg*33t'''I	( $/L/N ,77& -#/ 8 1M ,::& -#/ ; 1M  $66JJoo%%55h? P
\/
 o88-88:#
#E (- 87r ( '	(

3
3A
6

6
6((*4+++P 99;*33j;E&8+ 
P77;#::!,,.lg4+++.K%%r*+, , jj'$:M:MNP P
P 4+++& ,,!+GqLG))),I##%@%CDEF F JJ%g8K8KLN[[%%gmm4KKM7:$$&(I'	(G t'''V , DDQGl	s   B3P 5)H A<P B"H  H
P 
P ?O;#I;:O;;	LBLO;LO;)CO;6P ;P  P 	P:P55P:c                 T    | j                  ||||||||	t        j                  |

      S )r   )r3  r  r5  r  r  r   r  r  )r#  r  r  )r   r  r  r3  r  r5  r  r  rf   r   r  s              r   UploadObjectzGcsJsonApi.UploadObject  s?     m-)3#',90A/?%+0A0O0O+7  	9 	9r   c
                 T    | j                  |||||||t        j                  d|	
      S )r   N)r3  r5  r  r  r   r  r  r  r#  r  RESUMABLE_UPLOAD)
r   r  r  r3  r5  r  r  rf   r   r  s
             r   UploadObjectStreamingz GcsJsonApi.UploadObjectStreaming  s@     #+)+<<!  
# 
#r   c                 X    | j                  ||||||||||	t        j                  |      S )r   )
r3  r5  r   r  r  r  r  r  r  r  r6  )r   r  r  r3  r5  r  r  r  r  r  rf   r   r  s                r   UploadObjectResumablez GcsJsonApi.UploadObjectResumable  sF     #-)+)+<<!  # #r   c                    t        |       d}|r.t        j                  j                  | j	                  |            }|rt        |      }|s
t               }t        j                  j                  j                  }| j                  |      r$t        j                  j                  j                  }t        j                         }|rGt        g d      }|D ]  }|j                  d|z           dj                  t        |            |_        d}|	r)|	j                   t"        j$                  k(  r|	j&                  }d}|rX|j                   t"        j$                  k(  r|j&                  }n.|j                   t"        j(                  k(  r|j*                  |_        t/        |j0                  |j2                  |j0                  |j2                  d      }t5        |||||j6                  |j8                  |||||j                        }t;        ||      }| j=                  |	|      }d}	 t        d      }	 | j?                  |      5  t        j                  |j0                  |j2                  |j0                  |j,                  |j2                  ||||j6                  |j8                  |||| j@                  	      }| jB                  jD                  jG                  ||
      }ddd       t        jH                        }|r!|stK        t        |jL                        |      }|r|jO                  ||z
         |jP                  rn)|s#|jR                  }tU        |||jR                         |}8tW        |       |jX                  S # 1 sw Y   xY w# tZ        $ rv}t]        | j^                  |j0                  | j^                  |j0                  |j2                  |      }| ja                  ||j0                  |j2                  |       Y d}~yd}~ww xY w)r   N)done
objectSizerewriteTokentotalBytesRewrittenz	resource/r   JSON)src_generationr  r0  r3  max_bytes_per_callsrc_dec_key_sha256dst_enc_key_sha256r   )r~  r  r   )sourceBucketsourceObjectdestinationBucketdestinationKmsKeyNamedestinationObjectr  r   sourceGenerationr
  r$  destinationPredefinedAclr>  maxBytesRewrittenPerCallr   r   )src_providersrc_bucket_namesrc_object_namerA  r  )1r<   r   StorageObjectsRewriteRequest'DestinationPredefinedAclValueValuesEnumr/  r   r   r
  r  r   r  r   r   r_  r   r   r  rC   r  r  CMEKr  r  r0   r   rG  r1   r  r0  r2   r  r  rk   r   r   Rewriter?  r+   r=  Progressr<  r>  r3   r/   resourcer   rL   rf   r   )r   src_obj_metadatadst_obj_metadatarA  r3  r5  r  rB  r  r~  rf   r   r:  r  r   
new_fieldsfielddec_key_sha256enc_key_sha256tracker_file_namerewrite_params_hashresume_rewrite_tokencrypto_headersprogress_cb_with_timeoutlast_bytes_writtenr   rewrite_responsebytes_writtenr   r  s                                 r   
CopyObjectzGcsJsonApi.CopyObject	  s    ./N)FF?? @@LN  N+n#om#@@++EE ##F+%BB--dd %==?M  	j %{U*+  XXc*o6mN,88M<N<NN'99nN		%	%););	;);;''=+=+==&6&A&A# 22B2I2I2B2G2G2B2I2I2B2G2GQ 0%))$33!-))##% 22C2EG99)<L : NN  $4P7)).9.KK+22+00 0 7 7 %5$?$? 0 5 5#%- - 7 7$1$@$@'5/'9++!-
" "__44<<m = =
% :( -AAB%=%@#../1B&D
"#
"
+
+M<N,N
O  
%!1!>!>

!"35H"2"?"?A*G J )*&&&K :9L , PA
--

!
!}}*11*//') &&q3C3J3J3C3H3H;N ' P PPs3   #N
  BM>B"N
 >NN
 
	P	A,PP	c                 H   |s
t               }|rt        |      }t        j                  ||||j                  |j
                  | j                        }	 | j                  j                  j                  |      S # t        $ r}| j                  ||||       Y d}~yd}~ww xY w)r   )r   r   r   r
  r$  r   r   N)r   r   r   StorageObjectsDeleteRequestr  r0  rk   r   r   rT  r   r   )r   r   r   r5  r   rf   r   r   s           r   DeleteObjectzGcsJsonApi.DeleteObject  s     #om
#j(DD'11+::%%'>__$$++,<==+ >
&&q3>3>2< ' > >>s   $A9 9	B!BB!c           
         t        |       |j                  }d|_        |j                  }d|_        |j                  st        |_        |s
t               }t        j                         }	|rdj                  t        |            |	_
        t        j                  ||      }
| j                  |      }d}|r)|j                  t        j                  k(  r|j                   }| j#                  |      5  t        j$                  |
|||j&                  |j(                  | j*                  |      }	 | j,                  j.                  j1                  ||	      cddd       S # t2        $ rO}t5        |t6        j8                        r|j:                  dk(  rt=        d      | j?                  |       Y d}~nd}~ww xY w	 ddd       y# 1 sw Y   yxY w)	r   Nr   )sourceObjectsdestinationr  )composeRequestrG  rI  r
  r$  r   r  r     z)One of the source objects does not exist.) r<   rG  r   r  rN   r   r   r   r   r   r   ComposeRequestr  r  rC   rR  r  r  StorageObjectsComposeRequestr  r0  rk   r   r   Composer   r   apitools_exceptions	HttpErrorrW  r   r   )r   src_objs_metadatarW  r5  r  rf   r   dst_obj_namedst_bucket_namer   src_objs_compose_requestr!  r  r   r   s                  r   ComposeObjectzGcsJsonApi.ComposeObject  s    ./#((L &--O"''%9"#om%==?M XXc&k2m0??'5E G 99% : ' J		%	%););	;%00
		%	%&8	9*GG1+()33 - < <''!	.&&../?=J / L 
:	9 . .q-778MMS !"MN
N

*
*1
-
-. .% 
:	9	9s1   1;G-&E	F5&AF0+G0F55GGc                    t         j                  j                  j                  }t        j                  |||d      }t        j                  |||| j
                        }	t        j                         }
|rdj                  t        |            |
_	        	 | j                  j                  j                  |	|
      S # t        $ r}| j                  ||       Y d}~yd}~ww xY w)r   WEB_HOOK)addressidtokentype)r   channelr  r   r   r   r   N)r   StorageObjectsWatchAllRequestr
  r  Channelrk   r   r   r   r   r   r   WatchAllr   r   )r   r   ry  
channel_idr{  rf   r   r  r}  r   r   r   s               r   WatchBucketzGcsJsonApi.WatchBucket  s     $AA++DD   ''+5.3-79G
 )FF%%	' &==?M XXc&k2mC__$$--.><I . K K+ C
&&qk&BBCs   &B> >	C$CC$c                     t        j                  ||      }	 | j                  j                  j	                  |       y# t
        $ r}| j                  |       Y d}~yd}~ww xY w)r   )rz  
resourceIdN)r   r  r   channelsStopr   r   )r   r  resource_idrf   r}  r   s         r   StopChannelzGcsJsonApi.StopChannel   sR    '':+NG*
oo##G,+ *
&&q))*s   %? 	A#AA#c                     t        j                  || j                        }	 | j                  j                  j                  |      S # t        $ r}| j                  ||       Y d}~yd}~ww xY w)r   r   r   r   N)r   !StorageBucketsListChannelsRequestrk   r   r   ListChannelsr   r   )r   r   rf   r   r   r   s         r   r  zGcsJsonApi.ListChannels  sh    (JJ(9(9;C__$$112BCC+ C
&&qk&BBCs   $A 	A.A))A.c                     	 t        j                  t        j                  |            }| j                  j
                  j                  |      S # t        $ r}| j                  |       Y d}~yd}~ww xY w)r   )	projectIdN)	r   'StorageProjectsServiceAccountGetRequestr   	text_typer   projects_serviceAccountr  r   r   )r   project_numberr  r   s       r   GetProjectServiceAccountz#GcsJsonApi.GetProjectServiceAccount  s_    *!IIMM.13g__4488AA+ *
&&q))*s   AA 	A4A//A4c                    	 t        j                  ||      }|r||_        |r>|j                         D 	cg c]  \  }}	t	        ||	       }
}}	t        |
      |_        |r||_        t        j                  ||| j                        }| j                  j                  j                  |      S c c}	}w # t        $ r}| j                  |       Y d}~yd}~ww xY w)r   )topicpayload_format)r   value)additionalPropertiesr   notificationr   N)r   Notificationevent_typesrd  NotificationAdditionalProperty!NotificationCustomAttributesValuecustom_attributesobject_name_prefix!StorageNotificationsInsertRequestrk   r   notificationsrK  r   r   )r   r   pubsub_topicr  r  r  r  r  r   r  additional_propertiesr  r   s                r   CreateNotificationConfigz#GcsJsonApi.CreateNotificationConfig  s    *&33^=l	#. 	 0557!
7
U +s%@7 	 !
 *K!6*8&	*<'!CC#'')g __**11'::!
 , *
&&q))*s)   5B8 B2A#B8 2B8 8	CCCc                     	 t        j                  ||| j                        }| j                  j                  j                  |      S # t        $ r}| j                  |       Y d}~yd}~ww xY w)r   r  N)r   !StorageNotificationsDeleteRequestrk   r   r  rT  r   r   )r   r   r  r  r   s        r   DeleteNotificationConfigz#GcsJsonApi.DeleteNotificationConfig:  sd    *!CC#'')g __**11'::+ *
&&q))*s   AA	 		A-A((A-c              #     K   	 t        j                  || j                        }| j                  j                  j                  |      }|j                  D ]  }|  y# t        $ r}| j                  |       Y d}~yd}~ww xY ww)r   r  N)	r   StorageNotificationsListRequestrk   r   r  r  rd  r   r   )r   r   r  r   r  r   s         r   ListNotificationConfigsz"GcsJsonApi.ListNotificationConfigsE  st     *!AA$*;*;=g..33G<h".., )+ *
&&q))*s/   BAA  B 	B)A?:B?BBc                     	 t        j                  ||      }| j                  j                  j	                  |      S # t
        $ r}| j                  |       Y d}~yd}~ww xY w)r   )r  serviceAccountEmailN)r   $StorageProjectsHmacKeysCreateRequestr   hmacKeysCreater   r   )r   rL  r   r  r   s        r   CreateHmacKeyzGcsJsonApi.CreateHmacKeyP  sZ    *!FF4IKg__%%,,W55+ *
&&q))*   ;> 	A"AA"c                     	 t        j                  ||      }| j                  j                  j	                  |      S # t
        $ r}| j                  |       Y d}~yd}~ww xY wr   )r  accessIdN)r   $StorageProjectsHmacKeysDeleteRequestr   r  rT  r   r   r   rL  	access_idr  r   s        r   DeleteHmacKeyzGcsJsonApi.DeleteHmacKeyY  sX    *!FF4g__%%,,W55+ *
&&q))*r  c                     	 t        j                  ||      }| j                  j                  j	                  |      S # t
        $ r}| j                  |       Y d}~yd}~ww xY wr  )r   !StorageProjectsHmacKeysGetRequestr   r  r  r   r   r  s        r   
GetHmacKeyzGcsJsonApi.GetHmacKeyb  sX    *!CC4g__%%))'22+ *
&&q))*r  c              #   T  K   	 t        j                  |||t              }| j                  j                  j                  |      }|j                  D ]  }|  	 j                  rqt        j                  ||||j                  t              }	 | j                  j                  j                  |      }|j                  D ]  }|  |j                  rpyy# t        $ r}| j                  |       Y d}~d}~ww xY w# t        $ r}| j                  |       Y d}~jd}~ww xY ww)r   )r  r  showDeletedKeysrZ  N)r  r  r  r\  rZ  )
r   "StorageProjectsHmacKeysListRequestr^  r   r  r  rd  r   r   r[  )r   rL  r   show_deleted_keysr  r   r   r   s           r   ListHmacKeyszGcsJsonApi.ListHmacKeysk  s	    
*!DD3+.	0g ))..w7h#	  
 
 
 !DD3+**.0g,??++009 #	   
 
  , *
&&q))* . ,((++,s_   D(AC 5D(%D 6!D(D(	C>#C94D(9C>>D(	D%
D D( D%%D(c                     	 t        j                  ||      }t        j                         }||_        |r||_        ||_        | j                  j                  j                  |      S # t        $ r}| j                  |       Y d}~yd}~ww xY wr  )r   $StorageProjectsHmacKeysUpdateRequestHmacKeyMetadatastateetagrU  r   r  Updater   r   )r   rL  r  r  r  r  r  r   s           r   UpdateHmacKeyzGcsJsonApi.UpdateHmacKey  s    
*!FF4g"224hhn	!g__%%,,W55+ *
&&q))*s   A&A) )	B2BBc                 @    |t         v r	t         |   S t        d|z        )ai  Translates the input string to a bucket PredefinedAcl string.

    Args:
      canned_acl_string: Canned ACL string.

    Returns:
      String that can be used as a query parameter with the JSON API. This
      corresponds to a flavor of *PredefinedAclValueValuesEnum and can be
      used as input to apitools requests that affect bucket access controls.
    Invalid canned ACL %s)._BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr   r   canned_acl_strings     r   r-  z*GcsJsonApi._BucketCannedAclToPredefinedAcl  +     JJ;<MNN
36GG
HHr   c                 @    |t         v r	t         |   S t        d|z        )aj  Translates the input string to an object PredefinedAcl string.

    Args:
      canned_acl_string: Canned ACL string.

    Returns:
      String that can be used as a query parameter with the JSON API. This
      corresponds to a flavor of *PredefinedAclValueValuesEnum and can be
      used as input to apitools requests that affect object access controls.
    r  )._OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr   r  s     r   r/  z*GcsJsonApi._ObjectCannedAclToPredefinedAcl  r  r   c                     t        |t        j                  j                        r!|j                  dk(  s|j                  dk\  s y y y )N    )r   oauth2clientr  HttpAccessTokenRefreshErrorrR  )r   r   s     r   r  z/GcsJsonApi._ValidateHttpAccessTokenRefreshError  s=    1l))EEFXX_C !0_ 	Gr   c                     | j                   j                  t        j                        r.| j                   j	                  dt        j                                | j                  |||||      }|r| )a  Translates an HTTP exception and raises the translated or original value.

    Args:
      e: Any Exception.
      bucket_name: Optional bucket name in request that caused the exception.
      object_name: Optional object name in request that caused the exception.
      generation: Optional generation in request that caused the exception.
      not_found_exception: Optional exception to raise in the not-found case.

    Raises:
      Translated CloudApi exception, or the original exception if it was not
      translatable.
    zTranslateExceptionAndRaise: %s)r   r   r   r  )r   r  r  r  rg   r  r  rU  )r   r   r   r   r   r  translated_exceptions          r   r   z&GcsJsonApi._TranslateExceptionAndRaise  sn    & {{.
kk8!,,.0;;	/ < 1   r   c                    t        |t        j                        rVt        |dd       rH	 t	        j
                  t        j                  |j                              }d|v rd|d   v r|d   d   S y y y y # t        $ r Y y w xY w)Ncontenterrormessage)
r   rp  rq  r(  r  r  r   r  r  	Exception)r   
http_errorjson_objs      r   _GetMessageFromHttpErrorz#GcsJsonApi._GetMessageFromHttpError  s    *1;;<	Y	-	ZZz/A/A BC( Y(72C%CG$Y// &D  
. =  	
	s   AA2 2	A>=A>c                     	 |j                   d   }|j                  d      }|dk\  r||d  j                  d      d   }d|z  S y # t        $ r Y y w xY w)Nzwww-authenticatezscope="r   r   r  zAcceptable scopes: %s)r   findsplitr  )r   r  www_authenticate	scope_idxscopess        r   !_GetAcceptableScopesFromHttpErrorz,GcsJsonApi._GetAcceptableScopesFromHttpError  sr    #,,-?@ #''	2i	a!)*-33C8;&// 
  
s   A A 	AAc                 :   t        |t        j                        r| j                  |      }|j                  dk\  rt        |xs d|j                        S |j                  dk(  rt        |xs d|j                        S |j                  dk(  rt        |xs d|j                        S |j                  dk(  rt        |xs d|j                        S |j                  d	k\  rt        |xs d|j                        S t        |t        j                        rt        d
t        |      z        S t        |t        j                        rSdt        |      v sdt        |      v rt        t        |            S dt        |      v rt        dt        |      z        S y y )Nr  zServer ErrorrQ  r  zToo Many Requestsi  Bad Requestrl    za%s; if this issue persists, try deleting the tracker files present under ~/.gsutil/tracker-files/zAborting transferzNot enough bytes in streamzadditional bytes left in streamz?%s; this can happen if a file changes size while being uploaded)r   rp  rq  r  rW  r   r   r   StreamExhaustedstrTransferError)r   r   r  s      r   r-  z5GcsJsonApi._TranslateApitoolsResumableUploadException  s   !(223--a0g	
#	'(A>/0}}> 	>==C'(F3F/0}}> 	>==C01IM89G 	G==C01IM89G 	G==C,W-E45MMC 	C!(889*,.1!f56 6 !(667
Q
'
&#a&
0,SV44,A6,MF 	 7	 8r   c                 &   t        |t        j                        rE| j                  |      }|j                  dk(  rt        |xs d|j                        S |j                  dk(  radt        |      v rt        |xs d|j                        S dt        |      v rt        t        |j                  | j                  |            S |j                  d	k(  rcd
t        |      v rt        |xs d|j                        S dt        |      v rt        |xs d|j                        S dt        |      v rt        d|z  |j                        S dt        |      v rt        d|j                        S dt        |      v rt        d|j                        S dt        |      v rt        d|j                        S dt        |      v r+t        t        |j                  | j                  |            S dt        |      v rt        ||j                        S t        |xs t        |      |j                        S |j                  dk(  r|rt        |d|j                         |S |rG|r$t        |j                  | j                  |||      S t        |j                  | j                  |      S t!        |xs |j"                  |j                        S |j                  dk(  rC|rAdt        |      v rt        d|z  |j                        S t%        d|z  |j                        S |j                  dk(  rt'        ||j                        S t%        ||j                        S t        |t        j(                        rt%        dt        |      z        S y) a!  Translates apitools exceptions into their gsutil Cloud Api equivalents.

    Args:
      e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS.
      bucket_name: Optional bucket name in request that caused the exception.
      object_name: Optional object name in request that caused the exception.
      generation: Optional generation in request that caused the exception.
      not_found_exception: Optional exception to raise in the not-found case.

    Returns:
      ServiceException for translatable exceptions, None
      otherwise.
    r  r  rQ  i  zLogin RequiredzAccess denied: login required.insufficient_scope)rR  bodyi  z7The account for the specified project has been disabledzAccount disabled.z,Daily Limit for Unauthenticated Use Exceededz8Access denied: quota exceeded. Is your project ID valid?z-The bucket you tried to delete was not empty.zBucketNotEmpty (%s)zFThe bucket you tried to create requires domain ownership verification.zThe bucket you tried to create requires domain ownership verification. Please see https://cloud.google.com/storage/docs/naming?hl=en#verification for more details.zUser Rate Limit Exceededz5Rate limit exceeded. Please retry this request later.zAccess Not ConfiguredzAccess Not Configured. Please go to the Google Cloud Platform Console (https://cloud.google.com/console#/project) for your project, select APIs and Auth and enable the Google Cloud Storage JSON API.z,does not have permission to publish messagesrl  rR  )r   i  z,The bucket you tried to delete is not empty.zA Cloud Storage bucket named '%s' already exists. Try another name. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization.i  z.Transfer invalid (possible encoding error: %s)N)r   rp  rq  r  rW  r   r  r   "_INSUFFICIENT_OAUTH2_SCOPE_MESSAGEr  r   r   r*  rM   rf   rK   r   r  r   r   TransferInvalidError)r   r   r   r   r   r  r  s          r   rU  z&GcsJsonApi._TranslateApitoolsException	  s{   & !(223--a0g	
#	 #7#;m*+--9 	9==Cs1v%&w (H'G./mm= = "SV+ '0]]99!<> > ==CDAN&w'E2E./mm= =;s1vE&w (C(C./mm= = =AF"#8;#F*+--9 9!$Q(&6 ]]$ $ (3q61&]]$ $ %A.&/ ]]$ $ "SV+ '0]]99!<> > <s1vE1'!--P
P&w'8#a&O
O==C %x
?$
$0151<1<<F	H H
 /q}}dmm/:< < !5AIIammLL==CK9SVC"#8;#F*+--9 9 ** ==" 	" ==C$WQ]]CCgamm<<	A*??	@N!!f% & & 
Ar   )NNr   NNNNr  )NN)NNN)NNNNN)NNNN)NFr   NNN)NNNNNN)NNNNNNNF)NNNNNNF)
NNNNNNNNNF)	NNNNNNNNN)T)O__name__
__module____qualname____doc__r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r<  rA  rM  rX  rb  r`  ry  rp  rl  r   r  r  r  r  r  r  r  r  r  r  r  r   r  ONE_SHOTr  r  r  r  r  r  r#  r  r4  r8  r:  rd  rg  rv  r  r  r  r  r  r  r  r  r  r  r  r  r-  r/  r  r   r  r  r-  rU  __classcell__)r   s   @r   rc   rc      sv   E   $ MP^	:94
GPC" %)"& $>: %)"&@.C/0C0C	@ "!% $O,bC* #  #CJC0*X  #L\F"P	N < < 8<7;0 +/&*04*>^ :>59	'* $(!%#*>X,
 %):&)&H0 # $!%).'/'@'@'I'I !"'+(,#&*~>N ,049+,)-3715B*T #'+0"# $*.(,9&~ &*%)(,#'!%'+@>J  $"&%)!'+%)&*&7&E&E !&zPxDR #!%%)$(  %94 (,*..2-1%)#').#: (,*.!%/3-1.2-1%)#').#> !% ##'$("&"&BPN "&" >> #'%)!5.v C@*C* ,01526*>	*	**** &*B*II /3.2-16:B	(B /3.2-16:r&r   rc   )r  
__future__r   r   r   r   
contextlibr   r  	six.movesr   r  r  r   socketsslr  r  r   apitools.base.pyr	   r
   rp  r   r  r   r  apitools.base.py.utilr   botor   httplib2r  gslibr   gslib.cloud_apir   r   r   r   r   r   r   r   r   r   r   r   r   r   r   gslib.exceptionr   gslib.gcs_json_credentialsr    r!   gslib.gcs_json_mediar"   r#   r$   r%   r&   r'   r(   gslib.impersonation_credentialsr)   gslib.no_op_credentialsr*   gslib.progress_callbackr+   gslib.project_idr,   "gslib.third_party.storage_apitoolsr-   r   r.   r   gslib.tracker_filer/   r0   r1   r2   r3   gslib.utils.boto_utilr4   r5   r6   r7   r8   r9   r:   gslib.utils.cloud_api_helperr;   r<   gslib.utils.constantsr=   r>   r?   r@   rA   gslib.utils.encryption_helperrB   rC   rD   rE   gslib.utils.metadata_utilrF   gslib.utils.retry_utilrG   gslib.utils.signurl_helperrH   rI   gslib.utils.text_utilrJ   gslib.utils.translation_helperrK   rL   rM   rN   rO   rP   oauth2client.service_accountrQ   r  intr   r  CustomAttributesValuer  AdditionalPropertyr  r^  rq  r  r  r  r   TransferRetryErrorBadStatusCodeErrorRetryAfterErrorRequestErrorBadStatusLineIncompleteReadResponseNotReadyServerNotFoundErrorr  r  r  gaierrortimeoutSSLErrorr?  r  r   r   r  r  r  +FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONr   r   rr  r  r   r   rc   r  r   r   <module>r%     s   E & %  ' %  !   	  
   
 % > B : 7      1 - / $ / - - 1 ) < 6 9 4 = , , C 7 : G 7 2 E 8 6 D 3 ? . S W 0 8 4 5 6 . 6 ; 2 , / ? 8 B 4 ; 8 ; & M 7 A F C 6 A = H P H ? B = B77	$ !--$0$F$F !%88  ! $7$A$A$7$G$G$7$E$E$7$L$L$N   ****''$$    33
LL
OO
NNLL &    -'*2 . -*!9'2 . /].a.a / + + 2 224 Xy12    ; # ()	G#& G#&r   