
    9                         d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ dZ G d de      Zej,                  d        Z G d de      Zy)z(Wrapper for use in daisy-chained copies.    )absolute_import)division)print_function)unicode_literalsN)BadRequestException)CloudApi)	constants)parallelism_framework_util)CryptoKeyWrapperFromKeyi  @c                       e Zd ZdZddZd Zy)BufferWrapperz<Wraps the download file pointer to use our in-memory buffer.c                 \    || _         t        |d      r|j                  | _        y|| _        y)zProvides a buffered write interface for a file download.

    Args:
      daisy_chain_wrapper: DaisyChainWrapper instance to use for buffer and
                           locking.
    modeN)daisy_chain_wrapperhasattrr   )selfr   r   s      ,platform/gsutil/gslib/daisy_chain_wrapper.py__init__zBufferWrapper.__init__,   s+      3D"F+%**didi    c                    	 | j                   j                  5  | j                   j                  | j                   j                  k  r
	 ddd       n	 ddd       t	        j
                  d       mt        |      }|rd| j                   j                  5  | j                   j                  j                  |       | j                   xj                  |z  c_        ddd       yy# 1 sw Y   xY w# 1 sw Y   yxY w)z>Waits for space in the buffer, then writes data to the buffer.Nr   )	r   lockbytes_bufferedmax_buffer_sizetimesleeplenbufferappend)r   datadata_lens      r   writezBufferWrapper.write9   s    
##(($$33&&667
 )(7 )
 jjm  4yH##((  ''..t4  //8;/ )(  )( )(s   /C!AC-!C*-C6N)b)__name__
__module____qualname____doc__r   r!    r   r   r   r   )   s    D<r   r   c              #   ^   K   | j                  |      }| |r| j                          y y w)N)timeout)acquirerelease)r   r)   results      r   AcquireLockWithTimeoutr-   I   s,     <<<(&,LLN s   +-c                   Z    e Zd ZdZddedfdZd
dZddZd Ze	j                  fdZd	 Zy)DaisyChainWrappera  Wrapper class for daisy-chaining a cloud download to an upload.

  This class instantiates a BufferWrapper object to buffer the download into
  memory, consuming a maximum of max_buffer_size. It implements intelligent
  behavior around read and seek that allow for all of the operations necessary
  to copy a file.

  This class is coupled with the XML and JSON implementations in that it
  expects that small buffers (maximum of constants.TRANSFER_BUFFER_SIZE) in
  size will be used.
  FNc                 f   d| _         t        j                         | _        d| _        d| _        || _        d| _        d| _        t        j                         | _        t        j                         | _        || _        || _        || _        t!        |      | _        || _        d| _        d| _        || _        t-        j.                         | _        t-        j.                         | _        | j5                  | j*                         | j0                  j7                  d      st9        d      d| _        y)a`  Initializes the daisy chain wrapper.

    Args:
      src_url: Source CloudUrl to copy from.
      src_obj_size: Size of source object.
      gsutil_api: gsutil Cloud API to use for the copy.
      compressed_encoding: If true, source object has content-encoding: gzip.
      progress_callback: Optional callback function for progress notifications
          for the download thread. Receives calls with arguments
          (bytes_transferred, total_size).
      download_chunk_size: Integer number of bytes to download per
          GetObjectMedia request. This is the upper bound of bytes that may be
          unnecessarily downloaded if there is a break in the resumable upload.
      decryption_key: Base64-encoded decryption key for the source object,
          if any.
    Raises:
      Exception: if the download thread doesn't start within 60 seconds
    r   i   N)progress_callback<   z1Could not start download thread after 60 seconds.r   )positioncollectionsdequer   r   r   _download_chunk_sizelast_position	last_datar
   
CreateLockr   download_exception_locksrc_obj_sizesrc_urlcompressed_encodingr   decryption_tuple
gsutil_apidownload_exceptiondownload_threadr1   	threadingEventdownload_startedstop_downloadStartDownloadThreadwait	Exception_unused_data_from_previous_read)r   r<   r;   r?   r=   r1   download_chunk_sizedecryption_keys           r   r   zDaisyChainWrapper.__init__^   s   6 DM##%DKD&D 3D DDN +557DI $>#H#H#JD $DDL2D3NCD !DO #DD.D%OO-D"*Dt/E/EF  %%b)IJJ+.D(r   c                       fd}t        j                  |||f       _         j                  j                          y)zCStarts the download thread for the source object (from start_byte).c                    j                   j                          	 | j                  z   j                  k  r(j                  j                  j                  j                  j                  j                  t              j                  | | j                  z   dz
  j                  j                  j                  t        j                  j                  j                  j                  |j                          j"                  j%                         rj"                  j'                          y| j                  z  } | j                  z   j                  k  r(j                  j                  j                  j                  j                  j                  t              j                  | j                  j                  j                  t        j                  j                  j                  j                  |j                          y# t(        $ r+}j*                  5  |_         # 1 sw Y   nxY wY d}~yd}~ww xY w)a"  Downloads the source object in chunks.

      This function checks the stop_download event and exits early if it is set.
      It should be set when there is an error during the daisy-chain upload,
      then this function can be called again with the upload's current position
      as start_byte.

      Args:
        start_byte: Byte from which to begin the download.
        progress_callback: Optional callback function for progress
            notifications. Receives calls with arguments
            (bytes_transferred, total_size).
         )	r=   
start_byteend_byte
generationobject_sizedownload_strategyproviderr1   r>   N)r=   rO   rQ   rR   rS   rT   r1   r>   )rD   setr6   r;   r?   GetObjectMediar<   bucket_nameobject_namer   r=   rQ   r   DownloadStrategyONE_SHOTschemer>   rE   is_setclearrH   r:   r@   )rO   r1   er   s      r   PerformDownloadz>DaisyChainWrapper.StartDownloadThread.<locals>.PerformDownload   s   " !%4444t7H7HH
//
(
(ll&&ll&&D!"&":":#!D$=$==A00++ ( 9 9 B B||** 1#44 ) 6 &&($$&
11
1*% 4444t7H7HH& 	&&LL$$LL$$$ $ 8 8!||..))&77@@\\((/!22 	' 	4  ))$%$
!
 *))s7   DH 5+H "B5H 	I!I-	H66H?	;II)targetargsN)rB   ThreadrA   start)r   rO   r1   r_   s   `   r   rF   z%DaisyChainWrapper.StartDownloadThread   sA    7t %++?2<2C2EFD 	 r   c                    | j                   | j                  k(  s|dk(  ry||t        j                  kD  r t	        d|dt        j                  d      | j
                  r$| j
                  d| }| j
                  |d | _        nd}	 | j                  5  | j                  r
	 ddd       nqt        | j                  d      r| j                  r1| j                  | j                  j                         st        d	      ddd       t        j                  d       | j                  5  |s)| j                  j!                         }|d| }||d | _        | j                   | _        || _        t'        |      }| xj                   |z  c_         | xj(                  |z  c_        ddd       |kD  rt	        d
|d|d      |S # 1 sw Y   xY w# 1 sw Y   .xY w)z9Exposes a stream from the in-memory buffer to the upload.r    NzInvalid HTTP read size z+ during daisy chain operation, expected <= .r      zDownload thread died suddenly.z<Invalid read during daisy chain operation, got data of size z, expected size )r3   r;   r	   TRANSFER_BUFFER_SIZEr   rI   r   r   r-   r:   r@   rA   is_aliverH   r   r   popleftr7   r8   r   r   )r   amt
valid_datar   r    s        r   readzDaisyChainWrapper.read   s   }})))SAX 
{cI:::"I$B$BDE E ++77=j

.
.st
4 * jYY[[ Y $D$@$@"E&& +++''002>??  	

1  
 {{""$$3Z
/3CDz,==d!dnZh
mmxm
X%' 
( #~%-s45 5 K Y 
s    G+AG(A9GGGc                 ^    | j                   5  | j                  cd d d        S # 1 sw Y   y xY wN)r   r3   r   s    r   tellzDaisyChainWrapper.tell"  s    	]] 
s   #,c                 N   d}|t         j                  k(  rO|rt        d|z        | j                  5  | j                  | _        d| _        | j                  | _        ddd       y|t         j                  k(  r| j                  5  || j                  k(  rn||| j
                  k(  rk| j
                  | _        | j                  rP| j                  j                  | j                         | xj                  t        | j                        z  c_        nd}ddd       |rJ| j                  j                          | j                  j!                         r| j                  5  | j                  rC| xj                  t        | j                  j#                               z  c_        | j                  rCddd       t%        j&                  d       | j                  j!                         r| j                  5  || _        t)        j*                         | _	        d| _        d| _        d| _        | j                  j-                          ddd       | j/                  || j0                         yyt        d|z        # 1 sw Y   yxY w# 1 sw Y   rxY w# 1 sw Y   xY w# 1 sw Y   [xY w)z)Sets current read position of the stream.Fz_Invalid seek during daisy chain operation. Non-zero offset %s from os.SEEK_END is not supportedNTr   )rO   r1   z:Daisy-chain download wrapper does not support seek mode %s)osSEEK_ENDIOErrorr   r3   r7   r8   r;   SEEK_SETr   
appendleftr   r   rE   rU   rA   ri   rj   r   r   r4   r5   r]   rF   r1   )r   offsetwhencerestart_downloads       r   seekzDaisyChainWrapper.seek&  s&   	0289: 	: 99!]]))	 9
 
2;;	99T]]"
t))),,$-^^ KK""4>>23t~~#66 "
# & 
  ""++-yy%%!!S)<)<)>%??! %%  **Q-	 ""++- YY $-#))+$+ !$
 $
$.



"
"
$  	  F373I3I 	! 	K% 
*  #%+, - -] 9 92 y
 Ys2   *I6BJAJ1AJ6I?JJJ$c                      y)NTr'   rp   s    r   seekablezDaisyChainWrapper.seekable_  s    r   )r   Nro   )r#   r$   r%   r&   _DEFAULT_DOWNLOAD_CHUNK_SIZEr   rF   rm   rq   rs   rv   r{   r}   r'   r   r   r/   r/   Q   sB    
  $)!%#?"F/P@!D8t !# 7-rr   r/   )r&   
__future__r   r   r   r   r4   
contextlibrs   rB   r   gslib.cloud_apir   r   gslib.utilsr	   r
   gslib.utils.encryption_helperr   r~   objectr   contextmanagerr-   r/   r'   r   r   <module>r      sq    / &  % '   	   / $ ! 2 A  1 <F <@  O Or   