
    vs                     R   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 d dlm
Z
 d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ  G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de      Z G d de
      Z G d de	      Z G d de	      Zy)    N)six)seekablereadable)IN_MEMORY_UPLOAD_TAG)Task)SubmissionTask)CreateMultipartUploadTask)CompleteMultipartUploadTask)get_callbacks)get_filtered_dict)DeferredOpenFileChunksizeAdjusterc                   &    e Zd ZddZd Zd Zd Zy)AggregatedProgressCallbackc                 .    || _         || _        d| _        y)a  Aggregates progress updates for every provided progress callback

        :type callbacks: A list of functions that accepts bytes_transferred
            as a single argument
        :param callbacks: The callbacks to invoke when threshold is reached

        :type threshold: int
        :param threshold: The progress threshold in which to take the
            aggregated progress and invoke the progress callback with that
            aggregated progress total
        r   N)
_callbacks
_threshold_bytes_seen)self	callbacks	thresholds      $lib/third_party/s3transfer/upload.py__init__z#AggregatedProgressCallback.__init__   s     $#    c                     | xj                   |z  c_         | j                   | j                  k\  r| j                          y y N)r   r   _trigger_callbacks)r   bytes_transferreds     r   __call__z#AggregatedProgressCallback.__call__-   s7    --t.##% /r   c                 D    | j                   dkD  r| j                          yy)z@Flushes out any progress that has not been sent to its callbacksr   N)r   r   r   s    r   flushz AggregatedProgressCallback.flush2   s!    a##%  r   c                 Z    | j                   D ]  } || j                          d| _        y )N)r   r   )r   r   )r   callbacks     r   r   z-AggregatedProgressCallback._trigger_callbacks7   s&    Ht'7'78 (r   N)i   )__name__
__module____qualname__r   r   r"   r    r   r   r   r      s     &
&
r   r   c                   <    e Zd ZdZd Zd
dZd Zd Zd Zd Z	d	 Z
y)InterruptReadera  Wrapper that can interrupt reading using an error

    It uses a transfer coordinator to propagate an error if it notices
    that a read is being made while the file is being read from.

    :type fileobj: file-like obj
    :param fileobj: The file-like object to read from

    :type transfer_coordinator: s3transfer.futures.TransferCoordinator
    :param transfer_coordinator: The transfer coordinator to use if the
        reader needs to be interrupted.
    c                      || _         || _        y r   )_fileobj_transfer_coordinator)r   fileobjtransfer_coordinators      r   r   zInterruptReader.__init__J   s    %9"r   Nc                     | j                   j                  r| j                   j                  | j                  j                  |      S r   )r-   	exceptionr,   read)r   amounts     r   r2   zInterruptReader.readN   s:     %%//,,666}}!!&))r   c                 :    | j                   j                  |       y r   )r,   seek)r   wheres     r   r5   zInterruptReader.seekX   s    5!r   c                 6    | j                   j                         S r   )r,   tellr!   s    r   r8   zInterruptReader.tell[   s    }}!!##r   c                 8    | j                   j                          y r   )r,   closer!   s    r   r:   zInterruptReader.close^   s    r   c                     | S r   r(   r!   s    r   	__enter__zInterruptReader.__enter__a   s    r   c                 $    | j                          y r   )r:   )r   argskwargss      r   __exit__zInterruptReader.__exit__d   s    

r   r   )r%   r&   r'   __doc__r   r2   r5   r8   r:   r<   r@   r(   r   r   r*   r*   =   s*    :*"$r   r*   c                   X    e Zd ZdZddZed        Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zy)UploadInputManageraJ  Base manager class for handling various types of files for uploads

    This class is typically used for the UploadSubmissionTask class to help
    determine the following:

        * How to determine the size of the file
        * How to determine if a multipart upload is required
        * How to retrieve the body for a PutObject
        * How to retrieve the bodies for a set of UploadParts

    The answers/implementations differ for the various types of file inputs
    that may be accepted. All implementations must subclass and override
    public methods from this class.
    Nc                 .    || _         || _        || _        y r   )_osutilr-   _bandwidth_limiter)r   osutilr/   bandwidth_limiters       r   r   zUploadInputManager.__init__w   s    %9""3r   c                     t        d      )a  Determines if the source for the upload is compatible with manager

        :param upload_source: The source for which the upload will pull data
            from.

        :returns: True if the manager can handle the type of source specified
            otherwise returns False.
        zmust implement _is_compatible()NotImplementedErrorclsupload_sources     r   is_compatiblez UploadInputManager.is_compatible|   s     ""CDDr   c                     t        d      )a  Whether the body it provides are stored in-memory

        :type operation_name: str
        :param operation_name: The name of the client operation that the body
            is being used for. Valid operation_names are ``put_object`` and
            ``upload_part``.

        :rtype: boolean
        :returns: True if the body returned by the manager will be stored in
            memory. False if the manager will not directly store the body in
            memory.
        z%must implement store_body_in_memory())NotImplementedr   operation_names     r   stores_body_in_memoryz(UploadInputManager.stores_body_in_memory   s     DEEr   c                     t        d      )zProvides the transfer size of an upload

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request
        z&must implement provide_transfer_size()rJ   r   transfer_futures     r   provide_transfer_sizez(UploadInputManager.provide_transfer_size   s     ""JKKr   c                     t        d      )a  Determines where a multipart upload is required

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: boolean
        :returns: True, if the upload should be multipart based on
            configuartion and size. False, otherwise.
        z*must implement requires_multipart_upload()rJ   r   rW   configs      r   requires_multipart_uploadz,UploadInputManager.requires_multipart_upload   s     ""NOOr   c                     t        d      )a  Returns the body to use for PutObject

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type config: s3transfer.manager.TransferConfig
        :param config: The config associated to the transfer manager

        :rtype: s3transfer.utils.ReadFileChunk
        :returns: A ReadFileChunk including all progress callbacks
            associated with the transfer future.
        z$must implement get_put_object_body()rJ   rV   s     r   get_put_object_bodyz&UploadInputManager.get_put_object_body   s     ""HIIr   c                     t        d      )a  Yields the part number and body to use for each UploadPart

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The future associated with upload request

        :type chunksize: int
        :param chunksize: The chunksize to use for this upload.

        :rtype: int, s3transfer.utils.ReadFileChunk
        :returns: Yields the part number and the ReadFileChunk including all
            progress callbacks associated with the transfer future for that
            specific yielded part.
        z)must implement yield_upload_part_bodies()rJ   )r   rW   	chunksizes      r   yield_upload_part_bodiesz+UploadInputManager.yield_upload_part_bodies   s     ""MNNr   c                     t        || j                        }| j                  r(| j                  j                  || j                  d      }|S )NF)enabled)r*   r-   rF   get_bandwith_limited_stream)r   r.   s     r   _wrap_fileobjz UploadInputManager._wrap_fileobj   sL    !'4+E+EF""--II33U J DGr   c                 :    t        |d      }|rt        |      gS g S )Nprogress)r   r   )r   rW   r   s      r   _get_progress_callbacksz*UploadInputManager._get_progress_callbacks   s&    !/:>	 .y9::	r   c                 @    |D cg c]  }|j                    c}S c c}w r   )r"   )r   aggregated_progress_callbacksr$   s      r   _get_close_callbacksz'UploadInputManager._get_close_callbacks   s     /LM/L8/LMMMs   r   )r%   r&   r'   rA   r   classmethodrO   rT   rX   r\   r^   ra   re   rh   rk   r(   r   r   rC   rC   h   sP    4
 	E 	EFLPJO Nr   rC   c                   V    e Zd ZdZed        Zd Zd Zd Zd Z	d Z
d Zd	 Zd
 Zd Zy)UploadFilenameInputManagerzUpload utility for filenamesc                 6    t        |t        j                        S r   )
isinstancer   string_typesrL   s     r   rO   z(UploadFilenameInputManager.is_compatible   s    -)9)9::r   c                      y)NFr(   rR   s     r   rT   z0UploadFilenameInputManager.stores_body_in_memory   s    r   c                     |j                   j                  | j                  j                  |j                   j                  j
                               y r   )metarX   rE   get_file_size	call_argsr.   rV   s     r   rX   z0UploadFilenameInputManager.provide_transfer_size   s=    22LL&&$$..668	9r   c                 H    |j                   j                  |j                  k\  S r   )rt   sizemultipart_thresholdrZ   s      r   r\   z4UploadFilenameInputManager.requires_multipart_upload   s    ##((F,F,FFFr   c                     | j                  |      \  }}| j                  |      }| j                  |      }| j                  |      }|j                  j
                  }| j                  j                  |||||      S )Nr.   
chunk_sizefull_file_sizer   close_callbacks)&_get_put_object_fileobj_with_full_sizere   rh   rk   rt   rx   rE   #open_file_chunk_reader_from_fileobj)r   rW   r.   	full_sizer   r~   rx   s          r   r^   z.UploadFilenameInputManager.get_put_object_body   s    !HH $$W-00A	33I>##(( ||??Y @ B 	Br   c              #     K   |j                   j                  }| j                  ||      }t        d|dz         D ]  }| j	                  |      }| j                  |      }||dz
  z  }| j                  |j                   j                  j                  |||      \  }	}
| j                  |	      }	| j                  j                  |	||
||      }||f  y w)N   )
start_byte	part_sizer}   r{   )rt   rx   _get_num_partsrangerh   rk   '_get_upload_part_fileobj_with_full_sizerv   r.   re   rE   r   )r   rW   r`   r}   	num_partspart_numberr   r~   r   r.   r   read_file_chunks               r   ra   z3UploadFilenameInputManager.yield_upload_part_bodies  s     (--22''C	 IM2K44_EI"77	BO"kAo6J "&!M!M$$..66:#N "N "DGY ((1G #llNNI(I / O 1O ..) 3s   CCc                 J    t        ||| j                  j                        }|S )N)open_function)r   rE   open)r   r.   r   s      r   _get_deferred_open_filez2UploadFilenameInputManager._get_deferred_open_file  s#    "Zt||/@/@Br   c                     |j                   j                  j                  }|j                   j                  }| j	                  |d      |fS )Nr   )rt   rv   r.   rx   r   r   rW   r.   rx   s       r   r   zAUploadFilenameInputManager._get_put_object_fileobj_with_full_size#  sB    !&&0088##((++GQ7==r   c                 >    |d   }|d   }| j                  ||      |fS )Nr   r}   )r   )r   r.   r?   r   r   s        r   r   zBUploadFilenameInputManager._get_upload_part_fileobj_with_full_size(  s/    L)
+,	++GZ@)KKr   c                 ~    t        t        j                  |j                  j                  t        |      z              S r   )intmathceilrt   rx   float)r   rW   r   s      r   r   z)UploadFilenameInputManager._get_num_parts-  s3    IIo**//%	2BBCE 	Er   N)r%   r&   r'   rA   rl   rO   rT   rX   r\   r^   ra   r   r   r   r   r(   r   r   rn   rn      sH    &; ;9
GB&/2
>
L
Er   rn   c                   8    e Zd ZdZed        Zd Zd Zd Zd Z	y)UploadSeekableInputManagerz&Upload utility for an open file objectc                 2    t        |      xr t        |      S r   )r   r   rL   s     r   rO   z(UploadSeekableInputManager.is_compatible4  s    &B8M+BBr   c                     |dk(  ryy)N
put_objectFTr(   rR   s     r   rT   z0UploadSeekableInputManager.stores_body_in_memory8  s    \)r   c                    |j                   j                  j                  }|j                         }|j	                  dd       |j                         }|j	                  |       |j                   j                  ||z
         y )Nr      )rt   rv   r.   r8   r5   rX   )r   rW   r.   start_positionend_positions        r   rX   z0UploadSeekableInputManager.provide_transfer_size>  sg    !&&0088 !Q||~^$22>)	+r   c                 j    |j                  |d         }t        j                  |      t        |      fS )Nr   )r2   r   BytesIOlen)r   r.   r?   datas       r   r   zBUploadSeekableInputManager._get_upload_part_fileobj_with_full_sizeJ  s1     ||F;/0 {{4 #d)++r   c                     |j                   j                  j                  }|j                         |j                   j                  z   }||fS r   )rt   rv   r.   r8   rx   r   s       r   r   zAUploadSeekableInputManager._get_put_object_fileobj_with_full_sizeY  sA    !&&0088 ||~ 4 4 9 99}r   N)
r%   r&   r'   rA   rl   rO   rT   rX   r   r   r(   r   r   r   r   2  s-    0C C
+,r   r   c                   `     e Zd ZdZd fd	Zed        Zd Zd Zd Z	d Z
d Zdd	Zd
 Z xZS )UploadNonSeekableInputManagerz7Upload utility for a file-like object that cannot seek.c                 >    t         t        |   |||       d| _        y )Nr   )superr   r   _initial_data)r   rG   r/   rH   	__class__s       r   r   z&UploadNonSeekableInputManager.__init__c  s$    +T;(*;	= r   c                     t        |      S r   )r   rL   s     r   rO   z+UploadNonSeekableInputManager.is_compatibleh  s    &&r   c                      y)NTr(   rR   s     r   rT   z3UploadNonSeekableInputManager.stores_body_in_memoryl  s    r   c                      y r   r(   rV   s     r   rX   z3UploadNonSeekableInputManager.provide_transfer_sizeo  s     	r   c                 0   |j                   j                  #|j                   j                  |j                  k\  S |j                   j                  j                  }|j                  }| j                  ||d      | _        t        | j                        |k  ryy)NFT)rt   rx   ry   rv   r.   _readr   r   )r   rW   r[   r.   r   s        r   r\   z7UploadNonSeekableInputManager.requires_multipart_uploadt  s    $$0"'',,0J0JJJ "&&0088..	!ZZEBt!!"Y.r   c                     | j                  |      }| j                  |      }|j                  j                  j                  }| j                  | j                  |j                         z   ||      }d | _        |S r   )rh   rk   rt   rv   r.   
_wrap_datar   r2   )r   rW   r   r~   r.   bodys         r   r^   z1UploadNonSeekableInputManager.get_put_object_body  so    00A	33I>!&&0088/OM
 "r   c              #      K   |j                   j                  j                  }d}	 | j                  |      }| j	                  |      }|dz  }| j                  ||      }|sy | j                  |||      }d }||f Xw)Nr   r   )rt   rv   r.   rh   rk   r   r   )	r   rW   r`   file_objectr   r   r~   part_contentpart_objects	            r   ra   z6UploadNonSeekableInputManager.yield_upload_part_bodies  s     %**44<< 44_EI"77	BO1K::k9=L//i:K  L{** s   A<A>c                 T   t        | j                        dk(  r|j                  |      S |t        | j                        k  r'| j                  d| }|r| j                  |d | _        |S |t        | j                        z
  }| j                  |j                  |      z   }|rd| _        |S )a=  
        Reads a specific amount of data from a stream and returns it. If there
        is any data in initial_data, that will be popped out first.

        :type fileobj: A file-like object that implements read
        :param fileobj: The stream to read from.

        :type amount: int
        :param amount: The number of bytes to read from the stream.

        :type truncate: bool
        :param truncate: Whether or not to truncate initial_data after
            reading from it.

        :return: Generator which generates part bodies from the initial data.
        r   Nr   )r   r   r2   )r   r.   r3   truncater   amount_to_reads         r   r   z#UploadNonSeekableInputManager._read  s    $ t!!"a'<<'' S++,,%%gv.D %)%7%7%@"K
  #d&8&8"99!!GLL$@@ !$Dr   c                     | j                  t        j                  |            }| j                  j	                  |t        |      t        |      ||      S )a  
        Wraps data with the interrupt reader and the file chunk reader.

        :type data: bytes
        :param data: The data to wrap.

        :type callbacks: list
        :param callbacks: The callbacks associated with the transfer future.

        :type close_callbacks: list
        :param close_callbacks: The callbacks to be called when closing the
            wrapper for the data.

        :return: Fully wrapped data.
        r{   )re   r   r   rE   r   r   )r   r   r   r~   r.   s        r   r   z(UploadNonSeekableInputManager._wrap_data  sP      $$S[[%67||??D	#d) @ B 	Br   r   )T)r%   r&   r'   rA   r   rl   rO   rT   rX   r\   r^   ra   r   r   __classcell__)r   s   @r   r   r   a  sD    A!
 ' '
"+&(TBr   r   c                   L    e Zd ZdZg dZdgZd Z	 ddZd Zd Z	d	 Z
d
 Zd Zy)UploadSubmissionTaskz.Task for submitting tasks to execute an upload)SSECustomerKeySSECustomerAlgorithmSSECustomerKeyMD5RequestPayerr   c                     t         t        t        g}|j                  j                  j
                  }|D ]  }|j                  |      s|c S  t        d|dt        |      d      )ao  Retrieves a class for managing input for an upload based on file type

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future for the request

        :rtype: class of UploadInputManager
        :returns: The appropriate class to use for managing a specific type of
            input for uploads.
        zInput z
 of type: z is not supported.)	rn   r   r   rt   rv   r.   rO   RuntimeErrortype)r   rW   upload_manager_resolver_chainr.   upload_manager_clss        r   _get_upload_input_manager_clsz2UploadSubmissionTask._get_upload_input_manager_cls  sk     '&))
% "&&0088"?!//8)) #@ g() 	)r   Nc                     | j                  |      || j                  |      }|j                  j                  |j	                  |       |j                  ||      s| j                  ||||||       y| j                  ||||||       y)a  
        :param client: The client associated with the transfer manager

        :type config: s3transfer.manager.TransferConfig
        :param config: The transfer config associated with the transfer
            manager

        :type osutil: s3transfer.utils.OSUtil
        :param osutil: The os utility associated to the transfer manager

        :type request_executor: s3transfer.futures.BoundedExecutor
        :param request_executor: The request executor associated with the
            transfer manager

        :type transfer_future: s3transfer.futures.TransferFuture
        :param transfer_future: The transfer future associated with the
            transfer request that tasks are being submitted for
        N)r   r-   rt   rx   rX   r\   _submit_upload_request_submit_multipart_request)r   clientr[   rG   request_executorrW   rH   upload_input_managers           r   _submitzUploadSubmissionTask._submit  s    ( tAA 224E G
 $$, 66G $==)''(8/$& **(8/$&r   c                 "   |j                   j                  }| j                  |d      }| j                  j	                  |t        | j                  ||j                  |      |j                  |j                  |j                  dd      |       y )Nr   )r   r.   bucketkey
extra_argsT)r/   main_kwargsis_finaltag)
rt   rv   _get_upload_task_tagr-   submitPutObjectTaskr^   r   r   r   )	r   r   r[   rG   r   rW   r   rv   put_object_tags	            r   r   z+UploadSubmissionTask._submit_upload_request2  s    #((22	 22 ,0 	""))%)%?%?$3GG' )'..$=="+"6"6   	* 	
r   c                 z   |j                   j                  }| j                  j                  |t	        | j                  ||j
                  |j                  |j                  d            }g }	| j                  |j                        }
| j                  |d      }|j                   j                  }t               }|j                  |j                  |      }|j                  ||      }|D ]e  \  }}|	j                  | j                  j                  |t!        | j                  |||j
                  |j                  ||
dd|i      |             g | j#                  |j                        }| j                  j                  |t%        | j                  ||j
                  |j                  |d||	dd	
             y )N)r   r   r   r   )r/   r   upload_part)r   r.   r   r   r   r   	upload_id)r/   r   pending_main_kwargsr   )r   partsT)r/   r   r   r   )rt   rv   r-   r   r	   r   r   r   _extra_upload_part_argsr   rx   r   adjust_chunksizemultipart_chunksizera   appendUploadPartTask_extra_complete_multipart_argsr
   )r   r   r[   rG   r   rW   r   rv   create_multipart_futurepart_futuresextra_part_argsupload_part_tagrx   adjusterr`   part_iteratorr   r.   complete_multipart_extra_argss                      r   r   z.UploadSubmissionTask._submit_multipart_requestL  s    $((22	 #'"<"<"C"C%%)%?%?$'..$=="+"6"6	#
 66y7K7KL 33 -1 ##(($&--f.H.H$O	,EEY( %2 K**11$"-1-G-G&,'.&/&6&6#,==+6*9% ()@- (! 2  %2, )-(K(K  )"% 	""))'%)%?%?$'..$=="?	 "9)% 	
r   c                 .    t        || j                        S r   )r   UPLOAD_PART_ARGSr   r   s     r   r   z,UploadSubmissionTask._extra_upload_part_args  s     !T-B-BCCr   c                 .    t        || j                        S r   )r   COMPLETE_MULTIPART_ARGSr   s     r   r   z3UploadSubmissionTask._extra_complete_multipart_args  s     T-I-IJJr   c                 8    d }|j                  |      rt        }|S r   )rT   r   )r   r   rS   r   s       r   r   z)UploadSubmissionTask._get_upload_task_tag  s    55nE&C
r   r   )r%   r&   r'   rA   r   r   r   r   r   r   r   r   r   r(   r   r   r   r     sG    8 	)2 48%&N
4K
ZD
Kr   r   c                       e Zd ZdZd Zy)r   z Task to do a nonmultipart uploadc                 ^    |5 } |j                   d|||d| ddd       y# 1 sw Y   yxY w)aP  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.
        )BucketKeyBodyNr(   )r   )r   r   r.   r   r   r   r   s          r   _mainzPutObjectTask._main  s/     FNV4N:N WWs   #,Nr%   r&   r'   rA   r   r(   r   r   r   r     s    *
Or   r   c                       e Zd ZdZd Zy)r   z+Task to upload a part in a multipart uploadc           
      t    |5 } |j                   d|||||d|}	ddd       	d   }
|
|dS # 1 sw Y   xY w)a  
        :param client: The client to use when calling PutObject
        :param fileobj: The file to upload.
        :param bucket: The name of the bucket to upload to
        :param key: The name of the key to upload to
        :param upload_id: The id of the upload
        :param part_number: The number representing the part of the multipart
            upload
        :param extra_args: A dictionary of any extra arguments that may be
            used in the upload.

        :rtype: dict
        :returns: A dictionary representing a part::

            {'Etag': etag_value, 'PartNumber': part_number}

            This value can be appended to a list to be used to complete
            the multipart upload.
        )r   r   UploadId
PartNumberr   NETag)r   r   r(   )r   )r   r   r.   r   r   r   r   r   r   responseetags              r   r   zUploadPartTask._main  s[    * )v)) )3"{) ()H 
 K88 Ws   .7Nr   r(   r   r   r   r     s
    59r   r   )r   botocore.compatr   s3transfer.compatr   r   s3transfer.futuresr   s3transfer.tasksr   r   r	   r
   s3transfer.utilsr   r   r   r   objectr   r*   rC   rn   r   r   r   r   r   r(   r   r   <module>r     s      0 3 ! + 6 8 * . @ B(f (VvN vNrNE!3 NEb,!; ,^AB$6 ABH@> @FOD O9T 9r   