xdZddlmZddlmZddlmcmZ ddl mcm Z ddl Z ddlZddlZddlZddlZddlZddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZgd Zd Zd ZdZdZ dZ!dZ"dZ#Gdde$Z%Gdde%Z&ejNrGddejPZ)Gdde%Z*y)z)Upload and download support for apitools.)print_functionN) http_client)buffered_stream) compression) exceptions) http_wrapper) stream_slice)util)DownloadUploadRESUMABLE_UPLOAD SIMPLE_UPLOADDownloadProgressPrinterDownloadCompletePrinterUploadProgressPrinterUploadCompletePrinteriPsimple resumablecd|jvrtd|jdzytd|jzy)z*Print download progress based on response. content-rangez Received %szReceived %d bytesN)infoprintlength)responseunused_downloads ,lib/third_party/apitools/base/py/transfer.pyrr7s6(--' mhmmO<<= !HOO34ctdy)z-Print information about a completed download.zDownload completeNr)unused_responsers rrr?s  rc:td|jdzy)z(Print upload progress based on response.zSent %srangeN)rr)r unused_uploads rrrDs )hmmG, ,-rctdy)z+Print information about a completed upload.zUpload completeNr)r r#s rrrIs  rceZdZdZ ddZdZedZedZedZ e jdZ ed Z e jd Z ed Z ed Z d ZedZedZdZdZdZdZy) _Transferz-Generic bits common to Uploads and Downloads.Ncd|_||_||_||_d|_d|_||_tj|_ ||_ |xsd|_ y)Ni) _Transfer__bytes_http_Transfer__close_stream_Transfer__http_Transfer__stream_Transfer__url_Transfer__num_retries num_retriesr)HandleExceptionsAndRebuildHttpConnections retry_func auto_transfer chunksize)selfstream close_streamr3r2httpr/s r__init__z_Transfer.__init__Rs] *   &  B B *"-grct|SN)strr4s r__repr__z_Transfer.__repr__cs 4yrc|jSr:)r*r<s rr6z_Transfer.close_streamfs"""rc|jSr:)r+r<s rr7z_Transfer.httpjs {{rc6|jxs |jSr:)r)r7r<s r bytes_httpz_Transfer.bytes_httpns  -DII-rc||_yr:)r)r4values rrAz_Transfer.bytes_httprs !rc|jSr:)r.r<s rr/z_Transfer.num_retriesvs!!!rctj|tj|dkrt j d||_y)Nrz*Cannot have negative value for num_retries)r Typechecksix integer_typesrInvalidDataErrorr.rCs rr/z_Transfer.num_retrieszs; uc//0 19--<> >"rc|jSr:)r,r<s rr5z_Transfer.streams }}rc|jSr:)r-r<s rurlz _Transfer.urls zzrc|j|j|xstj|_||_y)aInitialize this download by setting self.http and self.url. We want the user to be able to override self.http by having set the value in the constructor; in that case, we ignore the provided http. Args: http: An httplib2.Http instance or None. url: The url for this transfer. Returns: None. Initializes self. N)EnsureUninitializedr7rGetHttpr+r-)r4r7rMs r _Initializez_Transfer._Initializes6   " 99 8,"6"6"8DK rc>|jduxr|jduSr:)rMr7r<s r initializedz_Transfer.initializedsxxt#= (==rc,t|jSr:)type__name__r<s r _type_namez_Transfer._type_namesDz"""rc`|js"tjd|jzy)NzCannot use uninitialized %srSrTransferInvalidErrorrWr<s rEnsureInitializedz_Transfer.EnsureInitializeds111-?A A rc`|jr"tjd|jzy)NzCannot re-initialize %srYr<s rrOz_Transfer.EnsureUninitializeds/   11)DOO;= = rcR|jr|jjyyr:)r*r,closer<s r__del__z_Transfer.__del__s    MM   ! rcX|(tj|||fjyy)N)targetargs) threadingThreadstart)r4callbackrs r_ExecuteCallbackz_Transfer._ExecuteCallbacks+     HHd3C D J J L r)FNTNr()rV __module__ __qualname____doc__r8r=propertyr6r7rAsetterr/r5rMrQrSrWr[rOr_rgrrr&r&Ns7=A<=."##..""""##&>>##A = "Mrr&czeZdZdZeej ejejejfZ edZ dfd Z e dZe dZeddZeddZe ddZe d Ze d Zd Zd Zd ZddZddZddZddZddZdZ ddZ ddZ ddZ!xZ"S)r zqData for a single download. Public attributes: chunksize: default chunksize to use for transfers. r2progress total_sizerMc |jdd}tt||fi|d|_d|_||_d|_||_||_ y)Nrqr) popsuperr r8_Download__initial_response_Download__progress_Download__total_size_Download__encodingprogress_callbackfinish_callback)r4r5ryrzkwdsrq __class__s rr8zDownload.__init__sUXXlD1  h&v66"&&!2.rc|jSr:)rvr<s rrpzDownload.progress rc|jSr:)rxr<s rencodingzDownload.encodingr~rc tjj|}tjj|r|st j d|z|t |dfd|d|S)z-Create a new download object from a filename.z*File %s exists and overwrite not specifiedwbT)r6r2)ospath expanduserexistsrInvalidUserInputErroropen)clsfilename overwriter2r{rs rFromFilezDownload.FromFilesoww!!(+ 77>>$  22IIl+-. ."3>>&1D1  $%2H "%)/%:H "  ,,T%[9Cu+C/j1AB14 3EF # rc|j|j|j|j|jdS)Nro)r[r2rprqrMr<s rserialization_datazDownload.serialization_datas6  !// //88   rc|jSr:)rwr<s rrqzDownload.total_size   rch|jsyd|j|j|jfzS)NzDownload (uninitialized)z1Download with %d/%s bytes transferred from url %srSrprqrMr<s r__str__zDownload.__str__s4-B MM4??DHHF66 6rcbd|jd<d|jdz fz|jd<y)Nmediaaltz bytes=0-%dRange) query_paramsr3headers)r4 http_request url_builders rConfigureRequestzDownload.ConfigureRequests5*1   ' )58J7L(L W%rcd|vr-|djd\}}}|dk7rt||_|jd|_yy)z;Sets the total size based off info if possible otherwise 0.r/*Nr) rpartitionintrwrq)r4r_totals r __SetTotalzDownload.__SetTotal#sQ d "/::3?KAq%|$'J! ?? " !D  #rc|j||tjd|xs |j}| |j |j |_|j }|j r|jd}|j|d|tj|jxs||}|j|jvrtjj|||_|j#|j$|j$j'd|j(}||j |}|j+|||j r|j-yy)aInitialize this download by making a request. Args: http_request: The HttpRequest to use to initialize this download. http: The httplib2.Http instance for this request. client: If provided, let this client process the final URL before sending any additional requests. If client is provided and http is not, client.http will be used instead. NMust provide client or http.rzcontent-location)rOr UserErrorr7rrMr2_Download__ComputeEndByte_Download__SetRangeHeaderr MakeRequestrA status_code_ACCEPTABLE_STATUSES HttpError FromResponseru_Download__SetTotalrget request_urlrQStreamInChunks)r4rr7rrMend_byters rInitializeDownloadzDownload.InitializeDownload0sN   " 8#,,X88B-DHx:L'NH* .%)%=%=eS%I"(*.'--h7H  'H!#'';>>9  33KMM)+h.>8#rc.|j|||dy)z%Stream the entire download in chunks.TrfrzrrN) StreamMediar4rfrzrs rrzDownload.StreamInChunkss! (O,>$(  *rcD|xs |j}|xs |j}|j |j|j}d|_n;|j |j |}|j |j ||}|j|j|j|j|}|j|||jtjk(s|j |jk\rn|j||y)a4Stream the entire download. Args: callback: (default: None) Callback to call as each chunk is completed. finish_callback: (default: None) Callback to call when the download is complete. additional_headers: (default: None) Additional headers to include in fetching bytes. use_chunks: (bool, default: True) If False, ignore self.chunksize and stream this download in a single request. Returns: None. Streams bytes into self.stream. N)rr)ryrzr[rurrprrqrrrrgrrr)r4rfrzrrrrs rrzDownload.StreamMedias"5t55)AT-A-A  &&222*.'008<&9rr c eZdZdZdZdZeZy)MultipartBytesGeneratoraoGenerates a bytes Message object tree for multipart messages This is a BytesGenerator that has been modified to not attempt line termination character modification in the bytes payload. Known to work with the compat32 policy only. It may work on others, but not tested. The outfp object must accept bytes in its write method. cT|jy|j|jyr:)_payloadr)r4msgs r _handle_textz$MultipartBytesGenerator._handle_text1s!||# JJs|| $rc&|jddS)Nasciisurrogateescape)encode)r4ss r_encodezMultipartBytesGenerator._encode8s88G%67 7rN)rVrhrirjrr _writeBodyrmrrrr)s  % 8" rrceZdZdZedZ d!fd ZedZe d"dZ e d"dZ e d#dZ edZ ed Zed Zd Zed Zej$d ZedZej$dZdZdZdZdZdZdZdZd$dZdZd%dZdZ d&dZ d'dZ d'dZ!dZ"d%dZ#d%d Z$xZ%S)(r aData for a single Upload. Fields: stream: The stream to upload. mime_type: MIME type of the upload. total_size: (optional) Total upload size for the stream. close_stream: (default: False) Whether or not we should close the stream when finished with the upload. auto_transfer: (default: True) If True, stream all bytes as soon as the upload is created. r2 mime_typerqrMc  tt| |f||||d| d|_d|_||_d|_d|_d|_d|_ | |_ ||_ | |_ ||_ y)N)r6r3r2r7Fr)rtr r8_Upload__complete_Upload__final_response_Upload__mime_type_Upload__progress!_Upload__server_chunk_granularity_Upload__strategy_Upload__total_size_Upload__gzip_encodedryrzrq) r4r5r rqr7r6r3r2ryrz gzip_encodedr{r|s rr8zUpload.__init__Os fd$  IIl+-. . 4 22AC C[(9C+/88L+A-9C=AC fbii (1B22HJ J  $#0F #'#8F  **4;7#U #* # **,  "     ! ! # rc|j|jtk7rtjd|j |j |j|jdS)Nz2Serialization only supported for resumable uploadsr ) r[r'r rrJr2r rqrMr<s rrzUpload.serialization_datas]  ==, ,--DF F"////88   rc|jSr:)rr<s rcompletezUpload.completer~rc|jSr:)rr<s rr zUpload.mime_typesrcp|jsyd|j|jxsd|jfzS)NzUpload (uninitialized)z.Upload with %d/%s bytes transferred for url %sz???rr<s rrzUpload.__str__s9+? MM4??3eTXXC?? ?rc|jSr:)rr<s rr'zUpload.strategyr~rc^|ttfvrtjd|z||_y)NzOInvalid value "%s" for upload strategy, must be one of "simple" or "resumable".)rr rrrrCs rr'zUpload.strategys9 (89 9&&+.3(45 5 rc|jSr:)rr<s rrqzUpload.total_sizerrc2|j||_yr:)rOrrCs rrqzUpload.total_sizes   "!rc|j t|_|jyt}|j|jtkDrt }|j r|jst }|jst }||_y)aDetermine and set the default upload strategy for this upload. We generally prefer simple or multipart, unless we're forced to use resumable. This happens when any of (1) the upload is too large, (2) the simple endpoint doesn't support multipart requests and we have metadata, or (3) there is no simple upload endpoint. Args: upload_config: Configuration for the upload endpoint. http_request: The associated http request. Returns: None. N) resumable_pathrr'rq_RESUMABLE_UPLOAD_THRESHOLDr bodysimple_multipart simple_path)r4 upload_configrr's r__SetDefaultUploadStrategyz!Upload.__SetDefaultUploadStrategysp  ' ' /)DM == $   OO '"=='H   ]%C%C'H(('H  rc|jrT|jrH|j|jkDr/tjd|jd|jt j |j |js/tjd|jd|j |j|||jtk(r|j|_ |jr!d|jd<|j|n d|jd<|j!||j"rXd|j$d <t'j(t+j,|jd j/|_ y y |j0|_ d |jd<|j3|y ) z.Configure the request and url for this upload.zUpload too big: z larger than max size z MIME type z) does not match any accepted MIME ranges multipart uploadTypergzipContent-EncodingrrN)rqmax_sizerrr AcceptableMimeTypeacceptr !_Upload__SetDefaultUploadStrategyr'rr8 relative_pathr6r"_Upload__ConfigureMultipartRequest_Upload__ConfigureMediaRequestrrrCompressStreamrHBytesIOreadr4"_Upload__ConfigureResumableRequest)r4r9rrs rrzUpload.ConfigureRequests OO 6 6-"8"8822OO]%;%;=> >&&}';';T^^L22NNM$8$8:; ; '' |D ==M )(5(A(AK %  9D ((600>9@ ((6,,\:"";A $$%78 .. L$5$56889;;?46!#)6(D(DK %5@K $ $\ 2  , ,\ :rc|j|jd<|jj|_d|_y)z;Configure http_request as a simple request for this upload. content-type N)r rr5rIr6 loggable_bodyr4rs r__ConfigureMediaRequestzUpload.__ConfigureMediaRequest"s3/3~~ ^, KK,,. %3 "rctjd}t|ddtj|j dj d}|j|j|j|tj|jj d}d|d<|j|jj|j|tj}tjrt }nt"j$}||d }|j'|d |j)|_|j+}d |z|j d<t-|tj.r|j1d }|jj |}|d j3d\} } } dj5| dg|d <|j5||_y)z>Configure http_request as a multipart request for this upload.related_write_headerscyr:rmr<s rz4Upload.__ConfigureMultipartRequest..-srrLrbinaryzContent-Transfer-EncodingF) mangle_from_)unixfromzmultipart/related; boundary=%rrs s --N)mime_multipart MIMEMultipartrmime_nonmultipartMIMENonMultipartrsplit set_payloadr6attachr r5rIrHrHPY3remail_generator Generatorflattengetvalue get_boundaryr# text_typer partitionrrN) r4rmsg_rootrfpgenerator_classgmultipart_boundarybody_componentsrrs r__ConfigureMultipartRequestz"Upload.__ConfigureMultipartRequest(s"// :*,=> 00  ! !. 1 7 7 <>  ))* 00$..2F2Fs2KL+3 '(  ((*+ [[] 775O-77O BU 3 (U +KKM %224 ,/A A ^, (#-- 8!3!:!:7!C &++112DE'+55g> A%llG5J+KL%7%<%<_%M "rc|j|jd<|j#t|j|jd<yy)NzX-Upload-Content-TypezX-Upload-Content-Length)r rrqr;rOs r__ConfigureResumableRequestz"Upload.__ConfigureResumableRequestRsE8< 45 ?? &-0-A  ) + 'rc|jtk7ry|jtj|j dddi}tj |j|d|j}|j|}|jtjtjfvrEd|_|j|_|j"j%|j&||_y|jtj*k(rI|d|_n|j-|d z|_|j"j%|j&yt.j0j3|) zTalk to the server and refresh the state of this resumable upload. Returns: Response if the upload is complete. NPUT Content-Rangez bytes */*)rM http_methodrr) redirectionsrTr)r'r r[rrrMrr7r/_GetRangeHeaderFromResponserrrCREATEDrrqrr5seekrprRESUME_INCOMPLETE_Upload__GetLastByterrr)r4refresh_requestrefresh_response range_headers rr(z"Upload.RefreshResumableUploadStateXs7 ==, ,   &..e$k24(33 IIQ$$&778HI  ' 'KNN,7,?,?,A A"DO"ooDO KK  T]] +%5D !  ) )\-K-K K#"#"&"4"4\"BQ"F KK  T]] +&&334DE Ercl|jjd|jjdS)Nrr")rrrs rrwz"Upload._GetRangeHeaderFromResponsezs(}}  (--*;*;G*DEErc|jtjd||tjd|jtk7ry|xs |j}| |j |j |_|jtj|||j}|jtjk7rtjj||j j#d|_|j d}||j |}|j'|||j(r|j+S|S)z3Initialize this upload from the given http_request.Nz6No upload strategy set; did you call ConfigureRequest?r)rzX-Goog-Upload-Chunk-Granularitylocation)r'rrr r7rrMrOrrr/rrrrrrrrrQr2r)r4rr7r http_responserMs rInitializeUploadzUpload.InitializeUpload}sA == &&HJ J .CallSendChunks!##*<$> >rc*j|Sr)_Upload__SendMediaBodyrs rCallSendMediaBodyz/Upload.__StreamMedia..CallSendMediaBodys!''*<(> >rz%Cannot gzip encode non-chunked uploadTrz?Failed to transfer all bytes in chunk, upload paused at byte %dryrz7Upload complete with %s additional bytes left in stream)%r'r rrryrzrr_Upload__ValidateChunksizer3r[r,r5tellrrrrxrrrz_Upload__IsRetryablerrr(rgr{rwrrpCommunicationErrorhasattrryrSEEK_ENDrZr) r4rfrzrrrrr send_func current_posend_poss ` ` r __StreamMediazUpload.__StreamMedias ==, ,2246 65t55)AT-A-A(( > >&0M5F d112279 9   $ $T^^ 4  -- !1!1!34H##  8K8K'LL"&##NNK$7$7 22,44 MM%55 ..x8$..;;HEE002%%h9"0000:.CheckResponse s3!--B(()** r)r1rcheck_response_funcr) rrrAr1r/rrzr{rwr5ry)r4rrrr last_bytes r__SendMediaRequestzUpload.__SendMediaRequest s * ++ OOW$$-I   <#A#A A**00: >".. KK502 &&488,79*...' DOO #'$//9L $05doo6I/3@L,8(  OO " "#5 6&&w@@rc|j|jdu}tj|jd}|j rXd|j d<tj|j|j\}}}||z}|j|r||_ n|jftj|j||j}|j}|jr||_ |j!|j}nFt#||jz|j}t%j&|j||z }||_|j*|j d<|rd|_|j d|d |d z d } n-||k(rd |jz} nd|d |d z d |j} | |j d<|r|j j/||j1||S)zSend the specified chunk.Nrs)rMrur>r?rrMrrrz/*rrrt)r[rqrrrMrrrrGr5r3rrBufferedStreamstream_end_positionstream_exhaustedrIrr rr6r rNrr) r4rer no_log_bodyrr read_length exhaustedrrs r __SendChunkzUpload.__SendChunk7s  oo- &&488G   28GOO. /2=2L2L T^^3- /Ki+%C&9$'! __ $*88 UDNN4K11C++$'! &**4>>:Kednn,doo>C&224;;e LK# *...' %3G ! ?? ".3S1W=L E\'$//9L 05cAgtOL+7(  OO " "#5 6&&w44r)NNFNTNNF)NTF)NFNrr:rr)&rVrhrirjrrr8rkrprrrrrr,r rr'rlrqrCrrFrErJr(rwrr{rrrrrrrrrrs@rr r ?s $'(;$< AECG9=#%(>B#66 JN %66=A,0""H      ? __  !!""!:+;Z4 (NTB FDF> 1& <@:>@D:>'+E&=A*.3&A265rr )+rj __future__remail.generator generatorrbemail.mime.multipartmimer<rZemail.mime.nonmultipart nonmultipartr\r$rrrrcrH six.movesrapitools.base.pyrrrrr r __all__r5rr rrrrobjectr&r raBytesGeneratorrr rmrrrs"0%)--33   !,('))! & 5 .  hMhMVl9yl9^ 77"/"@"@",n5Yn5r