qdZddlmZddlmZddlmZddlmZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl mZdd l mZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddlmZddl Z ddl m!Z!e jDre#Z$edZ%Gdde&Z'Gdde&Z(dZ)Gdde&Z*dZ+Gdde jXZ-Gdd e jXZ.y)!zEMedia helper functions and classes for Google Cloud Storage JSON API.)absolute_import)print_function)division)unicode_literalsN) http_client)urllib) cStringIO) exceptions)BadRequestException) LazyWrapper)ProgressCallbackWithTimeout)DEBUGLEVEL_DUMP_REQUESTS)SSL_TIMEOUT_SEC)TRANSFER_BUFFER_SIZE)UTF8) text_util) parse_uric,tjdS)Nz\d+)recompile'platform/gsutil/gslib/gcs_json_media.pyr2s RZZ%7rcJeZdZdZdZedZejdZy)BytesTransferredContainera Container class for passing number of bytes transferred to lower layers. For resumed transfers or connection rebuilds in the middle of a transfer, we need to rebuild the connection class with how much we've transferred so far. For uploads, we don't know the total number of bytes uploaded until we've queried the server, but we need to create the connection class to pass to httplib2 before we can query the server. This container object allows us to pass a reference into Upload/DownloadCallbackConnection. cd|_y)Nr-_BytesTransferredContainer__bytes_transferredselfs r__init__z"BytesTransferredContainer.__init__@s  Drc|jSNrr s rbytes_transferredz+BytesTransferredContainer.bytes_transferredCs  # ##rc||_yr$rr!values rr%z+BytesTransferredContainer.bytes_transferredGs $DrN)__name__ __module__ __qualname____doc__r"propertyr%setterrrrrr5s:! $ $%%rrc(eZdZdZeddddfdZdZy)$UploadCallbackConnectionClassFactoryzCreates a class that can override an httplib2 connection. This is used to provide progress callbacks and disable dumping the upload payload during debug statements. It can later be used to provide on-the-fly hash digestion during upload. rNcX||_||_||_||_||_||_yr$)bytes_uploaded_container buffer_size total_sizeprogress_callbackloggerdebug)r!r2r3r4r5r6r7s rr"z-UploadCallbackConnectionClassFactory.__init__Ts1%=D!"D DO.DDKDJrc|j|j|j|j|j|j Gfddt j}|S)z/Returns a connection class that overrides send.cjeZdZdZWZdZWZdZWZdZ dZ dZ dZ dZ d fd ZfdZd fd Zy) YUploadCallbackConnectionClassFactory.GetConnectionClass..UploadCallbackConnectionz&Connection class override for uploads.FNg?c\t|d<tjj|g|i|yNtimeoutrhttplib2HTTPSConnectionWithTimeoutr"r!argskwargss rr"zbUploadCallbackConnectionClassFactory.GetConnectionClass..UploadCallbackConnection.__init__z*+y++44TKDKFKrc8|jjdtjr |j}nYg}|jD]H}t |t r|j |%|j |jtJdj|}t|}tk(rrjd|z|jdd=t |tr||z }d}|j||||j|yy)zSend the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. Args: message_body: if specified, this is appended to the request. )rrs zsend: %sN)num_metadata_bytes)_bufferextendsixPY2 isinstancebytesappendencoderjoinlenrr7strsend) r! message_bodyencode_chunkeditemsitemmsgrG outer_debug outer_loggers r _send_outputzfUploadCallbackConnectionClassFactory.GetConnectionClass..UploadCallbackConnection._send_outputs J' 77,,%%lld$&ll4 ll4;;t,- # ll5! X 2 2|   Z#- . LLO lC (  #, #*< =  # ))L ! $rc |dk(rLdj|Dcgc] }t|c}}||_tk(r͉rˉj dn|dk(r: t dj|Dcgc] }t|c}}||_nz|dk(ru dj|Dcgc] }t|c}}tj|}t|dkDr%t |dt |dz dz|_ |jdk(r}|j rq|jre|jt|j z |_ d|_d |_d |_ tk(r rj d |jztjj ||g|y cc}wcc}w#t$rYwxYwcc}w#t$rYwxYw) aOverrides HTTPConnection.putheader. Send a request header line to the server. For example: h.putheader('Accept', 'text/html'). This override records the content encoding, length, and range of the payload. For uploads where the content-range difference does not match the content-length, progress printing will under-report progress. These headers are used to calculate a multiplier to correct the progress. For example: the content-length for gzip transport encoded data represents the compressed size of the data while the content-range difference represents the uncompressed size. Dividing the content-range difference by the content-length gives the ratio to multiply the progress by to correctly report the relative progress. Args: header: The header. *values: A set of values for the header. zcontent-encodingr;z4send: Using gzip transport encoding for the request.content-lengthz content-rangergzipNz&send: Setting progress modifier to %s.)rPrRheader_encodingrr7int header_length ValueError DECIMAL_REGEXfindallrQ header_rangefloat size_modifierrHTTPSConnection putheader)r!headervaluesvr(rangesrYrZs rrjzcUploadCallbackConnectionClassFactory.GetConnectionClass..UploadCallbackConnection.putheaders* ' '''626a3q6623%!&$  4 4   F H ' '  8AQ 89:E!&D  & GGV4VSVV45E"_,,U3F6{Q#&vay>Cq N#Ba"Gd  F *t/A/A   #0059K9K3LL$ !#$ #$ "$  4 4   G $ 2 2 4 5 ##--dFDVDO3!9    5   sNF:G-F? ?GG'G9AG?G GGG G$#G$c|jsId|_r@t|_|jj|jj t |tjr t|}n2t |tjrtj|}n|}|j|j}|rtjr!tj j#||ndt |t$r!tj j#||n3tj j#||j't(t+|}|r||kr||z}d}n||z}d}|jr3t-||j.z}|jj||j|j}|ryy)aOverrides HTTPConnection.send. Args: data: string or file-like object (implements read()) of data to send. num_metadata_bytes: number of bytes that consist of metadata (headers, etc.) not representing the data being uploaded. TrN)processed_initial_bytesr callback_processorProgressr2r%rLrJ text_typer binary_typeBytesIOreadGCS_JSON_BUFFER_SIZErKr@rArSrMrOrrQrarh)r!datarG full_bufferpartial_buffersent_data_bytesouter_progress_callbackouter_total_sizes rrSz^UploadCallbackConnectionClassFactory.GetConnectionClass..UploadCallbackConnection.sends++)-$ & $&A "9';D #  # # , ,--?? A dCMM *!$+ coo . D)++$))$*C*CD WW  / / 4 4T> J.%01166t^L1166--d35// !_4!33o#$  O3 !o  $ $"/D4F4F"FGO  # # , ,_ =&++D,E,EF.7r)NF)r)r)r*r+r,r2rprwrqsizer`rbrfrhr"r[rjrS)outer_buffer_sizeouter_bytes_uploaded_containerrYrZr|r}srUploadCallbackConnectionr:ksR2!?!&. domlmL%"N=E~3Grr)r2r3r4r5r6r7r@rA)r!rrrrYrZr|r}s @@@@@@rGetConnectionClassz7UploadCallbackConnectionClassFactory.GetConnectionClassbsi%)%B%B"(("44;;L**KoGoG8#F#FoGb $#rr)r*r+r,rr"rrrrr0r0Ls#0!% z$rr0c^|jdddtjdffd }||_y)zWraps upload_http so we only use our custom connection_type on PUTs. POSTs are used to refresh oauth tokens, and we don't want to process the data sent in those requests. Args: upload_http: httplib2.Http instance to wrap GETNc>|dk(s|dk(r|}nd}||||||S)NPUTPOSTmethodbodyheaders redirectionsconnection_typer)urirrrrroverride_connection_type request_origs r NewRequestz)WrapUploadHttpRequest..NewRequest*s? &F*!0!% %! '%1(@  BBr)requestr@DEFAULT_MAX_REDIRECTS) upload_httprrs @rWrapUploadHttpRequestrs7$$,&<Gfddtj}|S)z6Returns a connection class that overrides getresponse.ceZdZdZWjZWj ZWjZ WjZ dZ dZ dZddZy)]DownloadCallbackConnectionClassFactory.GetConnectionClass..DownloadCallbackConnectionz(Connection class override for downloads.FNc\t|d<tjj|g|i|yr=r?rBs rr"zfDownloadCallbackConnectionClassFactory.GetConnectionClass..DownloadCallbackConnection.__init__brErctjj}|jtjtj fvr|S|j dfd }||_|S)aWraps an HTTPResponse to perform callbacks and hashing. In this function, self is a DownloadCallbackConnection. Args: buffering: Unused. This function uses a local buffer. Returns: HTTPResponse object with wrapped read function. c.|r |tkDrtd|dtd|xst}jsgd_jrTt j j_j jjj|}t|}j rj j|jr/jD] }j|j|"|S)aOverrides HTTPConnection.getresponse.read. This function only supports reads of TRANSFER_BUFFER_SIZE or smaller. Args: amt: Integer n where 0 < n <= TRANSFER_BUFFER_SIZE. This is a keyword argument to match the read function it overrides, but it is required. Returns: Data read from HTTPConnection. zInvalid HTTP read size z during download, expected .T) rr rpr|r r}rqrr outer_bytes_downloaded_containerr%rQouter_digestersupdate)amtrx read_lengthalgorig_read_funcr!s rrvzwDownloadCallbackConnectionClassFactory.GetConnectionClass..DownloadCallbackConnection.getresponse..readws22%*,- ---C--+/D (++(C'')E)E)Gd%%%..77IIK $$D +  $ $  # # , ,[ 9  ! !++""3'..t4,+rr$)rHTTPConnection getresponsestatusOKPARTIAL_CONTENTrv)r! buffering orig_responservrs` @rrziDownloadCallbackConnectionClassFactory.GetConnectionClass..DownloadCallbackConnection.getresponsefsf$22>>tD    (3(C(C(E E &++# J" r)F)r)r*r+r,r4r}rrr5r|rrrprqr"rr srDownloadCallbackConnectionrYsH4o $ 6 6)-)H)H& %L8rr)r@rA)r!rs` rrz9DownloadCallbackConnectionClassFactory.GetConnectionClassVs EX%H%HEN &%rrrrrrr?s!0!% AJ&rrcd}|jdddtjdffd }tj|||_||_|S)zOverrides download request functions for an httplib2.Http object. Args: download_http: httplib2.Http.object to wrap / override. Returns: Wrapped / overridden httplib2.Http object. c z|jD cgc]'} | j||r| j|| f)} } | xrt| ddxsd} | r| j |||||j |||||\} } | rE| j | |r3| j |||||j |||||\} } d| _| jdk(r|j|||| | D]l}|j |||||j |||||\} } | jdk7s?|jj||j | |n|js|dvs| jdk(r|jrk| jdvr\|r>d| vr&| jd k7rtjd | | d| vr;| d}t|\}}}}}|#t j"j%||| d<| jd k(r8|dvr4| d| d <d | vr|| d <tj&|| | |j(| d|vr|d=d|vr|d=d|vr|j*s|d=d| vr| d}t-j.| }d |vr||d <|}| jdvrd}d}|j |||||dz |j0\} } || _| | fStj4d| | | | fS| jdvr0|dvr,d | vr|| d <tj&|| | |j(| | | fScc} w)zlDo the actual request using the connection object. Also follow one level of redirects if necessary. rr^Ni)rHEAD/),-.ri3locationrz:Redirected but the response is missing a Location: header.rz-x-permanent-redirect-urlzcontent-locationz if-none-matchzif-modified-since authorization)rrr)rrrrz4Redirected more times than redirection_limit allows.))authorizationsinscopedepthsortedr _conn_requestresponse _stale_digestr_auth_from_challengerNfollow_all_redirectsfollow_redirectsr@RedirectMissingLocationrrparseurljoin _updateCachecacheforward_authorization_headerscopydeepcopy __class__previous RedirectLimit)r!connhost absolute_uri request_urirrrrcachekeyauthauthsrcontentrrscheme authoritypathqueryfragment old_responseredirect_methods rOverrideRequestz0WrapDownloadHttpRequest..OverrideRequests ;?:M:M2:M$dK0zz+&-:ME2  (VE]1%a( 0DD  ll6;6,,T;-46Xw x & V['48"00{F15w@7!"#44  Wh9-fk7DA"00{F15w@7 ??c !    $ $] 3  4 0 9 !!f&? ??c !  8??7A$A  x 'HOOs,B22L'# #8 # +H9B89L 6VYeX %+\\%9%9,%Qhz" __ #/(A4.NewRequestsE #f4") *.00#f4") *9;;r)rr@rtypes MethodType_request) download_httprrrs @rWrapDownloadHttpRequestrsT*Z~&&,"t&<gaierrorcloser@ServerNotFoundErrorrsslSSLErrorerrorgetattrerrno ECONNREFUSEDr HTTPExceptionrrvResponse_decompressContent) r!rrrrreerrrrs rrzHttpWithNoRetries._conn_request9s v 499#4  ll6;g6*A!!#h g 6  --/""8,h 6 --h@ g G >>  ??4 jjl  ( ()J)-*3 44 <<   jjl << c F a #gg ++ +  ,  $ $  jjl  LL+33 4  jjl  s$|j*t"j,t"j.fvrd} t|dr|j1d } |} d} | j3t4} | rP|j6t9j:d t=j>|j6| | tA| z } nni| itC| tC| k7rR|jDjGtHjJd | | ||jLd =tO| |jLd <tj(|}|| fS|j3} tj(|}tjP|| } || fS#tj $rtj $r3|jtjd|jztjj$r|jtj$rS}d}t|drt|dd}n |j}|tjj k(rYd}~d}~wt"j$$r|jwxYw#tjt"j$f$r}|jd}~wwxYw) NrrrrCr;rrrXr]z5Cannot exercise HttpWithDownloadStream with no streamzlOnly got %s bytes out of content-length %s for request URI %s. Resetting content-length to match bytes read.))rrrrrr>rrr@rrrrrrrrrrrrrrr getheaderrvrr apitools_exceptionsInvalidUserInputErrorr write_to_fdrQlongrlogrDEBUGrXrRr)r!rrrrrrrrrcontent_length http_stream bytes_readnew_datas rrz$HttpWithDownloadStream._conn_requests v 499#4  ll6;g6.4A!!#h g 6  $$X.X g W U?x;>>3>3N3N3P P 8U ##--.>?.   %%&:;( {{"'==IKK  ! !$++x 8 #h- 'J   &  ^ 4 4 ,,  mm' Fll+,+.z?(,,' ($$X. g  --/$$X.--h@ g W >>  ??4 jjl  ( ()J)-*3 44 <<   jjl << c F a #gg ++ +  ,  $ $  jjl   LL+33 4  jjl  s0r#sL&%'  !>/*?:16&!77 $89 %%.P$6P$f#@a&Va&HEP7 7tkX]]kr