
                            d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%c m&Z& ddl'm(Z( ddl)m*Z* ddl)m+Z, ddl)m-Z- ddl)m.Z. ddl)m/Z/ ddl0m1Z2 ddl3m4Z4 ddl3m5Z5 ddl6m7Z7 ddl8m9Z9 ddl:m;Z; ddl:m<Z< ddl=m>Z> ddl=m?Z? dd lm@Z@mAZA  e@ eAd!d!d"             dd#lBmCZC d$ZDd%d&d'd(d)d*d+d,d-j                  e      g	ZF eG e!d.d/d0j                  eDej                  e      d1       e!d.d/d2j                  eDej                  e      d1       e!d.d/d3j                  eDej                  e      d1       e!d.d/d4j                  eDej                  e      d1      g      ZK e	j                  d5      ZMd6 ZNd7 ZO G d8 d9eP      ZQ e.d:       eCj                  d; eCj                  d<      =       G d> d?e&j                                      ZU G d@ dAeP      ZV G dB dCeP      ZW e.d:       G dD dEe&j                               ZYy)F)Unit tests for analytics data collection.    )absolute_import)print_function)division)unicode_literalsN)
exceptions)http_wrapper)BucketStorageUri)metrics)VERSION)ApiSelector)
GcsJsonApi)MetricsCollector)Metric)MockLoggingHandler)	SkipForS3)HAS_S3_CREDS)ObjectToURI)SetBotoConfigForTest)SkipForParFile)unittest)storage_v1_messages)FileMessage)RetryableErrorMessage)START_CALLBACK_PER_BYTES)LogAndHandleRetries)IS_LINUX)
IS_WINDOWS)ONE_KIB)ONE_MIB)add_moveMovedModulemockzunittest.mock)r#   zXa=b&c=d&cd1=cmd1+action1&cd10=0&cd2=x%2Cy%2Cz&cd3=opta%2Coptb&cd6=CommandException&cm1=0za=bzc=dzcd1=cmd1 action1z	cd2=x,y,zzcd3=opta,optbzcd6=CommandExceptionzcm1=0zev=0zel={0}https://example.comPOSTz,{0}&cm2=3&ea=cmd1+action1&ec={1}&el={2}&ev=0user-agent-007z){0}&cm2=2&ea=Exception&ec={1}&el={2}&ev=0z*{0}&cm2=1&ea=ValueError&ec={1}&el={2}&ev=0z*{0}&ea=CommandException&ec={1}&el={2}&ev=0z(\[Metric.*\])c                 $    	  | |i | y#  Y yxY w)aQ  Calls the given function with the arguments and ignores exceptions.

  In these tests, we often force a failure that doesn't matter in order to
  check that a metric was collected.

  Args:
    func: The function to call.
    *args: Any arguments to call the function with.
    **kwargs: Any named arguments to call the function with.
  N )funcargskwargss      +platform/gsutil/gslib/tests/test_metrics.py_TryExceptAndPassr-   h   s    	$&	s    c                  r   t        j                  ddgg dddg       t        t               d      } t        t	               d      }t        j
                  |        t        j
                  |        t        j
                  |       t        j                  t        j                  j                  d             y	)
z'Logs all the common metrics for a test.cmd1action1)z-yvalue)z-z )z-xr3   optbr3   optar3   )command_namesubcommandsglobal_optssub_optsr   testN)
r   LogCommandParamsr   	Exception
ValueErrorLogRetryableErrorLogFatalErrorgslib	exceptionCommandException)retry_msg_1retry_msg_2s     r,   _LogAllTestMetricsrG   y   s    	(1{(4%1<$@	B
 &ik15+%jlA6+	K(	K(	K(	88@A    c                       e Zd ZdZd Zy)RetryableErrorsQueuezGEmulates Cloud API status queue, processes only RetryableErrorMessages.c                 P    t        |t              rt        j                  |       y y N)
isinstancer   r   r@   )selfstatus_items     r,   putzRetryableErrorsQueue.put   s     +45, 6rH   N)__name__
__module____qualname____doc__rP   r(   rH   r,   rJ   rJ      s
    O-rH   rJ   z:Do not try spawning the interpreter nested in the archive.	time.timereturn_valuenewc                        e Zd ZdZ fdZ fdZd Zd Zd Zd Z	d Z
ej                  j                  ed	      d
        Zd Z xZS )TestMetricsUnitTestsr   c                    t         t        |           t        j                         | _        t        j                  ddddd       t        j                         | _        t               | _	        t        j                  d      j                  t        j                         t        j                  d      j                  | j                         y )Nr$   r&   bdacr   )superr[   setUpr   GetCollectororiginal_collector_instanceStartTestCollector	collectorr   log_handlerlogging	getLoggersetLevelDEBUG
addHandlerrN   	__class__s    r,   rc   zTestMetricsUnitTests.setUp   s    	
+- (8'D'D'FD$ ''(=?O1414)*+
 &224DN)+D i ))'--8i ++D,<,<=rH   c                 j    t         t        |           t        j                  | j
                         y N)original_instance)rb   r[   tearDownr   StopTestCollectorre   rn   s    r,   rs   zTestMetricsUnitTests.tearDown   s*    	
.0 &&::<rH   c           	      	   | j                  | j                  t        j                                t        j
                  j                  t        j                  ddd      5  t        j                          | j                  t        j                         | j                  | j                  t        j                                ddd       t	        j
                  dd      5  t        j                          | j                  t        j                         | j                  dt        j                                ddd       t        j
                  j                  t        j                  dd	d      5  t        j                          | j                  t        j                         | j                  dt        j                                ddd       t        j
                  j                  t        j                  d
d	i      5  t	        j
                  dd      5  t        j                          | j                  t        j                         | j                  dt        j                                ddd       ddd       t        j
                  j                  t        j                  d
d	i      5  t	        j
                  dd      5  t        j                  rd}nd}t	        j
                  |      5 }d |j                  _        t"        j$                  |j                  j&                  _        t        j                          | j                  t        j                         | j                  dt        j                                d|j                  j&                  _        t        j                          | j                  t        j                         | j                  | j                  t        j                                | j                  dt)        |j*                               | j                  dt)        |j                  j&                  j*                               ddd       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   TxY w# 1 sw Y   xY w# 1 sw Y   3xY w# 1 sw Y   8xY w# 1 sw Y   [xY w# 1 sw Y   _xY w# 1 sw Y   yxY w)z/Tests enabling/disabling of metrics collection.1555)CLOUDSDK_WRAPPERGA_CID)valuesNzboto.config.getboolTrV   r3   rx   zos.path.existsFz__builtin__.openzbuiltins.openc                     | S rL   r(   )ss    r,   <lambda>z4TestMetricsUnitTests.testDisabling.<locals>.<lambda>   s    qrH   mock_cid   )assertEqualrg   r   rd   r#   patchdictosenviron_CheckAndSetDisabledCacheassertFalse_disabled_cache
assertTruesixPY2rW   	__enter__r   _DISABLED_TEXTreadlencall_args_list)rN   builtin_open	mock_opens      r,   testDisablingz"TestMetricsUnitTests.testDisabling   se   T^^%5%B%B%DE 
14',! 
 

 002
'778
t~~'7'D'D'FG
 
)	=002
oo&667
t-::<= 
> 
14')! 
 

 002
oo&667
t-::<=
 
-?,D	E::&U;224(889/<<>? < 
F 
-?,D	E::&T:77+,(,ZZ%-8)
 
 
* 6=5K5K)
 
 
%
%
2

4
4
6
//*::
;


4!1!>!>!@
A 6@)
 
 
%
%
2

4
4
6


+;;
<


4>>+;+H+H+J
K 

1c)":":;
<


1c)"8"8"="="L"LM
N# & ; 
F	EE
 
 
>	=
 
 <; 
F	E &% ;: 
F	Es   !A"Q?"AR4ARR3AR&5R36S+S9E-S &S.S?R	RR#&R0	+R33R= S	SS	SS!c                   	 g d}g d	t        |	z         }t        j                  dd      5  | j                  d| j                  j                                ddd       t        j                  dd      5  | j                  dj                  |D cg c]  }|d	z   	 c}      | j                  j                                ddd       t        j                  dd
      5  | j                  dj                  |D cg c]  }|d	z   	 c}      | j                  j                                ddd       d }t        j                  d|      5  | j                  d| j                  j                                ddd       	fd}t        j                  d|      5  | j                  d| j                  j                                ddd       	fd}t        j                  d|      5  | j                  d| j                  j                                	fd}t        j                  d|      5  | j                  dj                  	D cg c]  }|d	z   	 c}      | j                  j                                ddd       d }t        j                  d|      5  | j                  d| j                  j                                ddd       ddd       y# 1 sw Y   xY wc c}w # 1 sw Y   .xY wc c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   TxY wc c}w # 1 sw Y   xY w# 1 sw Y   nxY w# 1 sw Y   yxY w)z6Tests the validation of potentially PII config values.)check_hashescontent_languagedisable_analytics_prompthttps_validate_certificatesjson_api_version(parallel_composite_upload_component_size#parallel_composite_upload_threshold
prefer_api%sliced_object_download_component_size sliced_object_download_thresholdtab_completion_time_logstoken_cacheuse_magicfile)debugdefault_api_versionhttp_socket_timeoutmax_retry_delaynum_retriesoauth2_refresh_retriesparallel_process_countparallel_thread_countresumable_thresholdrsync_buffer_lines%sliced_object_download_max_componentssoftware_update_check_periodtab_completion_timeouttask_estimation_thresholdzboto.config.get_valueNrV   r3   zinvalid string,z:INVALID   £c                 t    | dk(  r|dk(  ry|dk(  ry|dk(  ry|dk(  ry	|d
v ry| dk(  r|dk(  ry| dk(  r|dk(  ryy)NGSUtilr   if_fast_else_skipr   chir   v3r   xml)r   r   r   TrueOAuth2r   file_systemBotor   r3   r(   )sectioncategorys     r,   MockValidStringszHTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidStrings  sp    	H	~%$))))|# 4 4	H	]!:	F	x+HHrH   side_effectzcheck_hashes:if_fast_else_skip,content_language:chi,disable_analytics_prompt:True,https_validate_certificates:True,json_api_version:v3,prefer_api:xml,tab_completion_time_logs:True,token_cache:file_system,use_magicfile:Truec                     |v ryy)N1999r3   r(   _r   int_categoriess     r,   MockValidSmallIntszJTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidSmallInts2      	^	#rH   an  debug:1999,default_api_version:1999,http_socket_timeout:1999,max_retry_delay:1999,num_retries:1999,oauth2_refresh_retries:1999,parallel_process_count:1999,parallel_thread_count:1999,resumable_threshold:1999,rsync_buffer_lines:1999,sliced_object_download_max_components:1999,software_update_check_period:1999,tab_completion_timeout:1999,task_estimation_threshold:1999c                     |v ryy)N2001r3   r(   r   s     r,   MockValidLargeIntszJTestMetricsUnitTests.testConfigValueValidation.<locals>.MockValidLargeIntsB  r   rH   a  debug:INVALID,default_api_version:INVALID,http_socket_timeout:INVALID,max_retry_delay:INVALID,num_retries:INVALID,oauth2_refresh_retries:INVALID,parallel_process_count:INVALID,parallel_thread_count:INVALID,resumable_threshold:2001,rsync_buffer_lines:2001,sliced_object_download_max_components:INVALID,software_update_check_period:INVALID,tab_completion_timeout:INVALID,task_estimation_threshold:2001c                     |v ryy)Nz10.28r3   r(   r   s     r,   MockNonIntegerValuezKTestMetricsUnitTests.testConfigValueValidation.<locals>.MockNonIntegerValueT  s    ~%rH   c                     |dv ryy)N)r   r   r   r   10MiBr3   r(   )r   r   s     r,   MockDataSizeValuezITestMetricsUnitTests.testConfigValueValidation.<locals>.MockDataSizeValue_  s     < < rH   zparallel_composite_upload_component_size:10485760,parallel_composite_upload_threshold:10485760,sliced_object_download_component_size:10485760,sliced_object_download_threshold:10485760)sortedr#   r   r   rg   _ValidateAndGetConfigValuesjoin)
rN   string_and_bool_categoriesall_categoriesr   r   r   r   r   r   r   s
            @r,   testConfigValueValidationz.TestMetricsUnitTests.testConfigValueValidation   s   "N 6GHN 
+$	?
r4>>EEGH 
@ 
+:J	K

((.I.hHz).I
J
..
4
4
68 
L 
+$	?

((.I.hHz).I
J
..
4
4
68 
@& 
+9I	J
7 ..
4
4
68 
K
 
+9K	L
+ ..
4
4
68 
M
 
+9K	L
J ..
4
4
6	8
 ::-;NOHHNKNh+NKLNN668	: P ::-;LM8 NN668	: NA 
M	LA 
@	?
 J 
L	K J 
@	?2 
K	J 
M	LH L PO NMA 
M	Ls   +K9K0K+ $K0#L>K=
$L+L+L.AM5L.L)$L. "M"+L:MK(+K00K:=LLLL&)L..L7	3M:M	?MMc                 v   | j                  g | j                  j                         t                t	        j
                  d       | j                  g | j                  j                         | j                  j                          | j                  t        t        | j                  j                               y)z4Tests the collection of command and error GA events.cmd2)r8   N)	r   rg   _metricsrG   r   r=   _CollectCommandAndErrorMetricsCOMMAND_AND_ERROR_TEST_METRICSsetrN   s    r,   #testCommandAndErrorEventsCollectionz8TestMetricsUnitTests.testCommandAndErrorEventsCollectiono  s    R001&1 	R001NN11330013rH   c                 D   d| j                   j                  t        j                  d   <   t	        j
                  dddi      5  t        j                  d;i ddd	dd
dddddddddddddddddddddddddddg ddd       t        t        j                         d       }t        t        j                         d       }t        j                  |       t        j                  |       t        j                  |       t        d!d"d d#      }t        d!d"dd$      }d%x|_        |_        d%x|_        |_        t        j                  |&       t        j                  |&       | j!                  | j                   j"                  j$                  d'   j'                         d       t	        j
                  ddd(i      5  | j                   j)                          ddd       | j                   j*                  d    j,                  }d)t        j.                  fd*d+d,d-d.d/d0d1d2d3d4d5d6d7d8g}t0        r|j3                  d9       |D ]8  \  }}| j5                  d:j7                  t        j                  |   |      |       : y# 1 sw Y   xY w# 1 sw Y   xY w)<z4Test the collection of PerformanceSummary GA events.cpCommand Namez)gslib.metrics.system_util.GetDiskCountersz	fake-disk)r   r   r   r   r   r   rV   uses_fanT
uses_sliceavg_throughput
   is_daisy_chainhas_file_dstFhas_cloud_dsthas_file_srchas_cloud_srctotal_bytes_transferredd   total_elapsed_timethread_idle_time(   thread_execution_timenum_processesr   num_threads   num_objects_transferredprovider_typesgsNr   srcdst)size)finished   )file_message)r   r   )r   r   r   r   r   r   Event Category)Event ActionCloudToCloud%2CDaisyChain)zExecution Time10)Parallelism Strategyboth)Source URL Typecloud)Provider Typesr   )Num Processes2)Num Threads3)#Number of Files/Objects Transferredr  )!Size of Files/Objects Transferred100)Average Overall Throughputr   )Num Retryable Service Errorsrv   )zNum Retryable Network Errorsr  )Thread Idle Time Percentz0.8)Slowest Thread Throughputr   )Fastest Thread Throughputr   )Disk I/O Time20{0}={1}r(   )rg   	ga_paramsr   _GA_LABEL_MAPr#   r   LogPerformanceSummaryParamsr   apitools_exceptionsCommunicationErrorsocketerrorr@   r   	thread_id
process_idr   perf_sum_paramsthread_throughputsGetThroughput _CollectPerformanceSummaryMetricr   body_GA_PERFSUM_CATEGORYr   appendassertInformat)	rN   service_retry_msgnetwork_retry_msgstart_file_msgend_file_msgmetric_bodylabel_and_value_pairslabel	exp_values	            r,   %testPerformanceSummaryEventCollectionz:TestMetricsUnitTests.testPerformanceSummaryEventCollection~  s    GKDNNW22>BC 
?"-/A!B
D)) A4 A59A9;A :>A 8=	A
 9=A 8=A 9=A CFA >@A <>A ACA 9:A 78A CDA ;?A
D( ...0!5-fllna@/0/0/0 !qs;NueR$?L899N|5:;;N 7''^D''\B&&99 ;> 	??L}
 
?"-/C!D
F
nn557
F
 ..))!,11K	77785 ($ 44,--+++!$ ""#:;1y
mmI$$W%:%:5%A9M! 2E
D 
DP
F 
Fs   AJ	J	JJc                    t        | j                  j                  dddgd       | j                  d| j                  j
                  j                  t        j                  d                | j                  d| j                  j
                  j                  t        j                  d	                | j                  j
                  j                          | j                  j                  d
d       | j                  d| j                  j
                  j                  t        j                  d                | j                  d
| j                  j
                  j                  t        j                  d                | j                  j
                  j                          t        | j                  j                  dddgd       | j                  d| j                  j
                  j                  t        j                  d                y)z+Tests the collection of command parameters.aclr   -aT)collect_analyticszacl setr   r`   Command-Level OptionslistlsCommand Aliasiamgetdummy_bucketziam getN)
r-   command_runnerRunNamedCommandr   rg   r  r7  r   r  clearr   s    r,   testCommandCollectionz*TestMetricsUnitTests.testCommandCollection  s   d))99eT](,. 	  $$W%:%:>%JKM 	  $$!!"9:	<= 	NN""$''$'G  $$W%:%:>%JKM 	  $$W%:%:?%KLN 	NN""$d))99e^4(,. 	  $$W%:%:>%JKMrH   )HandleExceptionsAndRebuildHttpConnectionsc                    t               }t        j                  ddt               ddd      }t        j                  ddt	        j
                         ddd      }t        d|      }t        d|      } ||       | j                  | j                  j                  d   d        ||       | j                  | j                  j                  d   d        ||       t        j                  r*| j                  | j                  j                  d   d       n)| j                  | j                  j                  d	   d       t        ||       t        ||       | j                  | j                  j                  d   d
       t        j                  r*| j                  | j                  j                  d   d       y| j                  | j                  j                  d	   d       y)z@Tests the collection of a retryable error in the retry function.NF)is_data_transferstatus_queueTr?   r   r   SocketErrorOSErrorr   )rJ   r	   ExceptionRetryArgsr?   r  r  r   r   rg   retryable_errorsr   r   r-   )rN   mock_default_retry
mock_queuevalue_error_retry_argssocket_error_retry_argsmetadata_retry_funcmedia_retry_funcs          r,   testRetryableErrorCollectionz1TestMetricsUnitTests.testRetryableErrorCollection  s   
 &'J)<<dJL$d4*==dFLLND$6-u;EG*D8BD ./T^^44\BAF./T^^44\BAF/0
ww
t~~66}EqI
t~~66yA1E &(>?&(?@T^^44\BAF
ww
t~~66}EqI
t~~66yA1ErH   c                    t        j                  t        d      t                     }t	        j
                  |      } |        | j                  j                  d   }| j                  d|d          | j                  j                          | j                  d|j                         t        j                  t        d      t                     }t	        j
                  |      } |        | j                  d|j                         | j                  j                  d   }| j                  d|d          | j                  j                          t         j                  j                  t        d	d
      5  t	        j                           t	        j"                          t	        j$                          t	        j&                          t	        j(                          t	        j*                  d       | j                  j                  d   }d}dD ]  }| j                  d|z  ||          |dz  }! | j                  j                          ddd       y# 1 sw Y   yxY w)z>Tests the exception catching decorator CaptureAndLogException.mock_exc_fn)rQ   r   r   z;Exception captured in mock_exc_fn during metrics collectionr   r   mock_err_fnz;Exception captured in mock_err_fn during metrics collectionrd   znot a collectorrV   zinvalid argument)Shutdownr=   r@   rA   r  'CheckAndMaybePromptForAnalyticsEnablingz2Exception captured in %s during metrics collectionN)r#   	MagicMockstrr>   r   CaptureAndLogExceptionrh   messagesr#  resetr   
call_count	TypeErrorr   objectr   rO  r=   r@   rA   r  rP  )rN   rM  
wrapped_fndebug_messagesrN  message_index	func_names          r,   testExceptionCatchingDecoratorz3TestMetricsUnitTests.testExceptionCatchingDecorator  s    ..#m*<-6[:K//<JL%%..w7NMMO #%Q../..#m*<-6[:K//<JLQ../%%..w7NMMO #% 
		+)(9 
 
;  !))+556HI''009nmC) 	@9L=)	+ 	C +
; 
; 
;s   7CII)rQ   rR   rS   rT   rc   rs   r   r   r   r-  r<  r#   r   rX  r	   rK  r]  __classcell__ro   s   @r,   r[   r[      sb     2>(<?OB|:|3J!XMB ::\#NO F P FD1rH   r[   c                       e Zd ZdZd Zd Zy)&_JSONForceHTTPErrorCopyCallbackHandlerzDTest callback handler that raises an arbitrary HTTP error exception.c                 .    || _         || _        d| _        y )NF)_startover_at_byte_http_error_numstarted_over_once)rN   startover_at_bytehttp_error_nums      r,   __init__z/_JSONForceHTTPErrorCopyCallbackHandler.__init__G  s    /D)D"DrH   c                     || j                   k\  r7| j                  s*d| _        t        j                  d| j                  idd      yy)z<Forcibly exits if the transfer has passed the halting point.TstatusN)rc  re  r  	HttpErrorrd  rN   r   unused_total_sizes      r,   callz+_JSONForceHTTPErrorCopyCallbackHandler.callM  sP    4#:#::""#d))8T5I5I*J*.6 6 # 	;rH   NrQ   rR   rS   rT   rh  rn  r(   rH   r,   ra  ra  D  s    L#6rH   ra  c                        e Zd ZdZ	 ddZd Zy)_ResumableUploadRetryHandlerzFTest callback handler for causing retries during a resumable transfer.c                 J    || _         || _        || _        || _        d| _        y )Nr   )_retry_at_byte_exception_to_raise_exception_args_num_retries_retries_made)rN   retry_at_byteexception_to_raiseexc_argsr   s        r,   rh  z%_ResumableUploadRetryHandler.__init__Y  s,    
 (D1D#D#DDrH   c                     || j                   k\  rH| j                  | j                  k  r.| xj                  dz  c_         | j                  | j                   yy)z(Cause a single retry at the retry point.r   N)rs  rw  rv  rt  ru  rl  s      r,   rn  z!_ResumableUploadRetryHandler.callf  sX    4#6#66T...
A$D$$d&:&:;; 	/ 	7rH   N)r   ro  r(   rH   r,   rq  rq  V  s    N 	
<rH   rq  c                       e Zd ZdZ fdZ fdZddZd Zd Z e	j                  d e	j                  d	      
      d        Z ej                  d       e	j                  d e	j                  d	      
      d               Zd Zd Zd Zd Zd ZddZd Z ed      d        Z ed      d        Zd Z ej6                  ed      d        Z xZS )TestMetricsIntegrationTestsz0Integration tests for analytics data collection.c                     t         t        |           t        j                         | _        t        j                  ddddd       t        j                         | _        y )Nr$   r&   r]   r^   r_   )rb   r}  rc   r   rd   re   rf   rg   rn   s    r,   rc   z!TestMetricsIntegrationTests.setUpr  sW    	
%t24 (8'D'D'FD$ ''(=?O1414)*+
 &224DNrH   c                 j    t         t        |           t        j                  | j
                         y rq   )rb   r}  rs   r   rt   re   rn   s    r,   rs   z$TestMetricsIntegrationTests.tearDown  s*    	
%t57 &&::<rH   r   c                 ~    | j                  dg|z   d|ddi      }t        j                  |      j                         S )a  Runs the gsutil command to check for metrics log output.

    The env value is set so that the metrics collector in the subprocess will
    use testing parameters and output the metrics collected to the debugging
    log, which lets us check for proper collection in the stderr.

    Args:
      cmd: The command to run, as a list.
      expected_status: The expected return code.

    Returns:
      The string of metrics output.
    z-dTGSUTIL_TEST_ANALYTICSr  )return_stderrexpected_statusenv_vars)	RunGsUtilMETRICS_LOG_REsearchgroup)rN   cmdr  stderrs       r,   _RunGsUtilWithAnalyticsOutputz9TestMetricsIntegrationTests._RunGsUtilWithAnalyticsOutput  sJ     ^^TFSL*.,;&=s%C  EF   (..00rH   c                     t        j                  j                  |i |}| j                  |j                         |j                         S )at  Runs mock.patch.object with the given args, and returns the mock object.

    This starts the patcher, returns the mock object, and registers the patcher
    to stop on test teardown.

    Args:
      *args: The args to pass to mock.patch.object()
      **kwargs: The kwargs to pass to mock.patch.object()

    Returns:
      Mock, The result of starting the patcher.
    )r#   r   rX  
addCleanupstopstart)rN   r*   r+   patchers       r,   _StartObjectPatchz-TestMetricsIntegrationTests._StartObjectPatch  s9     jj00GOOGLL!==?rH   c                 j    | j                  dj                  t        j                  |   |      |       y)z;Checks for a correct key=value pair in a log output string.r  N)r#  r$  r   r  )rN   
param_namer,  metrics_to_searchs       r,   _CheckParameterValuez0TestMetricsIntegrationTests._CheckParameterValue  s,    MM..z:IFrH   rU   rV   rX   c                    | j                  t        d      }t        j                         }|j	                          | j                  t        d      }t        |j                  d      |_        | j                  j                          | j                  d|j                         t                t        j                          |j                  }| j                  dt!        |             |d   }| j#                  d|d   d          t%        t&        j(                        t%        |d   d   d   j+                  t,        j.                              z
  }| j                  t%               |       t        |j                  d      5 }t1        j2                  |      }d	d	d	       | j                  t4        t%                     y	# 1 sw Y   )xY w)
z5Tests the subprocess creation by Popen in metrics.py.PopenNamedTemporaryFilewbr   r   
PYTHONPATHenvrbN)r  
subprocesstempfiler  closeopennamerW   rg   ReportMetricsr   rV  rG   r   rO  r   r   r#  r   syspathsplitr   pathseppickleloadr   )rN   
popen_mockmetrics_filetemp_file_mock	call_listr*   missing_pathsreported_metricss           r,   testMetricsReportingz0TestMetricsIntegrationTests.testMetricsReporting  sg    ''
G<J ..0L++H6JKN"&|'8'8$"?N 	NN  "Q
--. ))IQI'Q<DMM,Q/ ]a5;;BJJGHIMSUM* 
l	&,\2 
'3S9I5JK 
'	&s   ?F==Gzexample.com is currently not accepting post requests, tracking issue: https://github.com/GoogleCloudPlatform/gsutil/issues/1800c                 h    t        j                         }|j                  }|j                          d } j	                  ||        fd}t        j                  g d        |t        j                  |j                         t        |j                  d      5 }|j                         }ddd       t        j                  r*dj                  t        t
        j                  t               }n8dj                  t        t
        j                  t               j#                  d	      } j%                  |        j%                  d
|        |t        j&                  |j                         t        |j                  d      5 }|j                         }ddd        j)                  |d        |t        j*                  |j                         t        |j                  d      5 }|j                         }ddd        j)                  |d       y# 1 sw Y   exY w# 1 sw Y   xY w# 1 sw Y   5xY w)zFTests the metrics posting process as performed in metrics_reporter.py.c                 N    	 t        j                  |        y # t        $ r Y y w xY wrL   )r   unlinkrB  )	file_paths    r,   MetricsTempFileCleanupzNTestMetricsIntegrationTests.testMetricsPosting.<locals>.MetricsTempFileCleanup  s%    
		) s    	$$c                 (   t        j                  ddgddg       t        j                  t        j                  j                  d             j                  j                  d| |       j                  g j                  j                         y )	Nr/   r0   r4   r6   )r8   r9   r;   r<   T)wait_for_report	log_levellog_file_path)
r   r=   rA   rB   rC   rD   rg   r  r   r   )r  r  rN   s     r,   CollectMetricAndSetLogLevelzSTestMetricsIntegrationTests.testMetricsPosting.<locals>.CollectMetricAndSetLogLevel  s~    F,5;)5|(DF EOO<<VDE nn""4-61> # @ r4>>223rH   r1   )r:   r  Ns   Metric(endpoint=u'https://example.com', method=u'POST', body='{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0', user_agent=u'user-agent-007')zMetric(endpoint='https://example.com', method='POST', body='{0}&cm2=0&ea=cmd1+action1&ec={1}&el={2}&ev=0', user_agent='user-agent-007')utf_8s   RESPONSE: 200rH   )r  r  r  r  r  r   r=   ri   rl   r  r   r   r   r$  GLOBAL_DIMENSIONS_URL_PARAMS_GA_COMMANDS_CATEGORYr   encoder#  INFOr   WARN)rN   r  metrics_file_namer  r  metrics_loglog_textexpected_responses   `       r,   testMetricsPostingz.TestMetricsIntegrationTests.testMetricsPosting  s    ..0L$)) 	OO*,=>
4  *M N  |/@/@A	l	&+!!#h 
'
ww--3V*G,I,I. ++162N292O2O29,;<BF7O  	MM#X.MM"H-l.?.?@	l	&+!!#h 
'Xs#l.?.?@	l	&+!!#h 
'Xs#7 
'	&( 
'	&
 
'	&s$   HH#H(HH%(H1c                    | j                  t        d      }t               |_        | j                  j
                  j                  d       | j                  j                          | j                  |j                         y)z?Tests that metrics reporting error does not throw an exception.r  zdummy metricN)
r  r  rB  r   rg   r   r"  r  r   called)rN   r  s     r,   testMetricsReportingWithFailz8TestMetricsIntegrationTests.testMetricsReportingWithFail  sX    ''
G<J$YJNN"">2NN  "OOJ%%&rH   c                    | j                  g dd      }| j                  dt        j                  |       | j                  dd|       | j                  dd|       | j                  d	d
|       | j                  dg      }| j                  dt        j                  |       | j                  dd|       | j                  dd|       y)z!Tests the collection of commands.)-mr/  r   r0  r   r  r   r   zacl+setzGlobal Optionszd%2Cmr2  r`   verversionr5  N)r  r  r   r  )rN   metrics_lists     r,   r<  z1TestMetricsIntegrationTests.testCommandCollection%  s    55"A 6 7L.0M0M*,niF.F5sLI55ug>L.0M0M*,niFoulCrH   c           	      b   | j                   t        j                  k7  rt        j                  d      S | j                         }| j                  |dd      }d| j                  j                  t        j                  d   <   t        t        t        j                         t               | j                         }d|j"                  _        d|j"                  _        |j)                         }t+        j,                  |j.                  |j0                  j.                  |j2                  	      }t+        j,                  |j0                  | j5                  d
      |j6                        }t8        j:                  j=                  t>        dtA        jB                               5  tE        |jF                  ||       ddd       tH        jJ                  r*| jM                  | j                  jN                  d   d       n)| jM                  | j                  jN                  d   d       t8        j:                  j=                  t>        dtQ        jR                  ddd            5  tE        |jT                  |jV                  |jX                         ddd       | jM                  | j                  jN                  d   d       | jM                  | j                  jZ                  j\                  d       | jM                  | j                  jZ                  j^                  d       y# 1 sw Y   fxY w# 1 sw Y   xY w)zFTests that retryable errors are collected on JSON metadata operations.+Retryable errors are only collected in JSONfoo   bar)
bucket_uriobject_namecontentsrsyncr   r   r   )r  bucketcontentTyperX  )r  r  r  _MakeRequestNoRetryr   NrA  rB  unusedrk  )0test_apir   JSONr   skipCreateBucketCreateObjectrg   r  r   r  r   r
   ri   rj   rJ   default_provider
api_clientr   max_retry_waitget_keyapitools_messagesObjectr  r  content_typeMakeTempNamer  r#   r   rX  r	   r  r  r-   
CopyObjectr   r   r   rD  r  rk  DeleteObjectbucket_namer  r  num_retryable_network_errorsnum_retryable_service_errors)rN   r  
object_uri
gsutil_apikeysrc_obj_metadatadst_obj_metadatas          r,   $testRetryableErrorMetadataCollectionz@TestMetricsIntegrationTests.testRetryableErrorMetadataCollection7  s    }}(((]]HII""$J""j/4,2 # 4J GNDNNW22>BC ,g.?.?.A02D4I4IKJ )*J%+,J( 


C(//SXX7:zz<?<L<LN )//&&x($002 
		<0'-||~ 
 
7 
--/?(*
7
 ww
t~~66}EqI t~~66yA1E 
		<0':'D'D'8(= 
 
> 
//1G1G"..0	
> 	T^^44[A1E 	&&CCQH&&CCQH/
7 
7
> 
>s   L,L%L"%L.c                    | j                   t        j                  k7  rt        j                  d      S ddt        t              fg}| j                         }t        dz  }| j                  d|z        }| j                  t        j                  t        dt        j                  d                  }t        |      5  | j!                  d	d
||t#        |      g      }| j%                  dt&        j(                  |       | j%                  dd|       | j%                  dd|       | j%                  dd|       ddd       | j                  t        j                  t+        dd                  }t        |      5  | j!                  d	d
||t#        |      g      }| j%                  dt&        j(                  |       | j%                  dd|       | j%                  dd|       | j%                  dd|       ddd       | j                  t        j                  t+        dd                  }t        |      5  | j!                  dd	d
||t#        |      g      }| j%                  dt&        j(                  |       | j%                  dd|       | j%                  dd|       | j%                  dd|       ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   yxY w)zCTests that retryable errors are collected on JSON media operations.r  r   r   r      a)r     )r  r  r  r   z--testcallbackfiler   r   BadStatusCodeErrorzRetryable Errorsrv   r  Ni  !ResumableUploadStartOverExceptionr  )r  r   r  r   r  rR  r   r  r   CreateTempFiler  dumpsrq  r  r  r   r  surir  r   _GA_ERRORRETRY_CATEGORYra  )rN   boto_config_for_testr  	halt_sizefpathtest_callback_filer  s          r,   !testRetryableErrorMediaCollectionz=TestMetricsIntegrationTests.testRetryableErrorMediaCollectiono  s    }}(((]]HII%'<c'lKL""$J )1,I	)9:E ,,fll$Q(;(N(N%C	E7F, G 
2	377
$&8%
z
9 	l  0 ' ? ?O
0D ,.
 2CF
 > ,. 
4 ,,DQLM - O	2	377
$&8%
z
9 	l  0 ' ? ?O
 C ,.  2CF
 > ,. 
4 ,,DQLM - O	2	377
*,>
z
9 	l  0 ' ? ?O
 C ,.  2CF
 > ,. 
4	3C 
4	3  
4	3" 
4	3s'   =A:J99A:K5A;K9KKKc                      fd} j                  dgd      } ||        j                  ddgd      } ||        j                         } j                  dt        |      t        |      gd      } ||       y)	z&Tests that fatal errors are collected.c                 n    j                  dt        j                  |        j                  dd|        y )Nr   r   rD   )r  r   _GA_ERRORFATAL_CATEGORY)
log_outputrN   s    r,   CheckForCommandExceptionzVTestMetricsIntegrationTests.testFatalErrorCollection.<locals>.CheckForCommandException  s3    
 0 ' ? ?M
0BJOrH   zinvalid-commandr   r  mbz-invalid-optionr   N)r  r  r  )rN   r  r  r  s   `   r,   testFatalErrorCollectionz4TestMetricsIntegrationTests.testFatalErrorCollection  s    P
 557H6IFG 6 IL\*55t=N6OFG 6 IL\*""$J55	tJj!12A 6 GL\*rH   c                 |     fd}|r, |d      } j                  |d        j                  |d        |d      } j                  |d        |d      } |d      } j                  ||        j                  |d        j                  |d       d}t        r |d	      } j                  |d       |||fS )
a<  Checks number metrics for PerformanceSummary tests.

    Args:
      metrics_to_search: The string of metrics to search.
      multithread: False if the the metrics were collected in a non-multithread
                   situation.

    Returns:
      (slowest_throughput, fastest_throughput, io_time) as floats.
    c                     t        j                  t        j                  |    dz         }|s+j	                  dt        j                  |    d| d       t        |j                  d            S )Nz=(\d+\.?\d*)&zCould not find z (z) in metrics string r   )rer  r   r  failfloatr  )r  extracted_matchr  rN   s     r,   _ExtractNumberMetriczVTestMetricsIntegrationTests._GetAndCheckAllNumberMetrics.<locals>._ExtractNumberMetric  sk    		




+.>
>
o 		"":.
<MO	P ?((+,,rH   r  r   r   r  r  r  Nr  )assertGreaterEqualassertLessEqualassertGreaterr   )	rN   r  multithreadr  r   
throughputslowest_throughputfastest_throughputio_times	   ``       r,   _GetAndCheckAllNumberMetricsz8TestMetricsIntegrationTests._GetAndCheckAllNumberMetrics  s    - -.HI
.2
+Q/%&BCJz1%-.IJ-.IJ.0BC)1-)1-G$_5g
gq)  2G<<rH   c                    | j                         }| j                         }t        }| j                  |d|z         t        rdnd}t	        ddt        |      fdg      5  | j                  dd	||g      }| j                  d
t        j                  |       | j                  dd|       | j                  dd|       | j                  dd|       | j                  dt        |      |       | j                  dd|       | j                  dd|       | j                  d||       | j                  dd|       | j                  |      \  }}}t        r| j                  |d       ddd       y# 1 sw Y   yxY w)z?Tests PerformanceSummary collection in a file-to-file transfer.r  tmpdirr  r      r   r   )r   r   7r  r  r   r   
FileToFiler   fanr  filer  r  r  r  r	  r  r   N)CreateTempDirr    r  r   r   rR  r  r  r   r!  r  r   r  )rN   tmpdir1tmpdir2	file_sizeprocess_countr  r   r  s           r,    testPerformanceSummaryFileToFilez<TestMetricsIntegrationTests.testPerformanceSummaryFileToFile  s     "G  "GIw	1AB $AM	*B #M 2 4GI 
J 77'7
+-l
 0'2N2N ,.
lK
 6|L
 16<H
]1C ,.
sLA
 0&,G
 CY ,.
 Eq ,. 99,Goq!W	 	+/
J 
J 
Js    DE..E7z$No slice parallelism support for S3.c                 $   | j                         }| j                         }d}| j                  |d|z         | j                  |d|z         t        rdnd}t	        ddt        |      fd	d
g      5  | j                  d|t        |      g      }| j                  dt        j                  |       | j                  dd|       | j                  dd|       | j                  dt        |      |       | j                  dd|       | j                  dd|j                  z   |       | j                  dd|z  |       | j                  dd|       | j                  |      \  }}}t        r| j                  |d       ddd       y# 1 sw Y   yxY w)z@Tests PerformanceSummary collection in a file-to-cloud transfer.r  r  r     br   r   r   r   )r   r   r  )r   r   rv   r  r   r   FileToCloudr   slicer  r  r  r  file%2Cr	  r  r   N)r  r  r  r   r   rR  r  r  r  r   r!  schemer  r   r  )rN   r  r  r"  r#  r  r   r  s           r,   !testPerformanceSummaryFileToCloudz=TestMetricsIntegrationTests.testPerformanceSummaryFileToCloud  s    ""$J!FIvy0@Avy0@A#AM		+S-?@0> 

 77FD,
-/l
 0'2N2N ,.
|L
 6N
]1C ,.
sLA
 0)j>O>O2O ,.
 C !I|=
 Eq ,.99,Goq!W	 	+1
 
 
s   3D
FFc                    | j                         }d}| j                  |d|z        }| j                         }t        rdnd}t	        ddt        |      fdd	d
g      5  | j                  ddt        |      |g      }| j                  dt        j                  |       | j                  dd|       | j                  dd|       | j                  dt        |      |       | j                  dd|       | j                  dd|j                  z   |       | j                  dd|       | j                  d||       | j                  |      \  }}}t        r| j                  |d       ddd       y# 1 sw Y   yxY w)z@Tests PerformanceSummary collection in a cloud-to-file transfer.r  r  r  r  r      r   r   )r   r   5)r   r   rv   )r   test_assume_fast_crcmodr   r  r   r   r   CloudToFiler   r   r  r  r/  r  r)  r  rv   r	  r   N)r  r  r  r   r   rR  r  r  r  r   r!  r*  r  r   r  )	rN   r  r"  r  r  r#  r  r   r  s	            r,   !testPerformanceSummaryCloudToFilez=TestMetricsIntegrationTests.testPerformanceSummaryCloudToFile:  s    ""$JI""j,09,< # >J !E#AM		+S-?@0;5	 
 77tJ'
/1l
 0'2N2N ,.
|L
 6M
]1C ,.
sLA
 0)j>O>O2O ,.
 Es ,.
 CY ,.99,Goq!W	 	+3
 
 
s   DE//E8c                    | j                         }| j                         }d}| j                  |d|z        }| j                  ddt        |      t        |      g      }| j	                  |d      \  }}}| j                  ||       | j                  dt        j                  |       | j                  d	d
|       | j                  dd|       | j                  dd|       | j                  dd|       | j                  dd|       | j                  d|j                  |       | j                  dd|       | j                  d||       y)zATests PerformanceSummary collection in a cloud-to-cloud transfer.r  r  r-  r   z-DF)r  r   r   r   r   noner  r  r  rv   r  r  r  r	  N)
r  r  r  r  r  r   r  r   r!  r*  )	rN   bucket1_uribucket2_urir"  key_urir  r  r  r   s	            r,   "testPerformanceSummaryCloudToCloudz>TestMetricsIntegrationTests.testPerformanceSummaryCloudToCloud`  se   ##%K##%KI;)-	)9   ;G 55	tT']	k		L
 
+
+Le
+
L+');<.0L0L*,n.I*,4flK/,GosLAmS,?.0B0B*,CS*,A9*,rH   z(Test requires both S3 and GS credentialsc                    | j                  d      }| j                  d      }| j                  |d      }| j                  |d      }| j                  dt        |      t        |      g      }| j	                  dd	|       | j	                  d
d|       | j                  dt        |      t        |      g      }| j	                  dd	|       | j	                  d
d|       y)z/Tests the collection of daisy-chain operations.s3)providerr   s   foor-  r  r  r   r   r  zgs%2Cs3r   N)r  r  r  r  r  )rN   	s3_bucket	gs_bucketunused_s3_keygs_keyr  s         r,   %testCrossProviderDaisyChainCollectionzATestMetricsIntegrationTests.testCrossProviderDaisyChainCollection  s     !!4!0I!!4!0I%%V%LM)fEF55	$y/4	?35Ln.I*,.	<H55	tF|T)_-/Ln.I*,.	<HrH   )r   )T)rQ   rR   rS   rT   rc   rs   r  r  r  r#   r   rQ  r  r   r  r  r  r<  r  r  r  r  r$  r   r+  r2  r8  
skipUnlessr   r@  r^  r_  s   @r,   r}  r}  n  s$   85<1(" 4::k~t~~1=> L ? LD 8==  S  T4::k~t~~1=>A$ ? TA$F	'D$6Hp>.@+*,=\!,F 34", 5",H 34#, 5#,J,B 8|%OPI QIrH   r}  )ZrT   
__future__r   r   r   r   ri   r   r  r	  r  r  r  r  pprintr   apitools.base.pyr   r  r	   boto.storage_urir
   rB   r   r   gslib.cs_api_mapr   gslib.exceptiongslib.gcs_json_apir   gslib.metricsr   gslib.metrics_tupler    gslib.tests.mock_logging_handlerr   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   gslib.tests.utilr   r   r  r   r   r   "gslib.third_party.storage_apitoolsr   r  gslib.thread_messager   r   gslib.utils.constantsr   gslib.utils.retry_utilr   gslib.utils.system_utilr   r   gslib.utils.unit_utilr   r    r!   r"   	six.movesr#   r  r$  GLOBAL_PARAMETERSr   r  r  r  r   compiler  r-   rG   rX  rJ   r   rQ  GsUtilUnitTestCaser[   ra  rq  GsUtilIntegrationTestCaser}  r(   rH   r,   <module>r\     sU    0 & %  '  	  	   
   
 > ) -   (  ) * & ? ' ' ? ) 0 1 + % W , 6 : 6 , . ) ) % VV_	5 6 ! 
 
5$k?GVX__W-E  "%
v6==('*G*G	&	(
 v3::('*I*I	&	(
 v4;;('*I*I	&	(
 v4;;('*I*I	&	(!& " 0 -.	"B-6 - LMK^T^^;<m866 m = Nmd6V 6$<6 <0 LMdI("D"D dI NdIrH   