
    ^                        d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlZddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmc m Z  ddl!m"Z" ddl#mZ ddl$m%Z% ddl$m&Z& ddl'm(Z( ddl'm)Z) dZ*d ejV                  d      z   dz   Z,d Z-d Z. G d de/      Z0d5dZ1d5dZ2d5d Z3d5d!Z4d" Z5d# Z6d5d$Z7d% Z8d5d&Z9d5d'Z:d( Z; G d) d*e	jx                        Z= G d+ d,e      Z> G d- d.e>      Z? G d/ d0e j                        ZA e&j                         ZC G d1 d2e j                        ZD G d3 d4eA      ZEy)6z,Unit tests for gsutil parallelism framework.    )absolute_import)print_function)division)unicode_literalsN)mock)BucketStorageUri)
StorageUri)
cs_api_map)command)Command)CreateOrGetGsutilLogger)DummyArgChecker)MockCloudApi)MockLoggingHandler)RequiresIsolation)unittest)$CheckMultiprocessingAvailableAndInit)multiprocessing_context)IS_OSX
IS_WINDOWSx   
z==> NOTE: You are performing a sequence of gsutil operations that may run significantly faster if you instead use gsutil -m fake ...
Please see the -m section under "gsutil help options" for further information about when gsutil -m can be advantageous.c                 B     t        j                          fd       }|S )z2Decorator used to provide a timeout for functions.c                      t         sAt        j                  t        j                  t               t        j                  t
               	  | i | t         st        j                  d       y y # t         st        j                  d       w w xY wNr   )r   signalSIGALRM_HandleAlarmalarm_TEST_TIMEOUT_SECONDS)argskwargsfuncs     9platform/gsutil/gslib/tests/test_parallelism_framework.pyWrapperzTimeout.<locals>.WrapperG   s^    mmFNNL1ll()
DFQ ZQ s   
A/ /B)	functoolswraps)r$   r&   s   ` r%   Timeoutr)   D   s'     ??4  
.    c                     t        d      )NzTest timed out.)	Exception)
signal_numcur_stack_frames     r%   r   r   V   s    #$$r*   c                        e Zd Z fdZ xZS )CustomExceptionc                 ,    t         t        |   |       y N)superr0   __init__)selfexception_str	__class__s     r%   r4   zCustomException.__init__\   s    	/4)-8r*   __name__
__module____qualname__r4   __classcell__r7   s   @r%   r0   r0   Z   s    9 9r*   r0   c                      yN    clsr"   thread_states      r%   _ReturnOneValuerE   `   s    	
r*   c                 f    t        j                         t        j                         j                  fS r2   )osgetpid	threadingcurrent_threadidentrB   s      r%   _ReturnProcAndThreadIdrL   d   s"    	i..066	66r*   c                 H    t        j                  d       t        | ||      S )N   )rD   )timesleeprL   rB   s      r%   _SleepThenReturnProcAndThreadIdrQ   h   s     **Q-	T	EEr*   c                     t        d      )NzFailing on purpose.)r0   rB   s      r%   _FailureFuncrS   p   s    -..r*   c                 B    | xj                   dz  c_         t        d      )Nr@   z%Exception handler failing on purpose.)failure_countr0   rC   es     r%   _FailingExceptionHandlerrX   t   s    q?@@r*   c                 d    | j                   j                  |       | xj                  dz  c_        y r?   )logger	exceptionrU   rV   s     r%   _ExceptionHandlerr\   y   s%    **qqr*   c                 @    | xj                   t        |      z  c_         y r2   )arg_length_sumlenrB   s      r%   _IncrementByLengthr`   ~   s    D	!r*   c                     t         ry| S r?   r   )process_counts    r%   _AdjustProcessCountIfWindowsrc      s    r*   c           	          |gdz  }t        |d         }|d   }| j                  t        |t        t        ||d      }t        |      }| j                  t        |t        t        ||d      }t        |      |z   S )a  Calls Apply with arguments repeated seven times.

  The first two elements of args should be the process and thread counts,
  respectively, to be used for the recursive calls.

  Args:
    cls: The Command class to call Apply on.
    args: Arguments to pass to Apply.
    thread_state: Unused, required by function signature.

  Returns:
    Number of values returned by the two calls to Apply.
     r   r@   Targ_checkerrb   thread_countshould_return_results)rc   Apply_PerformNRecursiveCallsr\   r   sumrE   r_   )rC   r"   rD   new_argsrb   rh   return_valuesrets           r%   _ReApplyWithReplicatedArgumentsrp      s     VaZ(.tAw7-a,))3$-(7*7)526  8- 	M#))O$-(7*7)526  8- 
]	c	!!r*   c           	          t        |d         }|d   }| j                  t        dg|d   z  t        t        ||d      }t        |      S )a  Calls Apply to perform N recursive calls.

  The first two elements of args should be the process and thread counts,
  respectively, to be used for the recursive calls, while N is the third element
  (the number of recursive calls to make).

  Args:
    cls: The Command class to call Apply on.
    args: Arguments to pass to Apply.
    thread_state: Unused, required by function signature.

  Returns:
    Number of values returned by the call to Apply.
  r   r@   rA      Trf   )rc   rj   rE   r\   r   r_   )rC   r"   rD   rb   rh   rn   s         r%   rk   rk      sX     /tAw7-a,))ObTDG^-(7*7)526  8- 
]	r*   c                     |dz  dk7  S )Nrr   r   rA   )rC   args     r%   _SkipEvenNumbersArgCheckerru      s    	qAr*   c                       e Zd Zd Zd Zd Zy)FailingIteratorc                 .    || _         || _        d| _        y r   )sizefailure_indicescurrent_index)r5   ry   rz   s      r%   r4   zFailingIterator.__init__   s    DI*DDr*   c                     | S r2   rA   r5   s    r%   __iter__zFailingIterator.__iter__   s    Kr*   c                    | j                   | j                  k(  rt        d      | j                   | j                  v r-| xj                   dz  c_         t	        d| j                   z        | xj                   dz  c_         | j                   dz
  S )N r@   z(Iterator failing on purpose at index %d.)r{   ry   StopIterationrz   r0   r}   s    r%   __next__zFailingIterator.__next__   s    TYY&"			t33	3
AF ../ 0 0 A!##r*   N)r9   r:   r;   r4   r~   r   rA   r*   r%   rw   rw      s    
	$r*   rw   c                   n    e Zd ZdZ ej
                  dg       Z ej                  dg dddi       Zd Z	y	)
FakeCommandz9Fake command class for overriding command instance state.fake)command_name_aliasescommand_helpzSomething to take up space.z Something else to take up space.)	help_namehelp_name_aliases	help_typehelp_one_line_summary	help_textsubcommand_help_textc                 l   t         | _        dgdgd}ddd}t        j                  j	                  t        j
                  ||      | _        t        d      | _        || _	        d| _
        t               | _        t               j                  | _        d| _        i | _        d | _        d | _        d | _        y )NJSONXML)gss3r   r   )r   bucket_storage_uri_classr
   GsutilApiMapFactory	GetApiMapGsutilApiClassMapFactorygsutil_api_mapr   rZ   parallel_operationsrU   r   
gsutil_apir   is_availablemultiprocessing_is_availabledebugnon_metadata_headersperf_trace_tokentrace_tokenuser_project)r5   do_parallelsupport_mapdefault_maps       r%   r4   zFakeCommand.__init__   s    $4D! %1Ku-K$88BB++[+GD)-8DK*DD"nDO,.;; 	%DJ "D DDDr*   N)
r9   r:   r;   __doc__r   CreateCommandSpeccommand_specHelpSpec	help_specr4   rA   r*   r%   r   r      sH    A***,
 g92)r*   r   c                        e Zd Z fdZ xZS )'FakeCommandWithoutMultiprocessingModulec                 :    t         t        |   |       d| _        y )NF)r3   r   r4   r   )r5   r   r7   s     r%   r4   z0FakeCommandWithoutMultiprocessingModule.__init__  s    	
14A+N(-D%r*   r8   r=   s   @r%   r   r     s    . .r*   r   c                   Z
   e Zd ZdZeZddddefdZed        Z	ed        Z
e ej                  ed      d               Ze ej                  ed      d	               Zed
        Zed        Zed        Ze ej                  ed      d               Ze ej                  ed      d               Zed        Ze ej                  ed      d               Zed        Ze ej                  ed      d               Zed        Zed        Zed        Ze ej                  ed      d               Ze ej                  ed      d               Zed        Zed        Z ed        Z!e ej                  ed      d               Z"e ej                  ed      d               Z#ed        Z$ed        Z%ed        Z&e ej                  ed      d                Z'e ej                  ed      d!               Z(ed"        Z)ed#        Z*ed$        Z+e ej                  ed      d%               Z,e ej                  ed      d&               Z-ed'        Z.eed(               Z/ed)        Z0ed*        Z1e ej                  ed      d+               Z2e ej                  ed      d,               Z3e ejh                  e5d-      d.               Z6e ej                  e5d/       ej                  ed      d0                      Z7ed1        Z8ed2        Z9ed3        Z:ed4        Z;e ej                  ed      d5               Z<e ej                  ed      d6               Z=ed7        Z>ed8        Z?ed9        Z@e ej                  ed      d:               ZAe ej                  ed      d;               ZBed<        ZCeDj                  j                  eGd=d>      eDj                  j                  eGd?d@A      dB               ZHeDj                  j                  eGd=d@      eDj                  j                  eGdCdD      eDj                  j                  eGd?d@A      dE                      ZIeDj                  j                  eGd=d@      eDj                  j                  eGdCdD      eDj                  j                  eGd?d>A      dF                      ZJeDj                  j                  eGd=d@      eDj                  j                  eGdCdD      eDj                  j                  eGd?dGA      dH                      ZKdI ZLy)JTestParallelismFrameworkz(gsutil parallelism framework test suite.NFc
                 t    |xs | j                  d      }|xs t        }
|j                  |||
|||	d||	      S )NT)rh   rb   rg   ri   shared_attrsfail_on_error)command_classr\   rj   )r5   r$   args_iteratorrb   rh   command_instr   r   thr_exc_handlerrg   exception_handlers              r%   	_RunApplyz"TestParallelismFramework._RunApply  sW      ;4#5#5d#;L'<+<d+/+7,9*548+7,9  ; ;r*   c                 (    | j                  dd       y r?   _TestBasicApplyr}   s    r%   'testBasicApplySingleProcessSingleThreadz@TestParallelismFramework.testBasicApplySingleProcessSingleThread*      Ar*   c                 (    | j                  dd       y Nr@      r   r}   s    r%   &testBasicApplySingleProcessMultiThreadz?TestParallelismFramework.testBasicApplySingleProcessMultiThread.  r   r*   +Multiprocessing is not supported on Windowsc                 (    | j                  dd       y Nr   r@   r   r}   s    r%   &testBasicApplyMultiProcessSingleThreadz?TestParallelismFramework.testBasicApplyMultiProcessSingleThread2       	Ar*   c                 (    | j                  dd       y Nr   r   r}   s    r%   %testBasicApplyMultiProcessMultiThreadz>TestParallelismFramework.testBasicApplyMultiProcessMultiThread7  r   r*   c                     dgd|z  |z  dz   z  }| j                  t        |||      }| j                  t        |      t        |             y )NrA      r@   )r   rE   assertEqualr_   r5   rb   rh   r"   resultss        r%   r   z(TestParallelismFramework._TestBasicApply<  sI    42%4q89Dnn_dM<PGSYG-r*   c                 (    | j                  dd       y r?   _TestApplyWithNoTasksr}   s    r%   $testNoTasksSingleProcessSingleThreadz=TestParallelismFramework.testNoTasksSingleProcessSingleThreadC      q!$r*   c                 (    | j                  dd       y r   r   r}   s    r%   #testNoTasksSingleProcessMultiThreadz<TestParallelismFramework.testNoTasksSingleProcessMultiThreadG  r   r*   c                 (    | j                  dd       y r   r   r}   s    r%   #testNoTasksMultiProcessSingleThreadz<TestParallelismFramework.testNoTasksMultiProcessSingleThreadK       	q!$r*   c                 (    | j                  dd       y r   r   r}   s    r%   "testNoTasksMultiProcessMultiThreadz;TestParallelismFramework.testNoTasksMultiProcessMultiThreadP  r   r*   c                     dg}t        ||z  dz         D ]  }| j                  t        |||        | j                  ||       y)zATests that calling Apply with no tasks releases locks/semaphores.rA   r@   N)ranger   rE   r   )r5   rb   rh   
empty_args_s        r%   r   z.TestParallelismFramework._TestApplyWithNoTasksU  sH     J=</!34
nn_j-N 5 	5r*   c                 (    | j                  dd       y r   /_TestApplySaturatesAvailableProcessesAndThreadsr}   s    r%   *testApplySaturatesMultiProcessSingleThreadzCTestParallelismFramework.testApplySaturatesMultiProcessSingleThread`       	88A>r*   c                 (    | j                  dd       y r   r   r}   s    r%   *testApplySaturatesSingleProcessMultiThreadzCTestParallelismFramework.testApplySaturatesSingleProcessMultiThreade  s    88A>r*   c                 (    | j                  dd       y r   r   r}   s    r%   )testApplySaturatesMultiProcessMultiThreadzBTestParallelismFramework.testApplySaturatesMultiProcessMultiThreadi  r   r*   c                 h   d}dg||z  |z  z  }|}| j                  d      j                  s||z  }| j                  t        |||      }i }|D ]!  \  }}	|j	                  ||	fd      dz   |||	f<   # t        j                  |      D ]*  \  }
}| j                  ||d|
d   d|
d   d|d	|       , y
)z<Tests that created processes and threads evenly share tasks.rr   rA   Tr   r@   zProcess z thread z completed z tasks. Expected: N)r   r   r   rQ   getsix	iteritemsr   )r5   rb   rh   calls_per_threadr"   expected_calls_per_threadr   
usage_dict
process_id	thread_idid_tuplenum_tasks_completeds               r%   r   zHTestParallelismFramework._TestApplySaturatesAvailableProcessesAndThreadsn  s     4=</2BBCD 0d#@@ #3]"Bnn<d*L:GJ#*Y-7^^y
!1.&().*j*i() $+ ,/==+D'&

8A;%8$&' ,Er*   c                 (    | j                  dd       y r?   _TestIteratorFailurer}   s    r%   ,testIteratorFailureSingleProcessSingleThreadzETestParallelismFramework.testIteratorFailureSingleProcessSingleThread      a#r*   c                 (    | j                  dd       y r   r   r}   s    r%   +testIteratorFailureSingleProcessMultiThreadzDTestParallelismFramework.testIteratorFailureSingleProcessMultiThread  r   r*   c                 (    | j                  dd       y r   r   r}   s    r%   +testIteratorFailureMultiProcessSingleThreadzDTestParallelismFramework.testIteratorFailureMultiProcessSingleThread       	a#r*   c                 (    | j                  dd       y r   r   r}   s    r%   *testIteratorFailureMultiProcessMultiThreadzCTestParallelismFramework.testIteratorFailureMultiProcessMultiThread  r   r*   c                 &   t        ddg      }| j                  t        |||      }| j                  dt	        |             t        ddg      }| j                  t        |||      }| j                  dt	        |             t        ddg      }| j                  t        |||      }| j                  dt	        |             ||z  dkD  rBt        ddg      }| j                  t        |||d      }| j                  dt	        |             t        dt        d            }| j                  t        |||      }| j                  dt	        |             t        dg       }| j                  t        |||      }| j                  dt	        |             y)	z$Tests apply with a failing iterator.
   r   	   rN   r@   Tr   N)rw   r   rE   r   r_   r   r   s        r%   r   z-TestParallelismFramework._TestIteratorFailure  sd   
 2s#Dnn_dM<PGQG%2s#Dnn_dM<PGQG%2s#Dnn_dM<PGQG%|#a'R!%d#,+-1	  3g
 q#g,'2uRy)Dnn_dM<PGQG%1b!Dnn_dM<PGQG%r*   c                 (    | j                  dd       y r?   _TestSharedAttrsWorkr}   s    r%   0testTestSharedAttrsWorkSingleProcessSingleThreadzITestParallelismFramework.testTestSharedAttrsWorkSingleProcessSingleThread  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   /testTestSharedAttrsWorkSingleProcessMultiThreadzHTestParallelismFramework.testTestSharedAttrsWorkSingleProcessMultiThread  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   /testTestSharedAttrsWorkMultiProcessSingleThreadzHTestParallelismFramework.testTestSharedAttrsWorkMultiProcessSingleThread  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   .testTestSharedAttrsWorkMultiProcessMultiThreadzGTestParallelismFramework.testTestSharedAttrsWorkMultiProcessMultiThread  r   r*   c                 B   | j                  d      }d|_        dddgg ddgg dg}| j                  t        ||||d	g
       d}|D ]  }|t	        |      z  } | j                  ||j                         t        ddg      dft        dg d      dft        ddg      dffD ]|  \  }}| j                  d      }|}| j                  t        ||||dg
       | j                  ||j                  d|d|j                  d|j                  d|j                         ~ y)z0Tests that Apply successfully uses shared_attrs.T   foobarbazxyabcdr^   )r   r   rN   r   r@   r   r@   r   rN   r      rU   z'Failure count did not match. Expected: z
, actual: z for failing iterator of size z, failing indices )msgN)r   r^   r   r`   r_   r   rw   rE   rU   ry   rz   )	r5   rb   rh   r   r"   expected_sumrt   failing_iteratorexpected_failure_counts	            r%   r  z-TestParallelismFramework._TestSharedAttrsWork  sa    %%d+L"$LE5>2Sz2v>DNN%  ,!1 2  4 Lc#hl \<#>#>?
 '6a!&=&'&)+:2y+I1*M&5a!&=q%A%C	!		 ''-ld
nn_"!".#2"3  5 
 

$
$ "<#=#=  "2"B"BD  E%Cr*   c                 (    | j                  dd       y r?   #_TestThreadsSurviveExceptionsInFuncr}   s    r%   ;testThreadsSurviveExceptionsInFuncSingleProcessSingleThreadzTTestParallelismFramework.testThreadsSurviveExceptionsInFuncSingleProcessSingleThread      ,,Q2r*   c                 (    | j                  dd       y r   r  r}   s    r%   :testThreadsSurviveExceptionsInFuncSingleProcessMultiThreadzSTestParallelismFramework.testThreadsSurviveExceptionsInFuncSingleProcessMultiThread  r  r*   c                 (    | j                  dd       y r   r  r}   s    r%   :testThreadsSurviveExceptionsInFuncMultiProcessSingleThreadzSTestParallelismFramework.testThreadsSurviveExceptionsInFuncMultiProcessSingleThread       	,,Q2r*   c                 (    | j                  dd       y r   r  r}   s    r%   9testThreadsSurviveExceptionsInFuncMultiProcessMultiThreadzRTestParallelismFramework.testThreadsSurviveExceptionsInFuncMultiProcessMultiThread  r"  r*   c           	          | j                  d      }dgdz  }| j                  t        ||||dgt               | j	                  t        |      |j                         y NTrA   rN   rU   )r   r   r   r   r   rS   rX   r   r_   rU   r5   rb   rh   r   r"   s        r%   r  z<TestParallelismFramework._TestThreadsSurviveExceptionsInFunc	  a    %%d+LD1HDNN<  ,!0 1#;  = 	SY : :;r*   c                 (    | j                  dd       y r?   &_TestThreadsSurviveExceptionsInHandlerr}   s    r%   >testThreadsSurviveExceptionsInHandlerSingleProcessSingleThreadzWTestParallelismFramework.testThreadsSurviveExceptionsInHandlerSingleProcessSingleThread      //15r*   c                 (    | j                  dd       y r   r+  r}   s    r%   =testThreadsSurviveExceptionsInHandlerSingleProcessMultiThreadzVTestParallelismFramework.testThreadsSurviveExceptionsInHandlerSingleProcessMultiThread  r.  r*   c                 (    | j                  dd       y r   r+  r}   s    r%   =testThreadsSurviveExceptionsInHandlerMultiProcessSingleThreadzVTestParallelismFramework.testThreadsSurviveExceptionsInHandlerMultiProcessSingleThread       	//15r*   c                 (    | j                  dd       y r   r+  r}   s    r%   <testThreadsSurviveExceptionsInHandlerMultiProcessMultiThreadzUTestParallelismFramework.testThreadsSurviveExceptionsInHandlerMultiProcessMultiThread#  r3  r*   c           	          | j                  d      }dgdz  }| j                  t        ||||dgt               | j	                  t        |      |j                         y r&  r'  r(  s        r%   r,  z?TestParallelismFramework._TestThreadsSurviveExceptionsInHandler(  r)  r*   c                       fd} fd} ||        fd} ||        fd} ||        fd} ||        fd} ||       y)zCTests that fail_on_error produces the correct exception on failure.c                     	  |         j                  d       y # t        $ r
}Y d }~y d }~wt        $ r'}j                  dt        |      z          Y d }~y d }~ww xY w)Nz=Setting fail_on_error should raise any exception encountered.zGot unexpected error: )failr0   r,   str)	test_funcrW   r5   s     r%   _ExpectCustomExceptionzLTestParallelismFramework.testFailOnErrorFlag.<locals>._ExpectCustomException:  sR    5		K	M  5		*SV3445s    	AAAAc            	      n    j                  d      } dgdz  }j                  t        |dd| dgd       y )NTrA   rN   r@   rU   )r   r   r   )r   r   rS   )r   r"   r5   s     r%   _RunFailureFunczETestParallelismFramework.testFailOnErrorFlag.<locals>._RunFailureFuncD  sF    ''-ldQhd
nn\".#2"3#'  )r*   c                      t        ddg      } j                  t        | ddd      }j                  dt	        |             y )Nr   r   r@   Tr   rw   r   rE   r   r_   r"   r   r5   s     r%    _RunFailingIteratorFirstPositionzVTestParallelismFramework.testFailOnErrorFlag.<locals>._RunFailingIteratorFirstPositionQ  =    R!%da$Og
q#g,'r*   c                      t        ddg      } j                  t        | ddd      }j                  dt	        |             y )Nr   rN   r@   Tr   r@  rA  s     r%   )_RunFailingIteratorPositionMiddlePositionz_TestParallelismFramework.testFailOnErrorFlag.<locals>._RunFailingIteratorPositionMiddlePositionX  rC  r*   c                      t        ddg      } j                  t        | ddd      }j                  dt	        |             y )Nr   r   r@   Tr   r@  rA  s     r%   _RunFailingIteratorLastPositionzUTestParallelismFramework.testFailOnErrorFlag.<locals>._RunFailingIteratorLastPosition_  rC  r*   c                      t        dg d      } j                  t        | ddd      }j                  dt	        |             y )Nr   r  r@   Tr   r@  rA  s     r%   $_RunFailingIteratorMultiplePositionszZTestParallelismFramework.testFailOnErrorFlag.<locals>._RunFailingIteratorMultiplePositionsf  s;    R+da$Og
q#g,'r*   NrA   )r5   r<  r>  rB  rE  rG  rI  s   `      r%   testFailOnErrorFlagz,TestParallelismFramework.testFailOnErrorFlag5  sW    
5	) ?+(
 ;<(
 DE(
 :;(
 ?@r*   c                 (    | j                  dd       y r?   *_TestRecursiveDepthThreeDifferentFunctionsr}   s    r%   BtestRecursiveDepthThreeDifferentFunctionsSingleProcessSingleThreadz[TestParallelismFramework.testRecursiveDepthThreeDifferentFunctionsSingleProcessSingleThreadm      33Aq9r*   c                 (    | j                  dd       y r   rL  r}   s    r%   AtestRecursiveDepthThreeDifferentFunctionsSingleProcessMultiThreadzZTestParallelismFramework.testRecursiveDepthThreeDifferentFunctionsSingleProcessMultiThreadq  rO  r*   c                 (    | j                  dd       y r   rL  r}   s    r%   AtestRecursiveDepthThreeDifferentFunctionsMultiProcessSingleThreadzZTestParallelismFramework.testRecursiveDepthThreeDifferentFunctionsMultiProcessSingleThreadu       	33Aq9r*   c                 (    | j                  dd       y r   rL  r}   s    r%   @testRecursiveDepthThreeDifferentFunctionsMultiProcessMultiThreadzYTestParallelismFramework.testRecursiveDepthThreeDifferentFunctionsMultiProcessMultiThreadz  rT  r*   z,This warning should only be printed on MacOSc                 .   t        d      }t               }|j                  |       | j                  dd       d}|j                  d   D cg c]  }|j                  |       }}| j                  t        |             |j                  |       y c c}w Nr   r   8If you experience problems with multiprocessing on MacOSinfo)	r   r   
addHandlerrM  messages
startswith
assertTrueanyremoveHandlerr5   rZ   mock_log_handlermacos_messagemessagecontains_messages         r%   #testMacOSLogsMultiprocessingWarningz<TestParallelismFramework.testMacOSLogsMultiprocessingWarning  s     %]3F)+
&'33Aq9NM (0088G 	=)8   	OOC()*
)*   Bz'This warning should be printed on MacOSc                 .   t        d      }t               }|j                  |       | j                  dd       d}|j                  d   D cg c]  }|j                  |       }}| j                  t        |             |j                  |       y c c}w rX  	r   r   r[  rM  r\  r]  assertFalser_  r`  ra  s         r%   ,testNonMacOSDoesNotLogMultiprocessingWarningzETestParallelismFramework.testNonMacOSDoesNotLogMultiprocessingWarning  s     %]3F)+
&'33Aq9NM (0088G 	=)8   	S)*+
)*rg  c                 .   t        d      }t               }|j                  |       | j                  dd       d}|j                  d   D cg c]  }|j                  |       }}| j                  t        |             |j                  |       y c c}w )Nr   r@   r   rY  rZ  ri  ra  s         r%   (testMultithreadingDoesNotLogMacOSWarningzATestParallelismFramework.testMultithreadingDoesNotLogMacOSWarning  s    $]3F)+
&'33Aq9NM (0088G 	=)8   	S)*+
)*rg  c                     g d}|D cg c]  }|||g	 }}| j                  t        |||      }| j                  dt        |      t	        |      z   z  t        |             yc c}w )zTests recursive application of Apply.

    Calls Apply(A), where A calls Apply(B) followed by Apply(C) and B calls
    Apply(C).

    Args:
      process_count: Number of processes to use.
      thread_count: Number of threads to use.
    )r   r@   r  r@   rN   re   N)r   rp   r   rl   r_   )r5   rb   rh   	base_argscountr"   r   s          r%   rM  zCTestParallelismFramework._TestRecursiveDepthThreeDifferentFunctions  sh      I>GHiU]L%0iDHnn<d*L:GQ#i.3y>9:CLI	 Is   A#c                 (    | j                  dd       y r?   +_TestExceptionInProducerRaisesAndTerminatesr}   s    r%   CtestExceptionInProducerRaisesAndTerminatesSingleProcessSingleThreadz\TestParallelismFramework.testExceptionInProducerRaisesAndTerminatesSingleProcessSingleThread      44Q:r*   c                 (    | j                  dd       y r   rr  r}   s    r%   BtestExceptionInProducerRaisesAndTerminatesSingleProcessMultiThreadz[TestParallelismFramework.testExceptionInProducerRaisesAndTerminatesSingleProcessMultiThread  ru  r*   c                 (    | j                  dd       y r   rr  r}   s    r%   BtestExceptionInProducerRaisesAndTerminatesMultiProcessSingleThreadz[TestParallelismFramework.testExceptionInProducerRaisesAndTerminatesMultiProcessSingleThread       	44Q:r*   c                 (    | j                  dd       y r   rr  r}   s    r%   AtestExceptionInProducerRaisesAndTerminatesMultiProcessMultiThreadzZTestParallelismFramework.testExceptionInProducerRaisesAndTerminatesMultiProcessMultiThread  rz  r*   c                 z    | }	 | j                  t        |||       | j                  d       y # t        $ r Y y w xY w)Nz!Did not raise expected exception.)r   rE   r9  	TypeError)r5   rb   rh   r"   s       r%   rs  zDTestParallelismFramework._TestExceptionInProducerRaisesAndTerminates  s?     D
nn_dM<H
ii34 
s   ). 	::c                 (    | j                  dd       y r?   _TestSkippedArgumentsr}   s    r%   -testSkippedArgumentsSingleThreadSingleProcesszFTestParallelismFramework.testSkippedArgumentsSingleThreadSingleProcess  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   ,testSkippedArgumentsMultiThreadSingleProcesszETestParallelismFramework.testSkippedArgumentsMultiThreadSingleProcess  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   ,testSkippedArgumentsSingleThreadMultiProcesszETestParallelismFramework.testSkippedArgumentsSingleThreadMultiProcess  r   r*   c                 (    | j                  dd       y r   r  r}   s    r%   +testSkippedArgumentsMultiThreadMultiProcesszDTestParallelismFramework.testSkippedArgumentsMultiThreadMultiProcess  r   r*   c                    d|z  |z  }t        d|dz         }| j                  t        |||t              }| j	                  |dz  t        |             | j	                  |dz  t        |             |D cg c]  }d|z  	 }}| j                  t        |||t              }| j	                  dt        |             y c c}w )Nrr   r@   )rg   r   )r   r   rE   ru   r   r_   rl   )r5   rb   rh   nr"   r   r  s          r%   r  z.TestParallelismFramework._TestSkippedArguments  s     	
ML(AAE?Dnn_!*))C	  EG
 	QUCL)QUCL)   4aAE4D nn_!*))C	  EG
 	QG% !s   6B>#OFFER_GSUTIL_M_SUGGESTION_THRESHOLDrr   GetTermLinesd   )return_valuec                 H   d|_         t        d      }t               }|j                  |       | j	                  t
        t        d      dd       |j                  d   D cg c]  }|t        k(   }}| j                  t        |             |j                  |       y c c}w )Nr  r   rr   r@   rb   rh   rZ  )r  r   r   r[  r   rE   r   r\  PARALLEL_PROCESSING_MESSAGEr^  r_  r`  r5   mock_get_term_linesrZ   rb  rd  re  s         r%   6testSequentialApplyRecommendsParallelismAfterThresholdzOTestParallelismFramework.testSequentialApplyRecommendsParallelismAfterThreshold
  s     (+$$]3F)+
&'NN?E!HAANN (0088G 	..8   	OOC()*
)*s   !B#OFFER_GSUTIL_M_SUGGESTION_FREQUENCYr   c                 <   t        d      }t               }|j                  |       | j                  t        t        d      dd       |j                  d   D cg c]  }|t        k(   }}| j                  t        |      d       |j                  |       y c c}w )Nr      r@   r  rZ  r   r   r   r[  r   rE   r   r\  r  r   rl   r`  r  s         r%   =testSequentialApplyRecommendsParallelismAtSuggestionFrequencyzVTestParallelismFramework.testSequentialApplyRecommendsParallelismAtSuggestionFrequency      
 %]3F)+
&'NN?E"IQQNO (0088G 	..8   	S)*A.
)*   Bc                 <   t        d      }t               }|j                  |       | j                  t        t        d      dd       |j                  d   D cg c]  }|t        k(   }}| j                  t        |      d       |j                  |       y c c}w )Nr      r@   r  rZ  r   r  r  s         r%   HtestSequentialApplyRecommendsParallelismAtEndIfLastSuggestionIsOutOfViewzaTestParallelismFramework.testSequentialApplyRecommendsParallelismAtEndIfLastSuggestionIsOutOfView.  r  r  r   c                 <   t        d      }t               }|j                  |       | j                  t        t        d      dd       |j                  d   D cg c]  }|t        k(   }}| j                  t        |      d       |j                  |       y c c}w )Nr   r  r@   r  rZ  rr   r  r  s         r%   ItestSequentialApplyDoesNotRecommendParallelismAtEndIfLastSuggestionInViewzbTestParallelismFramework.testSequentialApplyDoesNotRecommendParallelismAtEndIfLastSuggestionInView@  r  r  c                    t        j                  t        j                  j                  j
                        t        _        dt        j                  t        j                  j                  j
                        it        _        | j                  d      j                          | j                  t        j                         | j                  t        j                         y )N)specr   T)r   Mockbotor   
connectionS3Connectionr	   provider_poolr   _ResetConnectionPoolassertIsNonerj  r}   s    r%   -testResetConnectionPoolDeletesConnectionStatezFTestParallelismFramework.testResetConnectionPoolDeletesConnectionStateR  s     II477+=+=+J+JKJdiiTWW//<<= J 	t113j++,Z--.r*   )Mr9   r:   r;   r   r   r   r   r   r   r   r   r   skipIfr   r   r   r)   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r
  r  r  r  r!  r$  r  r-  r0  r2  r5  r,  rJ  rN  rQ  rS  rV  
skipUnlessr   rf  rk  rm  rM  rt  rw  ry  r|  rs  r  r  r  r  r  r   patchobjectr   r  r  r  r  r  rA   r*   r%   r   r     s   0- "!# $+;.     8??:LM N  8??:LM N  . . % % % % 8??:LM% N % 8??:LM% N % 6 6 8??:LM? N ? ? ? 8??:LM? N ? ' '4 $ $ $ $ 8??:LM$ N $ 8??:LM$ N $  &  &D $ $ $ $ 8??:LM$ N $ 8??:LM$ N $ $E $EL 3 3 3 3 8??:LM3 N 3 8??:LM3 N 3 
< 
< 6 6 6 6 8??:LM6 N 6 8??:LM6 N 6 
< 
< 
4A  4Al : : : : 8??:LM: N : 8??:LM: N : 8vMN+ O + 8??6DE8??:LM+ N F + + + J J$ ; ; ; ; 8??:LM; N ; 8??:LM; N ;   % % % % 8??:LM% N % 8??:LM% N % & &, ::WCQG::Wn3?+ @ H+  ::WCSI::WCRH::Wn3?+ @ I J+ ::WCSI::WCRH::Wn1=+ > I J+ ::WCSI::WCRH::Wn1=+ > I J+	/r*   r   c            	           e Zd ZdZeej                  j                  ed e	j                  ej                  d             ej                  ed      d                      Zy)+TestParallelismFrameworkWithMultiprocessingz1Tests that only run with multiprocessing enabled.r  N)side_effectr   c                     d}t        d      j                  t        g dt        |dt               t        |      D ]'  }| j                  t        j                  d             ) y )Nrr   T)r@   rr   r   r   )rb   rh   rg   g      ?)timeout)	r   rj   rE   r\   r   r   r  
call_queuer   )r5   mock_reset_connection_poolexpected_call_countr   s       r%   +testResetConnectionPoolCalledOncePerProcesszWTestParallelismFrameworkWithMultiprocessing.testResetConnectionPoolCalledOncePerProcessf  s]     OY-*=)*(7	  9 &'

s34 (r*   )r9   r:   r;   r   r   r   r  r  r   r'   partialr  putr   r  r   r  rA   r*   r%   r  r  c  si    9::[+!2!2!2:>>4!H  J 8??:LM
5 NJ 

5r*   r  c                       e Zd ZdZeZy).TestParallelismFrameworkWithoutMultiprocessingaE  Tests parallelism framework works with multiprocessing module unavailable.

  Notably, this test has no way to override previous calls
  to gslib.util.CheckMultiprocessingAvailableAndInit to prevent the
  initialization of all of the global variables in command.py, so this still
  behaves slightly differently than the behavior one would see on a machine
  where the multiprocessing functionality is actually not available (in
  particular, it will not catch the case where a global variable that is not
  available for the sequential path is referenced before initialization).
  N)r9   r:   r;   r   r   r   rA   r*   r%   r  r  x  s    	 :-r*   r  r2   )Fr   
__future__r   r   r   r   r'   rG   r   r   rI   textwraprO   r   r   r  boto.storage_urir   r	   gslibr
   r   gslib.commandr   r   r   gslib.tests.mock_cloud_apir    gslib.tests.mock_logging_handlerr   gslib.tests.testcaseteststestcasegslib.tests.testcase.baser   gslib.tests.util&gslib.utils.parallelism_framework_utilr   r   gslib.utils.system_utilr   r   r!   fillr  r)   r   r,   r0   rE   rL   rQ   rS   rX   r\   r`   rc   rp   rk   ru   Iteratorrw   r   r   GsUtilUnitTestCaser   Queuer  r  r  rA   r*   r%   <module>r     sV  , 3 & %  '  	  
      - '   ! 1 ) 3 ? ' ' 7 % W J * .
  #mhmm<'=  = @D	 D $%9i 97F/A

"""J4$cll $, '  F.k .M	/x:: M	/d +$**,
5(2M2M 5*:5M :r*   