DdZddlmZddlmZddlmZddlmZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddlmZddlmZddl m!Z!ddl"m#cm$Z$ddl"m%Z%ddl"m&Z&ddl'm(Z(ddl)m*Z*dZ+Gdde,Z-e%j\e$j^dGdde!j`Z1y)z*Contains gsutil base unit test case class.)absolute_import)print_function)division)unicode_literalsN)get_utf8able_str) project_id)wildcard_iterator)BotoTranslation)CloudApiDelegator) CommandRunner)ApiMapConstants) ApiSelector)DiscardMessagesQueue) GcsJsonApi)MockLoggingHandler)base)unittest)WorkingDirectory)UTF8) print_to_fdctjs|jy |jy#t$rYywxYw)zCSuppress IOError when closing sys.stdout or sys.stderr in tearDown.N)sixPY2closeIOError)fds 5platform/gsutil/gslib/tests/testcase/unit_testcase.py_AttemptToCloseSysFdr1s7 HHJ  HHJ   s4 AAc eZdZdZedZy) GsutilApiUnitTestClassMapFactoryzClass map factory for use in unit tests. BotoTranslation is used for all cases so that GSMockBucketStorageUri can be used to communicate with the mock XML service. ctjttjti}tjti}||d}|S)z*Returns a class map for use in unit tests.gss3)rXMLr JSON)cls gs_class_map s3_class_map class_maps r GetClassMapz,GsutilApiUnitTestClassMapFactory.GetClassMapGsC     L OO_5L#<8I N)__name__ __module__ __qualname____doc__ classmethodr+r,rr r @s   r,r zNot running integration tests.ceZdZdZedZfdZfdZ d dZed dZ ed dZ e ddZ dd Z dd ZxZS)GsUtilUnitTestCasez!Base class for gsutil unit tests.ctjjtj|_t |_tj|_ t|j |j|_ dt_y)N)bucket_storage_uri_classgsutil_api_class_map_factoryzmock-project-id-for-unit-tests)rGsUtilTestCase setUpClassutilGSMockBucketStorageUrimock_bucket_storage_urir !mock_gsutil_api_class_map_factorylogging getLoggerloggerr command_runnerrUNIT_TEST_PROJECT_ID)r's rr9zGsUtilUnitTestCase.setUpClass[se""$"&"="=C,LC)""$CJ&!$!r? root_logger isEnabledForDEBUG is_debugginghandlerslog_handlers_savelog_handler_filelog_handler_stream StreamHandlertemp_log_handler)selfr __class__s rrFzGsUtilUnitTestCase.setUphsd d)+DzzDzzD#++-B ww99R&cj99R8cj#++-B ww99R&cj99R8cj D D((*D((55gmmDD!--66D ( 0 0 2B iiD1D#11$2I2IJD!%!6!6 7Dr,c: tt| |j|j_|j j|j j|jjd|jj}|jjtj|jtj jdtj"jdt$j&r=tj j}tj"j}n= tj j}tj"j}|j,Dcgc]}t%j.|c}|j0Dcgc]}t%j.|c}t%j.t3|}t%j.t3|}|dj5|j0z }|dj5|j,z }t7tj t7tj"|j8t_|j:t_tj|j<tj|j>t%j.|jA}|jBrb|r`tEdjG|tj"tE|tj"tEdtj"|jBrb|r`tEdjG|tj"tE|tj"tEdtj"|jBrd|ratEdjG|tj"tE|tj"tEd tj"yyy#t($rtj jdtj"jdtj j*j}tj"j*j}YvwxYwcc}wcc}w) Nrz==== stdout {} ==== file==== end stdout ==== z==== stderr {} ==== ==== end stderr ==== z==== log output {} ==== z==== end log output ==== )$rEr4tearDownrZrUrYr^flushrr\seekreadrPunlinkr[rHrIrKrrUnicodeDecodeErrorbufferrT ensure_textrSrjoinrrJrLrOrRidrXrformat)r_ log_outputrIrKstring_idr`s rrgzGsUtilUnitTestCase.tearDownsD d,. $ 6 6D!!  #((--/J!!#IId##$JJOOAJJOOA wwzz fzz f*"" ,0+B+BC+BS__V+BC+/+B+BC+BS__V+BC __-f5 6F __-f5 6F bggd--..F bggd--..F$$!!CJ!!CJIIdIId //$'') $C V)005CJJG&szz**< V)005CJJG&szz**< Z-44S9 K*3::..SZZ@(7 *  ""'')""'') * DCsD} tj|jA}|jrW| rUt d j || |j t | |j t d |j |jrW| rUt d j || |j t | |j t d |j |jrW| rUt dj || |j t | |j t d|j tjjdtjjdtjjtjjg}|r|j| |r|j| |r|j| tC|dk(r|dStE|S#1swYZxYw#t4$rtjjdtjjdtjj6j} tjj6j} Y/wxYw#tjjdtjjdtj2r=tjj} tjj} n tjj} tjj} n#t4$rtjjdtjjdtjj6j} tjj6j} YnwxYwt#j$|j9| | j;djd tj<| j>D} tj|jA}|jrW| rUt d j || |j t | |j t d |j |jrW| rUt d j || |j t | |j t d |j |jrW| rUt dj || |j t | |j t d|j tjjdtjjdtjjtjjwxYw)aMethod for calling gslib.command_runner.CommandRunner. Passes parallel_operations=False for all tests, optionally saving/returning stdout output. We run all tests multi-threaded, to exercise those more complicated code paths. TODO: Change to run with parallel_operations=True for all tests. At present when you do this it causes many test failures. Args: command_name: The name of the command being run. args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). return_stdout: If True, will save and return stdout produced by command. return_stderr: If True, will save and return stderr produced by command. return_log_handler: If True, will return a MockLoggingHandler instance that was attached to the command's logger while running. cwd: The working directory that should be switched to before running the command. The working directory will be reset back to its original value after running the command. If not specified, the working directory is left unchanged. Returns: One or a tuple of requested return values, depending on whether return_stdout, return_stderr, and/or return_log_handler were specified. Return Types: stdout - str (binary in Py2, text in Py3) stderr - str (binary in Py2, text in Py3) log_handler - MockLoggingHandler  z RunCommand of {} rcrF)argsheadersdebugparallel_operations do_shutdownN c3TK|] \}}|rd|zdj|z"yw)z%s: z N)ro).0levelrecordss r z0GsUtilUnitTestCase.RunCommand..s2 HneW  e fkk'2 2 Hs&(z#==== logging RunCommand {} {} ==== z ==== end logging ==== z"==== stdout RunCommand {} {} ==== rez"==== stderr RunCommand {} {} ==== rf)#rrnrorXrrqrLrHrIrirKrjrSappendrTtruncaterr>r? addHandlersetLevelrWrrARunNamedCommandrrlrm removeHandlerr iteritemsmessagesrplentuple)r_ command_namerwrxry return_stdout return_stderrreturn_log_handlercwd command_linerIrKmock_log_handlerrrrt to_returns r RunCommandzGsUtilUnitTestCase.RunCommandsN :2D??388\NT,A#BCL (// ='') JJOOAJJOOA ZZ__ F ZZ__ F  $$V,  $$V,JJOOAJJOOAJJJJ)+ l#../?@   %..w}}=8 C  ++L154;27@E8= , ? ! jjooa jjooa "" ,::??$&::??$&  %334DE99 # .>.G.G Hj OODGGI &c  z:AA )) + JT%5%56/d6F6FG  v9@@ )) + F!1!12,43C3CD  v9@@ )) + F!1!12,43C3CD jjooa jjooa jj jjIvv'( 9~ q\  E ! "" , **//!  **//! ::$$))+&::$$))+&  , jjooa jjooa "" ,::??$&::??$&! , **//!  **//! ::$$))+&::$$))+&  ,  %334DE99 # .>.G.G Hj OODGGI &c  z:AA )) + JT%5%56/d6F6FG  v9@@ )) + F!1!12,43C3CD  v9@@ )) + F!1!12,43C3CD jjooa jjooa jj jjs^2 X"="U7X"3 > @##??K$4$45??#&  ##""//&  N S88+ ZZ13#(  **r,c|}t|dr |j}tj||j ||S)aConvenience method for instantiating a test instance of WildcardIterator. This makes it unnecessary to specify all the params of that class (like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri). Also, naming the factory method this way makes it clearer in the test code that WildcardIterator needs to be set up for testing. Args are same as for wildcard_iterator.wildcard_iterator(), except there are no class args for bucket_storage_uri_class or gsutil_api_class. Args: uri_or_str: StorageUri or string representing the wildcard string. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. debug: debug level to pass to the underlying connection (0..3) Returns: WildcardIterator, over which caller can iterate. uri) exclude_tuple)hasattrrr CreateWildcardIteratorr)r' uri_or_strrry uri_strings r_test_wildcard_iteratorz*GsUtilUnitTestCase._test_wildcard_iteratorVsD.Jz5!>>j  3 3J474E4Ee4LBO QQr,cPtj||||tjS)aConvenience method for instantiating a testing instance of StorageUri. This makes it unnecessary to specify bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri. Also naming the factory method this way makes it clearer in the test code that StorageUri needs to be set up for testing. Args, Returns, and Raises are same as for boto.storage_uri(), except there's no bucket_storage_uri_class arg. Args: uri_str: Uri string to create StorageUri for. default_scheme: Default scheme for the StorageUri debug: debug level to pass to the underlying connection (0..3) validate: If True, validate the resource that the StorageUri refers to. Returns: StorageUri based on the arguments. )boto storage_urir:r;)uri_strdefault_schemeryvalidates r_test_storage_uriz$GsUtilUnitTestCase._test_storage_urius(*   G^UH 77 99r,c|xs|jd}tj|d|jdtj }|j ||jj| t|t|D];\}}|j||dj|jt =|S#t$r0t|Dcgc]}|jdncc}w}}YwxYw) aaCreates a test bucket. The bucket and all of its contents will be deleted after the test. Args: bucket_name: Create the bucket with this name. If not provided, a temporary test bucket name is constructed. test_objects: The number of objects that should be placed in the bucket or a list of object names to place in the bucket. Defaults to 0. storage_class: storage class to use. If not provided we us standard. provider: string provider to use, default gs. Returns: StorageUri for the created bucket. bucketz://F)suppress_consec_slashesr6) storage_classobjztest {}) bucket_uri object_namecontents) MakeTempNamerrlowerr:r; create_bucketrGriter TypeErrorrange enumerate CreateObjectrqencoder) r_ bucket_name test_objectsrproviderr_inames r CreateBucketzGsUtilUnitTestCase.CreateBuckets*<!2!28!J=9J'M <\*4 :$(!*!1!1!!4!;!;D!AC+  M8=l8KL8K1d''.8KLlLMs= CD )DD  D c|xs|j|j}|xs|jd}|j|}||j ||S)a+Creates a test object. Args: bucket_uri: The URI of the bucket to place the object in. If not specified, a new temporary bucket is created. object_name: The name to use for the object. If not specified, a temporary test object name is constructed. contents: The contents to write to the object. If not specified, the key is not written to, which means that it isn't actually created yet on the server. Returns: A StorageUri for the created object. )rr)rdefault_providerrclone_replace_nameset_contents_from_string)r_rrrkey_uris rrzGsUtilUnitTestCase.CreateObjects`Pt00$:O:O0PJ9!2!25!9K++K8G &&x0 Nr,)NNrFFFN)r)Nr)rdrT)NrNr#)NNN)r-r.r/r0r1r9rFrgrrr staticmethodrrr __classcell__)r`s@rr4r4Ws) G G8:2Al$$$)FP**(QQ<990 $ !!% $Lr,r4)2r0 __future__rrrrr>rPrHrMrr boto.utilsrgslibrr gslib.boto_translationr gslib.cloud_api_delegatorr gslib.command_runnerr gslib.cs_api_mapr rgslib.discard_messages_queuergslib.gcs_json_apir gslib.tests.mock_logging_handlerrgslib.tests.testcasergslib.tests.utiltestsr:rrgslib.utils.constantsrgslib.utils.text_utilrrobjectr skipUnlessRUN_UNIT_TESTSr8r4r2r,rrs1&%'  '#27.,(=)?%%-&-  v.T((*JKo,,oLor,