
    eO                     d   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 dd	lm
Z
 dd
lmZ ddlmZ ddlmZ ddlmc mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlmZmZ  e eddd             ddlm Z   G d dejB                        Z"y)z#Tests for gsutil utility functions.    )absolute_import)print_function)division)unicode_literals)	boto_util)	constants)system_util)	ls_helper)
retry_util)	text_util)	unit_utilN)SetEnvironmentForTest)
TestParams)CompareVersions)DecimalShort)HumanReadableWithDecimalPlaces)
PrettyTime)add_moveMovedModulemockzunittest.mock)r   c                   ^   e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zej                  j                  ej                   d
      ej                  j                  ej"                  d      d               Zd Zd Zd Zd Zd Zd Zd Zd Zej                  j                  ed      d        Zy)TestUtilzTests for utility functions.c                    | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d	      d
       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       y)z7Tests converting byte counts to human-readable strings.r   0 B  z1023 B   1 KiB   1 MiB   @1 GiB333333Az5.3 GiB皙Bz2.7 TiB           z1 PiB            z1 EiBN)assertEqualr   MakeHumanReadableselfs    (platform/gsutil/gslib/tests/test_util.pytestMakeHumanReadablezTestUtil.testMakeHumanReadable9   s    Y003U;Y006AY006@Y0097CY0097CY00?KY00?KY0097CY0097C    c                    | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d	      d
       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       y)z6Tests converting bit counts to human-readable strings.r   z0 bitr   z1023 bitr   z1 Kibitr   z1 Mibitr    z1 Gibitr"   z	5.3 Gibitr#   z	2.7 Tibitr$   z1 Pibitr%   z1 EibitN)r&   r   MakeBitsHumanReadabler(   s    r*   testMakeBitsHumanReadablez"TestUtil.testMakeBitsHumanReadableE   s    Y44Q7AY44T:JGY44T:IFY44W=yIY44W=yIY44]C "Y44]C "Y44W=yIY44W=yIr,   c                    | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d	       | j                  t        j                  d
      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d      d       | j                  t        j                  d       d       | j                  t        j                  d!      d"       | j                  t        j                  d#      d"       | j                  t        j                  d$      d"       | j                  t        j                  d%      d"       | j                  t        j                  d&      d'       | j                  t        j                  d(      d'       | j                  t        j                  d)      d'       | j                  t        j                  d*      d'       y+),z7Tests converting human-readable strings to byte counts.1   15   z15.3z15.7   1023r   1kr   2048i   z1 kz1 Kz1 KBr   z1 mr   z1 Mz1 MBr   z1 gr    z1 Gz1 GBr!   1tl        1T1TB1TiBz1	   pr$   z1	   Pz1	   PBz1	   PiB1er%   1E1EB1EiBN)r&   r   HumanReadableToBytesr(   s    r*   testHumanReadableToBytesz!TestUtil.testHumanReadableToBytesS   sG   Y33C8!<Y33D92>Y33F;R@Y33F;R@Y33F;TBY33D94@Y33F;TBY33E:DAY33E:DAY33F;TBY33G<dCY33E:GDY33E:GDY33F;WEY33G<gFY33E:GDY33E:GDY33F;WEY33G<gFY33D97CY33D97CY33E:GDY33F;WEY33I>HY33I>HY33J?IY33K@'JY33D97CY33D97CY33E:GDY33F;WEr,   c                 $   t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd	      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        d
d
      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       t        dd      \  }}| j                  |       | j                  |       y)z,Tests CompareVersions for various use cases.z3.37z3.272z3.32z3.32prez3.31z3.4prez3.3preez3.3prefoobarbazz3.4z3.3z4.1z3.33z3.10z3.1N)r   
assertTrueassertFalse)r)   gms      r*   testCompareVersionszTestUtil.testCompareVersionsu   s   
 VU+FQOOAQS#&FQOOAOOAVY/FQOOAQY/FQOOAQXy1FQOOAQUF+FQQQS#&FQQQY/FQQQVY/FQQQXx0FQQQXu-FQQQVU+FQQQUE*FQOOAQUE*FQQQUF+FQOOAOOAVU+FQOOAQr,   c                    | j                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                         | j                  |j                  |j                         | j                  |j
                  |j
                         | j                  |j                  |j                         y N)r&   
proxy_type
proxy_host
proxy_port
proxy_rdns
proxy_user
proxy_pass)r)   pi1pi2s      r*   _AssertProxyInfosEqualzTestUtil._AssertProxyInfosEqual   s    S^^S^^4S^^S^^4S^^S^^4S^^S^^4S^^S^^4S^^S^^4r,   c                    t        dd      t        dd      t        dddid	
      t        dd      f}|D ]G  }t        j                  |j                  i |j                  }| j                  ||j                         I y )N)AFairlyShortKeyValuez!    AFairlyShortKey:        Value)argsexpected) rZ   z!    :                       Value)
AnotherKeyrZ   indent   z!        AnotherKey:         Value)r[   kwargsr\   )AKeyMuchLongerThanTheLastrZ   z#    AKeyMuchLongerThanTheLast:Value)r   r
   MakeMetadataLiner[   ra   r&   r\   )r)   test_paramsparamslines       r*   testMakeMetadataLinezTestUtil.testMakeMetadataLine   s    #?'JL='JL#:&.]'JL #I(MPQK ''&++J6==Jd
tV__- r,   c                    g d}g d}ddg}ddg}ddg}g d}t         j                  j                  t         j                  j                  t         j                  j                  t         j                  j                  d	}|D 	
cg c].  }|D ]'  }	|D ]   }
|D ]  }|D ]  }|D ]  }|||
||	|d
   " ) 0 }}}}
}	}}t        ddi      5  |D ]  }|j                  |j                  d            }|j                  d      }|j                  d      }|j                  d      }|j                  d      }t        |j                  d            }||d   k(  sd}t        j                  ||||||
      }|j                  r|j                  s5t        j                  |d   dd      }|j                  d      dk(  rd|_        | j                  t        j                  |      |        	 ddd       yc c}}}}
}	}w # 1 sw Y   yxY w)z<Tests SetProxyInfo for various proxy use cases in boto file.)socks4socks5http)hostname1.2.3.4Ni"  r   fooNBar)TFN)ri   rj   rk   https)rP   rO   rQ   rS   rT   rR   
http_proxyzhttp://host:50rO   rP   rQ   rS   rT   rR   rk   Fhost2   T)httplib2socksPROXY_TYPE_SOCKS4PROXY_TYPE_SOCKS5PROXY_TYPE_HTTPr   getbool	ProxyInforP   rQ   rR   rW   r   SetProxyInfo)r)   valid_proxy_typesvalid_proxy_hostvalid_proxy_portvalid_proxy_uservalid_proxy_passvalid_proxy_rdnsproxy_type_specp_hp_sp_pp_up_tp_dboto_proxy_config_test_valuestest_valuesrO   rP   rQ   rS   rT   rR   r\   s                          r*   testSetProxyInfozTestUtil.testSetProxyInfo   s3   44ayt}t}* ..22..22..//	O  1A%B %B 1A0@0@0@0A0@ & 1A& 1B& 1A& 1A& 1A& 1A " %B  
.>?	@6+$(()FG
 __\2
 __\2
 __\2
 __\2
+//,78
 of55*%%1;1;1;1;1;= ##(;(;''(?L(__\*d2"&H##I$:$:;$G$,	.5 7 
A	@!%B  
A	@s   	3G9DHHc                    ddg}t         j                  s|j                  d       i }|D ]  }d||<   	 t        |      5  |D ]E  }dD ]  }t        ||i      5  | j	                  t        j                  |      t        j                  t        j                  j                  d|j                         j                  d      rdnd	             |D ]Z  }||k(  r	| j	                  t        j                  |      t        j                  t        j                  j                  dd
             \ 	 ddd        dD ]j  }t        ||i      5  | j	                  t        j                  |      t        j                  t        j                  j                  dd             ddd       l dD ]m  }t        ||i      5  | j	                  t        j                  |      t        j                  t        j                  j                  dddd             ddd       o dD ]l  }t        ||i      5  | j	                  t        j                  |      t        j                  t        j                  j                  ddd             ddd       n H dD ]q  }dD ]j  }t        ||i      5  | j	                  t        j                  |      t        j                  t        j                  j                  dd
             ddd       l s 	 ddd       y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   #xY w# 1 sw Y   xY w# 1 sw Y   yxY w)z4Tests ProxyInfoFromEnvironmentVar for various cases.rq   https_proxyHTTPS_PROXYN)rl   zhttp://hostnamezhttps://hostnamerl   rp   i  P   r   )
1.2.3.4:50http://1.2.3.4:50zhttps://1.2.3.4:50rm   rs   )zfoo:bar@1.2.3.4:50zhttp://foo:bar@1.2.3.4:50zhttps://foo:bar@1.2.3.4:50rn   bar)rS   rT   )zbar@1.2.3.4:50zhttp://bar@1.2.3.4:50zhttps://bar@1.2.3.4:50)rS   )proxynoproxygarbage)r   r   )r	   
IS_WINDOWSappendr   rW   r   ProxyInfoFromEnvironmentVarrt   r{   ru   rx   lower
startswith)r)   valid_variables
clear_dictkeyenv_var
url_stringother_env_vars          r*   testProxyInfoFromEnvironmentVarz(TestUtil.testProxyInfoFromEnvironmentVar  s   #]3O!!]+Jjo 	z	*$'MJ$gz%:;''55g>""NN22J"==?55g>CBHI "1')))77F$$X^^%C%CT1MO "1 <; N
J %gz%:;''55g>""8>>#A#A9#%'( <;

J %gz%:;''55g>""8>>#A#A#,#%.3.3	56 <;	

J %gz%:;''55g>""8>>#A#A#,#%.356 <;
G %Z 5'=J$gz%:;''55g>""8>>#A#A4KM <; > 5] 
+	* <;  <; <; <; <;a 
+	*s   MCL	,MAL	M7AL%	M)AL2	>$M"AL?	5MLML"M%L/*M2L<7M?MMM)HandleExceptionsAndRebuildHttpConnectionsinfoc           	         t         j                  j                  d d d dd t        j                  dz         }t         j                  j                  d d d dd t        j                  dz
        } t        j
                         |       | j                  |j                         | j                  |j                          t        j
                         |       | j                  |j                  d       | j                  |j                         y )N   r2   r`   )r   http_wrapperExceptionRetryArgsr   LONG_RETRY_WARN_SECLogAndHandleRetriesrH   calledrI   r&   
call_count)r)   mock_log_info_fnmock_wrapped_fnretry_args_over_thresholdretry_args_under_thresholds        r*   testWarnAfterManyRetriesHandlerz(TestUtil.testWarnAfterManyRetriesHandlerC  s     !+ 7 7 J JdD!T9#@#@1#D!F!+!8!8!K!KdD!T9#@#@1#D"F %J""$%?@OOO**+%,,-$J""$%>?_//3OO$++,r,   c                    | j                  dt        d             | j                  dt        d             | j                  dt        d             | j                  dt        d             | j                  d	t        d
             | j                  dt        d             | j                  dt        d             | j                  dt        d             y)zTests DecimalShort for UI.z12.3bl   >? z123.5mi[z1.2ki  z1.0ki  432i  z43.2tg  zqBz43.2qg $Ѫ4cCz43250.0qg(=DN)r&   r   r(   s    r*   testUIDecimalShortzTestUtil.testUIDecimalShortX  s    Wl;78X|I67V\$/0V\$/0UL-.Wl>:;Wl>:;Zn!=>r,   c                    | j                  dt        d             | j                  dt        d             | j                  dt        d             | j                  dt        d             | j                  d	t        d
             y)zTests PrettyTime for UI.z25:02:10i` z01:00:00i  z00:59:59i  z100+ hrsi@~ z999+ hrsi Q%N)r&   r   r(   s    r*   testUIPrettyTimezTestUtil.testUIPrettyTimec  sk    ZE!23ZD!12ZD!12ZJ!78ZL!9:r,   c                    | j                  dt        dd             | j                  dt        d      d       | j                  dt        dd             | j                  dt        dd	             | j                  d
t        dd             | j                  dt        dd	             | j                  dt        dd             | j                  dt        dd             | j                  dt        dd             | j                  dt        dd             | j                  dt        dd             | j                  dt        dd             y)z,Tests HumanReadableWithDecimalPlaces for UI.z1.0 GiBi  @r2   r    z1.01 GiBr`   z	1.000 GiBi  P@r   z1.10 GiBi  @Fz	1.100 GiBz	10.00 MiBi   z2.01 GiBl     @ z2.0 GiBr   r   z0.00 Bz	0.00000 B   N)r&   r   r(   s    r*   $testUIHumanReadableWithDecimalPlacesz-TestUtil.testUIHumanReadableWithDecimalPlacesk  s;   Y34JANPY >w GKZ34JANP[34I1MOZ34KQOQ[34KQOQ[3L!DF 	Z!?
A!NOY >z1 MNU:1a@AX=aCD["@A"FGr,   c                     t        j                  d      }d}| j                  t        j                  |      |       | j                  t        j
                  |      |       y )N 9PpsRjBGjBh90IvIS96dgRc_UL6NyGqDl   Dqr9Ya*5LjSIdAvHHT98P9)six
ensure_strr&   r   DecodeLongAsStringEncodeStringAsLong)r)   amz_gen_as_stramz_gen_as_longs      r*    testAmzGenerationTypeConversionsz)TestUtil.testAmzGenerationTypeConversions  sR    ^^$FGNcOY11/B#%Y11.A$&r,   c                 V    t        j                  |||      }| j                  ||       y rN   )r   AddQueryParamToUrlr&   )r)   url
param_name	param_valexpected_urlnew_urls         r*   DoTestAddQueryParamToUrlz!TestUtil.DoTestAddQueryParamToUrl  s&    **3
IFGWl+r,   c                 ^    d}d}d}dj                  |||      }| j                  ||||       y )Nz+http://foo.bar/path/endpoint?a=1&a=2&b=3&c=newparamnevalue{}&{}={}formatr   r)   old_urlr   r   r   s        r*   )testAddQueryParamToUrlWorksForASCIIValuesz2TestUtil.testAddQueryParamToUrlWorksForASCIIValues  s9    ;GJI$$Wj)DL!!':y,Or,   c                 ^    d}d}d}dj                  |||      }| j                  ||||       y Nu.   http://foo.bar/path/êndpoint?Â=1&a=2&ß=3&c=u	   nêwparamu	   nêwvaluer   r   r   s        r*   (testAddQueryParamToUrlWorksForUTF8Valuesz1TestUtil.testAddQueryParamToUrlWorksForUTF8Values  9    >GJI$$Wj)DL!!':y,Or,   c                 ^    d}d}d}dj                  |||      }| j                  ||||       y r   r   r   s        r*   .testAddQueryParamToUrlWorksForRawUnicodeValuesz7TestUtil.testAddQueryParamToUrlWorksForRawUnicodeValues  r   r,   !GetMaxUploadCompressionBufferSizec                     d|_         | j                  t        j                         d       d|_         | j                  t        j                         d       y)z:Test GetMaxConcurrentCompressedUploads returns at least 1.r   r2   N)return_valuer&   r   !GetMaxConcurrentCompressedUploads)r)   mock_configs     r*   ,testGetMaxConcurrentCompressedUploadsMinimumz5TestUtil.testGetMaxConcurrentCompressedUploadsMinimum  sF      !KY@@BAF!KY@@BAFr,   N)__name__
__module____qualname____doc__r+   r/   rB   rL   rW   rg   r   r   r   patchobjectr   r   loggingr   r   r   r   r   r   r   r   r   r   r    r,   r*   r   r   6   s    $
DJ FD7r5.=.~=M@ ::Z,,@B::Z''0- 1B-$	?;H,&,PPP ::Y CDG EGr,   r   )#r   
__future__r   r   r   r   gslib.utilsr   r   r	   r
   r   r   r   gslib.tests.testcaseteststestcasegslib.tests.utilr   r   gslib.utils.text_utilr   gslib.utils.unit_utilr   r   r   rt   osr   r   r   	six.movesr   GsUtilUnitTestCaser   r   r,   r*   <module>r      sy   , * & %  ' ! ! # ! " ! ! ' ' 2 ' 1 . @ ,  	 
 % VV_	5 6 xGx** xGr,   