
    #                        d Z ddlmZ ddlZddlZddlmZ ddlZddlm	Z	 ddl
mZ ddlmZ ddlmZ d	Zd
ZdZdZdZdZdZdZd
ZdZdZdZddZddZ G d dej<                        Z G d dej@                        Z!y)Upload Throughput Diagnostic.    )annotationsN)List)storage_url)
diagnostic)log)scaled_integer   i   2CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_ENABLED4CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD9CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE50Mupload_throughputzUpload Throughput Diagnostic-ziProcess and/or thread count is set but streaming uploads dont support parallelism. Ignoring these values.zupload throughputc                >    d|  dt        j                  |d       dS )zDReturns the payload description for the given object count and size.zTransferred z& objects for a total transfer size of    decimal_places.r	   FormatBinaryNumber)object_countobject_sizes     Olib/googlecloudsdk/command_lib/storage/diagnose/upload_throughput_diagnostic.py_get_payload_descriptionr   5   s0     	\N #


+
+K
J	K1N    c                :    t        j                  | d      }| dS )z9Formats the upload throughput to a human readable format.r   r   z/secr   )r   scaled_upload_throughputs     r    _get_formatted_upload_throughputr   =   s(    +>> %%T	**r   c                      e Zd ZdZdZdZdZy)
UploadTypez;Enum class for specifying upload type for diagnostic tests.PARALLEL_COMPOSITE	STREAMINGFILEN)__name__
__module____qualname____doc__r"   r#   r$    r   r   r!   r!   E   s    C+)	$r   r!   c                       e Zd ZdZ	 	 	 d
	 	 	 	 	 	 	 	 	 ddZedd       Z fdZd Zd Z	d Z
 fdZedd	       Z xZS )UploadThroughputDiagnosticr   c                t   || _         || _        || _        || _        g | _        i | _        d| _        i | _        |r|| _        n7| j                  t        j                  k7  rt        gt        z  nt        g| _        t        | j                        | _        dt!        t#        j$                               z   | _        y)a\  Initializes the Upload Throughput Diagnostic.

    Args:
      test_bucket_url: The bucket to upload to.
      upload_type: The type of upload to perform.
      object_sizes: The sizes of the objects to upload.
      thread_count: The number of threads to use for the upload.
      process_count: The number of processes to use for the upload.
    Nupload_throughput_diagnostics_)
bucket_url_process_count_thread_count_upload_type_files_old_env_varstemp_dir_result_object_sizesr!   r#   _DEFAULT_OBJECT_SIZE_DEFAULT_OBJECT_COUNT_DEFAULT_STREAMING_SIZElen_object_countstruuiduuid4object_prefix)selftest_bucket_urlupload_typeobject_sizesthread_countprocess_counts         r   __init__z#UploadThroughputDiagnostic.__init__P   s    " &DO'D%D#DDKDDMDL'd *"6"66  
 #8
8'( 
 T//0D :C

<MMDr   c                    t         S )N)_DIAGNOSTIC_NAMEr@   s    r   namezUploadThroughputDiagnostic.namev   s    r   c                   t         t        |           | j                  t        j
                  k(  rUt        | _        t        | j                        dkD  rt        j                  dt                y| j                  d   | _        y| j                  | j                  | j                        st        j                  d      y)z1Prepares the environment for the diagnostic test.r   zgStreaming uploads do not support multiple objects. Ignoring the object count and using default size. : r   NzFailed to create test files.)superr+   _pre_processr1   r!   r#   r9   streaming_sizer:   r6   r   warning_create_test_filesr?   r   DiagnosticIgnorableError)r@   	__class__s    r   rM   z'UploadThroughputDiagnostic._pre_processz   s    	
$d8:J0003d	T	 1	$'(*	
  #003""4#5#5t7I7IJ//0NOO Kr   c                @   | j                          | j                  t        d       | j                  t        d       t	        | j
                        }|t        j                  t              k  r1t        j                  |dz  d      }| j                  t        |       yy)z>Sets the environment variables for parallel composite uploads.truer      r   N)_set_parallelism_env_vars_set_env_variable"_ENABLE_PARALLEL_COMPOSITE_ENV_VAR,_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD_ENV_VARminr6   r	   ParseBinaryInteger1_DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZEr   1_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE_ENV_VAR)r@   min_object_sizeoptimal_component_sizes      r    _set_parallel_composite_env_varsz;UploadThroughputDiagnostic._set_parallel_composite_env_vars   s    ""$=vF4	 $,,-O.;;9   .@@
A
a  
;
 r   c                \   | j                   t        j                  k(  r3| j                  | j                  t        j                  t               yy| j                   t        j                  k(  r| j                          y| j                   t        j                  k(  r| j                          yy)zOSets the environment variables for the diagnostic depending on the upload type.N)r1   r!   r#   r/   r0   r   rO   %_STREAMING_UPLOAD_PARALLELISM_WARNINGr"   r`   r$   rV   rI   s    r   _set_cloud_sdk_env_varsz2UploadThroughputDiagnostic._set_cloud_sdk_env_vars   s    J000				(D,>,>,J9: -K			j;;	;
++-			joo	-
$$& 
.r   c                   | j                          | j                  t        j                  k(  rt	        j
                  t        | j                        5  t        j                  j                  dj                  | j                  | j                               | j                  t        | j                  j                   | j"                  z   | j%                  | j                               ddd       y| j                  t        j&                  k(  s| j                  t        j(                  k(  rt        j                  j                  d| j*                   d| j                   d| j                  j,                          t	        j
                  t        | j                        5  | j                  | j.                  j0                  dz   | j"                  z   dz   | j                  j                          ddd       yt	        j2                  d	j                  | j4                  | j                              # 1 sw Y   yxY w# 1 sw Y   yxY w)
zRuns the diagnostic test.

    This involves running the gcloud command to upload the files and measuring
    the time it takes to upload the files.
    z-Starting streaming upload of {} bytes to : {})in_strNzStarting upload of z objects to : z with upload type: /*z{} : Unknown upload type: {})rc   r1   r!   r#   r   time_recorder_UPLOAD_THROUGHPUT_RESULT_KEYr5   r   statusPrintformatrN   r.   _run_cp_STREAMING_UPLOAD_SOURCE
url_stringr?   _generate_random_stringr"   r$   r;   valuer4   pathrQ   rJ   rI   s    r   _runzUploadThroughputDiagnostic._run   s    	  "J000##
' 	

;BB##T__	

 	$OO&&););;//0C0CD 	 	
  	Z:::
/	jj 2 23 4oo1$2C2C2I2I1JL ##
' 	MM$t'9'99C?OO&&	
  //
(
/
/		4;L;L
M ; * s   BH/AH;/H8;Ic                   t         t        |           | j                  r:	 | j                  j	                          t
        j                  j                  d       | j                  | j                  j                  | j                         y# t        $ r.}t        j                  | j                   d|        Y d}~cd}~ww xY w)z<Restores the environment variables and cleans up temp files.zCleaned up temp files.z" : Failed to clean up temp files. N)rL   r+   _post_processr4   Closer   rj   rk   OSErrorrO   rJ   _clean_up_objectsr.   ro   r?   )r@   erR   s     r   ru   z(UploadThroughputDiagnostic._post_process   s    	
$d9;}}I

12 	4??55t7I7IJ  Ityyk!CA3GHHIs   9B 	C$B==Cc                   | j                   rt        | j                   vry| j                   t           }t        | j                        }t	        j
                  |d      rt        j                  }nt        t        ||z  d            }t        j                  t        |t        | j                  |            }t        j                  | j                  |g      S )z:Returns the summarized result of the diagnostic execution.Ng        rU   )rJ   resultpayload_description)rJ   operation_results)r5   ri   sumr6   mathiscloser   PLACEHOLDER_METRIC_VALUEr   roundDiagnosticOperationResult_METRIC_NAMEr   r;   DiagnosticResultrJ   )r@   upload_timeupload_payload_sizer   operation_results        r   r{   z!UploadThroughputDiagnostic.result   s     <<8L,,<=Kd001||K%$==:
#k11
5 ";; 4 3
 &&YY+, r   )NNN)
rA   zstorage_url.CloudUrlrB   r!   rC   z	List[int]rD   intrE   r   )returnr<   )r   z"diagnostic.DiagnosticResult | None)r%   r&   r'   r(   rF   propertyrJ   rM   r`   rc   rs   ru   r{   __classcell__)rR   s   @r   r+   r+   M   s    % !%$N+$N $N 	$N
 $N $NL  P&('(T	K  r   r+   )r   r   r   r   r   r<   )r   floatr   r<   )"r(   
__future__r   enumr   typingr   r=   "googlecloudsdk.command_lib.storager   +googlecloudsdk.command_lib.storage.diagnoser   googlecloudsdk.corer   googlecloudsdk.core.utilr	   r8   r7   rX   rY   r]   r\   ri   rH   r9   rn   rb   r   r   r   Enumr!   
Diagnosticr+   r)   r   r   <module>r      s    $ "     : B # 3  " 8 # ; - @ 2 5: 1 3 1 %  3 & #+ l!6!6 lr   