
    "                       d Z ddlmZ ddlZddlZddlmZ ddlZddlm	Z	 ddlm
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ dZdZdZdZdZdZdZdZdZddZddZ G d dej<                        Z G d dej@                        Z!y)zDownload Throughput Diagnostic.    )annotationsN)List)optimize_parameters_util)storage_url)
diagnostic)log)files)scaled_integer   i   6CLOUDSDK_STORAGE_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE1CLOUDSDK_STORAGE_SLICED_OBJECT_DOWNLOAD_THRESHOLDdownload_throughputzDownload Throughput Diagnostic-zkProcess and/or thread count is set but streaming downloads dont support parallelism. Ignoring these values.zdownload throughputc                >    d|  dt        j                  |d       dS )zDReturns the payload description for the given object count and size.zTransferred z& objects for a total transfer size of    decimal_places.r
   FormatBinaryNumber)object_countobject_sizes     Qlib/googlecloudsdk/command_lib/storage/diagnose/download_throughput_diagnostic.py_get_payload_descriptionr   4   s0     	\N #


+
+K
J	K1N    c                :    t        j                  | d      }| dS )z;Formats the download throughput to a human readable format.r   r   z/secr   )r   scaled_download_throughputs     r   "_get_formatted_download_throughputr   <   s(    -@@!  ''t	,,r   c                      e Zd ZdZdZdZdZy)DownloadTypez=Enum class for specifying download type for diagnostic tests.	STREAMINGSLICEDFILEN)__name__
__module____qualname____doc__r!   r"   r#    r   r   r    r    D   s    E)&	$r   r    c                       e Zd ZdZ	 	 	 d
	 	 	 	 	 	 	 	 	 ddZedd       Z fdZd Zd Z	d Z
 fdZedd	       Z xZS )DownloadThroughputDiagnosticz*Diagnostic to measure download throughput.c                .   || _         || _        || _        |r|nt        gt        z  | _        t        | j
                        | _        || _        g | _	        i | _
        d| _        d| _        i | _        dt        t        j                                z   | _        y)a  Initializes the download throughput diagnostic.

    Args:
      test_bucket_url: Bucket url to use for the diagnostic.
      download_type: Type of download to use for the diagnostic.
      object_sizes: List of object sizes to use for the diagnostic.
      thread_count: Number of threads to use for the diagnostic.
      process_count: Number of processes to use for the diagnostic.
    N download_throughput_diagnostics_)
bucket_url_process_count_thread_count_DEFAULT_OBJECT_SIZE_DEFAULT_OBJECT_COUNT_object_sizeslen_object_count_download_type_files_old_env_varstemp_dir_download_dir_resultstruuiduuid4object_prefix)selftest_bucket_urldownload_typeobject_sizesthread_countprocess_counts         r   __init__z%DownloadThroughputDiagnostic.__init__O   s    " &DO'D%D  	"#&;; 	
 T//0D'DDKDDMDDL <c$**,>OODr   c                    t         S )N)_DIAGNOSTIC_NAMEr?   s    r   namez!DownloadThroughputDiagnostic.names   s    r   c                >   t         t        |           | j                  | j                  | j
                        }|st        j                  d      | j                  | j                  j                  dz   | j
                  z   dz   | j                  j                         y)z!Uploads test files to the bucket.zFailed to create test files./*N)superr*   _pre_process_create_test_filesr2   r>   r   DiagnosticIgnorableError_run_cpr8   pathr-   
url_string)r?   is_done	__class__s     r   rN   z)DownloadThroughputDiagnostic._pre_processw   s    	
&:<%%d&8&8$:L:LMG//0NOOLLS 4#5#55;""r   c                (   | j                          | j                  t        d       t        | j                        }|t        j                  t        j                        k  r1t        j                  |dz  d      }| j                  t        |       yy)z4Sets the environment variables for sliced downloads.r      r   r   N)_set_parallelism_env_vars_set_env_variable)_SLICED_OBJECT_DOWNLOAD_THRESHOLD_ENV_VARminr2   r
   ParseBinaryIntegerr   COMPONENT_SIZEr   ._SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE_ENV_VAR)r?   min_object_sizeoptimal_component_sizes      r   _set_sliced_download_env_varsz:DownloadThroughputDiagnostic._set_sliced_download_env_vars   s    ""$1	 $,,-O.;; //   .@@
A
a  
8
 r   c                \   | j                   t        j                  k(  r3| j                  | j                  t        j                  t               yy| j                   t        j                  k(  r| j                          y| j                   t        j                  k(  r| j                          yy)zQSets the environment variables for the diagnostic depending on the download type.N)r5   r    r!   r.   r/   r   warning'_STREAMING_DOWNLOAD_PARALLELISM_WARNINGr"   ra   r#   rX   rH   s    r   _set_cloud_sdk_env_varsz4DownloadThroughputDiagnostic._set_cloud_sdk_env_vars   s    l444				(D,>,>,J;< -K			 3 3	3
((*			 1 1	1
$$& 
2r   c                ~   | j                          | j                  t        j                  k(  rt        j
                  j                  d| j                   dt         d| j                  j                          t        j                  t        | j                        5  | j                  | j                  j                   | j"                  z   dz   t               ddd       y| j                  t        j$                  k(  s| j                  t        j&                  k(  rt)        j*                         | _        t        j
                  j                  d| j                   d| j,                  j.                   d| j                  j                          t        j                  t        | j                        5  | j                  | j                  j                   | j"                  z   dz   | j,                  j.                         ddd       yt        j0                  d| j                         # 1 sw Y   yxY w# 1 sw Y   yxY w)zRuns the diagnostic.zStarting download of z objects to path : z with download type:  rL   Nz with download type zUnknown download type: )re   r5   r    r!   r   statusPrintr4   _STREAMING_DOWNLOAD_DESTINATIONvaluer   time_recorder_DOWNLOAD_THROUGHPUT_RESULT_KEYr:   rQ   r-   rS   r>   r"   r#   
file_utilsTemporaryDirectoryr9   rR   rP   rH   s    r   _runz!DownloadThroughputDiagnostic._run   s     "l444	jj!$"4"4!5 6-. /!!''(*
 ##
)4<< 	OO&&););;cA+	
  	|222,"3"33%88:d	jj!$"4"4!5 6  %%& '!!''(*
 ##
)4<< 	OO&&););;cA##	
  //#D$7$7#8
9 3 " s   ;H'1AH3'H03H<c                   t         t        |           | j                  r:	 | j                  j	                          t
        j                  j                  d       | j                  r:	 | j                  j	                          t
        j                  j                  d       | j                  | j                  j                  | j                         y # t        $ r.}t        j                  | j                   d|        Y d }~d }~ww xY w# t        $ r.}t        j                  | j                   d|        Y d }~d }~ww xY w)NzCleaned up temp files.z" : Failed to clean up temp files. zCleaned up downloaded files.z- : Failed to clean up temp downloaded files. )rM   r*   _post_processr8   Closer   rg   rh   OSErrorrc   rI   r9   _clean_up_objectsr-   rS   r>   )r?   erU   s     r   rq   z*DownloadThroughputDiagnostic._post_process   s    	
&;=}}I

12 
  "

78
 	4??55t7I7IJ  Ityyk!CA3GHHI  
yykFqcJ	
 	

s/   9C '9D 	D$DD	E$D==Ec                   | j                   sy| j                   t           }t        | j                        }t	        j
                  |d      rt        j                  }nt        t        ||z  d            }t        j                  t        |t        | j                  |            }t        j                  | j                  |g      S )z0Returns the summarized result of the diagnostic.Ng        rW   )rI   resultpayload_description)rI   operation_results)r:   rl   sumr2   mathiscloser   PLACEHOLDER_METRIC_VALUEr   roundDiagnosticOperationResult_METRIC_NAMEr   r4   DiagnosticResultrI   )r?   download_timedownload_payload_sizer   operation_results        r   rw   z#DownloadThroughputDiagnostic.result   s     <<LL!@AM 2 23||M3'&??>
%5q
9 ";;"4 5
 &&YY+, r   )NNN)
r@   zstorage_url.CloudUrlrA   r    rB   z	List[int]rC   intrD   r   )returnr;   )r   z"diagnostic.DiagnosticResult | None)r$   r%   r&   r'   rE   propertyrI   rN   ra   re   ro   rq   rw   __classcell__)rU   s   @r   r*   r*   L   s    2 !%"P+"P ""P 	"P
 "P "PH  ,'%NK$  r   r*   )r   r   r   r   r   r;   )r   floatr   r;   )"r'   
__future__r   enumr{   typingr   r<   "googlecloudsdk.command_lib.storager   r   +googlecloudsdk.command_lib.storage.diagnoser   googlecloudsdk.corer   googlecloudsdk.core.utilr	   rm   r
   r1   r0   r^   rZ   rl   rG   ri   rd   r   r   r   Enumr    
Diagnosticr*   r(   r   r   <module>r      s     & #     G : B # 8 3  " < / 8 * #8 3 "% 3 ( %-499 k:#8#8 kr   