
    1                     T   d Z ddlZddlZddlmZ ddlZddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ g dZdZdZdZdZdZ dZ!dZ"dZ#dZ$dZ%dZ&de'de(fdZ)de*d e*de(fd!Z+ G d" d#ejX                        Z- G d$ d%e.      Z/y)&zLatency Diagnostic.    N)List)api_factory)	cloud_api)errors)request_config_factory)statistics_util)storage_url)
diagnostic)resource_reference)log)progress_tracker)files)scaled_integer)r   i   i  i      uploaddownloaddeletemetadataMeanzStandard deviationz90th percentilez50th percentilezLatency DiagnosticzSuccessful trialstime_in_secondsreturnc                     | dz  }|ddS )z*Formats a time in seconds as milliseconds.i  z.2fms )r   time_in_millisecondss     Elib/googlecloudsdk/command_lib/storage/diagnose/latency_diagnostic.py_format_as_millisecondsr   4   s    (4/ %R	((    object_sizeobject_numberc                 >    t        j                  |       }d| d| dS )zEReturns the payload description for the given object size and number.zobject size z at index [])r   FormatInteger)r   r    scaled_object_sizes      r   _get_payload_descriptionr%   :   s*    %33K@*+;}oQ	GGr   c                       e Zd ZdZ	 ddej
                  dee   fdZe	de
fd       Zd Zd	e
d
eddfdZ	 	 ddZd Z	 	 ddZ	 	 ddZd Zd Ze	dej*                  fd       Zy)LatencyDiagnosticzDiagnostic to measure the latency of various operations.

  This diagnostic test will upload, download, and delete objects of different
  sizes and record the latency of each operation.
  Ntest_bucket_urlobject_sizesc                 R   |r|nt         | _        t        | j                        | _        || _        d | _        g | _        t        j                  | j                  j                        | _
        t               | _        i | _        dt        t        j                                z   | _        y )Nlatency_diagnostics_)_DEFAULT_OBJECT_SIZESr)   lenobject_count
bucket_urltemp_dir_filesr   get_apischeme_api_client	DummyFile_discard_sink_resultstruuiduuid4object_prefix)selfr(   r)   s      r   __init__zLatencyDiagnostic.__init__G   s    
 )5:ODD--.D%DODMDK"**4??+A+ABD"D DL 0#djjl2CCDr   r   c                     t         S N)_DIAGNOSTIC_NAMEr<   s    r   namezLatencyDiagnostic.name[   s    r   c                 ~    | j                  | j                  | j                        }|st        j                  d      y)z4Creates the test files to be used in the diagnostic.zFailed to create test files.N)_create_test_filesr)   r;   r
   DiagnosticIgnorableError)r<   is_dones     r   _pre_processzLatencyDiagnostic._pre_process_   s:    %%d&7&79K9KLG//0NOO r   operation_titler    c                     | j                   j                  |      si | j                   |<   | j                   |   j                  |      si | j                   |   |<   yy)zCreates an entry in the result dictionary for the given operation.

    Args:
      operation_title: The title of the operation.
      object_number: The number of the object being operated on.
    N)r7   get)r<   rH   r    s      r   _create_result_entryz&LatencyDiagnostic._create_result_entryf   sS     <<O,&(dll?#<<(,,];57dll?#M2 <r   c                    | j                  t        |       | j                  t           |   }	 t        j                  ||      5  t        j                  |      5 }| j                  j                  |||       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w# t        j                  $ r	 ||v r||=  w xY w)aT  Uploads an object and records the latency.

    Args:
      object_number: The number of the object being uploaded.
      file_path: The path to the file to upload.
      object_resource: The object resource to upload.
      request_config: The request config to use for the upload.
      iteration: The iteration number of the upload.
    N)rK   _UPLOAD_OPERATION_TITLEr7   r
   time_recorder
file_utils
FileReaderr4   upload_object
api_errorsCloudApiError)r<   r    	file_pathobject_resourcerequest_config	iterationresults_dictfiles           r   _upload_objectz LatencyDiagnostic._upload_objectu   s     	5}E<< 78GL##I|<""9-



(
(
O . =<-- =< ## 	l	"#	sA   B! BB	8B B! 	B	BBB! B! !B=c                 R   | j                  t        |       | j                  t           |   }	 t        j                  ||      5  | j
                  j                  | j                  j                  |       ddd       y# 1 sw Y   yxY w# t        j                  $ r	 ||v r||=  w xY w)zFetches object metadata and records the latency.

    Args:
      object_number: The number of the object being fetched.
      object_name: The name of the object to fetch metadata for.
      iteration: The iteration number of the fetch.
    N)rK   _METADATA_OPERATION_TITLEr7   r
   rN   r4   get_object_metadatar/   bucket_namerR   rS   )r<   r    object_namerW   rX   s        r   _fetch_object_metadataz(LatencyDiagnostic._fetch_object_metadata   s     	7G<< 9:=IL##I|<,,OO''	
 =<< ## 	l	"#s)   B
 1A>5B
 >BB
 B
 
B&c                 t   | j                  t        |       | j                  t           |   }	 t        j                  ||      5  | j
                  j                  || j                  |t        j                  j                         ddd       y# 1 sw Y   yxY w# t        j                  $ r	 ||v r||=  w xY w)a-  Downloads an object and records the latency.

    Args:
      object_number: The number of the object being downloaded.
      object_resource: The object resource to download.
      request_config: The request config to use for the download.
      iteration: The iteration number of the download.
    )download_strategyN)rK   _DOWNLOAD_OPERATION_TITLEr7   r
   rN   r4   download_objectr6   r   DownloadStrategyONE_SHOTrR   rS   )r<   r    rU   rV   rW   rX   s         r   _download_objectz"LatencyDiagnostic._download_object   s     	7G<< 9:=IL##I|<(('88AA	 	) 	
 =<< ## 	l	"#s*   B ABB BB B B7c                 *   | j                  t        |       | j                  t           |   }	 t        j                  ||      5  | j
                  j                  ||       ddd       y# 1 sw Y   yxY w# t        j                  $ r	 ||v r||=  w xY w)a  Deletes an object and records the latency.

    Args:
      object_number: The number of the object being deleted.
      object_url: The object url to delete.
      request_config: The request config to use for the delete.
      iteration: The iteration number of the delete.
    N)	rK   _DELETE_OPERATION_TITLEr7   r
   rN   r4   delete_objectrR   rS   )r<   r    
object_urlrV   rW   rX   s         r   _delete_objectz LatencyDiagnostic._delete_object   s     	5}E<< 78GL##I|<&&z>B =<<## 	l	"#s)   A6 A*!A6 *A3/A6 3A6 6Bc                    t        t              D ]p  }t        j                  d|       5  t        | j                        D ]2  }| j
                  |   }| j                  |   }|j                  t        j                  j                        d   }t        j                  t        j                  j                  | j                  j                   |      }t#        j$                  ||      }t'        j(                  |j                  t&        j*                  |      }	 | j-                  |||||       | j/                  ||j0                  |       | j3                  ||||       | j5                  ||||       5 	 ddd       s y# t6        j8                  $ r3}	t;        j<                  d|j0                   d| d|	        Y d}	~	d}	~	wt>        $ r-}	tA        jB                  d	|j0                   d| d|	       d}	~	ww xY w# 1 sw Y   xY w)
zRuns the diagnostic.

    Uploads, downloads, and deletes objects of different sizes and records the
    latency of each operation.
    zRunning latency iteration )size)content_typero   z,Failed to run full operation set for object z in iteration z. NzUnexpected error for object )"range_ITERATION_COUNTr   ProgressTrackerr.   r1   r)   splitospathsepr	   CloudUrlProviderPrefixGCSr/   r^   r   ObjectResourcer   get_request_configDEFAULT_CONTENT_TYPErZ   r`   rB   rg   rl   rR   rS   r   debug	Exceptionr
   rE   )
r<   rW   r    rT   	file_sizer_   rk   rU   rV   es
             r   _runzLatencyDiagnostic._run   s    +,	++&yk
2 #4#4#45Mkk-0)''6) "4R8+"++((,,oo))* /==y/ 2DD))1FF.
 ''33Y !!	 z>9G 6	  -V )) II#(()	{"QCI   55#(()	{"QCI _ sC   C.HAF5HG?	(G	 HG?	(G:	:G?	?HH	c                 (   | j                   L	 | j                   j                          | j                  | j                  j                  | j                         y y # t        $ r.}t        j                  | j
                   d|        Y d }~dd }~ww xY w)Nz" : Failed to clean up temp files. )
r0   CloseOSErrorr   warningrB   _clean_up_objectsr/   
url_stringr;   )r<   r   s     r   _post_processzLatencyDiagnostic._post_process  s|    }} I T__779K9KL !  Ityyk!CA3GHHIs   A 	B#$BBc           
         g }| j                   j                         D ]  \  }}|j                         D ]  }| j                   |   |   j                         }| j                  |   }t        j                  |      }|st        j                  d| d| d       gt        |      }i }	| dt         |	t        <   t        |      |z  t              |	t        <   t        j                   t        fd|D              |z        }
t        |
      |	t"        <   t        t%        j&                  t)        |      d            |	t*        <   t        t%        j&                  t)        |      d            |	t,        <   t/        j0                  ||	t3        | j                  |   |            }|j5                  |         t/        j6                  | j8                  |      S )	NzNo successful trials for z on object size z. Skipping statistics./c              3   .   K   | ]  }|z
  d z    yw)   Nr   ).0xmeans     r   	<genexpr>z+LatencyDiagnostic.result.<locals>.<genexpr>4  s     0ATas   2   Z   )payload_description)r7   itemskeysvaluesr)   r   r#   r   r   r-   rr   _SUCCESSFUL_TRIALS_TITLEsumr   _MEAN_TITLEmathsqrt_STANDARD_DEVIATION_TITLEr   find_percentilelist_PERCENTILE_50TH_TITLE_PERCENTILE_90TH_TITLEr
   DiagnosticOperationResultr%   appendDiagnosticResultrB   )r<   operation_resultsrH   object_number_to_latency_dictr    trialsr   r$   
num_trialscumulative_result_dictstandard_deviationoperation_resultr   s               @r   resultzLatencyDiagnostic.result  s   :>,,:L:L:N668==?-o.}=DDF''6+99+F
++)/): ;)**@B [
!# l!,-. 	78 6{Z'.Ed.K{+!YY000:=

 $$67 	89
 $//VbA 	56 $//VbA 	56 &??" 8!!-0-!
 	  !12] @ ;Ob &&tyy2CDDr   r?   )r   N)__name__
__module____qualname____doc__r	   rx   r   intr=   propertyr8   rB   rG   rK   rZ   r`   rg   rl   r   r   r
   r   r   r   r   r   r'   r'   @   s     !%D"++D ID( C  P8 814882*6,:x	M 3Ej11 3E 3Er   r'   c                        e Zd ZdZdZd Zd Zy)r5   zCA dummy file-like object that throws away everything written to it.bwc                      y r?   r   )r<   agrskwargss      r   writezDummyFile.writeW      r   c                      y r?   r   rA   s    r   closezDummyFile.closeZ  r   r   N)r   r   r   r   moder   r   r   r   r   r5   r5   R  s    K	$		r   r5   )0r   r   ru   typingr   r9   googlecloudsdk.api_lib.storager   r   r   rR   r   "googlecloudsdk.command_lib.storager   r	   +googlecloudsdk.command_lib.storage.diagnoser
   ,googlecloudsdk.command_lib.storage.resourcesr   googlecloudsdk.corer   googlecloudsdk.core.consoler   googlecloudsdk.core.utilr   rO   r   r,   rr   rM   rc   ri   r\   r   r   r   r   r@   r   floatr8   r   r   r%   
Diagnosticr'   objectr5   r   r   r   <module>r      s       	   6 4 ? A > : B K # 8 8 3 ;  " & " & 0 * * ' . )U )s )H# Hc Hc HOE
-- OEd		 		r   