
                            d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZdd	lmZ dd
lmZ ddlZddlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl"m$Z$ dZ%dZ&dZ'dZ(dZ)dZ*dZ+ ejX                  d      Z-ej\                  j_                  ej\                  ja                  dd            Z1dZ2i dddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHZ3 G dI dJe4      Z5dK Z6dL Z7ejp                  e6dM               Z9e6	 	 	 	 	 dTdN       Z:e6dO        Z;e6dP        Z<e6dQ        Z=e6dR        Z>dUdSZ?y)Vz:Static data and helper functions for collecting user data.    )absolute_import)print_function)division)unicode_literalsN)defaultdictwraps)input)urllib)VERSION)Metric)system_util)CalculateThroughput)HumanReadableToBytesz(https://ssl.google-analytics.com/collectzUA-36037335-16zUA-36037335-17CommandRetryableError
FatalErrorPerformanceSummaryz.*google\.com$~z.gsutil/analytics-uuidDISABLEDzEvent CategoryeczEvent ActioneazEvent LabelelzEvent ValueevCommand Namecd1Global Optionscd2Command-Level Optionscd3Configcd4Command Aliascd5Fatal Errorcd6Parallelism Strategycd7Source URL Typecd8Provider Typescd9	Timestampcd10Execution Timecm1Retryable Errorscm2Is Google Corp Usercm3cm4cm5cm6cm7cm8cm9cm10cm11cm12cm13cm14)Num ProcessesNum Threads#Number of Files/Objects Transferred!Size of Files/Objects TransferredAverage Overall ThroughputNum Retryable Service ErrorsNum Retryable Network ErrorsThread Idle Time PercentSlowest Thread ThroughputFastest Thread ThroughputDisk I/O Timec                       e Zd ZdZeefdZdZdZd Z	e
efd       Ze
d        Zed        Ze	 	 	 dd       Zedd	       Ze
d
        Zd Zd ZeddfdZ G d de      Zd Zd Zd Zd Z	 	 	 ddZy)MetricsCollectorzA singleton class to handle metrics reporting to Google Analytics (GA).

  This class is not thread or process-safe, and logging directly to the
  MetricsCollector instance can only be done by a single thread.
  c                 N   t               | _        || _        t        j                         | _        t        j                         }t        j                  j                  dd      }|rt        }| j                         }d}t        j                  t        j                                rd}ddd|d|d	d
t"        d   |t"        d   |i| _        dj'                  t)        j*                         t)        j,                               | _        g | _        t3        t4              | _        d| _        y)a  Initialize a new MetricsCollector.

    This should only be invoked through the GetCollector or StartTestCollector
    functions.

    Args:
      ga_tid: The Google Analytics tracking ID to use for metrics collection.
      endpoint: The URL to send requests to.
    GSUtiluse_test_GA_propertyr      v1tidcidteventr!   r3   z{system}/{release})systemreleaseN)_GetTimeInMillis
start_timeendpointlogging	getLoggerloggerrL   _GetCIDbotoconfiggetbool_GA_TID_TESTING_ValidateAndGetConfigValues_GOOGLE_CORP_HOST_REmatchsocketgethostname_GA_LABEL_MAP	ga_paramsformatplatformrW   rX   
user_agent_metricsr   intretryable_errorsperf_sum_params)selfga_tidr[   rT   use_test_propertyconfig_valuesis_corp_users           platform/gsutil/gslib/metrics.py__init__zMetricsCollector.__init__x   s    '(DODM##%DK 
"
"
$C ++H6LMf 446M L!!&"4"4"67l 	SvsWh+,lDN +119J:B:J:J:L 2 NDO DM (,D  D    Nc                   
 g 
d
fd}dD ]  \  }} |||d         ddD ]  \  }} |||fd        d	D ]  \  }} |||d
         dD ]G  \  }}t         j                  j                  ||      }|s)	 t        |      }
j	                  ||f       I  |ddd         |ddd         |ddd         |ddd         |ddd        dj                  t        
D 	cg c]  }	dj                  |	d   |	d          c}	            S # t
        $ r 
j	                  |f       Y w xY wc c}	w )zParses the user's config file to aggregate non-PII config values.

    Returns:
      A comma-delimited string of config values explicitly set by the user in
      key:value pairs, sorted alphabetically by key.
    INVALIDc                     	 t         j                  j                  | |      }|r ||      rj                  ||f       y |rj                  |f       y y #  j                  |f       Y y xY wN)r`   ra   	get_valueappend)sectioncategoryvalidation_fnconfig_valueru   invalid_value_strings       rw   GetAndValidateConfigValuezOMetricsCollector._ValidateAndGetConfigValues.<locals>.GetAndValidateConfigValue   sr    ?{{,,Wh?M,7


,7
8


*>?
@ 
?h(<=>s   =A A A/))Botohttps_validate_certificates)rN   disable_analytics_prompt)rN   use_magicfile)rN   tab_completion_time_logsc                 8    t        |       j                         dv S )N)truefalse)strlowervals    rw   <lambda>z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>   s    #c(..:J 1;2ry   )r   r   r   i  ))r   debug)r   http_socket_timeout)r   num_retries)r   max_retry_delay)rN   default_api_version)rN   %sliced_object_download_max_components)rN   parallel_process_count)rN   parallel_thread_count)rN   software_update_check_period)rN   tab_completion_timeout)OAuth2oauth2_refresh_retriesc                 V    t        |       j                         xr t        |       k  S r}   )r   isdigitro   )r   small_int_thresholds    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>   s)    #c(:J:J:L ;? #C+> >;?ry   ))rN   resumable_threshold)rN   rsync_buffer_lines)rN   task_estimation_thresholdc                 4    t        |       j                         S r}   )r   r   r   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>   s    #c(:J:J:Lry   ))rN   (parallel_composite_upload_component_size)rN   #parallel_composite_upload_threshold)rN   %sliced_object_download_component_size)rN    sliced_object_download_thresholdrN   check_hashesc                 
    | dv S )N)if_fast_else_failif_fast_else_skipalwaysnever r   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>  s    #E#Fry   content_languagec                 B    | j                         xr t        |       dk  S )N   )isalphalenr   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>  s    #++-"ACHM"Ary   json_api_versionc                 X    | d   j                         dk(  xr | dd  j                         S )Nr   rQ   rP   )r   r   r   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>  s(    #a&,,.C"7"MCGOO<M"Mry   
prefer_apic                 
    | dv S )N)jsonxmlr   r   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>  s	    8Nry   r   token_cachec                 
    | dv S )N)file_system	in_memoryr   r   s    rw   r   z>MetricsCollector._ValidateAndGetConfigValues.<locals>.<lambda>  s    :9;ry   ,z{0}:{1}r   rP   )	r`   ra   r~   r   r   
ValueErrorjoinsortedrk   )rr   r   r   bool_categorysmall_int_categorylarge_int_categorydata_size_categoryr   size_in_bytesra   ru   r   r   s             @@@rw   rd   z,MetricsCollector._ValidateAndGetConfigValues   s    M %?#K  )6/23	#K (##  );/?@(&(G##  );.LN	(G(## [[**74FGl		K.|<-


 2MB
C(  FG #AC #MO h'3,NP h'4-;<
 88AN
ANvIVAYq	2
 	 7  	K


 24HI
J	K8
s    D0D<D98D9c                     t         j                         ryt         j                  st        |       t         _        t         j                  S )zDReturns the singleton MetricsCollector instance or None if disabled.N)rL   
IsDisabled	_instance)rs   s    rw   GetCollectorzMetricsCollector.GetCollector#  s6     ""$%%#3F#; %%%ry   c                  j    t         j                  t         j                          t         j                  S )z6Returns whether metrics collection should be disabled.)rL   _disabled_cache_CheckAndSetDisabledCacher   ry   rw   r   zMetricsCollector.IsDisabled-  s(     ''/002+++ry   c                 h   t         j                  j                  d      dk(  rd| _        y	t         j                  j                  d      dk(  rd| _        | j	                          y	t
        j                  j                  ddd      rd| _        y	t        j                         r&t         j                  j                  d       | _        y	t         j                  j                  t              r5t        t              5 }|j                         t        k(  | _        d	d	d	       y	d| _        y	# 1 sw Y   y	xY w)
z1Sets _disabled_cache based on user opt-in or out.GSUTIL_TEST_ANALYTICSrR   T2FrN   use_gcloud_storageGA_CIDN)osenvirongetr   StartTestCollectorr`   ra   rb   r   InvokedViaCloudSdkpathexists_UUID_FILE_PATHopenread_DISABLED_TEXT)clsfs     rw   r   z*MetricsCollector._CheckAndSetDisabledCache4  s     
zz~~-.#5 c 
/	0C	7!c	 
		X';U	C c		'	'	) "

x 88c		( A vvx>9 !  !c ! s   :D((D1c                    | j                         rdt        j                  d<   d| _         | t        |      | _        |ddd}|| j
                  _        || j
                  _        t        j                  d   dk7  rd	| j
                  _        yy)
aN  Reset the singleton MetricsCollector with testing parameters.

    Should only be used for tests, where we want to change the default
    parameters.

    Args:
      endpoint: str, URL to post to
      user_agent: str, User-Agent string for header.
      ga_params: A list of two-dimensional string tuples to send as parameters.
    0r   FNbd)acr   r   )	r   r   r   r   rc   r   rj   rm   rZ   )r   r[   rm   rj   s       rw   r   z#MetricsCollector.StartTestCollectorL  s~      ~~,/bjj()C2CM#&i'CMM)CMM	zz)*c1!"cmm 2ry   c                 F    dt         j                  d<   d| _        || _        y)zReset the MetricsCollector with default parameters after testing.

    Args:
      original_instance: The original instance of the MetricsCollector so we can
        set the collector back to its original state.
    rR   r   N)r   r   r   r   )r   original_instances     rw   StopTestCollectorz"MetricsCollector.StopTestCollectorg  s"     +.BJJ&'C%CMry   c                      t         j                  j                  t              r,t	        t              5 } | j                         }ddd       r|S t         j                  j                  d      S # 1 sw Y   ,xY w)zGets the client id from the UUID file or the SDK opt-in, or returns None.

    Returns:
      str, The hex string of the client id.
    Nr   )r   r   r   r   r   r   r   r   )r   rT   s     rw   r_   zMetricsCollector._GetCIDs  sT     
ww~~o& Affh !	
 ::>>(## ! s   A//A8c                 :    | j                   j                  |       y)a   Extends self.ga_params to include new parameters.

    This is only used to record parameters that are sent with every event type,
    such as global and command-level options.

    Args:
      new_params: A dictionary of key-value parameters to send.
    N)rj   update)rr   
new_paramss     rw   ExtendGAParamszMetricsCollector.ExtendGAParams  s     	NN*%ry   c                 F    | j                   j                  t        |         S )zConvenience function for getting a ga_param of the collector.

    Args:
      param_name: The descriptive name of the param (e.g. 'Command Name'). Must
        be a key in _GA_LABEL_MAP.

    Returns:
      The GA parameter specified, or None.
    )rj   r   ri   )rr   
param_names     rw   
GetGAParamzMetricsCollector.GetGAParam  s     >>mJ788ry   r   c                    d|fd|fd|fd|ft         d   t               fg}|j                  t        j                  |      D 	cg c]  \  }}	|		||	f c}	}       |j                  t        j                  | j
                        D 	cg c]  \  }}	|		||	f c}	}       |t               | j                  z
  }|j                  t         d   |f       t        j                  j                  t        |            }
| j                  j                  t        | j                  d|
| j                  	             yc c}	}w c c}	}w )
a  Adds a GA metric with the given parameters to the metrics queue.

    Args:
      category: str, the GA Event category.
      action: str, the GA Event action.
      label: str, the GA Event label.
      value: int, the GA Event value.
      execution_time: int, the execution time to record in ms.
      **custom_params: A dictionary of key, value pairs containing custom
          metrics and dimensions to send with the GA Event.
    r   r   r   r   r-   Nr/   POST)r[   methodbodyrm   )ri   rY   extendsix	iteritemsrj   rZ   r   r   parse	urlencoder   rn   r   r[   rm   )rr   r   actionlabelvalueexecution_timecustom_paramsparamskrQ   datas              rw   CollectGAMetricz MetricsCollector.CollectGAMetric  s.   $ Xvue}[)+;+=>@F
MM==7741a1=A7  MM==8841aAMA8 
 ')DOO;n
MM=!12NCD<<!!&.1DMM//	+,s   
D;
D;


E
E
c                   ,    e Zd ZdZd Z G d de      Zy))MetricsCollector._PeformanceSummaryParamszEThis class contains information to create a PerformanceSummary event.c                    d| _         d| _        d| _        d| _        t	               | _        t        j                  rt        j                         | _	        d| _
        d| _        d| _        d| _        t        | j                        | _        d | _        d | _        d | _        d| _        d| _        d| _        d| _        d| _        d| _        y )Nr   F)num_processesnum_threadsnum_retryable_service_errorsnum_retryable_network_errorssetprovider_typesr   IS_LINUXGetDiskCountersdisk_counters_startuses_fan
uses_slicethread_idle_timethread_execution_timer   _ThreadThroughputInformationthread_throughputsavg_throughputtotal_elapsed_timetotal_bytes_transferrednum_objects_transferredis_daisy_chainhas_file_dsthas_cloud_dsthas_file_srchas_cloud_srcrr   s    rw   rx   z2MetricsCollector._PeformanceSummaryParams.__init__  s    dd*+d'*+d'Ed 
		#.#>#>#@  dmdo  d#$d  !,D,M,M Nd !d $d%)d"%&d" "dd dd dry   c                   (    e Zd ZdZd Zd Zd Zd Zy)FMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformationzDA class to keep track of throughput information for a single thread.c                 <    d| _         d| _        d | _        d | _        y )Nr   )r  r  task_start_time	task_sizer   s    rw   rx   zOMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.__init__  s%    '($"#  $ ry   c                      || _         || _        y r}   )r$  r%  )rr   rZ   bytes_to_transfers      rw   LogTaskStartzSMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.LogTaskStart  s    )*ry   c                     | xj                   || j                  z
  z  c_         | xj                  | j                  z  c_        d | _        d | _        y r}   )r  r$  r  r%  )rr   end_times     rw   
LogTaskEndzQMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.LogTaskEnd   sB    8d.B.B#BB$$6$#ry   c                 B    t        | j                  | j                        S r}   )r   r  r  r   s    rw   GetThroughputzTMetricsCollector._PeformanceSummaryParams._ThreadThroughputInformation.GetThroughput  s     "4#?#?#'#:#:< 	<ry   N)__name__
__module____qualname____doc__rx   r(  r+  r-  r   ry   rw   r  r"    s    P	+<ry   r  N)r.  r/  r0  r1  rx   objectr  r   ry   rw   _PeformanceSummaryParamsr    s    O%!N<v <ry   r3  c                 l   | j                  d      dvry| j                  | j                         | _        d|v r| j                  |d          yt	        j
                  |      D ]  \  }}|dv r/t        | j                  |      }|st        | j                  ||       |dv r0t        | j                  |      }t        | j                  |||z          |dv r2t        | j                  |      }||k  rt        | j                  ||       |d	k(  s| j                  j                  j                  |        y)
ag  Updates the _PeformanceSummaryParams object.

    Args:
      params: A dictionary of keyword arguments.
        - uses_fan: True if the command uses fan parallelism.
        - uses_slice: True if the command uses slice parallelism.
        - avg_throughput: The average throughput of the data transfer.
        - is_daisy_chain: True if the transfer uses the daisy-chain method.
        - has_file_dst: True if the transfer's destination is a file URL.
        - has_cloud_dst: True if the transfer's destination is in the cloud.
        - has_file_src: True if the transfer has a file URL as a source.
        - has_cloud_src: True if the transfer has a cloud URL as a source.
        - total_elapsed_time: The total amount of time spent on Apply.
        - total_bytes_transferred: The total number of bytes transferred.
        - thread_idle_time: The additional amount of time that threads spent
                            idle in Apply.
        - thread_execution_time: The additional amount of time that threads
                                 spent executing in Apply.
        - num_retryable_service_errors: The additional number of retryable
                                        service errors that occurred.
        - num_retryable_network_errors: The additional number of retryable
                                        network errors that occurred.
        - num_processes: The number of processes used in a call to Apply.
        - num_threads: The number of threads used in a call to Apply.
        - num_objects_transferred: The total number of objects transferred, as
                                   specified by a ProducerThreadMessage.
        - provider_types: A list of additional provider types used.
        - file_message: A FileMessage used to calculate thread throughput and
                        number of objects transferred in the non-parallel case.
    r   )cprsyncNfile_message)r7  )r  r  r  r  r  r  r  r  r  r  r  )r  r  r
  r  )r  r	  r  )
r   rq   r3  _ProcessFileMessager   r   getattrsetattrr  r   )rr   r  r   param	cur_values        rw   UpdatePerformanceSummaryParamsz/MetricsCollector.UpdatePerformanceSummaryParams
  s@   > ~&o=#!::<d
 
F>,BC ]]62
E 
 N 
N D00*=	
$&&
E
: 
 8 
8 D00*=	$$j)e2CD 
7	7D00*=	u
$&&
E
: 
'	'++22597 3ry   c                    | j                   j                  |j                  |j                  f   }|j                  rg| j                   j
                  s5| j                   j                  s| j                   xj                  dz  c_        |j                  |j                         y|j                  |j                  |j                         y)aT  Processes FileMessages for thread throughput calculations.

    Update a thread's throughput based on the FileMessage, which marks the start
    or end of a file or component transfer. The FileMessage provides the number
    of bytes transferred as well as start and end time.

    Args:
      file_message: The FileMessage to process.
    rP   N)rq   r  
process_id	thread_idfinishedr  r  r  r+  timer(  size)rr   r7  thread_infos      rw   r8  z$MetricsCollector._ProcessFileMessageR  s     ''::!7!7<9 :K ""--1E1E1N1N4494\../|00,2C2CDry   c           	         | j                  d      }|rC | j                  dt        |dt        d   t	        | j
                  j                               i t        j                  | j
                        D ]'  \  }} | j                  dt        |dt        d   |i ) | j                  d      }|r| j                  t        |       yy)zDAggregates command and error info and adds them to the metrics list.r   )r   r   r1   r%   Nr   )r   r  _GA_COMMANDS_CATEGORYri   sumrp   valuesr   r   _GA_ERRORRETRY_CATEGORY_GA_ERRORFATAL_CATEGORY)rr   command_name
error_type
num_errorsfatal_error_types        rw   _CollectCommandAndErrorMetricsz/MetricsCollector._CollectCommandAndErrorMetricsh  s     ??>2Ld $9".  --?@#&t'<'<'C'C'E#F #&--0E0E"F
Jd N$;",N+,>?LN #G }5
$;"2  4 ry   c           
         | j                   yi }dD ]%  \  }}t        | j                   |      |t        |   <   ' t        j                  r| j                   j
                  }t        j                         }t        |j                         D cg c]  }|d   |d   z    c}      t        |j                         D cg c]  }|d   |d   z    c}      z
  |t        d   <   | j                   j                  r| j                   j                  rdnd}nd}||t        d	   <   | j                   j                  r| j                   j                  rdnd
}n| j                   j                  rdnd}||t        d   <   | j                   j                  | j                   j                  z   }	|	r5t        | j                   j                        t        |	      z  |t        d   <   | j                   j                   ri| j                   j                   j                         D 
cg c]  }
|
j#                          }}
t%        |      |t        d   <   t'        |      |t        d   <   dj)                  t+        | j                   j,                              |t        d   <   | j                   j                  xr | j                   j.                  | j                   j                  xr | j                   j0                  | j                   j2                  | j                   j                  xr | j                   j.                  | j                   j                  xr | j                   j0                  d}dj)                  t+        t5        j6                  |      D cg c]	  \  }}|r| c}}            }t9        | j                   j:                        } | j<                  dt>        ||d| yc c}w c c}w c c}
w c c}}w )zCAggregates PerformanceSummary info and adds the metric to the list.N))r  r@   )r	  rA   )r
  rE   )r  rF   )r  rD   )r  rB   )r  rC         rJ   bothcloudfiler)   fanslicenoner'   rG   rH   rI   r   r+   )CloudToCloudCloudToFile
DaisyChainFileToCloud
FileToFile)r   r   r   r   ) rq   r9  ri   r   r  r  r  rG  rH  r  r  r  r  r  r  floatr  r-  minmaxr   r   r  r  r  r  r   r   rY   r  r  _GA_PERFSUM_CATEGORY)rr   r   	attr_namer   
disk_startdisk_endstatsrc_url_typestrategy
total_timethreadthroughputstransfer_typestransfer_typecondr   apply_execution_times                    rw    _CollectPerformanceSummaryMetricz1MetricsCollector._CollectPerformanceSummaryMetric  s   #M	5 -4D4H4H4=-?mM%() '';;j,,.h X__->?->TtAwa ->?
@
Z->->-@A-@TtAwa -@A
BC M/23
 ))#33@@Vgll6BM- 123 $$//::h 00;;h;CM- 678 &&77&&<<=J
$$55
6z9J
J M"<=> .. ,,??FFHHf 


 H   CF
CmM"=>?BE
CmM"=>? 69XXt##22365M- 012   .. 3$$22  .. 2$$11  //  -- 3$$22  -- 2$$11N  XX'*}}^'D
'D#t 'D
 	F ,//1 D *"6 &(<* )*S @A2@
s   O:O
O"O'c                    | j                          | j                          | j                  sy|s| j                  j	                         }t
        j                  j                  d      dk(  rt        j                  }t        j                  d      }t        j                  |j                        }|5  t        j                   | j                  |       ddd       t        j"                  | j                         g | _        |t        j                  d|z        }t        j                  dj%                  |||            }t&        j(                  d|g}t
        j                  j+                         }t
        j,                  j/                  t&        j0                        |d	<   t3               }	t        j4                  |      D ]0  \  }
}t        j                  |      |	t        j                  |
      <   2 	 t7        j8                  ||	t        j:                  xr t<        j>                  
      }| j                  j#                  d       |r,|jA                          | j                  j#                  d       yy# 1 sw Y   xY w# tB        $ rF | j                  j#                  d       	 t        jD                  |j                         Y y#  Y Y yxY ww xY w)aZ  Reports the collected metrics using a separate async process.

    Args:
      wait_for_report: bool, True if the main process should wait for the
        subprocess to exit for testing purposes.
      log_level: int, The subprocess logger's level of debugging for testing
        purposes.
      log_file_path: str, The file that the metrics_reporter module should
        write its logs to. If not supplied, the metrics_reporter module will
        use a predetermined default path. This parameter is intended for use
        by tests that need to evaluate the contents of the file at this path.
    Nr   r   F)deletezr"%s"z_from gslib.metrics_reporter import ReportMetrics; ReportMetrics(r"{0}", {1}, log_file_path={2})z-c
PYTHONPATH)envshellz$Metrics reporting process started...z#Metrics reporting process finished.z*Metrics reporting process failed to start.)#rO  ro  rn   r^   getEffectiveLevelr   r   r   r\   WARNtempfileNamedTemporaryFiler   
ensure_strnamepickledumpr   rk   sys
executablecopypathsepr   r   dictr   
subprocessPopenPY3r   
IS_WINDOWScommunicateOSErrorunlink)rr   wait_for_report	log_levellog_file_pathtemp_metrics_filetemp_metrics_file_namereporting_codeexecution_argsexec_envsm_envr  rQ   ps                rw   ReportMetricszMetricsCollector.ReportMetrics  s$     	'')))+==++//1i	zz~~-.#5,,i 335A ^^,=,B,BC	kk$--!23 
MM$-- DM  nnW}%<=m^^	88>"I}9>?N nndN;Nzz HZZ__SXX6H\VFh'1"%.."3fS^^A ( 

>%"%''"Dk.D.DGa kk>?	 	
?@	 
; 
	D   kkDE
		#(()s1   3!I86B J 8J$K*KKKK)zhttps://example.comzuser-agent-007Nr}   )FNN)r.  r/  r0  r1  _GA_TID_GA_ENDPOINTrx   r   r   rd   staticmethodr   r   classmethodr   r   r   r_   r   r   r   r  r2  r3  r=  r8  rO  ro  r  r   ry   rw   rL   rL   q   s     $l 6 p )/m^ ! & & , , ! !. "7$4#'# #4 	& 	& $ $ 	&
9 $%)%,RD< D<LF:PE,40f*R %*""&Gry   rL   c                 .     t                fd       }|S )zFunction decorator to capture and log any exceptions.

  This is extra insurance that analytics collection will not hinder the command
  being run upon an error.

  Args:
    func: The function to wrap.

  Returns:
    The wrapped function.
  c                      	  | i |S # t         $ r<}t        j                  d      }|j                  dj                  |       Y d }~y d }~ww xY w)Nmetricsz6Exception captured in %s during metrics collection: %s)	Exceptionr\   r]   r   r.  )argskwdser^   funcs       rw   Wrapperz'CaptureAndLogException.<locals>.Wrapper?  sU    %4 4   %  +fllK==!% %%s    	A2AAr   r  r  s   ` rw   CaptureAndLogExceptionr  2  s"     	;% % 
.ry   c                 .     t                fd       }|S )a\  Function decorator to ignore an exception on collecting thread stats.

  An exception can happen if the thread_stats dictionary's manager gets shutdown
  before the thread's process is successfully killed. See
  _ThreadStat.AggregateStat for how we handle that case.

  Args:
    func: The function to wrap.

  Returns:
    The wrapped function.
  c                  $    	  | i |S #  Y y xY wr}   r   )r  r  r  s     rw   r  z+CaptureThreadStatException.<locals>.WrapperY  s"    4 4  
s    r   r  s   ` rw   CaptureThreadStatExceptionr  K  s"     	;  
.ry   c                  R    t         j                         } | r| j                          yy)z9Reports the metrics that were collected upon termination.N)rL   r   r  )	collectors    rw   Shutdownr  f  s&     ++-) ry   c           
      <   t         j                         }|sy| r+|j                  d      s|j                  t        d   | i       |re|j                  d      sTdj                  t        |D cg c]  }|d   j                  d       c}            }|j                  t        d   |i       |j                  d      } | sy|r;dj                  | dj                  |            }|j                  t        d   |i       |re|j                  d	      sTdj                  t        |D cg c]  }|d   j                  d       c}            }	|j                  t        d	   |	i       |r-|j                  d
      s|j                  t        d
   |i       yyyc c}w c c}w )a  Logs info about the gsutil command being run.

  This only updates the collector's ga_params. The actual command metric will
  be collected once ReportMetrics() is called at shutdown.

  Args:
    command_name: str, The official command name (e.g. version instead of ver).
    subcommands: A list of subcommands as strings already validated by
      RunCommand. We do not log subcommands for the help or test commands.
    global_opts: A list of string tuples already parsed by __main__.
    sub_opts: A list of command-level options as string tuples already parsed
      by RunCommand.
    command_alias: str, The supported alias that the user inputed.
  Nr   r   r   r   -z{0} {1} r   r#   )	rL   r   r   r   ri   r   r   striprk   )
rK  subcommandsglobal_optssub_optscommand_aliasr  optglobal_opts_stringfull_command_namesub_opts_strings
             rw   LogCommandParamsr  o  s   ( ++-)	
 )..~>mN;\JK--.>?[9[cAS![9:<	'	(*<=?
 %%n5,	
!((sxx7LMmN;=NOPi**+BChhvH&MHSs1v||C'8H&MNOO	.	/AC9//@mO<mLM A]! : 'Ns   ,F$Fc                     t         j                         }|rG|j                  | j                  xx   dz  cc<   | j                  rt        d       yt        d       yy)zLogs that a retryable error was caught for a gsutil command.

  Args:
    message: The RetryableErrorMessage posted to the global status queue.
  rP   )r
  )r  N)rL   r   rp   rL  is_service_errorLogPerformanceSummaryParams)messager  s     rw   LogRetryableErrorr    sO     ++-)w112a72!qA!qA ry   c                     t         j                         }|r/|j                  t        d   | j                  j
                  i       yy)z}Logs that a fatal error was caught for a gsutil command.

  Args:
    exception: The exception that the command failed on.
  r%   N)rL   r   r   ri   	__class__r.  )	exceptionr  s     rw   LogFatalErrorr    sB     ++-)	}	%y':':'C'CDF ry   c                  T    t         j                         }|r|j                  |        yy)a  Logs parameters necessary for a PerformanceSummary.

  gsutil periodically monitors its own threads; at the end of the execution of
  each cp/rsync command, it will present a performance summary of the command
  run.

  Args:
    **params_to_update: A dictionary. See UpdatePerformanceSummaryParams for
        details. The argument ambiguity at this level allows for flexibility in
        dealing with arguments that are processed similarly.
  N)rL   r   r=  )params_to_updater  s     rw   r  r    s)     ++-),,-=> ry   c                  P   t         j                  j                  dd      } t        j                  j                  t              s| st        j                         st        dt        j                  d      z   dz         }t        }|j                         d   dk(  rt        j                         j                   }t        j"                  t        j                  j%                  t                     t'        t        d      5 }|j)                  |       d	d	d	       y	y	y	y	# 1 sw Y   y	xY w)
zAsks a user to opt-in to data collection if a UUID file does not exist.

  If the user agrees, generates a UUID file. Will not prompt if part of SDK.
  rN   r   
zgsutil developers rely on user feedback to make improvements to the tool. Would you like to send anonymous usage statistics to help improve gsutil? [y/N]r  r   ywN)r`   ra   r~   r   r   r   r   r   r   r
   textwrapfillr   r   uuiduuid4hexCreateDirIfNeededdirnamer   write)disable_promptenable_analyticstext_to_writer   s       rw   'CheckAndMaybePromptForAnalyticsEnablingr    s     ;;((3MN.
''..
).

(
(
*THMM	 %! ! $'' (
 #M"c)jjl&&m!!"''///"BC	os	#qggm 
$	# + 3A
) 
$	#s   >DD%c                 J    | t        j                          } t        | dz        S )Ni  )rB  ro   )time_in_secs    rw   rY   rY     s#    ))+K	[4	  ry   )NNNNNr}   )@r1  
__future__r   r   r   r   atexitcollectionsr   	functoolsr	   r\   r   r{  rl   rerg   r  r}  rw  r  rB  r  r   	six.movesr
   r   r`   gslibr   gslib.metrics_tupler   gslib.utilsr   gslib.utils.unit_utilr   r   r  r  rc   rF  rI  rJ  ra  compilere   r   
expanduserr   r   r   ri   r2  rL   r  r  registerr  r  r  r  r  r  rY   r   ry   rw   <module>r     s{   A & %  '  #   	   	   
     
     & # 5 69
"! * & +  "rzz"34 ''$$RWW\\#2J&L M dD 4 4	
 E e U e U 5 E u e  e  !" 5#$ +0)."'$)$* &!'!'9@~
v ~
B26    "&!%!%"#'	.N .Nb B B" 	F 	F ? ?"  *!ry   