
    Y<                        d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlZddl	Z	ddl
mZ ddl
mZ ddl
mZ dd	l
mZ dd
lmZ ddlmZ ddlmZ ddlZdZ G d dej.                        Z G d dej.                        Z ej4                  dddg      Z G d d      Zd Z G d d ej<                  ej>                  e             Z! G d de!      Z" G d de!ejF                        Z$d Z%d$dZ& G d  d!      Z' G d" d#      Z(y)%z1Tools for monitoring and reporting task statuses.    )absolute_import)division)unicode_literalsN)errors)manifest_util)metrics_util)thread_messages)log)progress_tracker)scaled_integer   c                       e Zd ZdZdZdZdZy)OperationNameDownloadingzIntra-Cloud CopyingzDaisy Chain Copying	UploadingN)__name__
__module____qualname__DOWNLOADINGINTRA_CLOUD_COPYINGDAISY_CHAIN_COPYING	UPLOADING     ;lib/googlecloudsdk/command_lib/storage/tasks/task_status.pyr   r   *   s    +--)r   r   c                       e Zd ZdZdZy)IncrementTypeINTEGERFILES_AND_BYTESN)r   r   r   r   r   r   r   r   r   r   1   s    '%/r   r   ProgressManagerArgsincrement_typemanifest_pathc                       e Zd ZdZ	 	 	 ddZy)FileProgressa^  Holds progress information for file being copied.

  Attributes:
    component_progress (dict<int,int>): Records bytes copied per component. If
      not multi-component copy (e.g. "sliced download"), there will only be one
      component.
    start_time (datetime|None): Needed if writing file copy results to manifest.
    total_bytes_copied (int|None): Sum of bytes copied for each component.
      Needed because components are popped when completed, but we don't want to
      lose info on them if writing to the manifest.
    error_occurred (bool): Whether an error occurred during the operation.
  Nc                 v    t        |      D ci c]  }|d c}| _        || _        || _        || _        y c c}w Nr   )rangecomponent_progress
start_timetotal_bytes_copiederror_occurred)selfcomponent_countr)   r*   r+   is         r   __init__zFileProgress.__init__H   sC     .3?-CD-Cq!t-CDD DO0D(D Es   
6)NNF)r   r   r   __doc__r/   r   r   r   r$   r$   :   s      
)r   r$   c                 T    t        | |z  d      }t        j                  |d      dz   S )Nr      decimal_placesz/s)maxr   FormatBinaryNumber)bytes_processed
time_deltathroughput_bytess      r   _get_formatted_throughputr:   U   s4    :5q9		*	*q
*,0
1 1r   c                   j    e Zd ZdZej
                  d        Zd Zej
                  d        Zd Z	d Z
y)_StatusTrackerz>Abstract class for tracking and displaying operation progress.c                      y)z4Generates string to illustrate progress to the user.Nr   r,   s    r   _get_status_stringz!_StatusTracker._get_status_string^   s     	r   c                      y)z.Generates string for when StatusTracker exits.
r   r>   s    r   _get_done_stringz_StatusTracker._get_done_stringc   s    r   c                      y)zProcesses task status message for printing and aggregation.

    Args:
      status_message (thread_messages.*): Message to process.
    Nr   r,   status_messages     r   add_messagez_StatusTracker.add_messageg   s     	r   c                     t        j                  d| j                  | j                  d      | _        | j                  j                          | S )Nz  T)messagedetail_message_callbackdone_message_callback
no_spacing)r   ProgressTrackerr?   rB   _progress_tracker	__enter__r>   s    r   startz_StatusTracker.startp   sF    -== $ 7 7"33	D
 	$$&Kr   c                 X    | j                   r| j                   j                  |||       y y N)rM   __exit__r,   exc_typeexc_valexc_tbs       r   stopz_StatusTracker.stopy   s(    
%%h@ r   N)r   r   r   r0   abcabstractmethodr?   rB   rF   rO   rW   r   r   r   r<   r<   [   sG    F	 	 	 	Ar   r<   c                   .     e Zd ZdZ fdZd Zd Z xZS )_IntegerStatusTracker8See super class. Tracks both file count and byte amount.c                 F    t         t        |           d| _        d| _        y r&   )superr[   r/   
_completed_total_estimation)r,   	__class__s    r   r/   z_IntegerStatusTracker.__init__   s     	
/1DODr   c                     | j                   r'dj                  | j                  | j                         }n| j                  }dj                  |      S )See super class.{}/{}zCompleted {})r`   formatr_   )r,   file_progress_strings     r   r?   z(_IntegerStatusTracker._get_status_string   sH    $^^DOO,0,B,BD "__""#788r   c                     t        |t        j                        r | xj                  |j                  z  c_        yt        |t        j
                        r| xj                  dz  c_        yyrc   r2   N)
isinstancer	   WorkloadEstimatorMessager`   
item_countIncrementProgressMessager_   rD   s     r   rF   z!_IntegerStatusTracker.add_message   sM    ./"J"JK
 9 99	NO$L$L	M
ooo 
Nr   )r   r   r   r0   r/   r?   rF   __classcell__ra   s   @r   r[   r[   ~   s    @
9r   r[   c                   R     e Zd ZdZd
 fd	Zd Zd Zd Zd Zd Z	d Z
 fd	Z xZS )_FilesAndBytesStatusTrackerr\   c                    t         t        |           d| _        d| _        d| _        d| _        d | _        d | _        d| _	        d | _
        d| _        d | _        i | _        |rt        j                  |      | _        y d | _        y r&   )r^   rp   r/   _completed_files_processed_bytes_total_files_estimation_total_bytes_estimation_first_operation_time_last_operation_time_total_processed_bytes_window_start_time_window_processed_bytes_window_throughput_tracked_file_progressr   ManifestManager_manifest_manager)r,   r"   ra   s     r   r/   z$_FilesAndBytesStatusTracker.__init__   s    	
%t57DD#$D #$D  "&D $D"#D #D#$D "D #%D,<<]Kd#dr   c                    t        j                  | j                  d      }| j                  r'dj	                  | j
                  | j                        }n| j
                  }| j                  r4t        j                  | j                  d      }dj	                  ||      }n|}| j                  rd| j                  z   }nd}dj	                  |||      S )rc   r2   r3   rd   z |  zCompleted files {} | {}{})r   r6   rs   rt   re   rr   ru   r{   )r,   scaled_processed_bytesrf   scaled_total_bytes_estimationbytes_progress_stringthroughput_addendum_strings         r   r?   z._FilesAndBytesStatusTracker._get_status_string   s    +>>a1##$^^D,A,A,0,H,HJ "22##&4&G&G

&
&q':#%nn-C-JL 5#(4+B+B#B #% (//0D0E0JL Lr   c                 Z   | j                   #|j                  | _         |j                  | _        n|j                  | _        | xj                  |z  c_        |j                  | j                  z
  }|t
        kD  r4t        | j                  |      | _        |j                  | _        d| _        yy)z<Updates stats and recalculates throughput if past threshold.Nr   )rv   timery   rw   rz   $_THROUGHPUT_WINDOW_THRESHOLD_SECONDSr:   r{   )r,   rE   processed_bytesr8   s       r   _update_throughputz._FilesAndBytesStatusTracker._update_throughput   s    !!)#1#6#6d  . 3 3d"0"5"5d  O3 $$t'>'>>J88 9

&
&
!4d . 3 3d%&d"	 9r   c                     | xj                   |j                  z  c_         | xj                  |j                  z  c_        y)z?Adds WorloadEstimatorMessage info to total workload estimation.N)rt   rk   ru   sizerD   s     r   _add_to_workload_estimationz7_FilesAndBytesStatusTracker._add_to_workload_estimation   s0      N$=$==   N$7$77 r   c                 X   |j                   j                  }|| j                  vr|j                  r$t	        |j                        | j                  |<   nt	        d      | j                  |<   | j
                  rht        j                  j                  |j                  t        j                  j                        | j                  |   _        d| j                  |   _        | j                  |   j                  }|j                  r|j                  }nd}|j                  |j                   z
  }||j#                  |d      z
  }| xj$                  |z  c_        | j'                  ||       | j
                  r"| j                  |   xj                  |z  c_        |j(                  rd| j                  |   _        ||j*                  k(  r^|j-                  |d       |sI| j                  |   j(                  s| xj.                  dz  c_        | j
                  s| j                  |= yyy|||<   y)z-Track progress of a multipart file operation.)r-   r2   r   TN)
source_url
url_stringr|   total_componentsr$   r~   datetimefromtimestampr   timezoneutcr)   r*   r(   component_numbercurrent_byteoffsetgetrs   r   r+   lengthpoprr   )r,   rE   file_url_stringcomponent_trackerr   processed_component_bytesnewly_processed_bytess          r   _add_progressz)_FilesAndBytesStatusTracker._add_progress   s   $//::Od999		(	(7C*;;8=##O4 8D8##O4			++N,?,?,4,=,=,A,AC 	##O4? KL##O4G33++  &&'88 	##n&;&;;  	"$5$9$9:JA$NN 22N,AB
!!
 .--1FG-$$DHd!!/2A N$9$99,d3**?;JJ


1
$
%%))/: &	  -F()r   c                     | j                   st        j                  d      | j                  j	                  |j
                  j                  d      }| j                   j                  ||       y)z<Updates manifest file and pops file from tracking if needed.zRReceived ManifestMessage but StatusTracker was not initialized with manifest path.N)r~   r   Errorr|   r   r   r   	write_row)r,   rE   file_progresss      r   _add_to_manifestz,_FilesAndBytesStatusTracker._add_to_manifest  s`    !!LL  //33!!,,d4M$$^]Cr   c                    t        |t        j                        r| j                  |       yt        |t        j                        r#| j                  |       | j                  |       yt        |t        j                        r| xj                  dz  c_        yt        |t        j                        r| j                  |       yyrh   )ri   r	   rj   r   DetailedProgressMessage#_set_source_and_destination_schemesr   rl   rr   ManifestMessager   rD   s     r   rF   z'_FilesAndBytesStatusTracker.add_message'  s    ./"J"JK
&&~6	NO$K$K	L
..~>
(	NO$L$L	M
q 	NO$C$C	D
N+ 
Er   c                    t         t        |   |||       | j                  | j                  | j                  | j                  k7  r| j                  | j                  z
  }t
        j                  j                  dj                  t        | j                  |                   | j                  | j                  || j                         y y y y )Nz
Average throughput: {})r^   rp   rW   rv   rw   r
   statusPrintre   r:   rs   _report_metricsrr   )r,   rT   rU   rV   r8   ra   s        r   rW   z _FilesAndBytesStatusTracker.stop4  s    	
%t1(GVL"".!!-""d&?&??,,t/I/IIj	jj188
#D$9$9:
FH I 400*002 	@ 	. 	/r   rQ   )r   r   r   r0   r/   r?   r   r   r   r   rF   rW   rm   rn   s   @r   rp   rp      s7    @$6L4'"8
4Fl	D,2 2r   rp   c                     d}	 | j                         }|dk(  rn|r|j                  |       nd}-|rt        j                  d       yy)zGThread method for submiting items from queue to tracker for processing.FT	_SHUTDOWNzLStatus message submitted to task_status_queue without a manager to print it.N)r   rF   r
   warning)task_status_queuestatus_trackerunhandled_message_existsrE   s       r   status_message_handlerr   C  s[    "&**,N$  0!% 	 KK ( ) r   c                 2    | t        | |      S t               S )aj  Factory function that returns a ProgressManager instance.

  Args:
    task_status_queue (multiprocessing.Queue|None): Tasks can submit their
      progress messages here.
    progress_manager_args (ProgressManagerArgs|None): Determines what type of
      progress indicator to display.

  Returns:
    An instance of _ProgressManager or _NoOpProgressManager.
  )_ProgressManager_NoOpProgressManager)r   progress_manager_argss     r   progress_managerr   U  s"     "-/DEE!!r   c                   $    e Zd ZdZddZd Zd Zy)r   zContext manager for processing and displaying progress completing command.

  Ensure that this class is instantiated after all the child
  processes (if any) are started to prevent deadlock.
  Nc                 <    || _         d| _        d| _        || _        y)a  Initializes context manager.

    Args:
      task_status_queue (multiprocessing.Queue): Tasks can submit their progress
        messages here.
      progress_manager_args (ProgressManagerArgs|None): Determines what type of
        progress indicator to display.
    N)_progress_manager_args_status_message_handler_thread_status_tracker_task_status_queue)r,   r   r   s      r   r/   z_ProgressManager.__init__n  s$     #8D*.D'D/Dr   c                 
   | j                   r| j                   j                  t        j                  u rt	               | _        nJ| j                   j                  t        j                  u r$t        | j                   j                        | _        t        j                  t        | j                  | j
                  f      | _        | j                  j                          | j
                  r| j
                  j                          | S )N)targetargs)r   r!   r   r   r[   r   r   rp   r"   	threadingThreadr   r   r   rO   r>   s    r   rN   z_ProgressManager.__enter__|  s    ""		$	$	3	3}7L7L	L46''66))*:''55 7 +4*:*:%%%t';';<+>D' 	''--/
  "Kr   c                     | j                   j                  d       | j                  j                          | j                  r| j                  j                  |||       y y )Nr   )r   putr   joinr   rW   rS   s       r   rR   z_ProgressManager.__exit__  sN    ,'',,.
'6: r   rQ   )r   r   r   r0   r/   rN   rR   r   r   r   r   r   g  s    0$;r   r   c                       e Zd ZdZd Zd Zy)r   z}Progress Manager that does not do anything.

  Similar to contextlib.nullcontext, but it is available only for Python3.7+.
  c                     | S rQ   r   r>   s    r   rN   z_NoOpProgressManager.__enter__  s    Kr   c                  
    ~~~y rQ   r   rS   s       r   rR   z_NoOpProgressManager.__exit__  s    	7Fr   N)r   r   r   r0   rN   rR   r   r   r   r   r     s    
	r   r   )NN))r0   
__future__r   r   r   rX   collectionsr   enumr   "googlecloudsdk.command_lib.storager   r   r   r	   googlecloudsdk.corer
   googlecloudsdk.core.consoler   googlecloudsdk.core.utilr   sixr   Enumr   r   
namedtupler    r$   r:   with_metaclassABCMetaobjectr<   r[   MetricsReporterrp   r   r   r   r   r   r   r   <module>r      s    8 &  ' 
     5 < ; > # 8 3 

 () $DII &DII &
 -k,,,o>@ ) )61 A'S''V<  AFN 4h2.,2N2N h2V)$"$,; ,;^	 	r   