
    }                     V   d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlZddl	m
Z
 ddlmZ ddlmZ dd	lmZ d
 Z ej"                  dde      dd       Z ej"                  dde      d        Z G d dej(                        Z G d d      Zd Z	 	 ddZd Zd Z	 ddZy)zJFor managing the copy manifest feature (manifest = a file with copy info).    )absolute_import)division)unicode_literalsN)thread_messages)
properties)files)retryc                     ~~~| t         k(  S )z,Check if the exception is a PermissionError.)PermissionError)exc_type	exc_valueexc_tracebackstates       7lib/googlecloudsdk/command_lib/storage/manifest_util.py!_should_retry_if_permission_errorr       s     	_	$$       iX  )max_retrialssleep_msshould_retry_ifc                 2    t        j                  | ||      S )a  Returns the file handle for the given file path.

  We use a retry approach here to avoid failing early if
  another process, like an antivirus, has acquired the file.
  See https://github.com/python/cpython/issues/136965 for more details.

  Args:
    file_path (str): The path to the file to open.
    append (bool): Whether to open the file in append mode.
    newline (str|None): The line ending style to use, or None to use plaform
      default.
  appendnewline)r   
FileWriter)	file_pathr   r   s      r   get_file_write_handler   (   s    $ 
		)FG	DDr   c                 ,    t        j                  |       S )a.  Returns the file handle for the given file path.

  We use a retry approach here to avoid failing early if
  another process, like an antivirus, has acquired the file.
  See https://github.com/python/cpython/issues/136965 for more details.

  Args:
    file_path (str): The path to the file to open.
  )r   
FileReader)r   s    r   get_file_read_handler    =   s     
		)	$$r   c                       e Zd ZdZdZdZy)ResultStatuserrorOKskipN)__name__
__module____qualname__ERRORr$   SKIP r   r   r"   r"   O   s    
%"	$r   r"   c                       e Zd ZdZd ZddZy)ManifestManagerz*Handles writing copy statuses to manifest.c                    g dt         j                  j                  j                  j	                         rdgng z   g dz   | _        || _        t        j                  j                  |      r#t        j                  j                  |      dkD  ryt        |d      5 }t        j                  || j
                        j                          ddd       y# 1 sw Y   yxY w)z+Creates manifest file with correct headers.)SourceDestinationStartEndMd5UploadId)Source SizeBytes TransferredResultDescriptionr   N
)r   )r   VALUESstoragerun_by_gsutil_shimGetBool_manifest_column_headers_manifest_pathospathexistsgetsizer   csv
DictWriterwriteheader)selfmanifest_pathfile_writers      r   __init__zManifestManager.__init__X   s    	
   ((;;CCE L	

	
 	!* (D	ww~~m$)G!)K	t
		nn[$"?"?@LLN
 
 
s   /CCNc           
      0   |r)|j                   t        j                  u r|j                  }nd}|j                  j                  d      }|r|j                  j                  d      }n|}|j                  r-|j                  j                  dd      j                  dd      }nd}|j                  j                  |j                  j                  |||j                  xs d|j                  ||j                   j                  |d	}t         j"                  j$                  j&                  j)                         rd	|d
<   t+        | j,                  dd      5 }t/        j0                  || j2                        j5                  |       d	d	d	       y	# 1 sw Y   y	xY w)zWrites data to manifest file.r   z%Y-%m-%dT%H:%M:%S.%fZr9   z\nz\r )	r/   r0   r1   r2   r3   r5   r6   r7   r8   Nr4   Tr   )result_statusr"   r$   total_bytes_copiedend_timestrftime
start_timedescriptionreplace
source_url
url_stringdestination_urlversionless_url_stringmd5_hashsizevaluer   r:   r;   r<   r=   r   r?   rD   rE   r>   writerow)	rG   manifest_messagefile_progressbytes_copiedrP   rR   rS   row_dictionaryrI   s	            r   	write_rowzManifestManager.write_rowy   sa   )77<??J"55ll((112IJH ++445LMjj##$0088uEMM
k k #--88'77NN((.B',,)"0066"
N   33;;=#'nZ 	D$
8;F	nn[2244<H^4L
8 
8 
8s   0FFN)r&   r'   r(   __doc__rJ   ra   r+   r   r   r-   r-   U   s    2OB#Mr   r-   c                    | rt         j                  j                  |       s
t               S t               }t	        |       5 }t        j                  |      }|D ]P  }|d   t        j                  j                  t        j                  j                  fv s=|j                  |d          R 	 ddd       |S # 1 sw Y   |S xY w)z>Extracts set of completed or skipped copies from manifest CSV.r7   r/   N)r@   rA   rB   setr    rD   
DictReaderr"   r$   r[   r*   add)rH   resfile_reader
csv_readerrows        r   parse_for_completed_sourcesrl      s    
BGGNN=95L#M*k,J	X<??00,2C2C2I2IJ	JH  +
 
* +
 
*s   AB7B77Cc                     | j                  t        j                  |j                  |j                  t        j                  j                         |j                  |||             y)z9Send ManifestMessage to task_status_queue for processing.)rU   rW   rP   rZ   rN   rY   rS   N)putr   ManifestMessagestorage_urldatetimeutcnowrZ   )task_status_queuesource_resourcedestination_resourcerN   rY   rS   s         r   _send_manifest_messagerv      sX     %%$00.::$$++-##% !	r   c           
      T    t        | ||t        j                  dt        |             y)z;Send ManifestMessage for failed copy to central processing.NrY   rS   )rv   r"   r)   str)rs   rt   ru   r#   s       r   send_error_messagerz      s(     e*r   c                 B    t        | ||t        j                  d|       y)z<Send ManifestMessage for skipped copy to central processing.Nrx   )rv   r"   r*   )rs   rt   ru   messages       r   send_skip_messager}      s$     r   c                 >    t        | ||t        j                  |       y)z?Send ManifestMessage for successful copy to central processing.N)rv   r"   r$   )rs   rt   ru   rY   s       r   send_success_messager      s    
 *O-|Jr   )FN)NNrb   )rc   
__future__r   r   r   rD   rq   enumr@   "googlecloudsdk.command_lib.storager   googlecloudsdk.corer   googlecloudsdk.core.utilr   r	   r   RetryOnExceptionr   r    Enumr"   r-   rl   rv   rz   r}   r   r+   r   r   <module>r      s    Q &  ' 
   	 > * * *% 5
E
E  5

%

%499 GM GMT
" %)'+	,		 #'Jr   