
                  	       x   d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlmZ ddl	m
Z
 ddl	mZ dd	l	mZ dd
l	mZ ddl	mZ ddl	mZ ddl	mZ ddl	mZ ddl	mZ ddl	mZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z! ddl m"Z" ddl#m$Z$ ddl%Z%dZ&dZ'dZ(dZ)	 d=dZ*d=d Z+	 d>d!Z,d" Z-d# Z.d$ Z/d?d%Z0d& Z1	 	 d@d'Z2 G d( d)ejf                        Z4d*e5d+e5fd,Z6	 	 	 	 	 	 dAd-Z7d. Z8	 	 	 	 	 	 dBd/Z9d0 Z:d1 Z;d2 Z<d3 Z=d4 Z>d5 Z?d6 Z@	 	 	 	 dCd7e5d8e5d9e5d*e5fd:ZA	 	 	 	 	 	 dBd;ZB	 	 	 	 	 	 	 	 dDd<ZCy)EzUtils for the rsync command.    )absolute_import)division)unicode_literalsN)	cloud_api)errors)fast_crc32c_util)	hash_util)	path_util)plurality_checkable_iterator)
posix_util)progress_callbacks)storage_url)tracker_file_util)wildcard_iterator)resource_reference)resource_util)patch_file_posix_task)copy_task_factory)	copy_util)patch_object_task)delete_task)log)
properties)files,
   z&Did not find existing container at: {}managed_foldersc                    t        j                  | t        j                  j                  d|      }t        j                  |      }|j                         rht        j                  |       r+t        j                  dt        j                  |       z         t        j                  t        j                   |             S |j#                         r.t        j                  dj                  | t%        |                  t%        |      d   }|j'                         r|S t        j                  dj                  | |            )z>Returns existing valid container or UnknownResource or raises.T)fields_scopeget_bucket_metadataignore_symlinksz"Wildcard pattern matched nothing. z {} matched more than one URL: {}r   z {} matched non-container URL: {})r   get_wildcard_iteratorr   FieldsScopeSHORTr   PluralityCheckableIteratoris_emptycontains_wildcardr   InvalidUrlError_NO_MATCHES_MESSAGEformatr   UnknownResourcer   storage_url_from_string	is_plurallistis_container)pathr!   resource_iterator%plurality_checkable_resource_iteratorresources        <lib/googlecloudsdk/command_lib/storage/rsync_command_util.py0get_existing_or_placeholder_destination_resourcer5   :   s+    (==
((..%	 #==>OP ( +335**40""
.&&t,-  --++D1  +446

 
 *11$<=	
  78;(O(//h?	     c                     t        | |      }t        |t        j                        r(t	        j
                  t        j                  |             |S )z>Gets existing container resource at path and errors otherwise.)r5   
isinstancer   r+   r   r(   r)   r*   )r0   r!   r3   s      r4   get_existing_container_resourcer9   a   sF    =
O( ,<<=

 
 !4!;!;D!A
BB	/r6   c                    t        j                  |       }|rt        nd}t        j                  ||z         }|dj	                  |      }ndj	                  ||      }t        j
                  |       t        j                  j                  t        j                  j                  j                  j                         |      S )a  Hashes and returns a list file path.

  Args:
    list_file_name (str): The list file name prior to it being hashed.
    chunk_number (int|None): The number of the chunk fetched if file represents
      chunk of total list.
    is_managed_folder_list (bool): If True, the file will contain managed folder
      resources instead of object resources, and should have a different name.

  Returns:
    str: Final (hashed) list file path.

  Raises:
    Error: Hashed file path is too long.
   zFULL_{}zCHUNK_{}_{})r   get_delimiterless_file_path_MANAGED_FOLDER_PREFIXget_hashed_file_namer*   raise_exceeds_max_length_errorosr0   joinr   VALUESstoragersync_files_directoryGet)list_file_namechunk_numberis_managed_folder_listdelimiterless_file_namemanaged_folder_prefixhashed_file_namehashed_file_name_with_types          r4   get_hashed_list_file_pathrM   k   s    $ .II !7B  ';;55 !*!1!12B!C!.!5!5," 223MN	5599; 
 r6   c                     	 t        j                  |        y# t        $ r/}t        j                  dj                  | |             Y d}~yd}~ww xY w)z@Tries to delete file and debug logs instead of failing on error.zFailed to delete file {}: {}N)r@   remove	Exceptionr   debugr*   )r0   es     r4   try_to_delete_filerS      s?    >IIdO	 >II,33D!<==>s    	A%AAc                    | j                   j                  }t        | t        j                        r|S t        | t        j
                        rd}d}d}d}d}d}d}d}	d}
d}n| j                  }| j                  }| j                  }t        j                  |       \  }}}}}||}nt        j                  | j                        }|r|j                  nd}	| j                  t        j                   k(  rd}
n| j                  }
| j"                  }|||||||||	|
|g}t$        j'                  |D cg c]  }|dnt)        j*                  |       c}      S c c}w )a%  Builds a line for files listing the contents of the source and destination.

  Args:
    resource (FileObjectResource|ObjectResource|ManagedFolderResource): Contains
      item URL and metadata, which can be generated from the local file in the
      case of FileObjectResource.

  Returns:
    String formatted as "URL,etag,size,atime,mtime,uid,gid,mode,crc32c,md5".
      A missing field is represented as an empty string.
      "mtime" means "modification time", a Unix timestamp in UTC.
      "mode" is in base-eight (octal) form, e.g. "440".
  Nr;   )r   
url_stringr8   r   ManagedFolderResourceFileObjectResourceetagsizestorage_classr   (get_posix_attributes_from_cloud_resourcer   get_unix_timestamp_in_utccreation_timebase_eight_strcrc32c_hashNOT_SUPPORTED_DO_NOT_DISPLAYmd5_hashCSV_FIELD_SEPARATORrA   six	text_type)r3   urlrX   rY   rZ   atimemtimeuidgidmode_base_eightcrc32cmd5custom_metadata_mtimemodeline_valuesxs                   r4   get_csv_line_from_resourcerq      su    	''#,BBC J,??@DDMEE
C
COF
C==D==D**M;;HE 1E #sD (#e
 55h6L6LMe-1d))tO1NNNf##f


C 


			+ 
	!	!6ABkQYrCMM!,,kB
 Bs   - Ec                 R    | j                         j                  t        t              S )z.Splits and returns the fields from a CSV line.)rstriprsplitrb   _CSV_COLUMNS_COUNT)lines    r4   get_fields_from_csv_linerw      s     
		13E	FFr6   c                    | syt        |       }|d   }t        j                  |      }t        |t        j                        rt        j                  |      S |rt        j                  |      S | j                         j                  t        t              \  }}}}}	}
}}}}}t        j                  ||r|nd|rt        |      nd|r|nd|r|nd|r|ndi       }t        j                  |j                   t        j"                  |	rt        |	      nd|
rt        |
      nd|rt        |      nd|rt        |      nd|rt        j$                  j'                  |      nd             |S )a  Parses a line from files listing of rsync source and destination.

  Args:
    line (str|None): CSV line. See `get_csv_line_from_resource` docstring.
    is_managed_folder (bool): If True, returns a managed folder resource for
      cloud URLs. Otherwise, returns an object URL.

  Returns:
    FileObjectResource|ManagedFolderResource|ObjectResource|None: Resource
      containing data needed for rsync if data line given.
  Nr   )rX   rY   rZ   r_   ra   custom_fields)rf   rg   rh   ri   rn   )rw   r   r,   r8   FileUrlr   rW   rV   rs   rt   rb   ru   ObjectResourceintr   1update_custom_metadata_dict_with_posix_attributesry   PosixAttributes	PosixModefrom_base_eight_str)rv   is_managed_folderline_informationrU   
url_object_etag_stringsize_stringstorage_class_stringatime_stringmtime_string
uid_string
gid_stringmode_base_eight_stringcrc32c_string
md5_stringcloud_objects                    r4   parse_csv_line_to_resourcer      s_    
-d3"*22:>*
K//000<<33J?? kkm.0BC $22%;4*3{,@(d#0-d'zT, >>    %1L!t%1L!t!+c*o!+c*o# ##778NO 
r6   c           
         | j                   |j                   k7  ryt        j                  j                  j                  j                         }|t        j                  j                  j                  k(  ry| |fD ]u  }t        |t        j                        s|j                  |j                  cxu r:n =t        j                  dj!                  |j"                  j$                                y t        | t        j                        rt        |t        j                        r| j                  }|j                  }| j                  }|j                  }t        j&                  dj!                  | j"                  j$                  |||j"                  j$                  ||             ||||k(  S ||||k(  S yt        | t        j(                        }|r|}	| }
n| }	|}
|	j                  c|	j                  Wt+        j,                  |       t+        j.                  d      s(|t        j                  j0                  j                  k(  ryd}n%|	j                  t+        j.                  d      }nd}|r't2        j4                  j6                  }|	j                  }n&t2        j4                  j8                  }|	j                  }t3        j:                  t3        j<                  |
j"                  j>                  |            }||k(  S )z;Does minimal computation to compare checksums of resources.FTzbFound no hashes to validate on {}. Will not copy unless file modification time or size difference.zComparing hashes for two cloud objects. CRC32C checked first. If no comparable hash pairs, will not copy.
{}:
  CRC32C: {}
  MD5: {}
{}:
  CRC32C: {}
  MD5: {}
)warn_for_always)install_if_missing) rY   r   rB   rC   check_hashesrE   CheckHashesNEVERvaluer8   r   r{   r_   ra   r   warningr*   r   versionless_url_stringrQ   rW   r   log_or_raise_crc32c_issuescheck_if_will_use_fast_crc32cIF_FAST_ELSE_SKIPr	   HashAlgorithmCRC32CMD5get_base64_hash_digest_stringget_hash_from_fileresource_name)source_resourcedestination_resourcer   r3   source_crc32cdestination_crc32c
source_md5destination_md5	is_uploadcloud_resourcelocal_resourcecompare_crc32chash_algorithm
cloud_hash
local_hashs                  r4    _compute_hashes_and_return_matchr   ,  s   1666 ""**77;;=,Z++11777"$89h(.==> 1 199	kk3396""994  : )88');)J)JK#//M-99 ))J*33OII	 f''>> ,,CC
"  %7%C000/"=?** *<*O*OP))N$N$N)N+0G0G0O //	J::#
 J22DDJJJN!!- &CC N N,,33N++J,,00N((J66""

$
$
2
2N*
 
z	!!r6   c                 v    |xs |}|s%|#|!||k(  xr | j                   |j                   k(   S t        | |       S )zHCompares metadata and returns if source should be copied to destination.)rY   r   )r   r   source_mtimedestination_mtimecompare_only_hashesis_cloud_source_and_destinationskip_mtime_comparisons          r4   (_compare_metadata_and_return_copy_neededr     sh     .P1P


"

' 	)) 	>  $8$=$==  .+ 
 r6   c                       e Zd ZdZdZdZdZy)_IterateResourcez)Indicates what resources to compare next.sourcedestinationbothN)__name__
__module____qualname____doc__SOURCEDESTINATIONBOTH r6   r4   r   r     s    1&+	$r6   r   skip_unsupportedreturnc                     |r_t        j                  |       }|rHt        j                  j	                  t         j
                  j                  | |j                               yy)a  Checks if object type is unsupported and logs warning if so.

  Args:
    resource: The resource to check.
    skip_unsupported: Whether to skip unsupported object types.

  Returns:
    True if the object type is unsupported and needs to be skipped
    False otherwise.
  TF)r   get_unsupported_object_typer   statusPrint!UNSUPPORTED_OBJECT_WARNING_FORMATr*   r   )r3   r   unsupported_types      r4   $_should_skip_unsupported_object_typer     sU     $@@J	jj

9
9
@
@(..
 	r6   c           	      <   t        ||      ry|r|}nt        |||      }|r~t        |t        j                        r4	 t        j                  |j                  j                        5  	 ddd       t        j                  j                  dj                  ||             yt        |t        j                        r|t        |j                  t        j                        rX| j                   r1| j                   j"                  rt$        j&                  j(                  }	nt$        j&                  j*                  }	nd}	t-        j.                  ||d|	|| d      S # 1 sw Y   xY w#  t        j                  dj                  |j                  j                                xY w)z5Generates copy tasks with generic settings and logic.NzCould not open {}zWould copy {} to {}T)do_not_decompressfetch_source_fields_scopeposix_to_setuser_request_argsverbose)r   _get_copy_destination_resourcer8   r   rW   r   BinaryFileReaderr   r   r   errorr*   r   r   CloudResourceCloudUrlresource_argspreserve_aclr   r#   FULLNO_ACLr   get_copy_task)
r   r   r   source_containerr   destination_containerdry_runr   copy_destinationr   s
             r4   _get_copy_taskr     sj    */;KL
+ 6)+@ /#5#H#HI	##O$?$?$M$MN
 O JJ$$_6FG  "4"B"BC
%11;3G3G
H 	''++88**//l**11lL		(	( ,)
 ; ON		&&++99	

 	s#   )E  &E(E  EE   ;Fc                 N    t         j                  }| r|rt         j                  }|S )zEReturns the iterator instruction for no clobber and new mtimes flags.)r   r   r   )$delete_unmatched_destination_objects%use_gsutil_delete_unmatched_behaviouriteration_instructions      r4   7_get_iterator_instruction_for_no_clobber_and_new_mtimesr     s(     +11*
/,11	r6   c
           	      :   t        j                  |      }
|rA|j                  5|
j                  )|j                  |
j                  k  rt        ||	      }d|fS t	        |t
        j                        xr t	        |t
        j                        }t        |||j                  |
j                  ||      r"t        | |||
|||      t        j                  fS | j                  xr ||
k7  }| xr) |j                  duxr |j                  |
j                  k7  }|s|sdt        j                  fS |rq|r/t        j                  j                  dj                  |             n.t        j                  j                  dj                  |             dt        j                  fS t	        |t
        j                        r(t!        j"                  |||       t        j                  fS t%        j&                  t        j(                         ||||
      t        j                  fS )zDSimilar to get_task_and_iteration_instruction except for equal URLs.N)r   r   )r   r   r   z!Would set POSIX attributes for {}zWould set mtime for {})r   r   )r   "get_posix_attributes_from_resourcerg   r   r8   r   r{   r   r   r   r   preserve_posixr   r   r   r*   r   PatchObjectTaskr   PatchFilePosixTaskget_system_posix_data)r   source_objectdestination_objectr   r   r   /skip_if_destination_has_later_modification_timer   r   r   destination_posixr   r   need_full_posix_updateneed_mtime_updates                  r4   @_compare_equal_object_urls_to_get_task_and_iteration_instructionr   "  s?    !CC 6



(

!
!
-


066
6
 	@01	
  '(($.'66% %J%'9'H'HI " .-&E 	!3-	
 	  &&L<;L+L  *) 8


D
(8


 1 7 7
7 
 !$5"''((	jj
-
4
45G
H 
jj/667IJK"''(("$6$E$EF))%/	

 	  ..

*
*
,



 	
 	r6   c                    |j                   }|j                  d      j                  }| j                   j                  }|j                  |      s%t	        j
                  dj                  ||            |t        |      d S )z-Removes container URL prefix from object URL.r;   z6Received container {} that does not contain object {}.N)r   rA   r   
startswithr   Errorr*   len)object_resourcecontainer_resourcecontainer_url,container_url_string_with_trailing_delimiterobject_url_strings        r4   $_get_url_string_minus_base_containerr     s    $00-1>1C1C2 / &11HH		%	%2
 ,,@GG8:K	
 
 
3KLN	OOr6   c                     t        | |      }|j                  |j                  j                  t        j                        S )z8Gets URL to compare to decide if resources are the same.)r   replacer   	delimiterCLOUD_URL_DELIMITER)r   r   containerless_object_url_strings      r4   _get_comparison_urlr    s?    $H)%! 
)	0	0$$..0O0O
 r6   c                    t        | |      }|j                  | j                  j                  |j                  j                        }|j                  j	                  |      }t        j                  |      }t        j                  |      S )z0Gets destination resource needed for copy tasks.)	r   r   r   r  rA   r   r+   r
   "sanitize_file_resource_for_windows)r   r   r   containerless_source_string1destination_delimited_containerless_source_stringnew_destination_object_urlnew_destination_resources          r4   r   r     s     !E'! "))

%
%
/
/

+
+
5
5 4  5@@EE7  0??  
	5	56N	OOr6   c                 L    t        j                  dj                  |              y )NzSkipping symlink {})r   r   r*   r3   s    r4   _log_skipping_symlinkr    s    ++#**845r6   c                 `    t         j                  j                  dj                  |              y )NzWould remove {})r   r   r   r*   r  s    r4   _print_would_remover    s     **$++H56r6   c                     | j                   }t        |t         j                        rt        j                  ||      S t        j
                  ||      S )N)r   )r   r8   rz   r   DeleteFileTaskDeleteObjectTask)r3   r   re   s      r4   _get_delete_taskr    sQ    #[(()%%+ 
 ''+ r6   c                 (    |sy|rt        | |       S y)zCReturns True if the unmatched destination object should be deleted.FT)r   )r   r   r   r   s       r4   +_should_delete_unmatched_destination_objectr    s,     
. +3.   
r6   r   r   r   c                    t        |t        j                        rdt        j                  fS t        ||||      }|sdt        j                  fS |rt        |       dt        j                  fS t        ||       t        j                  fS )a  Gets task and iteration instruction for deleting/preserving an unmatched destination while used with delete_unmatched_destination_objects.

  Args:
    user_request_args: User flags.
    destination_resource: Destination resource to check for deletion.
    delete_unmatched_destination_objects: Clear objects at the destination that
      are not present at the source.
    use_gsutil_delete_unmatched_behaviour: Preserves objects at the destination
      that are not present at the source if delete_unmatched_destination_objects
      is set, if they would normally be preserved by other flags just as gsutil.
    dry_run: Print what operations rsync would perform without actually
      executing them.
    skip_unsupported: Skip copying unsupported object types.

  Returns:
    A pair of task and iteration instruction.
      - A delete task if applicable, or None.
      - An iteration instruction for the next step.
  N)r8   r   rV   r   r   r  r  r  )r   r   r   r   r   r   *should_delete_unmatched_destination_objects          r4   1_get_task_and_iteration_for_unmatched_destinationr    s    8 $&8&N&NO"..// 2

.
/
	 - 
4"..//,-"..// +->?""
 r6   c                    |s|st        j                  d      t        j                  j                  j
                  j                         }|st        | |||||
      S |r)|j                  rt        |       dt        j                  fS t        |t        j                        sHt        j                   |      }| j"                  r|}n&t        j$                  d|j&                  ddd      }nd}|s"t)        | ||||||
      t        j                  fS |r)|j                  rt        |       dt        j*                  fS t-        ||      }t-        ||      }||k  r"t)        | ||||||
      t        j                  fS ||kD  rt        | |||||
      S | j.                  rt1        ||      }d|fS t        |t        j                        r#t)        | |||||d|
      t        j2                  fS t5        | ||||||	|
||
      S )a  Compares resources and returns next rsync step.

  Args:
    user_request_args (UserRequestArgs): User flags.
    source_resource: Source resource for comparison, a FileObjectResource,
      ManagedFolderResource, ObjectResource, or None. `None` indicates no
      sources left to copy.
    source_container (FileDirectoryResource|PrefixResource|BucketResource):
      Stripped from beginning of source_resource to get comparison URL.
    destination_resource: Destination resource for comparison, a
      FileObjectResource, ManagedFolderResource, ObjectResource, or None. `None`
      indicates all remaining source resources are new.
    destination_container (FileDirectoryResource|PrefixResource|BucketResource):
      If a copy task is generated for a source item with no equivalent existing
      destination item, it will copy to this general container. Also used to get
      comparison URL.
    compare_only_hashes (bool): Skip modification time comparison.
    delete_unmatched_destination_objects (bool): Clear objects at the
      destination that are not present at the source.
    dry_run (bool): Print what operations rsync would perform without actually
      executing them.
    ignore_symlinks (bool): Skip operations involving symlinks.
    skip_if_destination_has_later_modification_time (bool): Don't act if mtime
      metadata indicates we'd be overwriting with an older version of an object.
    skip_unsupported (bool): Skip copying unsupported object types.

  Returns:
    A pair of with a task and iteration instruction.

    First entry:
    None: Don't do anything for these resources.
    DeleteTask: Remove an extra resource from the destination.
    FileDownloadTask|FileUploadTask|IntraCloudCopyTask|ManagedFolderCopyTask:
      Update the destination with a copy of the source object.
    PatchFilePosixTask: Update the file destination POSIX data with the source's
      POSIX data.
    PatchObjectTask: Update the cloud destination's POSIX data with the source's
      POSIX data.

    Second entry:
    _IterateResource: Enum value indicating what to compare next.

  Raises:
    errors.Error: Missing a resource (does not account for subfunction errors).
  z7Comparison requires at least a source or a destination.N)r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   rB   rC   >use_gsutil_rsync_delete_unmatched_destination_objects_behaviorGetBoolr  
is_symlinkr  r   r   r8   r   rV   r   r   r   r~   rg   r   r   r  
no_clobberr   r   r   )r   r   r   r   r   r   r   r   r!   r   r   r   source_posixr   
source_urldestination_urlr   s                    r4   #_get_task_and_iteration_instructionr!    sm   t 1
,,A 
 ^^ffh ( 
<,-  33/*"))**	O%7%M%M	N@@L ''!l//
""D$l L	%-"7-	
 	  -88./"..//"?4DE*'1/ /!%-"7-	
 	  /!<,-  !!?01	
  '((!3!I!IJ 	-!5"7-		
 	  
J-
9'+O
/
 r6   c              #     K   dx}}t        j                  |      5 }t        j                  |      5 }t        t        |d      |	      }t        t        |d      |	      }|s|rt	        | |||||||||
|      \  }}|r6|dz  }t        |t        j                        r||j                  xs dz  }| |t        j                  t        j                  fv rt        t        |d      |	      }|t        j                  t        j                  fv rt        t        |d      |	      }|r|rddd       ddd       |r|s|rt        j                  |||       yyy# 1 sw Y   2xY w# 1 sw Y   6xY ww)zBReturns task with next rsync operation (patch, delete, copy, etc).r   N)r   )r   r   r   r!   r   r      )
item_countrY   )r   
FileReaderr   nextr!  r8   r   ObjectCopyTaskrY   r   r   r   r   r   workload_estimator_callback)r   source_list_filer   destination_list_filer   r   r   r   r!   yield_managed_folder_operationsr   r   task_status_queueoperation_countbytes_operated_onsource_readerdestination_readerr   r   taskr   s                        r4   get_operation_iteratorr2    s      )*)/%()]E<L<L=0]D!9O 6&9
 1$G




12)=+%!d!" 
1dI445
338q
8

	

!
!


# 
 5%=
 


&
&


# 
  :#T*= 
G 1=)h O/@22o<M 0Ai= =))s:   E7E+C8E+E.E+6)E7E(	$E++E40E7)T)NF)F)FF)NNNNFF)FFFFFF)FFFF)FFFFFFFN)Dr   
__future__r   r   r   enumr@   googlecloudsdk.api_lib.storager   "googlecloudsdk.command_lib.storager   r   r	   r
   r   r   r   r   r   r   ,googlecloudsdk.command_lib.storage.resourcesr   r   (googlecloudsdk.command_lib.storage.tasksr   +googlecloudsdk.command_lib.storage.tasks.cpr   r   0googlecloudsdk.command_lib.storage.tasks.objectsr   +googlecloudsdk.command_lib.storage.tasks.rmr   googlecloudsdk.corer   r   googlecloudsdk.core.utilr   rc   rb   ru   r)   r=   r5   r9   rM   rS   rq   rw   r   r   r   Enumr   boolr   r   r   r   r   r  r   r  r  r  r  r  r!  r2  r   r6   r4   <module>r@     s   # &  '  	 4 5 ? 8 8 K 9 A : @ @ K F J I A N C # * * 
   > 
 +  $N ?D*Z>DNG<~`"P $)<tyy  $	8 =@2 49).*/dNP$P0672 2727"2 +/2 ,0	2
 2 2v ).49u| ).$)49Hr6   