
    =                     0   d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlmZ ddl	m
Z
 ddl	mZ dd	l	mZ dd
l	mZ ddl	mZ ddlmZ ddlmZ ddlmZ ddlmZ  G d dej,                        Z G d dej,                        Z G d d      Z G d d      Zy)z(Module for handling recursive expansion.    )absolute_import)division)unicode_literalsN)	cloud_api)errors)folder_util)plurality_checkable_iterator)storage_url)wildcard_iterator)resource_reference)log)
properties)debug_outputc                       e Zd ZdZdZdZdZy)BucketSettingz9An enum saying whether or not to include matched buckets._yes_no_no_with_errorN)__name__
__module____qualname____doc__YESNONO_WITH_ERROR     8lib/googlecloudsdk/command_lib/storage/name_expansion.pyr   r   $   s    A#""-r   r   c                       e Zd ZdZdZdZdZy)RecursionSettingz5An enum saying whether or not recursion is requested.r   _no_with_warningr   N)r   r   r   r   r   NO_WITH_WARNINGr   r   r   r   r    r    ,   s    ="&/#r   r    c                      e Zd ZdZej
                  j                  dej                  e	j                  j                  e	j                  j                  ej                  j                  dddej                   dfdZe	j                  j                  e	j                  j                  fdZed        Zd Zd	 Zd
 Zd Zd Zd Zy)NameExpansionIteratorav  Expand all urls passed as arguments, and yield NameExpansionResult.

  For each url, expands wildcards, object-less bucket names,
  subdir bucket names, and directory names, and generates a flat listing of
  all the matching objects/files.
  The resulting objects/files are wrapped within a NameExpansionResult instance.
  See NameExpansionResult docstring for more info.
  FTNc                 h   || _         t        j                  |      | _        || _        || _        || _        || _        || _        || _	        |	| _
        |
| _        || _        |t        j                         | _        n|| _        t        j                  | j!                               | _        d| _        y)a2  Instantiates NameExpansionIterator.

    Args:
      urls_iterable (Iterable[str]): The URLs to expand.
      fields_scope (cloud_api.FieldsScope): Determines amount of metadata
        returned by API.
      ignore_symlinks (bool): Skip over symlinks instead of following them.
      include_buckets (BucketSetting): Whether to fetch matched buckets and
        whether to raise an error.
      managed_folder_setting (folder_util.ManagedFolderSetting): Indicates how
        to deal with managed folders.
      folder_setting (folder_util.FolderSetting): Indicates how to deal with
        folders.
      object_state (cloud_api.ObjectState): Versions of objects to query.
      preserve_symlinks (bool): Preserve symlinks instead of following them.
      raise_error_for_unmatched_urls (bool): If True, raises an error if any url
        in `url_found_match_tracker` is unmatched after expansion.
      raise_managed_folder_precondition_errors (bool): If True, raises
        precondition errors from managed folder listing. Otherwise, suppresses
        these errors. This is helpful in commands that list managed folders by
        default.
      recursion_requested (RecursionSetting): Says whether or not recursion is
        requested.
      url_found_match_tracker (OrderedDict|None): Maps top-level URLs to a
        boolean indicating whether they matched resources. Mutated as resources
        are yielded. Reusing a tracker dict across NameExpansionIterators with
        different expansion criteria suppresses unmatched errors if any iterator
        expands a URL.
    N)object_stater	   PluralityCheckableIterator_urls_iterator_fields_scope_ignore_symlinks_include_buckets_managed_folder_setting_folder_setting_preserve_symlinks_raise_error_for_unmatched_urls)_raise_managed_folder_precondition_errors_recursion_requestedcollectionsOrderedDict_url_found_match_tracker_get_top_level_iterator_top_level_iterator!_has_multiple_top_level_resources)selfurls_iterablefields_scopeignore_symlinksinclude_bucketsmanaged_folder_settingfolder_settingr&   preserve_symlinksraise_error_for_unmatched_urls(raise_managed_folder_precondition_errorsrecursion_requestedurl_found_match_trackers                r   __init__zNameExpansionIterator.__init__=   s    X %D$??N 	 &D+D+D#9D )D/D+ID(0 	2 !4D&&1&=&=&?d#&=d# 	%??((*	
 	
 .2D*r   c                 L   t        j                  |t        j                  j                  j
                  j                         t        j                  j                  j                  k7  | j                  | j                  ||| j                  | j                  | j                  	      S )z>Returns get_wildcard_iterator with instance variables as args.)fetch_encrypted_object_hashesr:   r;   r=   r>   r&   r?   rA   )r   get_wildcard_iteratorr   VALUESstoragecheck_hashesGetCheckHashesNEVERvaluer)   r*   r&   r.   r0   )r8   urlr=   r>   s       r   _get_wildcard_iteratorz,NameExpansionIterator._get_wildcard_iterator   s     22%%22668%%++112 ''--5%&&11:: r   c                     | j                   ;| j                  j                         xs | j                  j                         | _         | j                   S )ag  Returns if the iterator yields plural items without recursing.

    Also returns True if the iterator was created with multiple URLs.
    This may not be true if one URL doesn't return anything, but it's
    consistent with gsutil and the user's probable intentions.

    Returns:
      Boolean indicating if iterator contains multiple top-level sources.
    )r7   r(   	is_pluralr6   r8   s    r    has_multiple_top_level_resourcesz6NameExpansionIterator.has_multiple_top_level_resources   sO     --5



'
'
) 2%%//1 , 111r   c              #   N  K   | j                   D ]  }t        j                  |      }t        |t        j                        rl|j                         r\| j                  t        j                  ur@| j                  t        j                  u r$t        j                  dj                  |            | j                  j!                  |d      | j                  |<   | j"                  t$        j&                  j(                  t$        j&                  j*                  hv r7| j                  t        j                  u rt$        j&                  j,                  }n| j"                  }| j.                  }| j1                  |||      D ]  }| j"                  t$        j&                  j*                  u rt        |t2        j4                        s@| j.                  t$        j6                  j*                  u rt        |t2        j4                        r|| j9                  ||j                  |      f   yw)z<Iterates over user-entered URLs and does initial processing.z!Expected object URL. Received: {}Fr=   r>   N)r(   r
   storage_url_from_string
isinstanceCloudUrl	is_bucketr1   r    r   r+   r   r   r   InvalidUrlErrorformatr4   getr,   r   ManagedFolderSettingLIST_WITH_OBJECTSLIST_WITHOUT_OBJECTSLIST_AS_PREFIXESr-   rP   r   ObjectResourceFolderSetting_get_name_expansion_result)r8   rO   original_storage_url(wildcard_iterator_managed_folder_setting wildcard_iterator_folder_settingresources         r   r5   z-NameExpansionIterator._get_top_level_iterator   s    ""(@@E
);+?+?
@",,.''/?/C/CC##}'B'BB$$/66s;
 	
 ,0+H+H+L+L
u,d##C(
 
&
&..@@..CC
 ''+;+?+??
 ,,== 	1 483O3O0)-)=)=&11
!I9 2 ( ((//DDE8%7%F%FG  ((==>8%7%F%FG
42283;3G3G3GI I 	IE #s   H#H%c              #   .  K   |j                   j                  j                  d      }| j                  |j                  | j
                  | j                        }|D ]5  }| j                  ||j                   j                  |j                         7 y w)Nz**rV   )	rh   r
   joinrP   
url_stringr,   r-   rd   original_url)r8   parent_name_expansion_resultnew_storage_urlchild_resourceschild_resources        r   _get_nested_objects_iteratorz2NameExpansionIterator._get_nested_objects_iterator   s     2;;GGLLO11""#;;++ 2 O
 *++
6??KK
&
3
35 5 *s   BBc                     | j                   t        j                  j                  u xs |j                  }|s0t        j                  |j
                  j                        }||_        t        |||      S )zEReturns a NameExpansionResult, removing generations when appropriate.)	r&   r   ObjectStateLIVE_AND_NONCURRENT
generationr
   rW   versionless_url_stringNameExpansionResult)r8   rh   expanded_urlrl   keep_generation_in_urlrn   s         r   rd   z0NameExpansionIterator._get_name_expansion_result   sl     	Y22FFF 	#""  "#;;



5
57o,hx|DDr   c                     | j                   sy | j                  j                         D cg c]	  \  }}|s| }}}|r3t        j                  dj                  dj                  |                  y c c}}w )Nz2The following URLs matched no objects or files:
{}
)r/   r4   itemsr   r[   r\   rj   )r8   rO   found_matchnon_matching_urlss       r   &_raise_no_url_match_error_if_necessaryz<NameExpansionIterator._raise_no_url_match_error_if_necessary   s    // %)$A$A$G$G$I$I [ 	$I   ""
?
F
Fii)*,- - 	s   A2c                     | j                   r=t        |t        j                        r"|j                  rt        j                  d      y y y )NzDMultiple URL strings are not supported when transferring from stdin.)r7   rX   r
   FileUrlis_stdior   Error)r8   rx   s     r   ._raise_error_if_multiple_sources_include_stdinzDNameExpansionIterator._raise_error_if_multiple_sources_include_stdin  sJ    ..:k))4+/;/D/DLL  0E4+.r   c              #     K   | j                   j                         | _        | j                   D ]'  \  }}| j                  |j                         | j
                  t        j                  u xr$ |j                  j                  j                         }| j                  t        j                  j                  t        j                  j                  hv xr$ t!        |j                  t"        j$                        }| j&                  t        j(                  j                  t        j(                  j*                  hv xr$ t!        |j                  t"        j,                        }t#        j.                  |j                        r|s|s|rd| j0                  |<   | t#        j.                  |j                        s| j2                  t4        j                  u r+| j7                  |      D ]  }d| j0                  |<   |  |r|r|r| j2                  t4        j8                  u st;        j<                  dj?                  |j                               * | jA                          yw)zIterates over each URL in self._urls_iterator and yield the expanded result.

    Yields:
      NameExpansionResult instance.

    Raises:
      InvalidUrlError: No matching objects found.
    TzDOmitting {} because it is a container, and recursion is not enabled.N)!r6   rR   r7   r   rx   r+   r   r   rh   r
   rZ   r,   r   r^   r_   r`   rX   r   ManagedFolderResourcer-   rc   LIST_AS_FOLDERSFolderResource!is_container_or_has_container_urlr4   r1   r    rq   r"   r   warningr\   r   )r8   	input_urlname_expansion_resultshould_return_bucketshould_return_managed_foldershould_return_foldernested_name_expansion_results          r   __iter__zNameExpansionIterator.__iter__  s%     .2-E-E-O-O .D*,0,D,D(	(
99

,
,. "22m6G6GG 

(
(
4
4
>
>
@  &*%A%A

*
*
<
<

*
*
?
?F & & 

(
(

2
2 # "11

#
#
8
8

#
#
3
36   

(
(

+
+  !BB#,, ")!37%%i0##		=	=

(
(
*$$(8(<(<<.2.O.O#/%*7;D)))4../% !+#&&*:*J*JJ KK ++161::,<=c -Ej 	//1s&   F3I*7A	I*I*I*
I*(AI*)r   r   r   r   r   FieldsScopeNO_ACLr   r   r   r^   DO_NOT_LISTrc   rs   LIVEr    r"   rD   rP   propertyrT   r5   rq   rd   r   r   r   r   r   r   r$   r$   3   s     ((//#&&(==II ..::((--%)/3*::"F2V )==II ..::	0 2 2"5In5
E-@2r   r$   c                   (    e Zd ZdZd Zd Zd Zd Zy)rw   a<  Holds one fully expanded result from iterating over NameExpansionIterator.

  This class is required to pass the expanded_url information to the caller.
  This information is required for cp and rsync command, where the destination
  name is determined based on the expanded source url.
  For example, let's say we have the following objects:
  gs://bucket/dir1/a.txt
  gs://bucket/dir1/b/c.txt

  If we run the following command:
  cp -r gs://bucket/dir* foo

  We would need to know that gs://bucket/dir* was expanded to gs://bucket/dir1
  so that we can determine destination paths (foo/a.txt, foo/b/c.txt) assuming
  that foo does not exist.

  Attributes:
    resource (Resource): Yielded by the WildcardIterator.
    expanded_url (StorageUrl): The expanded wildcard url.
    original_url (StorageUrl): Pre-expanded URL.
  c                 .    || _         || _        || _        y)a5  Initialize NameExpansionResult.

    Args:
      resource (resource_reference.Resource): Yielded by the WildcardIterator.
      expanded_url (StorageUrl): The expanded url string without any wildcard.
        This value should preserve generation even if not available in
        resource.storage_url. The versionless version of this should be same
        as resource.storage_url if recursion was not requested. This field is
        intended for only the cp and rsync commands.
      original_url (StorageUrl): Pre-expanded URL. Useful for knowing intention.
    N)rh   rx   rl   )r8   rh   rx   rl   s       r   rD   zNameExpansionResult.__init__m  s     DM$D$Dr   c                 ,    t        j                  |       S N)r   generic_reprrS   s    r   __repr__zNameExpansionResult.__repr__}  s    $$T**r   c                 B    | j                   j                  j                  S r   )rh   r
   rk   rS   s    r   __str__zNameExpansionResult.__str__  s    ==$$///r   c                     t        |t        |             st        S | j                  |j                  k(  xr | j                  |j                  k(  S r   )rX   typeNotImplementedrh   rx   )r8   others     r   __eq__zNameExpansionResult.__eq__  sF    eT$Z( MMU^^+ 8!!U%7%779r   N)r   r   r   r   rD   r   r   r   r   r   r   rw   rw   V  s    ,% +09r   rw   )r   
__future__r   r   r   r2   enumgooglecloudsdk.api_lib.storager   "googlecloudsdk.command_lib.storager   r   r	   r
   r   ,googlecloudsdk.command_lib.storage.resourcesr   googlecloudsdk.corer   r   googlecloudsdk.core.utilr   Enumr   r    r$   rw   r   r   r   <module>r      sn    / &  '   4 5 : K : @ K # * 1#DII #tyy `2 `2F	39 39r   