
    w                        d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 dd	l	mZ dd
l	mZ ddlmZ ddlmZ ddlmZ ddlmZ g dZdj+                  eddgz         Zdj+                  edgz         Zej0                  j2                  ej0                  j4                  ej0                  j6                  ej0                  j8                  ej0                  j:                  fZeej0                  j>                  fz   Z eZ!ej0                  j4                  gZ"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*d Z+d Z,d Z-d Z.	 d'd Z/d'd!Z0d" Z1d# Z2d$ Z3	 d'd%Z4d(d&Z5y))zeUtils for managng the many transfer job flags.

Tested through surface/transfer/jobs/create_test.py.
    )absolute_import)division)unicode_literalsN)errors)storage_url)
creds_util)jobs_flag_util)	name_util)log)
properties)
console_io)times)descriptionlogging_confignotification_configstatus,scheduletransfer_specreplication_specc                    | j                   t        j                  j                  u rt	        j
                         s.t        j                  dj                  | j                              t	        j                  |D cg c]  }|j                  dz    c}ddj                  |             }||   }t        j                  | |      S | S c c}w )z?Has user select a valid scheme from a list and returns new URL.zDid you mean "posix://{}"z://TzStorage Transfer does not support direct file URLs: {}
Did you mean to use "posix://"?
Run this command with "--help" for more info,
or select a valid scheme below.)cancel_optionmessage)schemer   ProviderPrefixFILEr   	CanPromptr   InvalidUrlErrorformatresource_namePromptChoicevalueswitch_scheme)urlvalid_schemesr   scheme_index
new_schemes        =lib/googlecloudsdk/command_lib/transfer/jobs_apitools_util.py!_prompt_user_and_add_valid_schemer)   @   s     	ZZ;--222!""
%
,
,S->->
?  **,9:M&	M:. &+	L |,J$$S*55	* 	;s   <Cc           
         |j                   t        j                  j                  u r"|j	                  |j
                        | _        y|j                   t        j                  j                  u r"|j                  |j
                        | _	        y|j                   t        j                  j                  u r-|j                  |j                  |j
                        | _        y|j                   t        j                  j                  u r|j                  rN|j!                  |j                  |j                  |j
                  |j"                  t%        ||            | _        y|j)                  |j                  |j
                        | _        yt-        |t        j.                        r8|j1                  |j                  |j
                  |j2                        | _        yy)a  Adds source url to transfer or replication spec.

  Args:
    specs:
      a submessage, must be one of [job.transferSpec, job.replicationSpec].
    args: argparse.namespace, the parsed arguments from the command line.
    messages: storagetransfer_v1_message instance.
    source_url:
      An instance of the storage_url variable specifying the source
      location for the data transfer.
  )pathrootDirectory
bucketNamer+   )r/   endpointr+   region
s3Metadata)	containerr+   storageAccountN)r   r   r   HDFSHdfsDatar    hdfsDataSourcePOSIXPosixFilesystemposixDataSourceGCSGcsDatabucket_namegcsDataSourceS3source_endpointAwsS3CompatibleDatasource_signing_region_get_s3_compatible_metadataawsS3CompatibleDataSource	AwsS3DataawsS3DataSource
isinstanceAzureUrlAzureBlobStorageDataaccountazureBlobStorageDataSource)specsargsmessages
source_urls       r(   add_source_urlrP   X   s    +44999#,,%% - 'EK66<<<$44 .. 5 0EK66:::"**))%% + E K66999

&
&#//++++//4T8D ' F % '00++'' 1 e *k223%% ,,))%-- 	& 	
 
$ 4    c                 :   |j                   t        j                  j                  u r-|j	                  |j
                  |j                        | _        y|j                   t        j                  j                  u r"|j                  |j                        | _
        yy)aN  Adds destination url to transfer or replication spec.

  Args:
    specs:
      a submessage, must be one of [job.transferSpec, job.replicationSpec]
    messages: storagetransfer_v1_message instance.
    destination_url:
      An instance of the storage_url variable specifying the destination
      location for the data transfer.
  r.   r,   N)r   r   r   r;   r<   r=   r    gcsDataSinkr8   r9   posixDataSink)rL   rN   destination_urls      r(   add_destination_urlrV      s     {99=== (("..** ) E !;!;!A!AA"22%33 3 5E BrQ   c                 8    t        ||      }t        | |||       y)a0  It validates the source url and adds it to transfer or replication spec.

  If no URL scheme is provided, prompt the user to add a valid one
  (e.g., 'gs://').

  Args:
    specs:
      a submessage, must be one of [job.transferSpec, job.replicationSpec].
    args: argparse.namespace, the parsed arguments from the command line.
    messages: storagetransfer_v1_message instance.
    source_url:
      An instance of the storage_url variable specifying the source
      location for the data transfer.
    valid_schemes: the schemes supported by the specs.
  N)r)   rP   )rL   rM   rN   rO   r%   s        r(   validate_and_add_source_urlrX      s    & 1]K*h
3rQ   c                 6    t        ||      }t        | ||       y)a  Adds destination url to transfer or replication spec.

  If no URL scheme is provided, prompt the user to add a valid one
  (e.g., 'gs://').
  Args:
    specs:
      a submessage, must be one of [job.transferSpec, job.replicationSpec]
    messages: storagetransfer_v1_message instance.
    destination_url:
      An instance of the storage_url variable specifying the destination
      location for the data transfer.
    valid_schemes: the schemes supported by the specs.
  N)r)   rV   )rL   rN   rU   r%   s       r(    validate_and_add_destination_urlrZ      s!    " 6}/ eX7rQ   c                    t        |dd      s(t        |dd      st        |dd      st        |dd      sy| j                  s|j                         | _        t        |dd      }|r=t        |j                  j                  |j	                               | j                  _        t        |dd      rzt        j                  |j                        }|t        j                  j                  u rd| j                  _
        n-|t        j                  j                  u rd| j                  _        |j                         }t        |dd      r|j                  D ]y  }t        j                  |      }|t        j                  j                   k(  r&|j                  j"                  j$                  |_        \|t        j                  j(                  k(  r&|j                  j*                  j,                  |_        |t        j                  j0                  k(  r&|j                  j2                  j4                  |_        |t        j                  j8                  k(  r'|j                  j:                  j<                  |_        &|t        j                  j@                  k(  r'|j                  jB                  jD                  |_#        j|t        j                  jH                  k(  r'|j                  jJ                  jL                  |_'        |t        j                  jP                  k(  r'|j                  jR                  jT                  |_+        |t        j                  jX                  k(  r'|j                  jZ                  j\                  |_/        6|t        j                  j`                  k(  sU|j                  jb                  jd                  |_3        | t        |dd      r@t        |j                  jJ                  d|jh                  j	                         z         |_'        ||j                         k7  r|| j                  _5        yy)z9Creates or modifies TransferOptions object based on args.overwrite_whenNdelete_frompreserve_metadatacustom_storage_classTSTORAGE_CLASS_)6getattrtransferOptionsTransferOptionsOverwriteWhenValueValuesEnumupperoverwriteWhenr	   DeleteOptionr]   SOURCE_AFTER_TRANSFER$deleteObjectsFromSourceAfterTransferDESTINATION_IF_UNIQUEdeleteObjectsUniqueInSinkMetadataOptionsr^   PreserveMetadataFieldACLAclValueValuesEnumACL_PRESERVEaclGIDGidValueValuesEnum
GID_NUMBERgidUIDUidValueValuesEnum
UID_NUMBERuidKMS_KEYKmsKeyValueValuesEnumKMS_KEY_PRESERVEkmsKeyMODEModeValueValuesEnumMODE_PRESERVEmodeSTORAGE_CLASSStorageClassValueValuesEnumSTORAGE_CLASS_PRESERVEstorageClassSYMLINKSymlinkValueValuesEnumSYMLINK_PRESERVEsymlinkTEMPORARY_HOLDTemporaryHoldValueValuesEnumTEMPORARY_HOLD_PRESERVEtemporaryHoldTIME_CREATEDTimeCreatedValueValuesEnum$TIME_CREATED_PRESERVE_AS_CUSTOM_TIMEtimeCreatedr_   metadataOptions)r   rM   rN   overwrite_when_argumentdelete_optionmetadata_optionsfield_value	field_keys           r(   "_create_or_modify_transfer_optionsr      s`   
$($
/7
M44!$+D2Et$L
$.
5
		&	&$,$<$<$>M!#D*:DA29  ==%%'3)M!!/ T=$'"//0@0@AM33IIIKOm##H	.55KK	K@Dm##=--/T&--- 66{Ci	n::>>	>$$77DD 	<<@@@$$77BB 	<<@@@$$77BB 	<<DDD$$::KK 	<<AAA$$88FF 	<<JJJ$$@@## 	% <<DDD$$;;LL 	 <<KKK$$AA$$ 	& <<III$$??dd 	$; .@ T)40$+  <<444::<<%>! 11334DM!!1 4rQ   c                    t        |dd      sBt        |dd      s5t        |dd      s(t        |dd      st        |dd      st        |dd      sy| j                  s|j                         | _        t        |dd      r|j                  | j                  _        t        |dd      r|j
                  | j                  _        t        |dd      rH|j                  j                  t        j                        j                         }|| j                  _        t        |dd      rH|j                  j                  t        j                        j                         }|| j                  _        t        |dd      r*dj                  |j                         | j                  _        t        |dd      r+dj                  |j$                        | j                  _        yy)	z3Creates or modifies ObjectConditions based on args.include_prefixesNexclude_prefixes include_modified_before_absoluteinclude_modified_after_absolute include_modified_before_relativeinclude_modified_after_relative{}s)ra   objectConditionsObjectConditionsr   includePrefixesr   excludePrefixesr   
astimezoner   UTC	isoformatlastModifiedBeforer   lastModifiedSincer   r   #minTimeElapsedSinceLastModificationr   #maxTimeElapsedSinceLastModification)r   rM   rN   modified_before_datetime_stringmodified_after_datetime_strings        r(   #_create_or_modify_object_conditionsr   
  s   
$*D
1
$*D
1
$:D
A
$94
@
$:D
A
$94
@
		'	'%-%>%>%@M"T%t,595J5JM""2T%t,595J5JM""2T5t<--88CMMO $8WM""5T4d;,,77		BLLN #7UM""4T5t<IN--J/M""FT4d;IN,,J.M""F <rQ   c                 f   | j                   rt        |dd      r$t        j                  |j                        \  }}}n.t        j                  d       t        j                         \  }}d}|r|s|st        j                  d       |j                  ||      | j                   _	        || j                   _
        y| j                  rkt        |dd      r%t        j                  |j                  dg      d   }nt        j                  d       d}|j                  |      | j                  _        yy)	z<Creates or modifies TransferSpec source creds based on args.source_creds_fileNzNNo --source-creds-file flag. Checking system config files for AWS credentials.zMissing AWS source creds.)accessKeyIdsecretAccessKeysasTokenzDNo Azure source creds set. Consider adding --source-creds-file flag.)r   )rF   ra   r   get_aws_creds_from_filer   r   warningget_default_aws_credsAwsAccessKeyawsAccessKeyroleArnrK   get_values_for_keys_from_fileAzureCredentialsazureCredentials)r   rM   rN   access_key_idsecret_access_keyrole_arn	sas_tokens          r(   _create_or_modify_credsr   *  s$   ""t($/

,
,T-C-C
D 1m& 
kk * +)3)I)I)K&m&h0X	kk-.191F1F!3D 2G 2FM!!.,4M!!)//t($/::

 
 :,00:<i 
kk / 0i!!9!5 ,,= 0rQ   c                 r   | j                   s%| j                  s| j                  s| j                  sy|j	                         }| j                   r2t        |j                  j                  d| j                   z         |_        | j                  r/t        |j                  j                  | j                        |_	        | j                  r2t        |j                  j                  d| j                  z         |_        | j                  r2t        |j                  j                  d| j                  z         |_        |S )z8Generates advanced settings for S3-compatible providers.NAUTH_METHOD_NETWORK_PROTOCOL_REQUEST_MODEL_)source_auth_methodsource_list_apisource_network_protocolsource_request_modelS3CompatibleMetadatara   AuthMethodValueValuesEnum
authMethodListApiValueValuesEnumlistApiProtocolValueValuesEnumprotocolRequestModelValueValuesEnumrequestModel)rM   rN   s3_compatible_metadatas      r(   rC   rC   I  s   

!
!T%9%9

&
&$*C*C#88:	(/%%??000)2% 
%,%%<<&" 
!!&-%%==d:::'<# 
*1%%AA4444+6' 
 rQ   c                 T    |j                   r|j                   | j                  _        yy)z&Adds additional options for S3 source.N)s3_cloudfront_domainrF   cloudfrontDomain)r   rM   s     r(   !_add_additional_s3_source_optionsr   c  s$    	595N5NM!!2 rQ   c                    | j                   s|j                         | _         t        |dd      rd| j                   _        d| j                   _        d| j                   _        d| j                   _        d| j                   _        d| j                   _        d| j                   _	        	 t        j                  |j                        }t        | j                   |||t               | j                   j                  rt-        | j                   |       t        |dd      r]d| j                   _        d| j                   _        t        j                  |j2                        }t5        | j                   ||t6               t        |dd      r.t9        j:                  |j<                        | j                   _        t        |dd      r.t9        j:                  |j@                        | j                   _!        t        |dd      rUt        j                  |jD                        }|jG                  |jH                  |jJ                        | j                   _&        t        |d	d      r+|jO                  |jP                  
      | j                   _)        tU        | j                   ||       tW        | j                   ||       tY        | j                   ||       y# t        j                   $ rp |j                  j#                  t        j$                  j&                  j(                        r.|j+                  |j                        | j                   _        d}n Y nw xY w)z/Creates or modifies TransferSpec based on args.sourceN)listUrldestinationdestination_agent_poolsource_agent_poolintermediate_storage_pathr.   manifest_file)location)-transferSpecTransferSpecra   httpDataSourcer:   r>   rD   rF   rK   r7   r   storage_url_from_stringr   rX   VALID_SOURCE_TRANSFER_SCHEMESr   r   
startswithr   HTTPr"   HttpDatar   rT   rS   r   rZ   "VALID_DESTINATION_TRANSFER_SCHEMESr
   add_agent_pool_prefixr   sinkAgentPoolNamer   sourceAgentPoolNamer   r<   r=   r    gcsIntermediateDataLocationTransferManifestr   transferManifestr   r   r   )jobrM   rN   rO   rU   intermediate_storage_urls         r(   _create_or_modify_transfer_specr   i  s   			,,.CT8T"&*C#'+C$%)C"15C.'+C$26C/&*C#66t{{Cj "






' 	%%%c&6&6=T=$'%)C"#'C !99$:J:JKO$*	 T+T2)2)H)H##*%C&T&-+4+J+J, C(T.5*BB&& (3;3C3C+77%33 4D 45C0 T?D)(0(A(A## )B )%C% #**D(;%c&6&6hG$S%5%5tXFe !! 			 : : ? ? E E	F*2*;*;DKK*;*P'
 	s   'K A?MMc                    t        |dd      }t        |dd      }t        |dd      }|s|s|sy| j                  s|j                         | _        || j                  _        || j                  _        || j                  _        y)zACreates or modifies event stream config. Returns if flag present.event_stream_nameNevent_stream_startsevent_stream_expiresFT)ra   eventStreamEventStreamnameeventStreamStartTimeeventStreamExpirationTime)r   rM   rN   r   event_stream_startevent_stream_expires         r(   ,_create_or_modify_event_stream_configurationr     s|    d$7>t%:DA&<dC
15H	**,CO*#//);#//&.A#//+	rQ   c                    t        |dd      }t        |dd      }t        |dd      }|xs |xs |}|r|s|j                  rt        d      |r|st        |dd      rt        d      |s|j                  s|s|r|sy| j                  s|j	                         | _        |r|j                  t        j                        }	|j                  |	j                  |	j                  |	j                  	      | j                  _        |j                  |	j                  |	j                  |	j                   
      | j                  _        na|s_t$        j&                  j)                         }
|j                  |
j                  |
j                  |
j                  	      | j                  _        |r dj+                  |      | j                  _        |r| j                  j,                  st        d      |j                  t        j                        }|j                  |j                  |j                  |j                  	      | j                  _        |j                  |j                  |j                  |j                   
      | j                  _        y|s=| j                  j,                  s&| j                  j                  | j                  _        yyy)z;Creates or modifies transfer Schedule object based on args.schedule_startsNschedule_repeats_everyschedule_repeats_untilz(Cannot set schedule and do-not-run flag.
do_not_runFz-Cannot set schedule on event-driven transfer.)daymonthyear)hoursminutessecondsr   zScheduling a job end time requires setting a frequency with --schedule-repeats-every. If no job end time is set, the job will run one time.)ra   r  
ValueErrorr   Scheduler   r   r   Dater  r  r  scheduleStartDate	TimeOfDayhourminutesecondstartTimeOfDaydatetimedatetodayr   repeatIntervalscheduleEndDateendTimeOfDay)r   rM   rN   	is_updateis_event_driven_transferr   r  r  has_schedule_flagstart
today_dateends               r(   _create_or_modify_scheduler    sU    D"3T:/"4)A4H"4)A4HI/I3I  ABB74u=
D
EE 	!
-
 	$$&CL&&uyy1E%-]]IIkkZZ &3 &CLL"
 #+"4"4jj #5 #CLL
 $$&J%-]]NN*"2"2 &3 &JCLL" "',,/E"FCLL <<&&  !
+
+EII
6C#+==GGiiXX $1 $CLL 
 !) 2 2hh



 !3 !CLL
 S\\88 $'<<#A#ACLL  99rQ   c                    t        |dd      }t        |dd      }t        |dd      }|s|s|sy|r5| j                  s|j                  |      | _        n|| j                  _        |s|r| j                  st	        d      |r@|j                         }t        |j                  j                  |      | j                  _        n1|s/|j                  j                  j                  | j                  _        |r_g }|D ]F  }	d|	j                         z   }
t        |j                  j                  |
      }|j                  |       H || j                  _        y|so|j                  j                  j                  |j                  j                  j                  |j                  j                  j                  g| j                  _        yy)zECreates or modifies transfer NotificationConfig object based on args.notification_pubsub_topicNnotification_event_typesnotification_payload_format)pubsubTopiczCCannot set notification config without --notification-pubsub-topic.TRANSFER_OPERATION_)ra   notificationConfigNotificationConfigr$  r
  re   PayloadFormatValueValuesEnumpayloadFormatJSON"EventTypesValueListEntryValuesEnumappend
eventTypesTRANSFER_OPERATION_SUCCESSTRANSFER_OPERATION_FAILEDTRANSFER_OPERATION_ABORTED)r   rM   rN   r  r!  r"  r#  payload_format_keyevent_typesevent_type_argevent_type_key
event_types               r(   %_create_or_modify_notification_configr6    s   %d,GN$T+EtL '.K(,!.
#'?
%
!!'::/  ;  1c ,Ec(!3+A+A
 5 6 6 !4::<+2##@@,C(  	##@@EE ( K2,~/C/C/EEn

%
%
H
H
j $ 3 )4C% 
	$	$	G	G
$
$		$	$	G	G
#
#		$	$	G	G
$
$)C% rQ   c                    t        |dd      }| j                  s,| j                  j                  s(| j                  j                  sd| j
                  _        y||| j
                  _        y|sd| j
                  _        y	 y)z)Sets enableOnpremGcsTransferLogs boolean.enable_posix_transfer_logsNFT)ra   replicationSpecr   r:   rT   loggingConfigenableOnpremGcsTransferLogs)r   rM   r  r8  s       r(    _enable_onprem_gcs_transfer_logsr<  E  s    &t-I4P	&&#*:*:*H*H49C1 	 "-4NC1 	 48C1 	 	rQ   c                    | j                   s|j                         | _         t        | ||       t        |dd      }t        |dd      }|s|sy| j                   xr | j                   j                  }| j                   xr | j                   j
                  }|r|s|r|r|r|st        d      |rWg }|D ]?  }	|j                  t        | j                   j                  |	j                                      A || j                   _        |rXg }
|D ]?  }|
j                  t        | j                   j                  |j                                      A |
| j                   _        yy)z@Creates or modifies transfer LoggingConfig object based on args.log_actionsNlog_action_stateszRBoth --log-actions and --log-action-states are required for a complete log config.)r:  LoggingConfigr<  ra   
logActionslogActionStatesr
  r,  "LogActionsValueListEntryValuesEnumre   'LogActionStatesValueListEntryValuesEnum)r   rM   rN   r  r>  r?  existing_log_actionsexisting_log_action_statesactionsactionaction_statesaction_states               r(    _create_or_modify_logging_configrK  Y  sY   			 ..0C #3i8mT2+d$7>
*
**Ks/@/@/K/K	=C--==  ,6"6
 3 4 4 Gnn
###FF,,."#  $+C M)
###KK$$&() * )6C% rQ   c                     |j                   }d|_         |j                  | j                         k(  rd|_        | j                  |j                  | j                  |||            S )z3Generates Apitools patch message for transfer jobs.N)	projectIdtransferJobupdateTransferJobFieldMask)jobNameupdateTransferJobRequest)rM  r   r  'StoragetransferTransferJobsPatchRequestr   UpdateTransferJobRequest)rN   r   
field_mask
project_ids       r(   #generate_patch_transfer_job_messagerV    sl    }}*#-\\X&&(( CL		9	9hh'@@%/  A   
: 
	 	rQ   c                    |rt        d      | j                  s|j                         | _        t        |dd      rtd| j                  _        t        j                  |j                        }|j                  t        vrt        j                  d      t        | j                  |||t               t        |dd      rsd| j                  _        t        j                  |j                        }|j                  t        vrt        j                  d      t        | j                  ||t               t!        | j                  ||       t#        | j                  ||       y)z2Adds/Updates the replication spec to transfer job.z:Not allowed to set event stream flags on replication jobs.r   NzQReplication feature is currently available for Google Cloud Storage buckets only.r   )r
  r9  ReplicationSpecra   r>   r   r   r   r   VALID_REPLICATON_SCHEMESr   ErrorrX   rS   r   rZ   r   r   )r   rM   rN   has_event_stream_flagrO   rU   s         r(   "_create_or_modify_replication_specr\    s<    
D  
		"224CT8T"(,C%44T[[AJ 88LL     T=$'&*C#!99$:J:JKO%==LL  %X8P &c&9&94J$S%8%8$IrQ   c                    |r|}n|j                         }|j                  s7t        j                  j                  j
                  j                         |_        t        | dd      r$t        j                  | j                        |_
        t        | dd      r| j                  |_        |rMt        | dd      re| j                  j                         }t        |j                   j                  |      |_        n%|j                   j                  j                  |_        t!        || |      }| xr t        | dd       xs |j"                  }|rt%        || |       nt'        || ||       |j(                  xs |j*                  }t-        || |t/        |      |       t1        || |t/        |             t3        || |t/        |             |r|rt4        nt6        }t9        |||      S |S )	z?Generates Apitools transfer message based on command arguments.r   Nr   r   replication)r[  )r  r  )r  )TransferJobrM  r   VALUEScoreprojectGetra   r
   add_job_prefixr   r   r   re   StatusValueValuesEnumENABLEDr   r   r   r\  r   r9  r  boolr6  rK  $UPDATE_FIELD_MASK_WITH_TRANSFER_SPEC'UPDATE_FIELD_MASK_WITH_REPLICATION_SPECrV  )	rM   rN   existing_jobr   
status_keyr[  is_transfer_jobr  update_masks	            r(   generate_transfer_job_messagern    s   
C



 C	%%**22668CMT64 ''		2CHT=$'&&COtXt$;;$$&j



4
4jcj %%;;CCCJF	4 

E74#EE 
			  #Cx8&T83H !__C0C0C	
\"7 (	4T,%7 #	4T,%7   	-4 
 /  
*rQ   )F)N)6__doc__
__future__r   r   r   r  "googlecloudsdk.command_lib.storager   r   #googlecloudsdk.command_lib.transferr   r	   r
   googlecloudsdk.corer   r   googlecloudsdk.core.consoler   googlecloudsdk.core.utilr   UPDATE_FIELD_MASKjoinrh  ri  r   r8   r;   r?   r   HTTPSCOMMON_VALID_TRANSFER_SCHEMESr5   r   r   rY  r)   rP   rV   rX   rZ   r   r   r   rC   r   r   r   r  r6  r<  rK  rV  r\  rn   rQ   r(   <module>r{     s  
 '  '  5 : : > 9 # * 2 *  (+xx_55( $ +.((+,,+ '
 $$""!!##$$!  !>##A !  &C "'66::; 0+\5*4.8.>EB.@7> 4OCGL$ >CKB\4n	(&6R	( 05,J^FrQ   