
    3                         d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlm	Z	 ddl
mZ ddl
mZ dd	l
mZ dd
lmZ ddlmZ ddlZddlZ G d d ej(                  ej*                  e            Zy)z1Utilities for building the dataproc clusters CLI.    )absolute_import)division)unicode_literalsN)encoding)	constants)
exceptions)storage_helpers)log)filesc                   B     e Zd ZdZ fdZd Zd ZddZd Zd Z	 xZ
S )	JobBasezBase class for Jobs.c                 V    t        t        | 
  |i | i | _        g | _        d | _        y N)superr   __init__files_by_typefiles_to_stage_staging_dir)selfargskwargs	__class__s      4lib/googlecloudsdk/command_lib/dataproc/jobs/base.pyr   zJobBase.__init__&   s/    	'4!4262DDD    c                    t         j                  j                  |      \  }}t        j                  j
                  j                  j                  |d      }|xs |j                   }|s|S t         j                  j                  |      s$t        j                  dj                  |            | j                  $t        j                  dj                  |            t         j                  j!                  |      }| j"                  j%                  |       t        j                  j
                  j                  j'                  | j                  |      }|S )z?Validate file URI and register it for uploading if it is local.F)allow_fragmentszFile Not Found: [{0}].zCould not determine where to stage local file {0}. When submitting a job to a cluster selected via --cluster-labels, either
- a staging bucket must be provided via the --bucket argument, or
- all provided files must be non-local.)ospath
splitdrivesixmovesurllibparseurlsplitschemeexistsr   Errorformatr   r   ArgumentErrorbasenamer   appendurljoin)r   file_strdrive_uriis_localr*   staged_files           r   _GetStagedFilezJobBase._GetStagedFile,   s   ww!!(+HE1
))


 
 
)
)(E
)
JC &CJJH o77>>(#KK077ABB  $$4 5;F84D	F F ww)Hx())""((001B1BHMKr   c                    t        j                  | j                        D ]n  \  }}|s	t        |t         j                        r| j                  |      | j                  |<   B|D cg c]  }| j                  |       }}|| j                  |<   p | j                  rdt        j                  dj                  | j                  | j                               t        j                  | j                  | j                         yyc c}w )z5Validate file URIs and upload them if they are local.zStaging local files {0} to {1}.N)r    	iteritemsr   
isinstancestring_typesr3   r   r
   infor(   r   r	   Upload)r   	file_typefile_or_filesfstaged_filess        r   ValidateAndStageFileszJobBase.ValidateAndStageFilesI   s    $'MM$2D2D$E 	=mS%5%56(,(;(;M(J9%8EF1++A.F(49% %F 	hh0778K8K8<8I8IK LT00$2C2CD  Gs   &Dc                    ||y|G|j                   r|j                   j                  }n$|j                  r|j                  j                  }nyd}||j                  }||}dj                  |t        j                  ||      }|S )z6Determine the GCS directory to stage job resources in.N
unresolvedz;gs://{bucket}/{prefix}/{environment}/jobs/{job_id}/staging/)bucketprefixenvironmentjob_id)configconfigBucketvirtualClusterConfigstagingBucketclusterUuidr(   r   GCS_METADATA_PREFIX)r   clustercluster_poolrD   rA   rC   staging_dirs          r   GetStagingDirzJobBase.GetStagingDirZ   s    ~'/~	,,''--;;
 K''k kELL00#	 	M 	  r   c           
      X   |sy|j                   j                  j                  j                  }t	        j
                  |j                         D cg c]  \  }}| ||      f c}}      }|j                  t        j                  ||j                   j                              S c c}}w )z$Build LoggingConfig from parameters.N)driverLogLevels)	LoggingConfigDriverLogLevelsValueAdditionalPropertyValueValueValuesEnumcollectionsOrderedDictitemsr   DictToAdditionalPropertyMessage)r   messagesdriver_logging
value_enumkeyvaluerE   s          r   BuildLoggingConfigzJobBase.BuildLoggingConfigv   s    ((==$$%9%9 $$4B4H4H4JK4Jjc5#z% 	!4JKMF!! @@""779 " : : 	Ls   B&
c                 X    | j                   j                  | j                  |             y r   )r   updateGetFilesByType)r   r   s     r   PopulateFilesByTypezJobBase.PopulateFilesByType   s!    d11$78r   r   )__name__
__module____qualname____doc__r   r3   r>   rN   r^   rb   __classcell__)r   s   @r   r   r   #   s%    :E"8:9r   r   )rf   
__future__r   r   r   abcrU   r   apitools.base.pyr   googlecloudsdk.api_lib.dataprocr   r   r	   googlecloudsdk.corer
   googlecloudsdk.core.utilr   r    six.moves.urllib.parsewith_metaclassABCMetaobjectr    r   r   <module>rs      sV    8 &  ' 
  	 % 5 6 ; # * 
 b9 c  f5 b9r   