
                             d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm	Z	 ddl
mZ dd	l
mZ  G d
 dej                        Zy)zBase class for Spark Job.    )absolute_import)division)unicode_literalsN)encoding)arg_parsers)base)utilc                   @    e Zd ZdZed        Zed        Zed        Zy)	SparkBasez.Submit a Java or Scala Spark job to a cluster.c                    | j                  dt        j                         dg d       | j                  dt        j                         dg d       | j                  dt        j                         d	g d
       | j                  dt        j                  d       | j                  dt        j
                         dd       | j                  dt        j                         | j                  dt        j
                         dd       y)z@Parses command-line arguments specific to submitting Spark jobs.z--jarsJARzWComma separated list of jar files to be provided to the executor and driver classpaths.)typemetavardefaulthelpz--filesFILEziComma separated list of files to be placed in the working directory of both the app driver and executors.z
--archivesARCHIVEzComma separated list of archives to be extracted into the working directory of each executor. Must be one of the following file formats: .zip, .tar, .tar.gz, or .tgz.job_argsz Arguments to pass to the driver.)nargsr   z--propertieszPROPERTY=VALUEzList of key value pairs to configure Spark. For a list of available properties, see: https://spark.apache.org/docs/latest/configuration.html#available-properties.)r   r   r   z--properties-file)r   z--driver-log-levelszPACKAGE=LEVELznList of package to log4j log level pairs to configure driver logging. For example: root=FATAL,com.example=INFON)add_argumentr   ArgListargparse	REMAINDERArgDictjob_utilPROPERTIES_FILE_HELP_TEXT)parsers    5lib/googlecloudsdk/command_lib/dataproc/jobs/spark.pyArgszSparkBase.Args"   s0      "0  2   "@  B   "  	   /  1   " 8	  9 //  1   "B	  D    c                 `    | j                   | j                  | j                  | j                  dS )z=Returns a dict of files by their type (jars, archives, etc.).main_jarjarsarchivesfilesr"   )argss    r   GetFilesByTypezSparkBase.GetFilesByTypeS   s*     MM		MM	 r    c           	      D   | j                  |j                  xs g |d   |d   |d   |j                  |d   |      }t        j                  |j
                  |j                        }|r1t        j                  || j                   j                  d      |_        ||_
        y)	z/Populates the sparkJob member of the given job.r%   r&   r$   r#   )r'   archiveUrisfileUrisjarFileUris	mainClassmainJarFileUriloggingConfigT)
sort_itemsN)SparkJobr   
main_classr   BuildJobProperties
propertiesproperties_filer   DictToAdditionalPropertyMessagePropertiesValuesparkJob)messagesjobfiles_by_typelogging_configr'   	spark_jobjob_propertiess          r   ConfigureJobzSparkBase.ConfigureJob\   s     !!]] b!*-w'!&)//$Z0$ " I 00--N %EE
(++;;i CLr    N)__name__
__module____qualname____doc__staticmethodr   r(   r?    r    r   r   r      sB    6.D .D`    r    r   )rC   
__future__r   r   r   r   apitools.base.pyr   googlecloudsdk.callioper   (googlecloudsdk.command_lib.dataproc.jobsr   job_baser	   r   JobBaser   rE   r    r   <module>rL      s6       &  '  % / E ES   Sr    