
                             d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddlm	Z	 ddl
mZ dd	l
mZ ddlZd
ZddZd ZddZd Zd Zd Zy)zJHelper functions for the ml-engine  client to use command_lib.logs.stream.    )absolute_import)division)unicode_literalsN)encoding)jobs)
properties)	resourcesz]value(severity,timestamp.date("%Y-%m-%d %H:%M:%S %z",tz="LOCAL"), task_name,trial_id,message)c                 p    ddj                  |       g}|r |j                  dj                  |             |S )zReturns filters for log fetcher to use.

  Args:
    job_id: String id of job.
    task_name: String name of task.

  Returns:
    A list of filters to be passed to the logging API.
  z7(resource.type="ml_job" OR resource.type="cloudml_job")zresource.labels.job_id="{0}"z;(resource.labels.task_name="{0}" OR labels.task_name="{0}"))formatappend)job_id	task_namefilterss      5lib/googlecloudsdk/command_lib/ml_engine/log_utils.py
LogFiltersr   *   sF     @$++F3' NNELL	 
.    c                     t        j                         t        j                  j                  j
                  j                  d      }t        j                  j                  d| |      fd}|S )zReturns a function to decide if log fetcher should continue polling.

  Args:
    job_id: String id of job.

  Returns:
    A one-argument function decides if log fetcher should continue.
  T)requiredzml.projects.jobs)jobsId
projectsIdc                 J    | dk  ryj                        j                  du S )a  Returns whether to continue polling the logs.

    Returns False only once we've checked the job and it is finished; we only
    check whether the job is finished once we've gone >1 interval without
    getting any new logs.

    Args:
      periods_without_logs: integer number of empty polls.

    Returns:
      True if we haven't tried polling more than once or if job is not finished.
       TN)GetendTime)periods_without_logsjob_refjobs_clients    r   ShouldContinuez,MakeContinueFunction.<locals>.ShouldContinueL   s*     q ??7#++t33r   )
r   
JobsClientr   VALUEScoreprojectr   r	   REGISTRYCreate)r   
project_idr   r   r   s      @@r   MakeContinueFunctionr&   ?   sf     !+  %%--1141@*%%J & @'4  
r   c              #      K   | D ]Q  }t        |      }|r| |d   j                         }|sdg}|D ]   }t        j                  |      }||d<   | " S yw)a  Splits the dict output of logs into multiple lines.

  Args:
    log_generator: iterator that returns a an ml log in dict format.
    allow_multiline: Tells us if logs with multiline messages are okay or not.

  Yields:
    Single-line ml log dictionaries.
  message N)_EntryToDict
splitlinescopydeepcopy)log_generatorallow_multilineloglog_dictmessagesr(   single_line_logs          r   SplitMultiliner4   _   si      cC Hn)$//1h4'--1%,	"  s   AAc                     i }| j                   r| j                   j                  nd|d<   | j                  |d<   t        |       }|d   |d<   d|v r|d   |d<   d|d<   | j                  ?t        | j                        }d|v r|d   r|dxx   |d   z  cc<   |d= d|v r|d= ||d	<   |S | j                  ,|dxx   t        j                  | j                        z  cc<   |S | j                  "t        j                  | j                        |d	<   |S )
z%Converts a log entry to a dictionary.DEFAULTseverity	timestampr   trial_idr)   r(   	levelnamejson)r7   namer8   _GetLabelAttributesjsonPayload_ToDicttextPayloadsix	text_typeprotoPayloadr   MessageToDict)	log_entryoutputlabel_attributes	json_datas       r   r*   r*   w   sG   &/8/A/AI&&++y 	!++&(3(5&##)*5F:&&	--.I I	9	yYy11
I
 i
K
 F6N
 
-	 (
9y'<'<== 
- )++I,B,BCF6N	-r   c                    ddi}t        | j                        }| j                  si nt        | j                  j                        }|j                  d      	|d   |d<   n3|j                  d      	|d   |d<   n|j                  d      |d   |d<   |j                  d      
|d   |d<   |S |j                  d      |d   |d<   |S )z2Reads the label attributes of the given log entry.r   unknown_taskzml.googleapis.com/task_namer9   zml.googleapis.com/trial_id)r?   labelsresourceget)rE   rG   rK   resource_labelss       r   r=   r=      s    !>29##$&'00Bg7!/%1$3K$@[!zz+*$*;$7[!zz/0<$*+H$I[!ZZ
'#)*#5Z  
 zz./;#)*F#GZ 	r   c                 X    | si S t        | t              r| S t        j                  |       S N)
isinstancedictr   rD   )r(   s    r   r?   r?      s*    	IN!!'**r   rP   )F)__doc__
__future__r   r   r   r,   apitools.base.pyr    googlecloudsdk.api_lib.ml_enginer   googlecloudsdk.corer   r	   rA   
LOG_FORMATr   r&   r4   r*   r=   r?    r   r   <module>rZ      sL    Q &  '  % 1 * ) 

*@0>,+r   