
    F                         d Z ddlZddlZddlZddlZddlZddlZddlmZ d Z	d Z
d Zd Zd Zd	 Zd
 Zd Zd Zd Zd Zd Zy)zSome file reading utilities.
    N)gcsioc                    | j                  d      ryt        j                  j                  |       ryt        j                  j	                  |       rt        d| z        	 t        j                  |        y#  t        d| z        xY w)zCreates a local directory.

  Does nothing if a Google Cloud Storage path is passed.

  Args:
    path: the path to create.

  Raises:
    ValueError: if path is a file or os.makedir fails.
  gs://Nz5Unable to create location. "%s" exists and is a file.zUnable to create location. "%s")
startswithospathisdirisfile
ValueErrormakedirsr   s    -lib/third_party/ml_sdk/cloud/ml/util/_file.pycreate_directoryr      sw     
__W
WW]]4
WW^^D
L  ?KK?
6=
>>s   !A7 7Bc                 P   | j                  d      r%	 t        j                         j                  | |      S t        | |      S # t        $ rY}t        j                  d|       t        j                  d       t        j                         j                  | |      cY d}~S d}~ww xY w)zOpens the given path.r   z-Retrying after exception reading gcs file: %s
   N)	r   r   GcsIOopen	Exceptionloggingerrortimesleep)r   modees      r   open_local_or_gcsr   7   s    	__W,[[]d++ d  ,mmCQG
jjn[[]d++	,s   #A 	B%AB B% B%c                     | j                  d      r#t        j                         j                  |       S t        j
                  j                  |       S )z Returns whether the file exists.r   )r   r   r   existsr   r   r   s    r   file_existsr   E   s8    	__W;;=%%77>>$    c                     t        | d      5 }t        |d      5 }t        j                  ||       ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zCopy a file from src to dest.

  Supports local and Google Cloud Storage.

  Args:
    src: source path.
    dest: destination path.
  rwN)r   shutilcopyfileobj)srcdesth_srch_dests       r   	copy_filer)   M   sI     c"e	4	%' 
& #"	%	% #"s!   AAAA	AAc                 h    t        | d      5 }|j                  |       ddd       y# 1 sw Y   yxY w)zWrites data to a file.

  Supports local and Google Cloud Storage.

  Args:
    path: output file path.
    data: data to write to file.
  r"   N)r   write)r   datar(   s      r   
write_filer-   [   s(     s#v
LL $##s   (1c                 v    | j                  d      rt        |       }nt        |       }|t        d| z        |S )Nr   zFile cannot be loaded from %s)r   _read_cloud_file_read_local_filer   )r   contents     r   	load_filer2   h   s=    	__Wt$Gt$G_
4t;
<<	.r   c                     | j                  d      r#t        j                         j                  |       S t        j                  |       S Nr   )r   r   r   globr   s    r   
glob_filesr6   t   s3    	__W;;=d##99T?r   c                 f    t        | d      5 }|j                         cd d d        S # 1 sw Y   y xY wNr!   )r   read)
local_pathfs     r   r0   r0   {   s#    J668    '0c                 f    t        | d      5 }|j                         cd d d        S # 1 sw Y   y xY wr8   )r   r9   )storage_pathr%   s     r   r/   r/      s#    s+s88: ,++r<   c              #      K   t        | t              s| n| gD ]<  }|j                  d      rt        |      D ]  }|  )t	        |      D ]  }|  > y wr4   )
isinstance
basestringr   _read_cloud_file_stream_read_local_file_stream)	file_listr   lines      r   read_file_streamrF      s_     ))Z@iG d w)$/$
 0 *$/$
 0s   AAc              #   d   K   t        |       5 }|D ]  }|  	 d d d        y # 1 sw Y   y xY ww)N)r   )r   file_inrE   s      r   rC   rC      s'     DzWj  zzs   0$	0-0c              #     K   t        j                  dd| dgt         j                  t         j                        }|j                  5 }|D ]  }|  	 d d d        |j	                         dk7  rt        d| z        y # 1 sw Y   +xY ww)Ngsutilcp-)stdoutstderrr   z3Unable to read data from Google Cloud Storage: "%s")
subprocessPopenPIPErM   waitIOError)r   	read_filerH   rE   s       r   rB   rB      s     tS!____) 7j   ^^
G$N
OO  s   AB
A>*B
>BB
)__doc__r5   r   r   r#   rO   r   apache_beam.ior   r   r   r   r)   r-   r2   r6   r0   r/   rF   rC   rB    r   r   <module>rX      s^      	     ?2 (
	

		Pr   