
    ,                         d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
Z
dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  G d dej,                        Zy)z"The BigQuery CLI truncate command.    )absolute_import)division)print_function)Optional)app)flagsN)
client_job)client_table)utils)bigquery_command)bq_cached_client)bq_error)bq_id_utils)bq_processor_utilsc                   ^    e Zd ZdZdedej                  f fdZddedee	   fdZ
dej                  j                  fd	Zd
ej                  j                  fdZdee	   fdZdede	fdZd
ej                  j                  dedefdZd
ej                  j                  dededefdZ xZS )Truncatezpbq truncate project_id:dataset[.table] [--timestamp] [--dry_run] [--overwrite] [--skip_fully_replicated_tables]
namefvc                    t         t        |   ||       t        j                  dd dd|       t        j
                  dd d|       t        j
                  dd	d
|       t        j
                  dddd|       | j                  |       y )N	timestampz^Optional timestamp to which table(s) will be truncated. Specified as milliseconds since epoch.t)
short_nameflag_valuesdry_runzlNo-op that simply prints out information and the recommended timestamp without modifying tables or datasets.)r   	overwriteFzZOverwrite existing tables. Otherwise timestamp will be appended to all output table names.skip_fully_replicated_tablesTzSkip tables that are fully replicated (synced) and do not need to be truncated back to a point in time. This could result in datasets that have tables synchronized to different points in time, but will require less data to be re-loadeds)superr   __init__r   DEFINE_integerDEFINE_boolean_ProcessCommandRc)selfr   r   	__class__s      (platform/bq/frontend/command_truncate.pyr   zTruncate.__init__    s    	(D"4,		$ 
	: 
	" 
&	, 	 	2    
identifierreturnc           
        
 t         j                  j                         
|r&t        j                  
|j                               }nt        j                  d      d| _        d| _	        d| _
        g }| j                  r| j                  st        d| j                  z         t        |t        j                   j"                        r|g}n\t        |t        j                   j$                        r8t'        t)        
fdt+        j,                  
j.                  |d                  }D ]8  }	 |j1                  | j3                  |t5        | j                        d             : nLt        |t        j                   j"                        r| j=                  |      }n5t        |t        j                   j$                        r| j?                  |      }	 tA        t'        t)        | jB                                    }|st        j                  d|z        t        d|z         D ]  }tI        |d      stK        d      	 t        j                   j"                  jM                  |jN                  |jP                  |d         }	|j1                  | j3                  |	t5        |      |d                 t        | j                  d| j                  d| j                  d       t        |ddi y# t6        j8                  $ rL}t        |       |j1                  | j;                  |d	             | xj                  d
z  c_
        Y d}~!d}~ww xY w# tD        t6        jF                  f$ r d}Y vw xY w# t6        j8                  $ rL}t        |       |j1                  | j;                  	d	             | xj                  d
z  c_
        Y d}~d}~ww xY w)a  Truncates table/dataset/project to a particular timestamp.

    Examples:
      bq truncate project_id:dataset
      bq truncate --overwrite project_id:dataset --timestamp 123456789
      bq truncate --skip_fully_replicated_tables=false project_id:dataset
    id_fallbacksr'   z-Must specify one of project, dataset or tabler   zQTruncating to user specified timestamp %s.(Not skipping fully replicated tables.)c                 8    t        j                  | d         S )Nidr*   )bq_client_utilsGetReference)xclients    r%   <lambda>z&Truncate.RunWithArgs.<locals>.<lambda>g   s    O88#)agr&   @B )	apiclient	referencemax_resultsFFailed   Nz:Unable to figure out a recovery timestamp for %s. Exiting.z*Recommended timestamp to truncate to is %s	datasetIdz!Missing `datasetId` on reference.r   	projectIdr9   tableIdfully_replicatedz tables truncated, z tables failed to truncate, z tables skippedsep
))r   ClientGetr.   r/   stripr   
UsageErrortruncated_table_countskipped_table_countfailed_table_countr   r   print
isinstancer   ApiClientHelperTableReferenceDatasetReferencelistmapr
   list_tablesr4   append_TruncateTablestrr   BigqueryError_formatOutputString_GetTableInfo_GetTableInfosFromDatasetmin_GetRecoveryTimestamp
ValueErrorBigqueryTypeErrorhasattrAttributeErrorCreater;   r9   )r#   r'   r5   status
all_tablesa_tableeall_table_infosrecovery_timestamptable_referencer1   s             @r%   RunWithArgszTruncate.RunWithArgsE   s\    $$((*F!..**:*:*<i NNJKK!"D DDF~~dll!NN
 
I{::II	J[
i!<!<!M!MN ** & 0 0 )"-		*  '	'
--!!'3t~~+>F   
I{::II	J,,Y7i!<!<!M!MN ::9E/" T//AB
  nnH
 	
 8;MMN$'y+.BC
C	''77FFMM!++!++fo N /
 --!!!(),- %( 
 &&##$$		
 
6tq %% 	'
(
--11'8D
F

!
!Q
&
!
!	' (445 "!"< %% 	'
(
--11/8L
N

!
!Q
&
!
!	'sE   .5K1(M  A1M51MAMMM21M25OAOOdataset_referencec                 D    d|j                   z  }| j                  |d      S )Na}  SELECT
  TABLE_NAME,
  UNIX_MILLIS(replicated_time_at_remote_site),
  CASE
    WHEN last_update_time <= min_latest_replicated_time THEN TRUE
  ELSE
  FALSE
END
  AS fully_replicated
FROM (
  SELECT
    TABLE_NAME,
    multi_site_info.last_update_time,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time DESC)[safe_OFFSET(1)] AS replicated_time_at_remote_site,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time ASC)[safe_OFFSET(0)] AS min_latest_replicated_time
  FROM
    %s.INFORMATION_SCHEMA.TABLES t,
    t.multi_site_info.site_info
  GROUP BY
    1,
    2)r3   )r9   _ReadTableInfo)r#   re   $recovery_timestamp_for_dataset_querys      r%   rU   z"Truncate._GetTableInfosFromDataset   s4    -
0 ((1,)(2 ,k r&   rc   c                 b    d|j                   d|j                  d}| j                  |d      S )Na%  SELECT
  TABLE_NAME,
  UNIX_MILLIS(replicated_time_at_remote_site),
  CASE
    WHEN last_update_time <= min_latest_replicated_time THEN TRUE
  ELSE
  FALSE
END
  AS fully_replicated
FROM (
  SELECT
    TABLE_NAME,
    multi_site_info.last_update_time,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time DESC)[safe_OFFSET(1)] AS replicated_time_at_remote_site,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time ASC)[safe_OFFSET(0)] AS min_latest_replicated_time
  FROM
    zX.INFORMATION_SCHEMA.TABLES t,
    t.multi_site_info.site_info
  WHERE
    TABLE_NAME = 'z'
  GROUP BY
    1,
    2 )r8   )	row_count)r9   r<   rg   )r#   rc   "recovery_timestamp_for_table_querys      r%   rT   zTruncate._GetTableInfo   s8    @ ((/*A*A5*C&6 AQOOr&   c                 ,    |d   rt        |d         S d S )Nrb   )int)r#   
table_infos     r%   rW   zTruncate._GetRecoveryTimestamp   s,     *+ 	J+,- r&   queryrj   c                    t         j                  j                         }	 t        j                  ||d      }g }t        j                  |      sqt        j                  ||d   d|      \  }}t        t        |            D ]9  }	i }
||	   d   |
d	<   ||	   d
   |
d<   ||	   d   dk(  |
d<   |j                  |
       ; |S y # t
        j                  $ r-}d|j                  d   v rt        j                  d      |d }~ww xY w)NF)use_legacy_sqlzName multi_site_info not foundmessagez:This functionality is not enabled for the current project.jobReferencer   )	start_rowmax_rowsr   r8   rb      truer=   )r   r@   rA   r	   Queryr   rR   errorr   rC   r.   IsFailedJobReadSchemaAndJobRowsrangelenrO   )r#   ro   rj   r1   jobr`   ra   _rowsirn   s              r%   rg   zTruncate._ReadTableInfo   s   $$((*FVU5Ac O&&s+//
#n%Yga SY!
!!WQZ
6+/71:
'()-av)=
%&z*    , !! 	 
*QWWY-?	?nnH
 	
 	s   C D(C<<Dr]   c                     d||fz  S )Nz%s %200s )r#   rc   r]   s      r%   rS   zTruncate._formatOutputString  s    
 &111r&   rb   is_fully_replicatedc           	      (   t         j                  j                         }i }| j                  s]t        j
                  j                  j                  |j                  |j                  dj                  |j                  d|g            }n|}| j                  r)|r'| xj                  dz  c_        | j                  |d      S | j                  r| j                  |d|z        S ddd	d
}t         j"                  j$                  rt         j"                  j$                  |d<   t'        j(                  ||d|      }d}	 t+        j,                  ||g|fi |}	|	'| xj.                  dz  c_        | j                  |d      S t1        j2                  |	      }| xj4                  dz  c_        | j                  |d|z        S # t6        j8                  $ r?}
t;        |
       | xj.                  dz  c_        | j                  |d|z        cY d }
~
S d }
~
ww xY w)Nr   TRUNCATED_ATr:   r8   zFully replicated...Skippedzwill be Truncated@%sWRITE_TRUNCATEFCOPY)write_dispositionignore_already_existsoperation_typelocation@r*    r7   zSuccessful %s z
Failed %s )r   r@   rA   r   r   rI   rJ   r\   r;   r9   joinr<   r   rE   rS   r   bq_flagsLOCATIONvaluer.   GetTableReferencer	   	CopyTablerF   r   ConstructObjectReferencerD   r   rR   rG   )r#   rc   rb   r   r1   kwdsdestsource_tablejob_refr~   r`   s              r%   rP   zTruncate._TruncateTable  s    $$((*FD>>((77>>#--#--((&&8JK ? d d((-@
!#%%
7  ||%%
&);;  .!& D
 !**00d:"44-/ABL GD  ,FFc	1$''h77";;C@g
  A% %%d,<w,FGG!! DAh
"%%dL7,BCCDs%   >AF?  >F? ?H4HHH) )__name__
__module____qualname__usagerQ   r   
FlagValuesr   r   rm   rd   r   rI   rK   rU   rJ   rT   rW   rg   rS   boolrP   __classcell__)r$   s   @r%   r   r      s   %#3 #E$4$4 #JiC i# iV!*::KK!F!P(88GG!PF# # # 62"22AA2 2 	23D"22AA3D 3D  	3D
 3Dr&   r   )__doc__
__future__r   r   r   typingr   abslr   r   r   clientsr	   r
   r   r.   frontendr   r   r   r   r   BigqueryCmdr   r   r&   r%   <module>r      sL    ( &  %        , % %   $vD++ vDr&   