
    `                     d   U d Z ddlmZ ddlmZ ddlmZ ddlZddlmZ ddl	Z	ddl
Z
ddlZddlZddlZddlmZmZmZmZ ddlZddlmZ ddlZddlZdd	lmZ ddlZdd
lmZ ddlZddlZddlZddlm Z  ddl!m"Z" ddl!m#Z$ ddl!m%Z% ddl&m'Z' ddl&m(Z( ddl#m)Z) ddl#m*Z* ddl#m+Z+ 	 ddl,m-Z. dZ/	 ddl1Z1dZ2 e3       Z4ee jj                  ejl                  f   Z7e)jp                  Z8eejr                  f   Z9ee:d<   eejr                  df   Z;ee:d<    G d d      Z<y# e0$ r dZ/Y tw xY w# e0$ r dZ2Y zw xY w)zBigqueryClient class.    )absolute_import)division)print_functionN)client)CallableListOptionalUnion)flags)	discovery)	TypeAlias)main_credential_loader)bigquery_http)utils)wait_printer)discovery_document_cache)discovery_document_loader)bq_api_utils)bq_error)
bq_logging)credentialsTFHttpz#google_auth_httplib2.AuthorizedHttpAuthorizedHttpc                       e Zd ZdZ G d deej                        Zddeddde	j                   ej                   ej                               ddddddd	ed
edee   dee   deeedf   dedee   dedeeg e	j(                  f      dej*                  dee   dee   dedee   def fdZg dZg dZdefdZ	 	 d5ded
edee   dee   def
d Zded!edefd"Z ded#ee   d
edeeeeef      fd$Z!ded!ed#edeeeeef      fd%Z"	 	 	 	 	 d6ded#ee   d&ee   d
ee   dee   dee   de#jH                  fd'Z%e&de#jH                  fd(       Z'de#jH                  fd)Z(de#jH                  fd*Z)de#jH                  fd+Z*de#jH                  fd,Z+de#jH                  fd-Z,	 d7d.ee   de#jH                  fd/Z-	 d7d0ee   de#jH                  fd1Z.	 d7d2ee   de#jH                  fd3Z/deeef   dedee   fd4Z0 xZ1S )8BigqueryClientz:Class encapsulating interaction with the BigQuery service.c                       e Zd ZdZdZdZy)BigqueryClient.JobCreationModezEnum of job creation mode.JOB_CREATION_REQUIREDJOB_CREATION_OPTIONALN)__name__
__module____qualname____doc__r   r        &platform/bq/clients/bigquery_client.pyJobCreationModer   P   s    $33r%   r'    NTF)
project_id
dataset_iddiscovery_documentjob_propertytracesyncwait_printer_factoryjob_id_generatormax_rows_per_requestquota_project_iduse_google_authr   enable_resumable_uploadsapiapi_versionr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r   r4   c                0   t         |           || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        |
| _        || _        || _        || _        || _        || _        |j#                         D ]  \  }}t%        | ||        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        d| _        dt8        i}|j#                         D ]  \  }}t;        | |      rt%        | ||       ! y)a8  Initializes BigqueryClient.

    Required keywords:
      api: the api to connect to, for example "bigquery".
      api_version: the version of the api to connect to, for example "v2".

    Optional keywords:
      project_id: a default project id to use. While not required for
        initialization, a project_id is required when calling any
        method that creates a job on the server. Methods that have
        this requirement pass through **kwds, and will raise
        bq_error.BigqueryClientConfigurationError if no project_id can be
        found.
      dataset_id: a default dataset id to use.
      discovery_document: the discovery document to use. If None, one
        will be retrieved from the discovery api. If not specified,
        the built-in discovery document will be used.
      job_property: a list of "key=value" strings defining properties
        to apply to all job operations.
      trace: a tracing header to include in all bigquery api requests.
      sync: boolean, when inserting jobs, whether to wait for them to
        complete before returning from the insert request.
      wait_printer_factory: a function that returns a WaitPrinter.
        This will be called for each job that we wait on. See WaitJob().

    Raises:
      ValueError: if keywords are missing or incorrectly specified.
    Niam_policy_discovery_document)super__init__r5   r6   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r   r4   itemssetattr
_apiclient_routines_apiclient_row_access_policies_apiclient_op_transfer_client_op_reservation_client_op_bi_reservation_client_models_apiclient_op_connection_service_client_iam_policy_apiclient_DEFAULThasattr)selfr5   r6   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r   r4   kwdskeyvaluedefault_flag_valuesflagnamedefault	__class__s                         r&   r:   zBigqueryClient.__init__V   s(   h 
GDH"D DO DO0D$DDJDI 4D,D 4D,D*D"D$<D!jjl
UdC #DO#D*.D'#D"&D%)D"!D)-D&!%D' 1668'T8$h( 9r%   )
updateTimeschedulerunTimescheduleTimeparamsendTimedataSourceIddestinationDatasetIdstate	startTimename)rQ   rU   rY   returnc                 X   t         j                  }t        j                  j                  rt        j                  j
                  r	 t        t        j                  j
                        }t        j                  dt        j                  j                  |t        j                  j                  xs dt        j                  j                  xs d      }t        j                  |t        j                  j                  xs t        j                         t        j                  j                         }t#        |d      rt%        |j&                        dhz
  |_        t        j                  j(                  rvt+        j,                         \  }| _        t+        j,                         \  }| _        t3        j4                  |t3        j6                         | j.                  | j0                         |S # t        $ r3 t        dj                  t        j                  j
                              w xY w)z!Returns the httplib2 Http to use.z Invalid value for proxy_port: {}   N)
proxy_type
proxy_host
proxy_port
proxy_user
proxy_pass)
proxy_infoca_certs"disable_ssl_certificate_validationredirect_codesi4  )httplib2proxy_info_from_environmentr   FLAGSproxy_addressr`   int
ValueErrorformat	ProxyInfoproxy_usernameproxy_passwordr   ca_certificates_filecertifiwheredisable_ssl_validationrG   setrf   mtlstempfilemkstemp
_cert_file	_key_filer   add_mtls_credsget_client_options)rH   rc   porthttp_s        r&   GetHttpzBigqueryClient.GetHttp   s   
 55J{{  U[[%;%;
5;;))*
 %%[[..[[//74[[//74j ==11DW]]_+0;;+M+MD t%& 3 34u<d{{#++-a"**,a
	,,. K7  
.55ekk6L6LM
 	

s   #G- -<H)servicedomain_rootlabelsc                 v    d}|s4t        j                  |xs t        j                  |t              |      }|S )z7Returns the url to the discovery document for bigquery.Nr   inputted_flagsr6   )r   get_discovery_url_from_root_urlget_tpc_root_url_from_flagsbq_flags)rH   r   r6   r   r   discovery_urls         r&   GetDiscoveryUrlzBigqueryClient.GetDiscoveryUrl   sD     M"BB
 55h "m r%   r~   c                    | j                   rKt        st        j                  d       t	        |t
        j                        s+t        j                  d       nt        j                  d       t        rBt	        |t
        j                        r(t        st        d      t        j                  ||      S t        |d      r|j                  |      S t        d      )zEReturns an http client that is authorized with the given credentials.z8System is set to use `google.auth`, but it did not load.zVThe system is using `google.auth` but the parsed credentials are of an incorrect type.z'System is set to not use `google.auth`.zCredentials from google.auth specified, but google-api-python-client is unable to use these credentials unless google-auth-httplib2 is installed. Please install google-auth-httplib2.)r~   	authorizez0Unsupported credential type: {type(credentials)})r3   _HAS_GOOGLE_AUTHloggingerror
isinstancegoogle_credentialsCredentialsdebug_HAS_GOOGLE_AUTH_HTTPLIB2rl   google_auth_httplib2r   rG   r   	TypeError)rH   r   r~   s      r&   GetAuthorizedHttpz BigqueryClient.GetAuthorizedHttp   s     F	
 %7%C%CD%	

 mm=> J'33 '$
 	
 "004HH {K(""4((HIIr%   r   c                    d}| j                   t        k7  r%| j                   }t        j                  d||       |S |t        j                  d|       |S t        j
                  j                  rt        j                  d|       |S 	 t        j                  || j                  |      }|rt        j                  d|       |S # t        $ r"}t        j                  d||       Y d}~|S d}~ww xY w)a?  Loads the local discovery document for the given service.

    Args:
      service: The BigQuery service being used.
      discovery_url: The URL to load the discovery doc from.
      api_version: The API version for the targeted discovery doc.

    Returns:
      discovery_document The loaded discovery document.
    NzTSkipping local "%s" discovery document load since discovery_document has a value: %szOSkipping the local "%s" discovery document load since discovery_url has a valuez^Skipping local "%s" discovery document load since the bigquery_discovery_api_key flag was used)r   r5   r6   z(The "%s" discovery doc is already loadedz:Failed to load the "%s" discovery doc from local files: %s)r+   rF   r   infor   BIGQUERY_DISCOVERY_API_KEY_FLAGpresentr   %load_local_discovery_doc_from_servicer5   FileNotFoundErrorwarning)rH   r   r   r6   r+   es         r&   _LoadDiscoveryDocumentLocalz*BigqueryClient._LoadDiscoveryDocumentLocal$  s
   " (*22ll

	H = 
	"ll
: 1 
	1	1	9	9ll6
. !
%KKHH' 	 
,,A7
K   
H	
 	

 
s   :C 	C/C**C/c                 x   d}d}d}t        | d      r| j                  rddind}||k  r||dkD  rt        j                  d|z         |dz  }	 t	        j
                  d	||       |r|j                  ||
      \  }}n|j                  |      \  }}|j                  d      }t        |j                  d            dk\  rCd|j                  d      d|}	t	        j                  d|	|       t        j                  |	      	 ||k  r||S # t        j                  t        j                   j"                  t$        j&                  f$ r}
t        |
d      r:||k(  rdd}t        |
d      r|
j(                  }t        j                  d|
d|      ||k(  r*t        j                  dt+        j,                               Y d}
~
d}
~
wt.        $ r'}
||k(  rt        j                  d|
      Y d}
~
d}
~
wt        j                   j0                  $ r&}
t        j                  dt3        |
            d}
~
ww xY w)a  Loads the discovery document from the provided URL.

    Args:
      service: The BigQuery service being used.
      http: Http object to be used to execute request.
      discovery_url: The URL to load the discovery doc from.

    Returns:
      discovery_document The loaded discovery document.

    Raises:
      bq_error.BigqueryClientError: If the request to load the discovery
      document fails.
    Nr]   r   use_uber_mintz#X-ESF-Use-Cloud-UberMint-If-Enabled1      z*Requesting "%s" discovery document from %s)headerszutf-8statusi  zGot z response from discovery url: z%s:
%scontentr(   z0Cannot contact server. Please try again.
Error: z

Content: z4Cannot contact server. Please try again.
Traceback: zInvalid API name or version: )rG   r   timesleepr   r   requestdecoderk   getr   r   BigqueryCommunicationErrorrg   HttpLib2Errorgoogleapiclienterrors	HttpErrorhttp_client_libHTTPExceptionr   	traceback
format_excIOErrorUnknownApiNameOrVersionstr)rH   r   r~   r   r+   max_retries
iterationsr   response_metadatamsgr   r   s               r&   _LoadDiscoveryDocumentUrlz(BigqueryClient._LoadDiscoveryDocumentUrl_  sZ   *  KJ 4)d.@.@ 
/4 
 {
"'9'A	a

1j=!Aoj4
8	

 26,,W 3? 3
/
/ 37,,}2M
/
//66w? $$X./36##H-# --	3(:
;33C8
8 7% {
"'9'At A 
 
 

 
 
*
*

'
'  1i ;&Gq)$		g55#$g/ 
 ;&55''),   $33GHJ  % ##;; 
 1114Q9
 	

s2   B5D <H9A5GH9G33 H9!H44H9discovery_root_urlc           	         t        j                  d||       || j                  }t        j                  | j
                  t        j                  | j                  | j                  | j                  | j                              }t        j                  j                  || j                        }| j                  r't        | j                  d      rd| j                  _        | j!                         }	| j#                  | j                  |	      }
d}|| j%                  |||      }|du }||| j'                  ||||      }|s| j)                  ||
|      }| j+                  ||	      }t-        j.                  d
|t0        j2                  j4                         	 t7        j8                  ||
||      }|S # t:        $ r t        j<                  d||        w xY w)zABuild and return BigQuery Dynamic client from discovery document.z8BuildApiClient discovery_url: %s, discovery_root_url: %sN)r2   r)   r3   r   )r-   r2   _quota_project_id)r   r   r6   )r   r6   r   r   )r   r~   r   )r+   r   r+   )file_prefixr   apilog)r~   modelrequestBuilderz3Error building from the "%s" discovery document: %s)r   r   r6   r   BigqueryModelr-   bq_utils'GetEffectiveQuotaProjectIDForHTTPHeaderr2   r)   r3   r   BigqueryHttpFactoryrG   r   r   r   r   r   r   OverrideEndpointr   #SaveStringToLogDirectoryIfAvailabler   APILOGrK   r   build_from_document	Exceptionr   )rH   r   r   r   r6   r   r   bigquery_modelbq_request_builderhttp_clientr~   r+   discovery_url_not_provided"discovery_document_to_build_clientbuilt_clients                  r&   BuildApiClientzBigqueryClient.BuildApiClient  s    LLB
 $$k #00jj!II!22 00((	
N '33;; (8(8:M N+/d(,,.K!!$"2"2K@D!;;%! <  "/$!6!m&;**!!	 + m 99% :  *.)>)>-w *? *& 22(2$$ 22
,+	l   mm
?


 s   F9 9"Gc                     | j                   r!t        j                  d       | j                   S | j                  t        j
                        | _         | j                   S )z>Returns a singleton ApiClient built for the BigQuery core API.z$Using the cached BigQuery API clientr   )r=   r   r   r   ServiceBIGQUERYrH   s    r&   	apiclientzBigqueryClient.apiclient'  sK     ll9: ?? ++G4D4D+Edo??r%   c                 |    | j                   %| j                  t        j                        | _         | j                   S )z'Returns the apiclient attached to self.r   )rC   r   r   r   r   s    r&   GetModelsApiClientz!BigqueryClient.GetModelsApiClient0  s6    %#227;K;K2Ld!!!r%   c                 |    | j                   %| j                  t        j                        | _         | j                   S z&Return the apiclient attached to self.r   )r>   r   r   r   r   s    r&   GetRoutinesApiClientz#BigqueryClient.GetRoutinesApiClient6  s6    '!%!4!4W=M=M!4!Nd###r%   c                 |    | j                   %| j                  t        j                        | _         | j                   S r   )r?   r   r   r   r   s    r&   GetRowAccessPoliciesApiClientz,BigqueryClient.GetRowAccessPoliciesApiClient<  s>    **2,0,?,?"" -@ -d) ...r%   c                 |    | j                   %| j                  t        j                        | _         | j                   S r   )rE   r   r   BQ_IAMr   s    r&   GetIAMPolicyApiClientz$BigqueryClient.GetIAMPolicyApiClientD  s;    !!)#'#6#6.. $7 $d  %%%r%   c                     | j                   }|S )z4Return the apiclient that supports insert operation.)r   )rH   insert_clients     r&   GetInsertApiClientz!BigqueryClient.GetInsertApiClientL  s    NNMr%   transferserver_addressc                 H   t        j                  d|       | j                  r!t        j                  d       | j                  S |xs) t        j                  t
        j                  t              }| j                  |dt
        j                        | _        | j                  S )z9Return the apiclient that supports Transfer v1 operation.z1GetTransferV1ApiClient transferserver_address: %sz$Using the cached Transfer API clientr   v1)r   r6   r   )	r   r   r@   r   r   r   DTSr   r   )rH   r   paths      r&   GetTransferV1ApiClientz%BigqueryClient.GetTransferV1ApiClientQ  s     LL;
 ll9: ### $ |'O'O++h(d "&!4!4++ "5 "d
 ###r%   reservationserver_addressc                 &   | j                   r!t        j                  d       | j                   S |xs) t        j                  t
        j                  t              }d}d}| j                  t
        j                  |||      | _         | j                   S )z:Return the apiclient that supports reservation operations.z(Using the cached Reservations API clientr   r   N)r   r   r6   r   )	rA   r   r   r   r   r   RESERVATIONSr   r   )rH   r   r   reservation_versionr   s        r&   GetReservationApiClientz&BigqueryClient.GetReservationApiClientg  s     ""ll=>" &&& $ 55**%  !f$($7$7&&)	 %8 %d! &&&r%   connection_service_addressc                    | j                   r!t        j                  d       | j                   S |xs) t        j                  t
        j                  t              }t        j                  |d      }t        j                  |t        j                  j                  t              }| j                  |t
        j                        | _         | j                   S )z:Return the apiclient that supports connections operations.z'Using the cached Connections API clientr   r   r   )r   universe_domainr   )r   r   )rD   r   r   r   r   r   CONNECTIONSr   r   add_api_key_to_discovery_urlUNIVERSE_DOMAINrK   r   )rH   r   r   r   s       r&   GetConnectionV1ApiClientz'BigqueryClient.GetConnectionV1ApiClient  s     ))ll<=* ---% % 55))%  #BB
Dm #??%"2288!m
 ,0+>+>%%% ,? ,d( ---r%   c                 x   ||S t        j                  |      }t        j                  d|d   |d   |d          d}|rt        j                  |t
              |d<   t        j                  j                  |d   |d         |d<   t        j                  d|d   |d   |d          t        j                  |      S )	aQ  Override rootUrl for regional endpoints.

    Args:
      discovery_document: BigQuery discovery document.
      service: The BigQuery service being used.

    Returns:
      discovery_document updated discovery document.

    Raises:
      bq_error.BigqueryClientError: if location is not set and
        use_regional_endpoints is.
    zgDiscovery doc routing values being considered for updates: rootUrl: (%s), basePath: (%s), baseUrl: (%s)rootUrlbasePathbaseUrlTr   servicePathzWDiscovery doc routing values post updates: rootUrl: (%s), basePath: (%s), baseUrl: (%s))r   parse_discovery_docr   r   r   r   urllibparseurljoinjsondumps)rH   r+   r   is_prods       r&   r   zBigqueryClient.OverrideEndpoint  s      !%99:LMLL	/9%:&9% G&2&N&N('#
 %+LL$8$89%'9-'H%y! LL	9%:&9% ::())r%   )NN)NNNNN)N)2r    r!   r"   r#   r   enumEnumr'   rF   r   TransitionWaitPrinterbq_client_utilsJobIdGeneratorIncrementingJobIdGeneratorRandomr	   r
   bytesobjectboolr   WaitPrinterJobIdGeneratorrk   'LegacyAndGoogleAuthCredentialsUnionTyper:   #columns_to_include_for_transfer_run&columns_excluded_for_make_transfer_runr   r   r   r   r   r   r   r   r   Resourcer   propertyr   r   r   r   r   r   r   r   r   r   __classcell__)rO   s   @r&   r   r   M   s   B4TYY 4 #%"$7?! 
,
,9c9c9c
./
.
.
0: -1(,#GK'++U) U) 	U)
 3-U) 3-U)  vt 34U) U) c]U) U) %
2|///
0U) (66U)" %SM#U)$ !%U)& 'U)( CD)U)* !%+U)n)%  ,P($$T $("  C=	
 sm &%J:%J %J 	%JX88 c]8 	8
 c5&()*8vYY Y 	Y
 c5&()*Y| &**.#'#'"kk c]k #3-	k
 C=k C=k smk kZ ++  ")"4"4 "$I$6$6 $/Y-?-? /&Y%7%7 &)"4"4  59$$,SM$$. 8<''/}''2 9=.(0..81* %c5j 11*<C1*}1*r%   r   )=r#   
__future__r   r   r   r
  r~   r   r   r  r   rw   r   r   typingr   r   r	   r
   r  abslr   rr   r   r   rg   typing_extensionsr   r   r   credential_loaderauthr   clientsr   r   r  r   discovery_documentsr   r   r   r   r   google.authr   r   r   ImportErrorr   r   r  rF   GoogleAuthCredentialsUnionTypeCredentialsFromFlagsUnionTyper  r   r   __annotations__r   r   r$   r%   r&   <module>r(     s3    &  %  *      2 2     %  '    ' ! ,   8 9   ;
$" 8*/99335+ '
 

MMNi  "MM)+	 	* 	*A    $#$s$   %D .D% D"!D"%D/.D/