
    &s                        d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ  ddl!m"Z" ddl#m$Z$ ddl#m%Z% ddl#m&Z& ddl'm(Z( ddl'm)Z) ddl*Z* G d de+      Z, G d de+      Z- G d dej\                        Z/ G d de+      Z0 G d d e+      Z1 G d! d"e+      Z2 G d# d$e+      Z3y)%zFUtility file that contains helpers for the Cloud TPU Execution groups.    )absolute_import)division)unicode_literalsN)
list_pager)HttpNotFoundError)base_classes)poller)apis)waiter)base)scope)	ssh_utils)flags)util)ssh)log)
properties)	resources)retry)timesc                   0    e Zd ZdZed        Zed        Zy)DefaultArgszCHelper to check if required flags are set and sets defaults if not.c                     t         j                  j                  j                  j	                  d      }|j                  d      dk(  r|}n|d|j                  d       }| j                  xs || _        y)z?Validates the name arg and sets defaults if values are not set.Trequired@r   N)r   VALUEScoreaccountGetfindname)argsr    usernames      Dlib/googlecloudsdk/command_lib/compute/tpus/execution_groups/util.pyValidateNamezDefaultArgs.ValidateName2   sd     $$,,00$0?G||CBh7<<,-h		%XDI    c                     | j                   xs4 t        j                  j                  j                   j	                  d      | _         y)z?Validates the zone arg and sets defaults if values are not set.Tr   N)zoner   r   computer!   )r$   s    r&   ValidateZonezDefaultArgs.ValidateZone=   s4     		NZ..66;;???NDIr(   N)__name__
__module____qualname____doc__staticmethodr'   r,    r(   r&   r   r   /   s-    K& & O Or(   r   c                   X    e Zd ZdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zy)TPUNodez&Helper to create and modify TPU nodes.c                     |t         j                  j                  k(  rd| _        nd| _        t	        j
                  d| j                        | _        t	        j                  d| j                        | _        y )Nv1alpha1v1tpu)	r   ReleaseTrackALPHA_api_versionr
   GetClientInstanceclientGetMessagesModulemessages)selfrelease_tracks     r&   __init__zTPUNode.__init__F   sZ    ))///$dd((0A0ABDK**5$2C2CDDMr(   c                     | j                   j                         }||_        ||_        ||_        | j                   j                  |      |_        |S )Npreemptible)r?   NodeacceleratorTypenetworktensorflowVersionSchedulingConfigschedulingConfig)r@   accelerator_type
tf_versionrE   rH   nodes         r&   _CreateDefaultNodezTPUNode._CreateDefaultNodeN   sO    ==D+DDL'D MM:: ; !DKr(   c                 X    t         j                  j                  |j                  d      S )5Get a resource reference to a long running operation.z!tpu.projects.locations.operations
collection)r   REGISTRYParseRelativeNamer#   r@   	operations     r&   _GetTpuOperationRefzTPUNode._GetTpuOperationRefX   s,    //#F 0 H Hr(   c           
         t         j                  j                  j                  j	                  d      }t
        j                  j                  |d|id      }| j                  j                  |j                         || j                  ||||            }	| j                  j                  j                  |	      }
| j                  |
      S )a  Create builds and issues a request to create a TPU node.

    Args:
      name: Name of the TPU Node to be created.
      accelerator_type: Slice type of TPU accelerator like 'v2-8', 'v2-32'.
      tf_version: Tensorflow Version like '1.1', '1.5'.
      zone: Zone to create the TPU Node in.
      preemptible: Boolean argument, to create a Preemptible node.
      network: The network to create the node in
    Returns:
      A TPU Create response which needs to be polled on.
    Tr   
projectsIdtpu.projects.locationsparamsrS   )parentnodeIdrN   )r   r   r   projectr!   r   rT   Parser?   &TpuProjectsLocationsNodesCreateRequestRelativeNamerO   r=   projects_locations_nodesCreaterX   )r@   r#   rL   rM   r*   rE   rH   r`   
parent_refrequestrW   s              r&   re   zTPUNode.Create]   s     $$,,00$0?G##))g&+ * -J mmBB&&($$j+w@ C AG
 44;;GDI##I..r(   c                     t        j                  | j                  j                  | j                  j                        }t        j
                  |||      S N)r   CloudOperationPollerr=   rd   projects_locations_operationsWaitForr@   operation_refmessageoperation_pollers       r&   WaitForOperationzTPUNode.WaitForOperationx   sC    22,,113 >>*M7CCr(   c                     t        j                  | j                  j                        }t        j                  |||      S ri   )r   CloudOperationPollerNoResourcesr=   rk   rl   rm   s       r&   WaitForOperationNoResourcesz#TPUNode.WaitForOperationNoResources~   s4    ==113>>*M7CCr(   c                 t   t         j                  j                  j                  j	                  d      }t
        j                  j                  |||dd      }| j                  j                  |j                               }| j                  j                  j                  |      }| j                  |      S )z)Deletes the TPU node with the given name.Tr   locationsIdrZ   tpu.projects.locations.nodesr\   r#   )r   r   r   r`   r!   r   rT   ra   r?   &TpuProjectsLocationsNodesDeleteRequestrc   r=   rd   DeleterX   )r@   r#   r*   r`   fully_qualified_node_name_refrg   rW   s          r&   r{   zTPUNode.Delete   s    $$,,00$0?G$-$6$6$<$<!
 2 %= %
! mmBB*779 C ;G44;;GDI##I..r(   c                 b   t         j                  j                  j                  j	                  d      }t
        j                  j                  |d|id      }| j                  j                  |j                               }t        j                  | j                  j                  |ddd	
      S )zRetrieves all TPU Nodes.Tr   rZ   r[   r\   r^   ListpageSizenodes)servicerg   methodbatch_size_attributefield)r   r   r   r`   r!   r   rT   ra   r?   $TpuProjectsLocationsNodesListRequestrc   r   YieldFromListr=   rd   )r@   r*   r`   rf   rg   s        r&   r   zTPUNode.List   s    $$,,00$0?G##))g&+ * -J mm@@&&( A *G##44'
 
r(   c                 R   t         j                  j                  j                  j	                  d      }t
        j                  j                  |||dd      }| j                  j                  |j                               }| j                  j                  j	                  |      S )z)Retrieves the TPU node in the given zone.Tr   rv   rx   r\   ry   )r   r   r   r`   r!   r   rT   ra   r?   #TpuProjectsLocationsNodesGetRequestrc   r=   rd   )r@   r#   r*   r`   r|   rg   s         r&   r!   zTPUNode.Get   s    $$,,00$0?G$-$6$6$<$<!
 2 %= %
! mm??*779 @ ;G;;//33G<<r(   c                 z   t         j                  j                  j                  j	                  d      }t
        j                  j                  |d|id      }| j                  j                  |j                               }t        j                  | j                  j                  |dd	      }g }|D ]0  }|j                  t         j#                  |j$                               2 t'        |      }|D ]9  }	|	j(                  rt+        d
dd      |	j,                  r)|	j/                         c S  t+        d
dd      )zEParses available Tensorflow versions to find the most stable version.Tr   rZ   r[   r\   r~   r   tensorflowVersions)r   rg   r   r   zNo stable release foundN)r   r   r   r`   r!   r   rT   ra   r?   1TpuProjectsLocationsTensorflowVersionsListRequestrc   r   r   r=   %projects_locations_tensorflowVersionsappendTensorflowVersionParserParseVersionversionsorted
is_nightlyr   modifierVersionString)
r@   r*   r`   rf   rg   tf_versionsparsed_tf_versionsrM   sorted_tf_versionsr   s
             r&   LatestStableTensorflowVersionz%TPUNode.LatestStableTensorflowVersion   s0   $$,,00$0?G##))g&+ * -J mmMM&&( N 
G **AA'"	$K
 !

!
.
.z/A/A
BD "   23%			 94FF$$&& & 5tT
BBr(   c                     |j                   | j                  j                  j                  j                  k(  xsE |j                   | j                  j                  j                  j
                  k(  xr |j                  S ri   )stater?   rF   StateValueValuesEnumREADYCREATING	ipAddress)r@   rN   s     r&   	IsRunningzTPUNode.IsRunning   s\    ::++@@FFF 

dmm((==FFF 	r(   c                     d}t        j                  ||j                  t         j                        }|r|j	                  d      S y)Nz'projects/(.*)/locations/(.*)/nodes/(.*)    )researchr#   
IGNORECASEgroup)r@   rN   patternmatchs       r&   NodeNamezTPUNode.NodeName   s5    7GIIgtyy"--8E[[^r(   N)r-   r.   r/   r0   rB   rO   rX   re   rq   rt   r{   r   r!   r   r   r   r2   r(   r&   r4   r4   C   sF    .EH
/6DD
/ 
"=C<
r(   r4   c                   *     e Zd ZdZd fd	Zd Z xZS )ComputePollerNoResourcesz:Compute operations poller that does not create a resource.c                 0    t         t        |   ||       y )N)resource_service
target_ref)superr   rB   )r@   r   r   	__class__s      r&   rB   z!ComputePollerNoResources.__init__   s    	
"D2)j 3 Br(   c                      y)z
Overrides.Nr2   rV   s     r&   	GetResultz"ComputePollerNoResources.GetResult   s    r(   ri   )r-   r.   r/   r0   rB   r   __classcell__)r   s   @r&   r   r      s    BBr(   r   c                       e Zd ZdZ G d de      Z G d de      Z ej                  d      Z
 ej                  d      Z ej                  d      Zed	        Zy
)r   z$Helper to parse tensorflow versions.c                       e Zd ZdZy)"TensorflowVersionParser.ParseErrorz?Error raised with input is unabled to be parse as a TF version.N)r-   r.   r/   r0   r2   r(   r&   
ParseErrorr      s    Ir(   r   c                   @    e Zd ZdZ	 	 	 	 	 d	dZd Zd Zd Zd Zd Z	y)
TensorflowVersionParser.Resultz3Helper to capture result of parsing the TF version.c                 J    || _         || _        || _        || _        || _        y ri   majorminorpatchr   r   )r@   r   r   r   r   r   s         r&   rB   z'TensorflowVersionParser.Result.__init__   s'     djdjdj"dodmr(   c                 `    | j                   dk(  xr | j                  dk(  xr | j                   S )Nr   )r   r   r   r@   s    r&   	IsUnknownz(TensorflowVersionParser.Result.IsUnknown   s)    ZZ1_HqH5HHr(   c                    | j                   rdj                  | j                        S | j                  dk(  r| j                  dk(  r| j                  S dj                  | j                  | j                  | j                        S )Nz	nightly{}r   z{}.{}{})r   formatr   r   r   r   s    r&   r   z,TensorflowVersionParser.Result.VersionString  s^    	!!$--00	qTZZ1_}}djj$**dmmDDr(   c                     t        | j                        t        | j                        z   t        | j                        z   t        | j                        z   t        | j
                        z   S ri   )hashr   r   r   r   r   r   s    r&   __hash__z'TensorflowVersionParser.Result.__hash__	  sT    $**TZZ 004

3CCd
//G !$--01 1r(   c                    | j                   |j                   k(  xrj | j                  |j                  k(  xrO | j                  |j                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S ri   r   r@   others     r&   __eq__z%TensorflowVersionParser.Result.__eq__  sq    jjEKK' .jjEKK'.jjEKK'. oo!1!11. mmu~~-	/r(   c                    | j                   s|j                   s| j                         s|j                         s| j                  |j                  k7  r| j                  |j                  kD  S | j                  |j                  k7  r| j                  |j                  kD  S | j                  |j                  k7  r| j                  |j                  kD  S | j
                  sy|j
                  sy| j                   r&|j                   r| j
                  sy|j
                  sy| j                         r)|j                         r| j
                  |j
                  k  S | j                         ry|j                         ry| j                   ryy)NTF)r   r   r   r   r   r   r   s     r&   __lt__z%TensorflowVersionParser.Result.__lt__  s   __U%5%5dnn ?!::$ekk)
)::$ekk)
)::$ekk)
)}}~~ 
U--}}~~ 
	eoo/}}u~~-- 
				r(   N)r   r   r   Fr   )
r-   r.   r/   r0   rB   r   r   r   r   r   r2   r(   r&   Resultr      s6    = !
IE1/#r(   r   z^(\d+)\.(\d+)(.*)$z^nightly(.*)$z	^\.(\d+)$c                 J   | st         j                  d      t         j                  j                  |       }t         j                  j                  |       }||t         j                  |       S |&|$t         j                  dj                  |             |rt        |j                  d            }t        |j                  d            }t         j                  ||      }|j                  d      rkt         j                  j                  |j                  d            }|r%t        |j                  d            }|r||_
        |S |j                  d      |_        |S |r?t         j                  d	
      }|j                  d      r|j                  d      |_        |S y)z?Helper to parse the tensorflow version into it's subcomponents.z!Bad argument: tf_version is emptyN)r   z!TF version error: bad version: {}      )r   r   r   T)r   )r   r   _VERSION_REGEXr   _NIGHTLY_REGEXr   r   intr   _PATCH_NUMBER_REGEXr   r   )rM   version_matchnightly_matchr   r   resultpatch_matchmatched_patchs           r&   r   z$TensorflowVersionParser.ParseVersion=  s    #.. 0E F F ,::@@LM+::@@LM!6$++Z+@@ ]%>#..
-
4
4Z
@B B-%%a()e-%%a()e&--E-Gf			Q	-AAGG"$k//23-(FL m *//2&/m&--->f			Q	'--a0m	 r(   N)r-   r.   r/   r0   	Exceptionr   objectr   r   compiler   r   r   r1   r   r2   r(   r&   r   r      sh    ,J9 JGv GR 2::56.2::o.."

=1" "r(   r   c                   n    e Zd ZdZd Zd Zd Z	 ddZd Zd Z	d	 Z
d
 Zd Zd Zd Zd Zd Zd Zd Zy)Instancez?Helper to create the GCE VM required to work with the TPU Node.c                     t        j                  |      }|j                  j                  | _        |j                  j                  | _        y ri   )r   ComputeApiHolderr=   apitools_clientr?   r@   rA   holders      r&   rB   zInstance.__init__f  s4    **=9F--//DKMM**DMr(   c                 T   |dk(  ryt         j                  |      }|j                  r$t         j                  dj	                  |            |r[|j
                  dk(  r&dj	                  |j
                  |j                        S dj	                  |j
                  |j                        S |j                  s|j
                  dk\  r@|j                  dk\  r1dj	                  |j
                  |j                  |j                        S d	j	                  |j
                  |j                        S )
z7Generates the image family from the tensorflow version.nightlyz
tf-nightlyzSInvalid tensorflow version:{} (non-empty modifier); please set the --gce-image flagr   ztf2-{}-{}-cpuztf-{}-{}-cpu   ztf-{}-{}-{}ztf-{}-{})r   r   r   r   r   r   r   r   )r@   rM   use_dl_imageparseds       r&   !_ImageFamilyFromTensorflowVersionz*Instance._ImageFamilyFromTensorflowVersionk  s    Y$11*=F#.. 06 7=fZ6HJ J
 		%%fllFLLAA$$V\\6<<@@ ||)flla.?!!&,,fllKKV\\6<<88r(   c                     d}|rd}| j                  ||      }| j                  j                  ||      }| j                  j                  j                  |      }|xr |j                  S )z=Queries GCE to find the right image for the given TF version.z	ml-imageszdeeplearning-platform-release)familyr`   )r   r?   !ComputeImagesGetFromFamilyRequestr=   imagesGetFromFamilyselfLink)r@   rM   r   r`   image_familyrg   images          r&   !ResolveImageFromTensorflowVersionz*Instance.ResolveImageFromTensorflowVersion  sk    G/g99L"Lmm==W > .GKK,,W5E#U^^#r(   Nc	           
      b   | j                   j                  dd| j                   j                  ||            }	t        j                  t
        j                  j                  j                  j                  d            }
| j                   j                  dj                  |
|      | j                   j                  d| j                   j                  j                  j                        g      }| j                   j                  j!                  d	|
      g}|r6|j#                  | j                   j                  j!                  dd
             | j                   j%                  dg d      }| j                   j&                  j)                  | j                   j&                  j(                  j+                  d	|
      g      }| j                   j'                  || j                   j                  |      dj                  ||      |	g| j                   j-                  |      |g||g      S )z9Builds an instance spec to be used for Instance creation.T)sourceImage
diskSizeGb)boot
autoDeleteinitializeParamsr   zprojects/{}/global/networks/{}zExternal NAT)r#   type)rH   accessConfigsctpu)keyvaluez
proxy-modeproject_editorsdefault)z5https://www.googleapis.com/auth/devstorage.read_writez-https://www.googleapis.com/auth/logging.writez0https://www.googleapis.com/auth/monitoring.writez.https://www.googleapis.com/auth/cloud-platform)emailscopes)additionalProperties)itemszzones/{}/machineTypes/{}rD   )r#   metadatamachineTypedisks
schedulingnetworkInterfaceslabelsserviceAccounts)r?   AttachedDiskAttachedDiskInitializeParamsp_utilGetProjectNumberr   r   r   r`   r!   NetworkInterfacer   AccessConfigTypeValueValuesEnumONE_TO_ONE_NATMetadataItemsValueListEntryr   ServiceAccountr   LabelsValueAdditionalProperty
Scheduling)r@   r#   r*   machine_type	disk_sizerE   rH   use_with_notebooksource_imagediskproject_numbernetwork_interfacer  service_accountr
  s                  r&   BuildInstanceSpeczInstance.BuildInstanceSpec  s    ==%%CC$  D 
 & D ,,&&**D*9;N66077G%}}11++??NN 2 P Q 7 
 &&:: ;  H oo
--
 
 
4
4&7 5 9: mm22
 3 O ]]##//**==d 	> 	$F/ F
 ==!!''h'7.55dLIf==+++D,-() " + +r(   c                 X    t         j                  j                  |j                  d      S )rQ   zcompute.zoneOperationsrR   )r   rT   ra   r   rV   s     r&   _GetComputeZoneOperationRefz$Instance._GetComputeZoneOperationRef  s.    ##'? $ A Ar(   c	                 :   | j                   j                  t        j                  j                  j
                  j                  d      || j                  ||||||||            }	| j                  j                  j                  |	      }
| j                  |
      S )z$Issue request to create an Instance.Tr   r`   r*   instance)r?   ComputeInstancesInsertRequestr   r   r   r`   r!   r"  r=   	instancesInsertr$  )r@   r#   r*   r  r  rE   	gce_imagerH   r  rg   rW   s              r&   re   zInstance.Create  s     mm99!!&&..22D2A''$igy* : +G %%,,W5I++I66r(   c                    t         j                  j                  j                  j	                  d      }| j
                  j                  |||      }| j                  j                  j                  |      }| j                  |      S )z#Issue request to stop the Instance.Tr   r'  r`   r*   )r   r   r   r`   r!   r?   ComputeInstancesStopRequestr=   r)  Stopr$  r@   r#   r*   r`   rg   rW   s         r&   r/  zInstance.Stop  sv    $$,,00$0?Gmm77 8 
G
 %%**73I++I66r(   c                    t         j                  j                  j                  j	                  d      }| j
                  j                  |||      }| j                  j                  j                  |      }| j                  |      S )z$Issue request to start the Instance.Tr   r-  )r   r   r   r`   r!   r?   ComputeInstancesStartRequestr=   r)  Startr$  r0  s         r&   r3  zInstance.Start  sv    $$,,00$0?Gmm88 9 
G
 %%++G4I++I66r(   c                     t        j                  | j                  j                        }t	        j
                  |||      S )z(Wait for Instance operation to complete.)r	   Pollerr=   r)  r   rl   rm   s       r&   rq   zInstance.WaitForOperation  s/    }}T[[%:%:;>>*M7CCr(   c                 n    t        | j                  j                        }t        j                  |||      S ri   )r   r=   r)  r   rl   rm   s       r&   rt   z$Instance.WaitForOperationNoResources  s+    /0E0EF>>*M7CCr(   c                 Z   t         j                  j                  j                  j	                  d      }| j
                  j                  ||      }t        j                  | j                  j                  |dd      }g }|D ]%  }| j                  |      s|j                  |       ' |S )z3Retrieves all Instances created by Execution Group.Tr   )r*   r`   r   r  )r   rg   r   r   )r   r   r   r`   r!   r?   ComputeInstancesListRequestr   r   r=   r)  _VMCreatedByExecGroupr   )r@   r*   r`   rg   r)  
result_setr'  s          r&   r   zInstance.List  s    $$,,00$0?Gmm777 8 $G((%%	I J		#	#H	-(#  r(   c                 N   t         j                  j                  j                  j	                  d      }| j
                  j                  |||      }| j                  j                  j	                  |      }| j                  |      r|S t        dj                  |      dd      )zRetrieves the Instance data.Tr   )r*   r`   r'  zInstance:{} not foundN)r   r   r   r`   r!   r?   ComputeInstancesGetRequestr=   r)  r9  r   r   )r@   instance_namer*   r`   rg   r'  s         r&   r!   zInstance.Get  s    $$,,00$0?Gmm667] 7 <G{{$$((1H!!(+o
&&}5tTC Cr(   c                 x    |r8|j                   r,|j                   j                  D ]  }|j                  dk(  s y y)Nr   TF)r
  r  r   )r@   r'  labels      r&   r9  zInstance._VMCreatedByExecGroup  s3    HOO??77%99 8 r(   c                 p    |j                   | j                  j                  j                  j                  k(  S ri   )statusr?   r   StatusValueValuesEnumRUNNING)r@   r'  s     r&   r   zInstance.IsRunning!  s'    ??dmm44JJRRRRr(   c                    | j                   j                  t        j                  j                  j
                  j                  d      ||      }| j                  j                  j                  |      }| j                  |      S )z=Deletes the specified instance in the given zone and project.Tr   r&  )r?   ComputeInstancesDeleteRequestr   r   r   r`   r!   r=   r)  r{   r$  )r@   r#   r*   rg   rW   s        r&   r{   zInstance.Delete$  sq    mm99!!&&..22D2A : 
G
 %%,,W5I++I66r(   ri   )r-   r.   r/   r0   rB   r   r   r"  r$  re   r/  r3  rq   rt   r   r!   r9  r   r{   r2   r(   r&   r   r   c  s[    G+
94$* &*9+vA

7	7	7D
D$	CS7r(   r   c                   8    e Zd ZdZd Zd Zd Zd Z	 d	dZd Z	y)
SSHz;Helper class to SSH to the VM associated with the TPU node.c                     t        j                  |      }|| _        |j                  | _        |j                  | _        y ri   )r   r   rA   r=   r   r   s      r&   rB   zSSH.__init__2  s2    **=9F&D--DK%%DNr(   c                 >    d |_         d|_        d |_        d |_        |S )Nno)plainstrict_host_key_checkingforce_key_file_overwritessh_key_file)r@   r$   s     r&   _DefaultArgsForSSHzSSH._DefaultArgsForSSH8  s(     DJ$(D!$(D!DKr(   c           	         t         j                  j                  |j                  gt        j
                  j                  || j                  t        j                  | j                              d   }|j                  | j                  |j                        }|j                  | j                  |||      }|!|st        j                  j                  d       |S )z6Wrapper around SSH Utils to get the host keys for SSH.)scope_listerr   z@Unable to retrieve host keys from instance metadata. Continuing.)instance_flagsSSH_INSTANCE_RESOLVERResolveResourcesr#   compute_scope	ScopeEnumZONEr   GetInstanceZoneScopeListerr=   
GetProjectr`   GetHostKeysFromGuestAttributesr   rA  Print)r@   r*   
ssh_helperr'  instance_refr`   	host_keyss          r&   _GetHostKeyFromInstancezSSH._GetHostKeyFromInstanceA  s    !77HH	0055t#>>t{{K I M NOPL ##DKK1E1EFG99\8W6I Y 
jj % &r(   c                     |j                  t        j                  |      d|      }|t        j                  d<   d|d<   |S )NrJ  )rL  host_keys_to_addTPU_NAMESendEnv)	GetConfigr   HostKeyAliasosenviron)r@   r#   r\  r'  r^  optionss         r&   _GetSSHOptionszSSH._GetSSHOptionsS  sI    ""9#9#9(#C<@4= # ?G "BJJz#GINr(   c                    |j                  | j                  |||j                  | j                  t        j                  j
                  j                  j                  d            t        j                         t        j                  d      z          t        j                  |||d      }	 |j                  |j                  |       y# t         j"                  $ r t%        j&                         w xY w)	z@Waits for SSH keys to propagate in order to SSH to the instance.Tr   i,  )secondsi )remoteidentity_filerh  max_wait_msputty_force_connectN)EnsureSSHKeyExistsr=   rY  r   r   r   r`   r!   r   Nowdatetime	timedeltar   	SSHPollerPollenvr   WaitExceptionr   NetworkError)	r@   r\  rl  rm  userr'  rh  rp  
ssh_pollers	            r&   _WaitForSSHKeysToPropagatezSSH._WaitForSSHKeysToPropagate[  s     !!T8KK**//77;;T;J	L		h((55	7
 #W(LJ%oo
..1  3  %""$$%s   /C (C5c           
         | j                  |      }t        j                  |      }t        j                  j                  dj                  |             t        j                  j                  |_
        t        j                         }|j                  |       |j                  j                  }t        j                  |j                         \  }}| j#                  |j$                  ||      }| j'                  |j                   |||      }	|j                  j)                         j+                  d      }
t        j,                  ||j/                  | j0                  t2        j4                  j6                  j8                  j;                  d            ||
d| j<                  d| j0                  j>                        }|j@                  }t        jB                  ||      }|jD                   xr2 t2        j4                  j                  jF                  jI                         }|jJ                  s| jM                  ||||||	|       g }|jN                  r|jQ                  g d       |||	|d	}t        jR                  di |}d
}d}tU        |      D ]f  }	 t        j                  j                  dj                  |             |j                  |jV                  |      }|rtY        jZ                  |        y y# t        j\                  $ rj}||dz
  k(  r|t        j                  j                  dj                  t_        j`                  |                   tc        jd                  |       Y d}~d}~ww xY w)z@Helper to manage authentication followed by SSH to the instance.z"Trying to SSH to VM with NAT IP:{}T)include_commentr   NF)username_requestedr?   )z-A-Lz6006:localhost:6006r  z8888:localhost:8888)rl  rm  rh  extra_flags
      zSSH Attempt #{}...ro  r   zRetrying: SSH command error: {}r2   )3rO  r   GetExternalIPAddressr   rA  r[  r   r   KeysDEFAULT_KEY_FILErN  BaseSSHCLIHelperRunkeyskey_fileGetUserAndInstancer#   r_  r*   ri  GetPublicKeyToEntryGetOsloginStaterY  r=   r   r   r   r`   r!   rA   r?   rz  Remoteoslogin_2fa_enabledrp  GetBooloslogin_enabledr|  forward_portsextend
SSHCommandrangerw  sysexitCommandErrorsix	text_typetimesleep)r@   r$   r'  external_natr\  rm  rz  _r^  rh  
public_keyoslogin_staterl  rp  r  ssh_cmd_argscmdmax_attemptssleep_intervalireturn_codees                         r&   SSHToInstancezSSH.SSHToInstancen  s   ""4(D11(;LJJ,33LAC11D++-JNN4OO,,M**4995GD!,,TYY
HMI!!$))Z"*I7G --/777MJ''KK**//77;;T;J	L %%	'M DZZd+F --- 	<1199;  ((
%%j&-&.9LN K 
JL &"	L ..
(<
(CLN
 < 

-44Q78ggNN 3  5  ((;
 ' !  q  '

-44S]]15EF	H

>"s   A"K99M6A M11M6N)F)
r-   r.   r/   r0   rB   rO  r_  ri  r|  r  r2   r(   r&   rG  rG  /  s)    C&$  %&Rr(   rG  c                   .    e Zd ZdZdZdZdZd Zd Zd Z	y)	ResourceManagerz@Helper to interact with Cloud Resource Manager and related ACLs.zroles/logging.logWriterzroles/storage.adminzroles/tpu.serviceAgentc                     d| _         t        j                  d| j                         | _        t        j                  d| j                         | _        y )Nr7   cloudresourcemanager)r;   r
   r<   r=   r>   r?   r   s    r&   rB   zResourceManager.__init__  sD    D(( 1 13DK** 1 13DMr(   c                    t         j                  j                  j                  j	                  d      }| j
                  j                  |      }| j                  j                  j                  |      }| j                  ||      }|/t        j                  j                  dj                  |             y| j
                  j                  || j
                  j!                  |            }| j                  j                  j#                  |       t        j                  j                  dj                  |             y)	zXAddTPUUserAgent adds the TPU user agent to enable Cloud Storage access and send logging.Tr   )resourceNz/TPU Service account:{} has already been enabled)policy)r  setIamPolicyRequestz?Added Storage and Logging permissions to TPU Service Account:{})r   r   r   r`   r!   r?   /CloudresourcemanagerProjectsGetIamPolicyRequestr=   projectsGetIamPolicy_AddAgentToPolicyr   rA  r[  r   /CloudresourcemanagerProjectsSetIamPolicyRequestSetIamPolicyRequestSetIamPolicy)r@   tpu_user_agentr`   get_iam_policy_requestr  set_iam_policy_requests         r&   AddTpuUserAgentzResourceManager.AddTpuUserAgent  s   $$,,00$0?G!]]ZZ [ [[!!../EFF##FN;F~	jjH~.0  $}}\\"mm?? @   ]  
 kk''(>?	jj
K6.!#r(   c                    d}d}dj                  |      }|j                  D ]k  }|j                  | j                  k(  r|}|j                  | j                  k(  r|}|j                  | j
                  k7  sS|j                  D ]
  }||k(  s	  y m |A| j                  j                  | j                        }|j                  j                  |       |A| j                  j                  | j                        }|j                  j                  |       |j                  j                  |       |j                  j                  |       |S )z2Adds the tpuUserAgent to the policy and return it.NzserviceAccount:{})role)
r   bindingsr  logging_rolestorage_roletpu_service_agentmembersr?   Bindingr   )r@   r  r  logging_bindingstorage_bindingtpu_member_strbindingmembers           r&   r  z!ResourceManager._AddAgentToPolicy  s    OO(//?N??	**	*!	**	*! 
//	/ooF~% 	 & # --43D3D-Eooo_---43D3D-Eooo_-"">2"">2Mr(   N)
r-   r.   r/   r0   r  r  r  rB   r  r  r2   r(   r&   r  r    s&    H*,&, /3#* r(   r  )4r0   
__future__r   r   r   rs  rf  r   r  r  apitools.base.pyr   apitools.base.py.exceptionsr   googlecloudsdk.api_lib.computer   )googlecloudsdk.api_lib.compute.operationsr	   googlecloudsdk.api_lib.utilr
   r   googlecloudsdk.callioper   "googlecloudsdk.command_lib.computer   rU  r   ,googlecloudsdk.command_lib.compute.instancesr   rR  #googlecloudsdk.command_lib.projectsr   r  #googlecloudsdk.command_lib.util.sshr   googlecloudsdk.corer   r   r   googlecloudsdk.core.utilr   r   r  r   r   r4   r5  r   r   r   rG  r  r2   r(   r&   <module>r     s    M &  '  	 	 
  ' 9 7 < , . ( E 8 P > 3 # * ) * * 
O& O(Xf Xv	v}} 	vf vrI7v I7XQ& QhEf Er(   