
    !h5                        S r SSKJr  SSKrSSKrSSKrSSKJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  \(       a  SSKJr  SSKJr  \R4                  " \5      r " S S5      rg)zH
MemoryUsage extension

See documentation in docs/topics/extensions.rst
    )annotationsN)import_module)pformat)TYPE_CHECKING)task)signals)NotConfigured)
MailSender)get_engine_status)Self)Crawlerc                  x    \ rS rSrSS jr\SS j5       rSS jrSS jrSS jr	SS jr
SS jrSS	 jrSS
 jrSrg)MemoryUsage!   c                   UR                   R                  S5      (       d  [        e [        S5      U l        Xl        SU l        UR                   R                  S5      U l	        UR                   R                  S5      S-  S-  U l        UR                   R                  S5      S-  S-  U l        UR                   R                  S5      U l        [        R                   " U5      U l        UR$                  R'                  U R(                  [$        R(                  S	9  UR$                  R'                  U R*                  [$        R*                  S	9  g ! [
         a    [        ef = f)
NMEMUSAGE_ENABLEDresourceFMEMUSAGE_NOTIFY_MAILMEMUSAGE_LIMIT_MB   MEMUSAGE_WARNING_MBMEMUSAGE_CHECK_INTERVAL_SECONDS)signal)settingsgetboolr	   r   r   ImportErrorcrawlerwarnedgetlistnotify_mailsgetintlimitwarninggetfloatcheck_intervalr
   from_crawlermailr   connectengine_startedengine_stopped)selfr   s     a/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/extensions/memusage.py__init__MemoryUsage.__init__"   s.   ''(:;;	 )*5DM !(!'.'7'7'?'?@V'W!**112EFMPTT
#,,334IJTQTXX%,%5%5%>%>-&
 !+ 7 7 @	 3 3G<R<RS 3 3G<R<RS  	 	 s   E E"c                    U " U5      $ N )clsr   s     r,   r&   MemoryUsage.from_crawler7   s    7|    c                    U R                   R                  U R                   R                  5      R                  n[        R
                  S:w  a  US-  nU$ )Ndarwinr   )r   	getrusageRUSAGE_SELF	ru_maxrsssysplatform)r+   sizes     r,   get_virtual_sizeMemoryUsage.get_virtual_size;   s@    MM++DMM,E,EFPP<<8#DLDr4   c                   U R                   R                  (       d   eU R                   R                  R                  SU R                  5       5        / U l        [
        R                  " U R                  5      nU R                  R                  U5        UR                  U R                  SS9  U R                  (       aU  [
        R                  " U R                  5      nU R                  R                  U5        UR                  U R                  SS9  U R                  (       aV  [
        R                  " U R                  5      nU R                  R                  U5        UR                  U R                  SS9  g g )Nmemusage/startupT)now)r   stats	set_valuer=   tasksr   LoopingCallupdateappendstartr%   r"   _check_limitr#   _check_warningr+   tsks     r,   r)   MemoryUsage.engine_startedB   s   ||!!!!$$%79N9N9PQ-/
t{{+

#		$%%4	0::""4#4#45CJJc"IId))tI4<<""4#6#67CJJc"IId))tI4 r4   c                p    U R                    H&  nUR                  (       d  M  UR                  5         M(     g r0   )rD   runningstoprK   s     r,   r*   MemoryUsage.engine_stoppedR   s"    ::C{{{
 r4   c                    U R                   R                  (       d   eU R                   R                  R                  SU R                  5       5        g )Nmemusage/max)r   rB   	max_valuer=   )r+   s    r,   rF   MemoryUsage.updateW   s8    ||!!!!$$^T5J5J5LMr4   c                   U R                   R                  (       d   eU R                   R                  (       d   eU R                  5       nXR                  :  Gac  U R                   R                  R                  SS5        U R                  S-  S-  n[        R                  SSU0SU R                   0S9  U R                  (       av  U R                   R                  S    S	U S
[        R                  " 5        3nU R                  U R                  U5        U R                   R                  R                  SS5        U R                   R                  R                  bE  U R                   R                  R                  U R                   R                  R                  S5        g U R                   R                  5         g [        R!                  SSUS-  S-  05        g )Nzmemusage/limit_reached   r   z>Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...memusager   extraBOT_NAMEz# terminated: memory usage exceeded MiB at zmemusage/limit_notifiedmemusage_exceededz'Peak memory usage is %(virtualsize)dMiBvirtualsize)r   enginerB   r=   r"   rC   loggererrorr    r   socketgethostname_send_reportspiderclose_spiderrP   info)r+   peak_mem_usagememsubjs       r,   rI   MemoryUsage._check_limit[   s   ||""""||!!!!..0JJ&LL(()A1E**t#d*CLLPS! $,,/  
   ||,,Z89 :--0E9K9K9M8NP  !!$"3"3T:"",,-FJ||""))5##00LL''..0C !!#KK9 5 <=r4   c                r   U R                   (       a  g U R                  R                  (       d   eU R                  5       U R                  :  a  U R                  R                  R                  SS5        U R                  S-  S-  n[        R	                  SSU0SU R                  0S9  U R                  (       av  U R                  R                  S    S	U S
[        R                  " 5        3nU R                  U R                  U5        U R                  R                  R                  SS5        SU l         g g )Nzmemusage/warning_reachedrW   r   z$Memory usage reached %(memusage)dMiBrX   r   rY   r[   z warning: memory usage reached r\   zmemusage/warning_notifiedT)r   r   rB   r=   r#   rC   r`   r    r   rb   rc   rd   )r+   ri   rj   s      r,   rJ   MemoryUsage._check_warning{   s   ;;||!!!!  "T\\1LL(()CQG,,%,CNN6S! $,,/  
   ||,,Z89 :,,/58J8J8L7MO  !!$"3"3T:"",,-H!LDK 2r4   c                   U R                   R                  (       d   eU R                   R                  (       d   eU R                   R                  nSUR                  S5      S-  S-   S3nUSUR                  S5      S-  S-   S3-  nUSU R	                  5       S-  S-   S3-  nUS-  nUS	-  nU[        [        U R                   R                  5      5      -  nUS	-  nU R                  R                  XU5        g
)z7send notification mail with some additional useful infoz!Memory usage at engine startup : r@   r   zM
z Maximum memory usage          : rS   z Current memory usage          : zHENGINE STATUS ------------------------------------------------------- 
z
N)	r   r_   rB   	get_valuer=   r   r   r'   send)r+   rcptssubjectrB   ss        r,   rd   MemoryUsage._send_report   s   ||""""||!!!!""/@R0SVZ0Z]a0a/bbgh	/0ORV0VY]0]/^^cdd	/0E0E0G$0NQU0U/VV[\\	X	
 	
V	W&t||':':;<<	V		uq)r4   )	r%   r   r"   r'   r    r   rD   r   r#   N)r   r   )r   r   returnr   )ru   int)ru   None)rq   z	list[str]rr   strru   rw   )__name__
__module____qualname____firstlineno__r-   classmethodr&   r=   r)   r*   rF   rI   rJ   rd   __static_attributes__r1   r4   r,   r   r   !   sA    T*  5 
N@**r4   r   )__doc__
__future__r   loggingrb   r:   	importlibr   pprintr   typingr   twisted.internetr   scrapyr   scrapy.exceptionsr	   scrapy.mailr
   scrapy.utils.enginer   typing_extensionsr   scrapy.crawlerr   	getLoggerry   r`   r   r1   r4   r,   <module>r      sW    #   
 #    !  + " 1&& 
		8	$~* ~*r4   