
    !h                       S r SSKJr  SSKJrJr  SSKJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJr  \(       a$  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  \" \5      rSSS\SS.               SS jjr " S S5      rg)a  
An extension to retry failed requests that are potentially caused by temporary
problems such as a connection timeout or HTTP 500 error.

You can change the behaviour of this middleware by modifying the scraping settings:
RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry

Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non-failed) pages.
    )annotations)Logger	getLogger)TYPE_CHECKING)NotConfigured)load_object)global_object_name)response_status_message)Self)Crawler)Response)Request)BaseSettings)SpiderunspecifiedNretry)reasonmax_retry_timespriority_adjustloggerstats_base_keyc               6   UR                   R                  nUR                   R                  (       d   eUR                   R                  nU R                  R	                  SS5      S-   n	Uc/  U R                  R	                  S5      nUc  UR                  S5      nX::  a  UR                  SX	US.S	U0S
9  U R                  5       n
XR                  S'   SU
l        Uc  UR                  S5      nU R                  U-   U
l	        [        U5      (       a  U" 5       n[        U[        5      (       a  [        UR                  5      nUR                  U S35        UR                  U SU 35        U
$ UR                  U S35        UR!                  SX	US.S	U0S
9  g)aC  
Returns a new :class:`~scrapy.Request` object to retry the specified
request, or ``None`` if retries of the specified request have been
exhausted.

For example, in a :class:`~scrapy.Spider` callback, you could use it as
follows::

    def parse(self, response):
        if not response.text:
            new_request_or_none = get_retry_request(
                response.request,
                spider=self,
                reason='empty',
            )
            return new_request_or_none

*spider* is the :class:`~scrapy.Spider` instance which is asking for the
retry request. It is used to access the :ref:`settings <topics-settings>`
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
:func:`logging.debug`).

*reason* is a string or an :class:`Exception` object that indicates the
reason why the request needs to be retried. It is used to name retry stats.

*max_retry_times* is a number that determines the maximum number of times
that *request* can be retried. If not specified or ``None``, the number is
read from the :reqmeta:`max_retry_times` meta key of the request. If the
:reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
is read from the :setting:`RETRY_TIMES` setting.

*priority_adjust* is a number that determines how the priority of the new
request changes in relation to *request*. If not specified, the number is
read from the :setting:`RETRY_PRIORITY_ADJUST` setting.

*logger* is the logging.Logger object to be used when logging messages

*stats_base_key* is a string to be used as the base key for the
retry-related job stats
retry_timesr      Nr   RETRY_TIMESz?Retrying %(request)s (failed %(retry_times)d times): %(reason)s)requestr   r   spider)extraTRETRY_PRIORITY_ADJUSTz/countz/reason_count/z/max_reachedzGGave up retrying %(request)s (failed %(retry_times)d times): %(reason)s)crawlersettingsstatsmetagetgetintdebugcopydont_filterprioritycallable
isinstance	Exceptionr	   	__class__	inc_valueerror)r   r   r   r   r   r   r   r!   r"   r   new_requests              i/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/downloadermiddlewares/retry.pyget_retry_requestr2   %   s   d ~~&&H>>NN  E,,""=!4q8K!,,**+<="&oom<O%MvNV$ 	 	

  '||~*5'"&"&oo.EFO&///AFXFfi(('(8(89F>*&12>*.AB	OO~&l34
LLQ6J   
     c                      \ rS rSrS	S jr\S
S j5       r        SS jr        SS jr        SS jr	Sr
g)RetryMiddleware}   c                J   UR                  S5      (       d  [        eUR                  S5      U l        UR	                  S5       Vs1 s H  n[        U5      iM     snU l        UR                  S5      U l        [        S UR	                  S5       5       5      U l	        g s  snf )NRETRY_ENABLEDr   RETRY_HTTP_CODESr   c              3  f   #    U  H'  n[        U[        5      (       a  [        U5      OUv   M)     g 7fN)r+   strr   ).0xs     r1   	<genexpr>+RetryMiddleware.__init__.<locals>.<genexpr>   s,      )
9 )C00KNa79s   /1RETRY_EXCEPTIONS)
getboolr   r%   r   getlistintretry_http_codesr   tupleexceptions_to_retry)selfr!   r>   s      r1   __init__RetryMiddleware.__init__~   s    00'}=191A1ABT1U V1UAQ1U V'/FG#( )
%%&89)
 $
  !Ws   B c                &    U " UR                   5      $ r;   )r!   )clsr    s     r1   from_crawlerRetryMiddleware.from_crawler   s    7##$$r3   c                    UR                   R                  SS5      (       a  U$ UR                  U R                  ;   a0  [	        UR                  5      nU R                  XU5      =(       d    U$ U$ N
dont_retryF)r#   r$   statusrE   r
   _retry)rH   r   responser   r   s        r1   process_response RetryMiddleware.process_response   s[     <<L%00O??d333,X__=F;;w7C8Cr3   c                    [        X R                  5      (       a3  UR                  R                  SS5      (       d  U R	                  XU5      $ g rP   )r+   rG   r#   r$   rS   )rH   r   	exceptionr   s       r1   process_exception!RetryMiddleware.process_exception   sH     i!9!9::7<<CSCS%D
 D
 ;;w6::r3   c                    UR                   R                  SU R                  5      nUR                   R                  SU R                  5      n[	        UUUUUS9$ )Nr   r   )r   r   r   r   )r#   r$   r   r   r2   )rH   r   r   r   r   r   s         r1   rS   RetryMiddleware._retry   sZ     ",,**+<d>R>RS!,,**+<d>R>RS ++
 	
r3   )rG   r   r   rE   N)r!   r   )r    r   returnr   )r   r   rT   r   r   r   r]   zRequest | Response)r   r   rX   r,   r   r   r]   zRequest | Response | None)r   r   r   !str | Exception | type[Exception]r   r   r]   Request | None)__name__
__module____qualname____firstlineno__rI   classmethodrM   rU   rY   rS   __static_attributes__ r3   r1   r5   r5   }   s    	
 % %*2<B	+4>D	"

 2
 	

 

r3   r5   )r   r   r   r   r   r^   r   
int | Noner   rg   r   r   r   r<   r]   r_   ) __doc__
__future__r   loggingr   r   typingr   scrapy.exceptionsr   scrapy.utils.miscr   scrapy.utils.pythonr	   scrapy.utils.responser
   typing_extensionsr   scrapy.crawlerr   scrapy.httpr   scrapy.http.requestr   scrapy.settingsr   scrapy.spidersr   r`   retry_loggerr2   r5   rf   r3   r1   <module>rw      s   
 # %   + ) 2 9&&$+,% " 1>"&"&!!UU U .	U
  U  U U U Up1
 1
r3   