
    !h/                    
   S SK Jr  S SKrS SKrS SKJrJrJr  S SKJ	r	  S SK
Jr  \(       a  S SKJr  S SKJr  S SKJr  S S	KJr  \R(                  " \5      rSS
 jr " S S\5      r " S S5      r " S S5      r " S S5      rg)    )annotationsN)TYPE_CHECKINGProtocolcast)Request)build_from_crawler)Iterable)Self)
Downloader)Crawlerc                    SR                  U  Vs/ s H!  oR                  5       (       d  US;   a  UOSPM#     sn5      n[        R                  " U R	                  S5      5      R                  5       nU SU 3$ s  snf )a  
Return a filesystem-safe version of a string ``text``

>>> _path_safe('simple.org').startswith('simple.org')
True
>>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
True
>>> _path_safe('some@symbol?').startswith('some_symbol_')
True
 z-.__utf8-)joinisalnumhashlibmd5encode	hexdigest)textcpathable_slotunique_slots       U/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/pqueues.py
_path_safer      sm     GGdSd))++eQDdSTM ++dkk&12<<>K_Ak]++	 Ts   (A9c                  @    \ rS rSrSrS	S jrS
S jrSS jrSS jrSr	g)QueueProtocol(   z:Protocol for downstream queues of ``ScrapyPriorityQueue``.c                    g N selfrequests     r   pushQueueProtocol.push+   s    c    c                    g r"   r#   r%   s    r   popQueueProtocol.pop-   s    Sr)   c                    g r"   r#   r+   s    r   closeQueueProtocol.close/   s    Sr)   c                    g r"   r#   r+   s    r   __len__QueueProtocol.__len__1   s    cr)   r#   Nr&   r   returnNoner5   zRequest | Noner5   r6   r5   int)
__name__
__module____qualname____firstlineno____doc__r'   r,   r/   r2   __static_attributes__r#   r)   r   r   r   (   s    D1( !r)   r   c                      \ rS rSrSr\ SSS.           SS jjj5       r SSS.         SS jjjrSS jrSS jr	SS	 jr
SS
 jrSS jrSS jrSS jrSS jrSS jrSS jrSrg)ScrapyPriorityQueue4   aR  A priority queue implemented using multiple internal queues (typically,
FIFO queues). It uses one internal queue for each priority value. The internal
queue must implement the following methods:

    * push(obj)
    * pop()
    * close()
    * __len__()

Optionally, the queue could provide a ``peek`` method, that should return the
next object to be returned by ``pop``, but without removing it from the queue.

``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
argument, which is a class used to instantiate a new (internal) queue when
a new priority is allocated.

Only integer priorities should be used. Lower numbers are higher
priorities.

startprios is a sequence of priorities to start with. If the queue was
previously closed leaving some priority buckets non-empty, those priorities
should be passed in startprios.
Nstart_queue_clsc                   U " UUUUUS9$ NrD   r#   clscrawlerdownstream_queue_clskey
startpriosrE   s         r   from_crawler ScrapyPriorityQueue.from_crawlerM          +
 	
r)   c                   Xl         X l        XPl        X0l        0 U l        0 U l        S U l        U R                  U5        g r"   )rJ   rK   _start_queue_clsrL   queues_start_queuescurprio
init_prios)r%   rJ   rK   rL   rM   rE   s         r   __init__ScrapyPriorityQueue.__init___   s<     !(9M!<K0279#'
#r)   c                   U(       d  g U Hd  nU R                  U5      nU(       a  X0R                  U'   U R                  (       d  M<  U R                  U5      nU(       d  MV  X0R                  U'   Mf     [        U5      U l        g r"   )qfactoryrS   rR   
_sqfactoryrT   minrU   )r%   rM   priorityqs       r   rV   ScrapyPriorityQueue.init_priosq   si    "Hh'A()H%$$$OOH-134&&x0 # :r)   c                v    [        U R                  U R                  U R                  S-   [	        U5      -   5      $ )N/)r   rK   rJ   rL   strr%   rL   s     r   rZ   ScrapyPriorityQueue.qfactory   s3    !%%LLHHsNSX%
 	
r)   c                    U R                   c   e[        U R                   U R                  U R                   SU S35      $ )Nra   s)rR   r   rJ   rL   rc   s     r   r[   ScrapyPriorityQueue._sqfactory   sE    $$000!!!LLxxj#a 
 	
r)   c                    UR                   * $ r"   )r]   r$   s     r   r]   ScrapyPriorityQueue.priority   s        r)   c                   U R                  U5      nUR                  R                  SS5      nU(       aN  U R                  (       a=  X R                  ;  a  U R                  U5      U R                  U'   U R                  U   nO<X R                  ;  a  U R                  U5      U R                  U'   U R                  U   nUR                  U5        U R                  b  X R                  :  a  X l	        g g )Nis_start_requestF)
r]   metagetrR   rT   r[   rS   rZ   r'   rU   )r%   r&   r]   rk   r^   s        r   r'   ScrapyPriorityQueue.push   s    ==)"<<++,>F 5 5111/3x/H""8,""8,A{{*(,h(?H%H%A	w<<8ll#:#L $;r)   c                   U R                   b{   U R                  U R                      nUR                  5       nU(       dH  U R                  U R                   	 UR                  5         U R                  (       d  U R                  5         U$ g ! [         a     Of = fU R                  (       a   U R                  U R                      nUR                  5       nU(       d7  U R                  U R                   	 UR                  5         U R                  5         U$ ! [         a    U R                  5          Of = fU R                  5         U R                   b  GMG  Nr"   )rU   rS   r,   r/   rT   _update_curprioKeyError)r%   r^   ms      r   r,   ScrapyPriorityQueue.pop   s   ll&KK- EEGDLL1GGI--,,. 1   !!
**4<<8A A ..t||<	,,.H   +((*+ $$&5 ll&s#   B
 

BB,D D21D2c                    U R                   U R                  4 VVVs1 s H(  nUR                  5         H  u  p#U(       d  M  UiM     M*     nnnnU(       a  [        U5      U l        g S U l        g s  snnnf r"   )rS   rT   itemsr\   rU   )r%   rS   pr^   prioss        r   rp   #ScrapyPriorityQueue._update_curprio   se      ;;(:(:;
; & ; 	 
 &+s5z
s   !A0
A0c                    U R                   c  g U R                  U R                      n[	        [
        UR                  5       5      $ ! [         a    U R                  U R                      n NCf = f)Returns the next object to be returned by :meth:`pop`,
but without removing it from the queue.

Raises :exc:`NotImplementedError` if the underlying queue class does
not implement a ``peek`` method, which is optional for queues.
N)rU   rT   rq   rS   r   r   peek)r%   queues     r   r{   ScrapyPriorityQueue.peek   sb     <<	.&&t||4E GUZZ\**  	.KK-E	.s   A #A-,A-c                    [        5       nU R                  U R                  4 H=  nUR                  5        H&  u  p4UR	                  U5        UR                  5         M(     M?     [        U5      $ r"   )setrS   rT   ru   addr/   list)r%   activerS   rv   r^   s        r   r/   ScrapyPriorityQueue.close   sT    5{{D$6$67F

1	 ' 8 F|r)   c                    U R                   (       d  U R                  (       a(  [        S U R                   U R                  4 5       5      $ S$ )Nc              3  h   #    U  H(  nUR                  5         H  n[        U5      v   M     M*     g 7fr"   )valueslen).0rS   xs      r   	<genexpr>.ScrapyPriorityQueue.__len__.<locals>.<genexpr>   s0      ?FA A( ?s   02r   )rS   rT   sumr+   s    r   r2   ScrapyPriorityQueue.__len__   sH     {{d00  #{{D,>,>? 	
 	
r)   )rR   rT   rJ   rU   rK   rL   rS   r#   )rJ   r   rK   type[QueueProtocol]rL   rb   rM   Iterable[int]rE   type[QueueProtocol] | Noner5   r
   )
rJ   r   rK   r   rL   rb   rM   r   rE   r   )rM   r   r5   r6   )rL   r:   r5   r   )r&   r   r5   r:   r4   r7   r8   )r5   z	list[int]r9   )r;   r<   r=   r>   r?   classmethodrN   rW   rV   rZ   r[   r]   r'   r,   rp   r{   r/   r2   r@   r#   r)   r   rB   rB   4   s    0  %'
 7;

 2
 	

 "
 4
 

 
, %'$ 7;$$ 2$ 	$
 "$ 4$$'

!$<5+ 	
r)   rB   c                  <    \ rS rSrSS jrS	S jrS
S jrSS jrSrg)DownloaderInterface   c                `    UR                   (       d   eUR                   R                  U l        g r"   )engine
downloader)r%   rJ   s     r   rW   DownloaderInterface.__init__   s    ~~~&-nn&?&?r)   c                R    U Vs/ s H  o R                  U5      U4PM     sn$ s  snf r"   )_active_downloads)r%   possible_slotsslots      r   statsDownloaderInterface.stats   s(    AOP''-t4PPPs   $c                8    U R                   R                  U5      $ r"   )r   get_slot_keyr$   s     r   r    DownloaderInterface.get_slot_key   s    ++G44r)   c                    XR                   R                  ;  a  g[        U R                   R                  U   R                  5      $ )z<Return a number of requests in a Downloader for a given slotr   )r   slotsr   r   r%   r   s     r   r   %DownloaderInterface._active_downloads   s5    ,,,4??((.5566r)   )r   N)rJ   r   )r   zIterable[str]r5   zlist[tuple[int, str]])r&   r   r5   rb   )r   rb   r5   r:   )	r;   r<   r=   r>   rW   r   r   r   r@   r#   r)   r   r   r      s    @Q57r)   r   c                      \ rS rSrSr\ SSS.           SS jjj5       r SSS.         SS jjjr S     SS jjrSS jr	SS	 jr
SS
 jrSS jrSS jrSS jrSrg)DownloaderAwarePriorityQueue   zPriorityQueue which takes Downloader activity into account:
domains (slots) with the least amount of active downloads are dequeued
first.
NrD   c                   U " UUUUUS9$ rG   r#   rH   s         r   rN   )DownloaderAwarePriorityQueue.from_crawler  rP   r)   c                  UR                   R                  S5      S:w  a  [        SU R                   S35      eU(       a/  [	        U[
        5      (       d  [        SUR                  < S35      e[        U5      U l        X l        XPl	        X0l
        Xl        0 U l        U=(       d    0 R                  5        H#  u  pgU R                  Xg5      U R                  U'   M%     g )NCONCURRENT_REQUESTS_PER_IPr   "z-" does not support CONCURRENT_REQUESTS_PER_IPzDDownloaderAwarePriorityQueue accepts ``slot_startprios`` as a dict; z instance is passed. Most likely, it means the state is created by an incompatible priority queue. Only a crawl started with the same priority queue class can be resumed.)settingsgetint
ValueError	__class__
isinstancedictr   _downloader_interfacerK   rR   rL   rJ   pqueuesru   	pqfactory)r%   rJ   rK   rL   slot_startpriosrE   r   rM   s           r   rW   %DownloaderAwarePriorityQueue.__init__  s     ""#?@AEDNN##PQ  :ot#D#D2",,/ 0..  ;Ng:V"9M!<K '79!0!6B = = ?D!%!ADLL !@r)   c                    [        U R                  U R                  U R                  S-   [	        U5      -   UU R
                  S9$ )Nra   rD   )rB   rJ   rK   rL   r   rR   )r%   r   rM   s      r   r   &DownloaderAwarePriorityQueue.pqfactory9  sB     #LL%%HHsNZ-- 11
 	
r)   c                    U R                   R                  U R                  5      nU(       d  g [        U5      S   nU R                  U   nUR	                  5       n[        U5      S:X  a  U R                  U	 U$ )N   r   )r   r   r   r\   r,   r   )r%   r   r   r|   r&   s        r   r,    DownloaderAwarePriorityQueue.popD  sd    **00>5z!}T"))+u:?T"r)   c                    U R                   R                  U5      nX R                  ;  a  U R                  U5      U R                  U'   U R                  U   nUR	                  U5        g r"   )r   r   r   r   r'   )r%   r&   r   r|   s       r   r'   !DownloaderAwarePriorityQueue.pushQ  sS    ))66w?||#!%!5DLLT"

7r)   c                    U R                   R                  U R                  5      nU(       d  g[        U5      S   nU R                  U   nUR	                  5       $ )rz   Nr   )r   r   r   r\   r{   )r%   r   r   r|   s       r   r{   !DownloaderAwarePriorityQueue.peekX  sK     **00>5z!}T"zz|r)   c                    U R                   R                  5        VVs0 s H  u  pXR                  5       _M     nnnU R                   R                  5         U$ s  snnf r"   )r   ru   r/   clear)r%   r   r|   r   s       r   r/   "DownloaderAwarePriorityQueue.closef  sK    9=9K9K9MN9M+$$%9MN Os   Ac                |    U R                   (       a*  [        S U R                   R                  5        5       5      $ S$ )Nc              3  8   #    U  H  n[        U5      v   M     g 7fr"   )r   )r   r   s     r   r   7DownloaderAwarePriorityQueue.__len__.<locals>.<genexpr>l  s     9#8a3q66#8s   r   )r   r   r   r+   s    r   r2   $DownloaderAwarePriorityQueue.__len__k  s+    =A\\s94<<#6#6#899PqPr)   c                    XR                   ;   $ r"   )r   r   s     r   __contains__)DownloaderAwarePriorityQueue.__contains__n  s    ||##r)   )r   rR   rJ   rK   rL   r   r"   )rJ   r   rK   r   rL   rb   rM   dict[str, Iterable[int]] | NonerE   r   r5   r
   )
rJ   r   rK   r   rL   rb   r   r   rE   r   r   )r   rb   rM   r   r5   rB   r7   r4   )r5   zdict[str, list[int]]r9   )r   rb   r5   bool)r;   r<   r=   r>   r?   r   rN   rW   r   r,   r'   r{   r/   r2   r   r@   r#   r)   r   r   r      s    
  7;
 7;

 2
 	

 4
 4
 

 
, <@!B 7;!B!B 2!B 	!B
 9!B 4!BH 68	
	
%2	
		

Q$r)   r   )r   rb   r5   rb   )
__future__r   r   loggingtypingr   r   r   scrapyr   scrapy.utils.miscr   collections.abcr	   typing_extensionsr
   scrapy.core.downloaderr   scrapy.crawlerr   	getLoggerr;   loggerr   r   rB   r   r   r#   r)   r   <module>r      so    "   0 0  0( '1&			8	$,$	"H 	"u
 u
p7 7$q$ q$r)   