
    !h7                       S SK Jr  S SKrS SKrS SKrS SKrS SKJrJrJ	r	J
r
  S SKJr  S SKJrJr  S SKJr  S SKJr  S SKJr  S S	KJrJr  S S
KJr  S SKJr  S SKJrJr  S SK J!r!  S SK"J#r#  S SK$J%r%  \(       a"  S SK&r&S SK'J(r(J)r)J*r*J+r+  S SK,J-r-  S SK.J/r/  S SK0J1r1  \Rd                  " \35      r4\	" S5      r5 " S S\5      r6g)    )annotationsN)TYPE_CHECKINGAnyTypeVaroverload)ItemAdapter)DeferredmaybeDeferred)is_url)BaseRunSpiderCommand)
UsageError)RequestResponse)display)collect_asyncgen)aiter_errbackdeferred_from_coro)failure_to_exc_info)arg_to_iter)spidercls_for_request)AsyncGeneratorAsyncIterator	CoroutineIterable)Failure)	CallbackT)Spider_Tc                    ^  \ rS rSr% SrSrS\S'   0 rS\S'   0 rS\S	'   S
\S'   Sr	S'S jr
S'S jrS(U 4S jjr\S)S j5       rS*S jr\    S+S j5       r\S,S j5       rS-S jrS.S jrS/S jrS0S1S jjrS0S1S jjrS2S jr            S3S jr S4       S5S jjr      S6S jrS7S jrS7S jr    S8S jrSS.       S9S  jjr        S:S! jrS;U 4S" jjrS2S# jr S2S$ jr!S;S% jr"S&r#U =r$$ )<Command&   TNzSpider | Nonespiderzdict[int, list[Any]]itemszdict[int, list[Request]]requestsztype[Spider] | None	spiderclsc                    g)Nz[options] <url> selfs    \/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/commands/parse.pysyntaxCommand.syntax0   s         c                    g)Nz2Parse URL (using its spider) and print the resultsr'   r(   s    r*   
short_descCommand.short_desc3   s    Cr-   c           	       > [         TU ]  U5        UR                  SSS SS9  UR                  SSSS9  UR                  S	S
SSS9  UR                  SSSSS9  UR                  SSSSS9  UR                  SSSSSS9  UR                  SSSSS9  UR                  SSSSS9  UR                  S S!S"S9  UR                  S#S$S%[        S&S'S(9  UR                  S)S*S+SS,S9  g )-Nz--spiderr"   z'use this spider without looking for one)destdefaulthelpz--pipelines
store_truezprocess items through pipelines)actionr4   z	--nolinksnolinksz/don't show links to follow (extracted requests))r2   r6   r4   z	--noitemsnoitemszdon't show scraped itemsz
--nocolournocolourz+avoid using pygments to colorize the outputz-rz--rulesrulesz.use CrawlSpider rules to discover the callbackz-cz
--callbackcallbackz=use this callback for parsing, instead looking for a callback)r2   r4   z-mz--metametazFinject extra meta into the Request, it must be a valid raw json stringz
--cbkwargscbkwargszQinject extra callback kwargs into the Request, it must be a valid raw json stringz-dz--depthdepth   z9maximum depth for parsing requests [default: %(default)s])r2   typer3   r4   z-vz	--verboseverbosez!print each depth level one by one)superadd_optionsadd_argumentint)r)   parser	__class__s     r*   rC   Command.add_options6   s~   F#:	 	 	
 	,5V 	 	
 	B	 	 	
 	+	 	 	
 	>	 	 	
 	A 	 	
 	P	 	 	
 	Y	 	 	
 	d 	 	

 	L 	 	
 	4 	 	
r-   c                    Su  pU R                   (       a  [        U R                   5      nU R                  (       a  [        U R                  5      n[        X5      $ )N)r   r   )r#   maxr$   )r)   	max_itemsmax_requestss      r*   	max_levelCommand.max_level{   s>    "&	::DJJI==t}}-L9++r-   c                >    [         R                  S[        U5      S9  g )Nz5An error is caught while iterating the async iterable)exc_info)loggererrorr   )r)   _failures     r*   handle_exceptionCommand.handle_exception   s    C(2 	 	
r-   c                    g Nr'   r)   results     r*   iterate_spider_outputCommand.iterate_spider_output   s     r-   c                    g rW   r'   rX   s     r*   rZ   r[      s    BEr-   c                n   [         R                  " U5      (       aD  [        [        [	        XR
                  5      5      5      nUR                  U R                  5        U$ [         R                  " U5      (       a(  [        U5      nUR                  U R                  5        U$ [        [        U5      5      $ rW   )
inspect
isasyncgenr   r   r   rT   addCallbackrZ   iscoroutiner   )r)   rY   ds      r*   rZ   r[      s    f%%" v7L7L!MNA MM$445Hv&&"6*AMM$445H-f566r-   c                ^    U R                   R                  U/ 5      nX2-   U R                   U'   g rW   )r#   get)r)   lvl	new_items	old_itemss       r*   	add_itemsCommand.add_items   s&    JJNN3+	#/

3r-   c                ^    U R                   R                  U/ 5      nX2-   U R                   U'   g rW   )r$   rd   )r)   re   new_reqsold_reqss       r*   add_requestsCommand.add_requests   s)    ==$$S"-%0cr-   c                T   Uc5  U R                   R                  5        VVs/ s H  o3  H  oDPM     M     nnnOU R                   R                  U/ 5      n[        SS5        [        R
                  " U Vs/ s H  n[        U5      R                  5       PM     snUS9  g s  snnf s  snf )Nz# Scraped Items z<------------------------------------------------------------colorize)r#   valuesrd   printr   pprintr   asdict)r)   re   colourlstitemr#   xs          r*   print_itemsCommand.print_items   s    ;%)ZZ%6%6%8I%8cSTTST%8EIEJJNN3+E (+?AA--/?&Q J
 @s   B5#B%c                    Ub  U R                   R                  U/ 5      nO6U R                   (       a#  U R                   [        U R                   5         nO/ n[        SS5        [        R
                  " X2S9  g )Nz# Requests zA-----------------------------------------------------------------rp   )r$   rd   rJ   rs   r   rt   )r)   re   rv   r$   s       r*   print_requestsCommand.print_requests   sV    ?}}((b1H]]}}S%78HHmX&x1r-   c                   UR                   (       + nUR                  (       av  [        SU R                  S-   5       HX  n[	        SU S35        UR
                  (       d  U R                  X25        UR                  (       a  MG  U R                  X25        MZ     g [	        SU R                   S35        UR
                  (       d  U R                  US9  UR                  (       d  U R                  US9  g g )Nr?   z
>>> DEPTH LEVEL: z <<<z
>>> STATUS DEPTH LEVEL )rv   )	r9   rA   rangerM   rs   r8   rz   r7   r}   )r)   optsrv   levels       r*   print_resultsCommand.print_results   s    ]]"<<q$..1"45+E7$78||$$U3|||''6 6 -dnn-=TBC<<   /<<##6#2  r-   c                    / / pvU H<  n[        U[        5      (       a  UR                  U5        M+  UR                  U5        M>     XgX#XE4$ rW   )
isinstancer   append)	r)   spider_outputr   r>   r"   r;   r#   r$   ry   s	            r*   _get_items_and_requestsCommand._get_items_and_requests   sH     bxA!W%%"Q	 
 V==r-   c                R    U=(       d    0 n[        U R                  U" U40 UD65      $ rW   )r
   rZ   )r)   responser;   	cb_kwargss       r*   run_callbackCommand.run_callback   s/     O	&&(GY(G
 	
r-   c                   [        USS 5      (       aU  UR                   HD  nUR                  R                  UR                  5      (       d  M/  UR
                  =(       d    Ss  $    g [        R                  SSUR                  05        g )Nr:   parsez]No CrawlSpider rules found in spider %(spider)r, please specify a callback to use for parsingr"   )	getattrr:   link_extractormatchesurlr;   rQ   rR   name)r)   r"   r   rules       r*   get_callback_from_rulesCommand.get_callback_from_rules   st     67D))&&..x||<<==3G3 %  LL?6;;'
 r-   c                  ^ ^^ T R                   (       d   eT R                   R                  nTR                  (       a"   UR                  TR                  5      T l        OC[        U[        T5      5      T l        T R                  (       d  [        R                  SST05        SUU U4S jjnT R                  (       a  UT R                  l
        g g ! [
         a%    [        R                  SSTR                  05         N\f = f)Nz!Unable to find spider: %(spider)sr"   z"Unable to find spider for: %(url)sr   c               L   >#    TR                  U [        T5      T5      7v   g 7frW   )prepare_requestr   )r"   r   r)   r   s    r*   start$Command.set_spidercls.<locals>.start  s     &&vws|TBBs   !$)r"   r   returnzAsyncIterator[Any])crawler_processspider_loaderr"   loadr%   KeyErrorrQ   rR   r   r   r   )r)   r   r   r   r   s   ```  r*   set_spiderclsCommand.set_spidercls   s    ####,,::;;!.!3!3DKK!@ 3='#,ODN>>AE3<P	C 	C >>#(DNN    7(DKK9Ps    C ,D ?D c                   U R                   (       d   eU R                  (       d   eU R                   R                  " U R                  40 UR                  D6  [	        [        U R                   R                  5      5      U l        U R                   R                  5         U R                  (       d  [        R                  SSU05        g g )Nz#No response downloaded for: %(url)sr   )r   r%   crawlspargsnextitercrawlerspcrawlerr   first_responserQ   rR   )r)   r   r   s      r*   start_parsingCommand.start_parsing  s    ####~~~""4>>AT[[AT$"6"6"?"?@A""$""LL>M #r-   c                   Uu  p#pEpgUR                   (       aD  U R                  R                  R                  R                  nU H  n	UR                  X5        M     U R                  XR5        U R                  XS5        UR                  (       a  UO/ n
XTR                  :  a>  U H4  nUS-   UR                  S'   UR                  UR                  S'   X{l        M6     X-  n
U
$ )Nr?   _depth	_callback)	pipelinesr   enginescraperitemprocprocess_itemrh   rm   outputr>   r<   r;   )r)   argsr#   r$   r   r>   r"   r;   r   rx   scraped_datareqs               r*   r   Command.scraped_data  s     :>6f>>}}++33<<H%%d3 u$%* $u::%*QY"(+%'   $Lr-   )r   c                  S nU(       a  UR                   S   nU(       d  UR                  (       a  UR                  nOiU(       a`  UR                  (       aO  U R                  U:X  a?  U R	                  X5      nU(       d&  [        SUR                  < SUR                   35      eOSn[        U5      (       d@  Uc   e[        XS 5      n[        U5      (       a  UnO[        SU< SUR                   35      e[        U5      (       d   eU$ )Nr   z Cannot find a rule that matches z in spider: r   zCannot find callback )
r<   r;   r:   r   r   
ValueErrorr   r   callabler   )r)   r"   r   r   cb	cb_methods         r*   _get_callbackCommand._get_callback-  s     &*{+B}}]]djjT-@-@H-L11&C$:8<<:J,!;;-)   ||>!>D1I	"" +B6fkk]K  |||	r-   c                  ^ ^^^ SUUU U4S jjmTR                   (       a%  UR                   R                  TR                   5        TR                  (       a%  UR                  R                  TR                  5        SUR                   S'   UR                  UR                   S'   UR                  (       d8  TR
                  (       d'  T R                  TTS9n[        R                  " TU5        TUl        U$ )Nc                  > TR                   (       d  U Tl         TR                  TTU S9nU R                  S   nTR                  XU5      nUR	                  TR
                  TUTT5        UR	                  TR                  5        U$ )N)r"   r   r   r   )r   r   r<   r   r`   r   r   )	r   r   r   r>   rb   r;   r   r)   r"   s	        r*   r;   )Command.prepare_request.<locals>.callbackS  s~    &&&.###6x#PB "x0E!!(	:AMM$66eVXVMM$++,Hr-   r?   r   r   )r"   r   )r   r   r   r   r   zDeferred[list[Any]])	r<   updater=   r   r;   r:   r   	functoolsupdate_wrapper)r)   r"   requestr   r   r;   s   `` ` @r*   r   Command.prepare_requestP  s    	 	  99LL		* ==$$T]]3!"X$+$4$4[!

##6#=B$$Xr2#r-   c                h   > [         TU ]  X5        U R                  U5        U R                  U5        g rW   )rB   process_optionsprocess_request_metaprocess_request_cb_kwargs)r)   r   r   rG   s      r*   r   Command.process_optionsr  s,    +!!$'&&t,r-   c                    UR                   (       a'   [        R                  " UR                   5      Ul         g g ! [         a    [	        SSS9ef = f)NzdInvalid -m/--meta value, pass a valid json string to -m or --meta. Example: --meta='{"foo" : "bar"}'F
print_help)r<   jsonloadsr   r   r)   r   s     r*   r   Command.process_request_metax  sM    99 JJtyy1	    :$ 	   %: Ac                    UR                   (       a'   [        R                  " UR                   5      Ul         g g ! [         a    [	        SSS9ef = f)NzgInvalid --cbkwargs value, pass a valid json string to --cbkwargs. Example: --cbkwargs='{"foo" : "bar"}'Fr   )r=   r   r   r   r   r   s     r*   r   !Command.process_request_cb_kwargs  sM    == $

4== 9    >$ r   c                
   [        U5      S:X  a  [        US   5      (       d  [        eUS   nU R                  X25        U R                  (       a4  UR
                  S:  a#  U R                  X25        U R                  U5        g g g )Nr?   r   )lenr   r   r   r%   r>   r   r   )r)   r   r   r   s       r*   runCommand.run  sl    4yA~VDG__1g 	3%>>djj1ns)t$ ->r-   )r   r   r%   )r   str)rF   zargparse.ArgumentParserr   None)r   rE   )rS   r   r   r   )rY   z,AsyncGenerator[_T] | Coroutine[Any, Any, _T]r   zDeferred[_T])rY   r   r   Iterable[Any])rY   r   r   zIterable[Any] | Deferred[Any])re   rE   rf   	list[Any]r   r   )re   rE   rk   zlist[Request]r   r   )NT)re   z
int | Nonerv   boolr   r   )r   argparse.Namespacer   r   )r   r   r   r   r>   rE   r"   r   r;   r   r   Ktuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]rW   )r   r   r;   r   r   zdict[str, Any] | Noner   zDeferred[Any])r"   r   r   r   r   zCallbackT | str | None)r   r   r   r   r   r   )r   r   r   r   )r"   r   r   r   r   zResponse | Noner   r   )r"   r   r   r   r   r   r   r   )r   z	list[str]r   r   r   r   )%__name__
__module____qualname____firstlineno__requires_projectr"   __annotations__r#   r$   r   r+   r/   rC   propertyrM   rT   r   rZ   rh   rm   rz   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   __static_attributes____classcell__)rG   s   @r*   r    r    &   s    FM "$E$)+H&+""N!DC
J , ,
 B	  E E701R	23">$> !> 	>
 > > 
U>( ,0		
	
 	
 )		

 
	
(0	)*N

 
: %)! ! !	!
 "! 
!F  '. 6H 	 D-		% %r-   r    )7
__future__r   r   r^   r   loggingtypingr   r   r   r   itemadapterr   twisted.internet.deferr	   r
   	w3lib.urlr   scrapy.commandsr   scrapy.exceptionsr   scrapy.httpr   r   scrapy.utilsr   scrapy.utils.asyncgenr   scrapy.utils.deferr   r   scrapy.utils.logr   scrapy.utils.miscr   scrapy.utils.spiderr   argparsecollections.abcr   r   r   r   twisted.python.failurer   scrapy.http.requestr   scrapy.spidersr   	getLoggerr   rQ   r   r    r'   r-   r*   <module>r     s}    "     8 8 # :  0 ( )   2 @ 0 ) 5RR.-% 
		8	$T]s%" s%r-   