
    !h2                       S SK Jr  S SKrS SKrS SKrS SKJrJr  S SKJ	r	  S SK
JrJrJrJrJrJrJr  S SKJr  S SKJrJr  S SKJr  S S	KJr  S S
KJr  S SKJrJ r   S SK!J"r"  S SK#J$r$  S SK%J&r&J'r'  S SK(J)r)  S SK*J+r+  S SK,J-r-J.r.  \(       a$  S SK/J0r0  S SK1J2r2  S SK3J4r4  S SK5J6r6  S SK7J8r8  S SK9J:r:   " S S\5      r;\\<\S   \;4   \<\S   \4   4   r=\R|                  " \?5      r@ " S S\5      rAg)    )annotationsN)ABCabstractmethod)defaultdict)TYPE_CHECKINGAnyLiteralNoReturn	TypedDictUnioncast)version)DeferredDeferredList)Failure)Version)ScrapyDeprecationWarning)NO_CALLBACKRequest)Settings)SequenceExclude)defer_resultmustbe_deferred)failure_to_exc_info)arg_to_iter)get_func_argsglobal_object_name)Callable)Self)Spider)Crawler)Response)RequestFingerprinterProtocolc                  >    \ rS rSr% S\S'   S\S'   S\S'   S\S'   Srg	)
FileInfo$   strurlpath
str | Nonechecksumstatus N)__name__
__module____qualname____firstlineno____annotations____static_attributes__r-       ]/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/pipelines/media.pyr%   r%   $   s    	H
IKr4   r%   TFc                  "   \ rS rSr% S\S'   S\S'   SrSrS\S	'    " S
 S5      r  S"SS.     S#S jjjrS$S jr	S%S jr
  S"       S&S jjr\S'S j5       rS(S jr      S)S jr        S*S jrS+S jrS,S jr          S-S jr        S.S jr\SS.       S/S jj5       r\S0S j5       r\SS.         S1S jj5       r\        S2S j5       r        S3S jr\  S"SS.         S4S  jjj5       rS!rg)5MediaPipeline0   r!   crawlerr#   _fingerprinterFTboolLOG_FAILED_RESULTSc                      \ rS rSrSS jrSrg)MediaPipeline.SpiderInfo7   c                d    Xl         [        5       U l        0 U l        [	        [
        5      U l        g N)spidersetdownloading
downloadedr   listwaitingselfrB   s     r5   __init__!MediaPipeline.SpiderInfo.__init__8   s(    "(K+.5D?ADOITJDLr4   )rE   rD   rB   rG   N)rB   r    )r.   r/   r0   r1   rJ   r3   r-   r4   r5   
SpiderInfor>   7   s    	r4   rL   Nr9   c               0   Xl         UbA  Ub1  [        R                  " S[        U R                  5       S3[
        SS9  UR                  nO#[        U[        5      (       d  Uc  [        U5      n[        R                  " U R                  SUS9nUR                  U" S5      S5      U l        U R                  U R                  5        U(       a  U R!                  U5        S	U l        g [        R                  " S
[        U R                  5       S3[
        SS9  g )NzbMediaPipeline.__init__() was called with a crawler instance and a settings instance when creating z. The settings instance will be ignored and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.   )category
stacklevelr7   )base_class_namesettingsMEDIA_ALLOW_REDIRECTSFTzOMediaPipeline.__init__() was called without the crawler argument when creating zQ. This is deprecated and the argument will be required in future Scrapy versions.)download_funcwarningswarnr   	__class__r   rS   
isinstancedictr   	functoolspartial_key_for_pipegetboolallow_redirects_handle_statuses_finish_init_modern_init)rI   rU   rS   r9   resolves        r5   rJ   MediaPipeline.__init__@   s    +#&&8&H%I J|} 6  ''H$''8+;)H##(
 &.%5%5+,e&
 	d223g& $DMM""4T^^"D!E Fcd 2r4   c                X    Xl         UR                  (       d   eUR                  U l        g rA   )r9   request_fingerprinterr:   )rI   r9   s     r5   ra   MediaPipeline._finish_initi   s&    
 ,,,,%;;r4   c                V    S U l         U(       a  [        [        SS5      5      U l         g g )Ni,  i  )handle_httpstatus_listr   range)rI   r_   s     r5   r`   MediaPipeline._handle_statusesr   s%    &*#*9%S/*JD' r4   c                    U R                   R                  nUR                  5        SU 3nU(       a"  XB:X  d  U(       a  UR                  U5      (       d  U$ U$ )N_)rX   r.   upperget)rI   keyrR   rS   
class_nameformatted_keys         r5   r]   MediaPipeline._key_for_pipew   sR     ^^,,
%++-.au5,m!<!<Jr4   c                   [        U S5      (       aA  U R                  UR                  5      n[        R                  " [        U 5       S3[        S9  OLS[        U R                  5      ;   a  U " US9nO,U " 5       n[        R                  " [        U 5       S3[        S9  UR                  (       d  UR                  U5        U$ )Nfrom_settingsaF   has from_settings() and either doesn't have from_crawler() or calls MediaPipeline.from_crawler() from it, so from_settings() was used to create the instance of it. This is deprecated and calling from_settings() will be removed in a future Scrapy version. Please move the initialization code into from_crawler() or __init__().)rP   r9   rM   z|.__init__() doesn't take a crawler argument. This is deprecated and the argument will be required in future Scrapy versions.)hasattrru   rS   rV   rW   r   r   r   rJ   rb   ra   )clsr9   pipes      r5   from_crawlerMediaPipeline.from_crawler   s     3(($$W%5%56DMM%c*+ ,1 1 2 -55w'D5DMM%c*+ ,c c1
   g&r4   c                0    U R                  U5      U l        g rA   )rL   
spiderinforH   s     r5   open_spiderMediaPipeline.open_spider   s    //&1r4   c                    U R                   n[        U R                  X5      5      nU Vs/ s H  oPR                  XSU5      PM     nn[	        S[        USS95      nUR                  U R                  X5      $ s  snf )NDeferred[list[FileInfoOrError]]T)consumeErrors)r|   r   get_media_requests_process_requestr   r   addCallbackitem_completed)rI   itemrB   inforequestsrdlistdfds           r5   process_itemMediaPipeline.process_item   sw     t66tBC?GHx!&&q5xH-|EQU/V
 t22D??	 Is   A8c                  ^	 U R                   R                  U5      nUR                  n[        Ul        S Ul        XBR
                  ;   a2  [        UR
                  U   5      nU(       a  UR                  U5        U$ [        5       m	U(       a  T	R                  U5        UR                  U   R                  T	5        XBR                  ;   a  T	$ UR                  R                  U5        [        U R                  XUS9nUR                  U R                   XUS9nUR                  U R"                  5        UR%                  U R&                  XB5        UR%                  U	4S j5      $ )Nr   c                   > T$ rA   r-   )rm   wads    r5   <lambda>0MediaPipeline._process_request.<locals>.<lambda>   s    cr4   )r:   fingerprinterrbackr   callbackrE   r   
addErrbackr   rG   appendrD   addr   media_to_downloadr   _check_media_to_download_log_exceptionaddBoth!_cache_result_and_execute_waiters)
rI   requestr   r   fpebdr   dfd2r   s
            @r5   r   MediaPipeline._process_request   s.      ,,W5__&  T__R01AR H #+*NN2R$ !!!J 	R )8""G*
 $'??))7t $3 $
 	++,T;;RF||M**r4   c                0    [         R                  U5        U$ rA   )logger	exception)rI   results     r5   r   MediaPipeline._log_exception   s     r4   c                x    U R                   (       a  U R                   UR                  S'   g SUR                  S'   g )Nri   Thandle_httpstatus_all)ri   meta)rI   r   s     r5   _modify_media_request#MediaPipeline._modify_media_request   s/    &&595P5PGLL1248GLL01r4   c                   Ub  U$ U R                   (       a!  [        U R                   X#R                  5      nOSU R                  U5        U R                  R
                  (       d   eU R                  R
                  R                  U5      nUR                  U R                  X#US9nUR                  U R                  X#5        U$ )Nr   )rU   r   rB   r   r9   enginedownloadr   media_downloadedr   media_failed)rI   r   r   r   r   r   r   s          r5   r   &MediaPipeline._check_media_to_download   s     M!$"4"4g{{KC&&w/<<&&&&,,%%..w7C#&??!!7t $3 $
 	))79r4   c                   [        U[        5      (       aq  UR                  5         / Ul        [        [        SSSS5      :  a  / Ul        [        UR                  SS 5      n[        U[        5      (       a  S UR                  l
        UR                  R                  U5        XR                  U'   UR                  R                  U5       H  n[!        U5      R#                  U5        M     g )Ntwisted   
   r   __context__)rY   r   cleanFailureframestwisted_versionr   stackgetattrvalueStopIterationr   rD   removerE   rG   popr   chainDeferred)rI   r   r   r   contextr   s         r5   r   /MediaPipeline._cache_result_and_execute_waiters   s     fg&&!FMBA!>>!& fllM4@G'=11+/(#$<<##B'C ..s3 (r4   r   c                   [         e)z&Check request before starting downloadNotImplementedError)rI   r   r   r   s       r5   r   MediaPipeline.media_to_download  
    
 "!r4   c                    [         e)z&Returns the media requests to downloadr   )rI   r   r   s      r5   r    MediaPipeline.get_media_requests  s
     "!r4   c                   [         e)zHandler for success downloadsr   )rI   responser   r   r   s        r5   r   MediaPipeline.media_downloaded#  
     "!r4   c                    [         e)zHandler for failed downloadsr   )rI   failurer   r   s       r5   r   MediaPipeline.media_failed/  r   r4   c                   U R                   (       am  U Hg  u  pEU(       a  M  [        U[        5      (       d   e[        R	                  SU R
                  R                  US.[        U5      SUR                  0S9  Mi     U$ )z:Called per item when all media requests has been processedz*%(class)s found errors processing %(item)s)classr   rB   )exc_infoextra)	r<   rY   r   r   errorrX   r.   r   rB   )rI   resultsr   r   okr   s         r5   r   MediaPipeline.item_completed6  sr     ""$	r%eW5555LLD"&.."9"94H!4U!;'5	 !  % r4   c                   [         e)z8Returns the path where downloaded media should be storedr   )rI   r   r   r   r   s        r5   	file_pathMediaPipeline.file_pathF  r   r4   )r:   rb   r_   r9   rU   ri   r|   )NN)rU   z,Callable[[Request, Spider], Response] | NonerS   z Settings | dict[str, Any] | Noner9   zCrawler | None)r9   r!   returnNone)r_   r;   r   r   )rp   r'   rR   r*   rS   zSettings | Noner   r'   )r9   r!   r   r   )rB   r    r   r   )r   r   rB   r    r   r   )r   r   r   rL   r   r   r   zDeferred[FileInfo])r   r   r   r   )r   r   r   r   )
r   zFileInfo | Noner   r   r   rL   r   r   r   zFileInfo | Deferred[FileInfo])r   zFileInfo | Failurer   bytesr   rL   r   r   )r   r   r   rL   r   r   r   zDeferred[FileInfo | None])r   r   r   rL   r   zlist[Request])
r   r"   r   r   r   rL   r   r   r   r%   )r   r   r   r   r   rL   r   r
   )r   zlist[FileInfoOrError]r   r   r   rL   r   r   )
r   r   r   zResponse | Noner   zSpiderInfo | Noner   r   r   r'   )r.   r/   r0   r1   r2   rb   r<   rL   rJ   ra   r`   r]   classmethodry   r}   r   r   r   r   r   r   r   r   r   r   r   r   r   r3   r-   r4   r5   r7   r7   0   s   00L##  GK59'
 #''C' 3'
  'R<K '+$(	 $ "	
 
   42	@	@!'	@	(	@#+#+&0#+8;#+	#+J9%07?IQT	&&"4("4.3"4;E"4	"4J AE""&0";>"	"" " " "  	"	" 	" 		" 	" 
	" 	" "")0"8B"	" ",47?I	   %)"&		" 	"	" "	"  		" 	" 
	" 	"r4   r7   )B
__future__r   r[   loggingrV   abcr   r   collectionsr   typingr   r   r	   r
   r   r   r   r   r   r   twisted.internet.deferr   r   twisted.python.failurer   twisted.python.versionsr   scrapy.exceptionsr   scrapy.http.requestr   r   scrapy.settingsr   scrapy.utils.datatypesr   scrapy.utils.deferr   r   scrapy.utils.logr   scrapy.utils.miscr   scrapy.utils.pythonr   r   collections.abcr   typing_extensionsr   scrapyr    scrapy.crawlerr!   scrapy.httpr"   scrapy.utils.requestr#   r%   tupleFileInfoOrError	getLoggerr.   r   r7   r-   r4   r5   <module>r      s    "    # # P P P . 9 * + 6 4 $ 2 < 0 ) A( '&$Ay  gdmX56geng>U8VVW			8	$`"C `"r4   