
    !h$                    V   S r SSKJr  SSKrSSKrSSKrSSKJr  SSKJ	r	  SSK
JrJr  SSKJr  SSKJrJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJr  SSKJrJrJ r   \(       a.  SSK!J"r"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  SSK*J+r+  SSK,J-r-  SSK.J/r/J0r0   " S S\5      r1 " S S\5      r2g)zA
Images Pipeline

See documentation in topics/media-pipeline.rst
    )annotationsN)suppress)BytesIO)TYPE_CHECKINGAny)ItemAdapter)NotConfiguredScrapyDeprecationWarning)RequestResponse)NO_CALLBACK)FileExceptionFilesPipeline_md5sum)Settings)get_func_argsglobal_object_nameto_bytes)CallableIterable)PathLikeImage)Self)Spider)Crawler)FileInfoOrErrorMediaPipelinec                      \ rS rSrSrSrg)ImageException'   zGeneral image error exception N)__name__
__module____qualname____firstlineno____doc____static_attributes__r"       ^/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/pipelines/images.pyr    r    '   s    'r)   r    c                    ^  \ rS rSr% SrSrS\S'   SrS\S'   SrS\S	'   S
r	S\S'   0 r
S\S'   SrSr  SSS.       SU 4S jjjjr\S S j5       rSS.         S!S jjrSS.         S!S jjrSS.         S"S jjr S#       S$S jjr      S%S jr        S&S jr  SSS.         S'S jjjr  SSS.           S(S jjjrSrU =r$ ))ImagesPipeline+   zEAbstract pipeline that implement the image thumbnail generation logicimagestr
MEDIA_NAMEr   int	MIN_WIDTH
MIN_HEIGHTZ   EXPIRESzdict[str, tuple[int, int]]THUMBS
image_urlsimagesNcrawlerc                 >  SSK Jn  XPl        [
        TU ]  UU(       d  UOS UUS9  UbA  Ub1  [        R                  " S[        U R                  5       S3[        SS9  UR                  nO#[        U[        5      (       d  Uc  [        U5      n[         R"                  " U R$                  S	US
9nUR'                  U" S5      U R(                  5      U l        [-        U S5      (       d  U R.                  U l        [-        U S5      (       d  U R2                  U l        UR7                  U" S5      U R4                  5      U l        UR7                  U" S5      U R0                  5      U l        UR'                  U" S5      U R<                  5      U l        UR'                  U" S5      U R@                  5      U l!        UR7                  U" S5      U RD                  5      U l#        g ! [         a    [	        S5      ef = f)Nr   r   z8ImagesPipeline requires installing Pillow 8.0.0 or later)settingsdownload_funcr:   zcImagesPipeline.__init__() was called with a crawler instance and a settings instance when creating z. The settings instance will be ignored and crawler.settings will be used. The settings argument will be removed in a future Scrapy version.   )category
stacklevelr,   )base_class_namer<   IMAGES_EXPIRESIMAGES_RESULT_FIELDIMAGES_URLS_FIELDIMAGES_MIN_WIDTHIMAGES_MIN_HEIGHTIMAGES_THUMBS)$PILr   _ImageImportErrorr	   super__init__warningswarnr   	__class__r
   r<   
isinstancedictr   	functoolspartial_key_for_pipegetintr5   expireshasattrDEFAULT_IMAGES_RESULT_FIELDrC   DEFAULT_IMAGES_URLS_FIELDrD   getimages_urls_fieldimages_result_fieldr2   	min_widthr3   
min_heightr6   thumbs)self	store_urir=   r<   r:   r   resolverO   s          r*   rL   ImagesPipeline.__init__9   s   	!K 	%,X$'	 	 	
 #&&8&H%I J|} 6  ''H$''8+;)H##,

 %OOG4D,Et||Tt233,0,L,LD$t011*.*H*HD"&.ll'($*@*@'
 )1)*D,D,D)
  'oo&'
  ('($// 
 3;,,O$dkk3
c  	J 	s   G G4c                    U R                  U5        US   nS[        U R                  5      ;   a  U " X2S9nU$ U " X1S9nU(       a  UR                  U5        [        R
                  " [        U 5       S3[        S9  U$ )NIMAGES_STOREr:   r9   )r<   z|.__init__() doesn't take a crawler argument. This is deprecated and the argument will be required in future Scrapy versions.)r?   )_update_storesr   rL   _finish_initrM   rN   r   r
   )clsr<   r:   ra   os        r*   _from_settingsImagesPipeline._from_settingsz   s    8$^,	cll33I/A  I1Aw'MM%c*+ ,c c1
 r)   itemc               "    U R                  XX4S9$ )Nrl   )image_downloaded)r`   responserequestinform   s        r*   file_downloadedImagesPipeline.file_downloaded   s     $$X$HHr)   c          	         S nU R                  XX4S9 HT  u  pgnUc  UR                  S5        [        U5      nUR                  u  pU R                  R                  UUUXS.SS0S9  MV     Uc   eU$ )Nrl   r   )widthheightzContent-Typez
image/jpeg)metaheaders)
get_imagesseekr   sizestorepersist_file)r`   rp   rq   rr   rm   checksumpathr.   bufrv   rw   s              r*   ro   ImagesPipeline.image_downloaded   s      $ $4 SD"3<!JJMEJJ##$7'6 $  !T ###r)   c             #  &  #    U R                  X!X4S9nU R                  R                  [        UR                  5      5      nUR
                  u  pxXpR                  :  d  XR                  :  a,  [        SU SU SU R                   SU R                   S3	5      eU R                  U[        UR                  5      S9u  pXYU
4v   U R                  R                  5        H.  u  pU R                  X+XUS9nU R                  XU
S9u  pXU4v   M0     g 7f)N)rp   rr   rm   zImage too small (xz < ))response_body)	file_pathrI   openr   bodyr|   r]   r^   r    convert_imager_   items
thumb_path)r`   rp   rq   rr   rm   r   
orig_imagerv   rw   r.   r   thumb_idr|   r   thumb_image	thumb_bufs                   r*   rz   ImagesPipeline.get_images   s     ~~gt~O[[%%ghmm&<=
">>!Voo%= 7!F83>>"!DOO#4A7  ''ghmm&< ( 

 3"kk//1NHHd ) J &*%7%7SV%7%W"K944 2s   DDc               2   UR                   S;   aZ  UR                  S:X  aJ  U R                  R                  SUR                  S5      nUR                  X5        UR                  S5      nOUR                  S:X  a[  UR                  S5      nU R                  R                  SUR                  S5      nUR                  X5        UR                  S5      nO!UR                  S:w  a  UR                  S5      nU(       aC  UR                  5       n U R                  R                  R                  nUR                  X%5        OUR                   S:X  a  X4$ [        5       nUR                  US5        X4$ ! [         a    U R                  R                  n Nff = f)N)PNGWEBPRGBA)   r   r   RGBPJPEG)formatmoderI   newr|   pasteconvertcopy
ResamplingLANCZOSAttributeError	ANTIALIAS	thumbnailr   save)r`   r.   r|   r   
backgroundresampling_filterr   s          r*   r   ImagesPipeline.convert_image   sK    <<?*uzzV/C_MJU*&&u-EZZ3MM&)E_MJU*&&u-EZZ5 MM%(EJJLE: %)KK$:$:$B$B! OOD4\\V#''i

3z " :$(KK$9$9!:s    E3 3 FFc                    [        U5      R                  U R                  / 5      nU Vs/ s H  n[        U[        S9PM     sn$ s  snf )N)callback)r   rZ   r[   r   r   )r`   rm   rr   urlsus        r*   get_media_requests!ImagesPipeline.get_media_requests   s@     4 $$T%;%;R@:>?$QK0$???s   Ac                    [        [        5         U VVs/ s H  u  pEU(       d  M  UPM     snn[        U5      U R                  '   S S S 5        U$ s  snnf ! , (       d  f       U$ = fN)r   KeyErrorr   r\   )r`   resultsrm   rr   okr   s         r*   item_completedImagesPipeline.item_completed   sS     hJQ:X'UW1':XKd667   ;Y  s    AAAAA
A&c               |    [         R                  " [        UR                  5      5      R	                  5       nSU S3$ )Nzfull/.jpghashlibsha1r   url	hexdigest)r`   rq   rp   rr   rm   
image_guids         r*   r   ImagesPipeline.file_path   s3     \\(7;;"78BBD
zl$''r)   c                   [         R                  " [        UR                  5      5      R	                  5       nSU SU S3$ )Nzthumbs//r   r   )r`   rq   r   rp   rr   rm   
thumb_guids          r*   r   ImagesPipeline.thumb_path  s9     \\(7;;"78BBD

!J<t44r)   )	rC   rD   rI   rV   r\   r[   r^   r]   r_   )NN)ra   zstr | PathLike[str]r=   z,Callable[[Request, Spider], Response] | Noner<   z Settings | dict[str, Any] | Noner:   Crawler | None)r<   r   r:   r   returnr   )
rp   r   rq   r   rr   MediaPipeline.SpiderInform   r   r   r/   )
rp   r   rq   r   rr   r   rm   r   r   z*Iterable[tuple[str, Image.Image, BytesIO]]r   )r.   zImage.Imager|   ztuple[int, int] | Noner   r   r   ztuple[Image.Image, BytesIO])rm   r   rr   r   r   zlist[Request])r   zlist[FileInfoOrError]rm   r   rr   r   r   r   )
rq   r   rp   Response | Nonerr   MediaPipeline.SpiderInfo | Nonerm   r   r   r/   )rq   r   r   r/   rp   r   rr   r   rm   r   r   r/   )r#   r$   r%   r&   r'   r0   __annotations__r2   r3   r5   r6   rY   rX   rL   classmethodrj   rs   ro   rz   r   r   r   r   r   r(   __classcell__)rO   s   @r*   r,   r,   +   sj   OJ IsJGS)+F&+ ,"*
 GK59	?
 #'?
&?
 D?
 3	?
  ?
 ?
B  , II I '	I I 
I    '	  
< 55 5 '	5 5 
45D (,"" %"
 " 
%"H@@7@	@,47?W	 %)04		( 	(	( "	( .		( 	( 
	( %)04
5 
5
5 
5 "	
5
 .
5 
5 

5 
5r)   r,   )3r'   
__future__r   rR   r   rM   
contextlibr   ior   typingr   r   itemadapterr   scrapy.exceptionsr	   r
   scrapy.httpr   r   scrapy.http.requestr   scrapy.pipelines.filesr   r   r   scrapy.settingsr   scrapy.utils.pythonr   r   r   collections.abcr   r   osr   rH   r   typing_extensionsr   scrapyr   scrapy.crawlerr   scrapy.pipelines.mediar   r   r    r,   r"   r)   r*   <module>r      sp    #      % # E ) + H H $ K K2 '&E(] (g5] g5r)   