
    !h                       % S SK Jr  S SKJr  S SKJr  S SKJrJr  S SK	J
r
JrJr  S SKJrJr  S SKJrJr  S SKJr  S S	KJrJrJrJr  S S
KJr  \(       a  S SKJr  S SKJr  S SK J!r!  \" \"5      r#SS/r$S\%S'     S SK&r&\$RS                  S5         S SK*r*\$RS                  S5         " S S5      r+g! \' a    S SK(r( N>f = f! \' a     N9f = f! \' a     N/f = f)    )annotations)chain)	getLogger)TYPE_CHECKINGAny)RequestSpidersignals)IgnoreRequestNotConfigured)ResponseTextResponse)responsetypes)_DecompressionMaxSizeExceeded_inflate	_unbrotli_unzstd)gunzip)Self)Crawler)StatsCollector   gzip   deflatelist[bytes]ACCEPTED_ENCODINGSN   br   zstdc                      \ rS rSrSr SSS.   SS jjjr\SS j5       rSS jr      SS jr	        SS	 jr
        SS
 jr\    SS j5       r\SS j5       r      SS jrSrg)HttpCompressionMiddleware1   z[This middleware allows compressed (gzip, deflate) traffic to be
sent/received from websitesNcrawlerc               F   U(       d  Xl         SU l        SU l        g UR                   U l         UR                  R	                  S5      U l        UR                  R	                  S5      U l        UR
                  R                  U R                  [
        R                  5        g )Ni   @i   DOWNLOAD_MAXSIZEDOWNLOAD_WARNSIZE)	stats	_max_size
_warn_sizesettingsgetintr
   connectopen_spiderspider_opened)selfr&   r"   s      s/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/downloadermiddlewares/httpcompression.py__init__"HttpCompressionMiddleware.__init__5   sz     J'DN&DO]]
 ))001CD!**112EF 0 0'2G2GH    c                Z    UR                   R                  S5      (       d  [        eU " US9$ )NCOMPRESSION_ENABLEDr!   )r)   getboolr   )clsr"   s     r/   from_crawler&HttpCompressionMiddleware.from_crawlerE   s+    ''(=>>7##r2   c                    [        US5      (       a  UR                  U l        [        US5      (       a  UR                  U l        g g )Ndownload_maxsizedownload_warnsize)hasattrr:   r'   r;   r(   )r.   spiders     r/   r,   %HttpCompressionMiddleware.open_spiderK   s=    6-..#44DN6.//$66DO 0r2   c                b    UR                   R                  SSR                  [        5      5        g )NzAccept-Encodings   , )headers
setdefaultjoinr   )r.   requestr=   s      r/   process_request)HttpCompressionMiddleware.process_requestQ   s&     	""#4ejjAS6TUr2   c           
     N   UR                   S:X  a  U$ [        U[        5      (       Ga  UR                  R	                  S5      nU(       Ga  UR
                  R                  SU R                  5      nUR
                  R                  SU R                  5      n U R                  UR                  XE5      u  pt[        UR                  5      Us=:  a  [        U5      ::  a*  O  O'[        R                  U S	[        U5       S
U S35        U(       a  U R!                  X$5        XBR                  S'   U R"                  (       a>  U R"                  R%                  S[        U5      US9  U R"                  R%                  SUS9  [&        R(                  " UR                  UR*                  US9nSU0n	[-        U[.        5      (       a  S U	S'   UR0                  " SSU0U	D6nU(       d  UR                  S	 U$ ! [         a)    [        SU S[        UR                  5       SU S35      ef = f)NHEADzContent-Encodingr:   r;   zIgnored response z because its body (z* B compressed) exceeded DOWNLOAD_MAXSIZE (z B) during decompression.z  body size after decompression (z. B) is larger than the download warning size (z B).zhttpcompression/response_bytes)r=   zhttpcompression/response_count)r@   urlbodyrI   encodingr6    )method
isinstancer   r@   getlistmetagetr'   r(   _handle_encodingrI   r   r   lenloggerwarning_warn_unknown_encodingr&   	inc_valuer   	from_argsrH   
issubclassr   replace)
r.   rC   responser=   content_encodingmax_size	warn_sizedecoded_bodyrespclskwargss
             r/   process_response*HttpCompressionMiddleware.process_responseW   s    >>V#Oh))'//778JK"<<++,>O#LL,,-@$//R	
595J5J '762L x}}%	FS5FFNN#* %-. /22;DB
 $//K7G  !34::JJ((8L)% ) 
 JJ((8 )  (11$,,(,,\ +1,)?g|44 *.F:&#++BB6B' (();<K 5 '+H: 6./ 0--5J 7)* s   G1 13H$c                d    U R                  U5      u  pEU H  nU R                  XU5      nM     X4$ N)_split_encodings_decode)r.   rI   r[   r\   	to_decodeto_keeprJ   s          r/   rQ   *HttpCompressionMiddleware._handle_encoding   s:     "223CD	!H<<9D "}r2   c                P   1 [         kSin[        R                  " S U  5       5       Vs/ s H!  nUR                  5       R	                  5       PM#     nn/ nU(       aC  UR                  5       nX!;  a  UR                  U5        XC4$ UR                  U5        U(       a  MC  XC4$ s  snf )N   x-gzipc              3  B   #    U  H  oR                  S 5      v   M     g7f)   ,N)split).0	encodingss     r/   	<genexpr>=HttpCompressionMiddleware._split_encodings.<locals>.<genexpr>   s      07G)%%7Gs   )r   r   from_iterablestriplowerpopappend)r[   supported_encodingsrJ   rh   rg   s        r/   re   *HttpCompressionMiddleware._split_encodings   s     ? 2>I> "// 07G0  
 NN""$ 	  
 "$	{{}H2x( ))X& g !! 
s   (B#c                ~    US;   a	  [        XS9$ US:X  a	  [        XS9$ US:X  a	  [        XS9$ US:X  a	  [        XS9$ U $ )N>   r   rk   )r\   r   r   r   )r   r   r   r   )rI   rJ   r\   s      r/   rf   !HttpCompressionMiddleware._decode   sT    ++$22z!D44uT55w433r2   c                    SR                  U5      R                  5       nU R                  R                   SUR                   SU S3nSU;   a  US-  nSU;   a  US-  n[
        R                  U5        g )	Nrm   z  cannot decode the response for z from unsupported encoding(s) 'z'.r   z9 You need to install brotli or brotlicffi to decode 'br'.r   z0 You need to install zstandard to decode 'zstd'.)rB   decode	__class____name__rH   rS   rT   )r.   rZ   rp   encodings_strmsgs        r/   rU   0HttpCompressionMiddleware._warn_unknown_encoding   s     		),335~~&&''G~ V--:O2? 	 INNCiEECsr2   )r'   r(   r&   rd   )r&   zStatsCollector | Noner"   zCrawler | None)r"   r   returnr   )r=   r	   r   None)rC   r   r=   r	   r   zRequest | Response | None)rC   r   rZ   r   r=   r	   r   zRequest | Response)rI   bytesr[   r   r\   intr   ztuple[bytes, list[bytes]])r[   r   r   ztuple[list[bytes], list[bytes]])rI   r   rJ   r   r\   r   r   r   )rZ   r   rp   r   r   r   )r   
__module____qualname____firstlineno____doc__r0   classmethodr7   r,   rD   ra   rQ   staticmethodre   rf   rU   __static_attributes__rK   r2   r/   r   r   1   s   #
 (,I #'	I$I  	I  $ $
7(.	"33*23<B3	3j-8DG	" "%"	(" "& 
 
 -8	r2   r   ),
__future__r   	itertoolsr   loggingr   typingr   r   scrapyr   r	   r
   scrapy.exceptionsr   r   scrapy.httpr   r   scrapy.responsetypesr   scrapy.utils._compressionr   r   r   r   scrapy.utils.gzr   typing_extensionsr   scrapy.crawlerr   scrapy.statscollectorsr   r   rS   r   __annotations__brotliImportError
brotlicffirw   	zstandardr   rK   r2   r/   <module>r      s    "   % + + : . .  #&&5 
8	#*J"7 K 7% e$' g&P P   		  		s<   :B0 C 0
B=:C  <B==C   C	C	CC