
    !hA                       S SK Jr  S SKrS SKrS SKrS SKJrJr  S SKJ	r	  S SK
Jr  S SKJr  S SKJrJrJrJr  S SKJr  S S	KJrJr  S S
KJrJr  S SKJr  S SKJr  S SKJr  S SK J!r!J"r"  \(       a.  S SK#r#S SK$J%r%  S SK&J'r'  S SK(J)r)  S SK*J+r+  S SK,J-r-  S SK.J/r/  S SK0J1r1  \Rd                  " \35      r4 " S S5      r5 " S S5      r6 " S S5      r7 " S S5      r8S S jr9S!S jr:g)"    )annotationsN)	mktime_tzparsedate_tz)import_module)Path)time)IOTYPE_CHECKINGAnycast)WeakKeyDictionary)headers_dict_to_rawheaders_raw_to_dict)HeadersResponse)responsetypes)urlparse_cached)	data_path)to_bytes
to_unicode)Callable)
ModuleType)Concatenate)Request)BaseSettings)Spider)RequestFingerprinterProtocolc                  b    \ rS rSrS	S jrS
S jrSS jr      SS jr        SS jrSr	g)DummyPolicy&   c                    UR                  S5      U l        UR                  S5       Vs/ s H  n[        U5      PM     snU l        g s  snf )NHTTPCACHE_IGNORE_SCHEMESHTTPCACHE_IGNORE_HTTP_CODES)getlistignore_schemesintignore_http_codes)selfsettingsxs      b/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/extensions/httpcache.py__init__DummyPolicy.__init__'   sH    )1)9)9:T)U$,,-JK-
KqCFK-
 -
s   Ac                F    [        U5      R                  U R                  ;  $ N)r   schemer%   )r(   requests     r+   should_cache_request DummyPolicy.should_cache_request-   s    w'..d6I6III    c                4    UR                   U R                  ;  $ r/   )statusr'   )r(   responser1   s      r+   should_cache_response!DummyPolicy.should_cache_response0   s    d&<&<<<r4   c                    gNT )r(   cachedresponser1   s      r+   is_cached_response_fresh$DummyPolicy.is_cached_response_fresh3        r4   c                    gr;   r<   )r(   r=   r7   r1   s       r+   is_cached_response_valid$DummyPolicy.is_cached_response_valid8   r@   r4   )r'   r%   Nr)   r   r1   r   returnboolr7   r   r1   r   rF   rG   r=   r   r1   r   rF   rG   r=   r   r7   r   r1   r   rF   rG   )
__name__
__module____qualname____firstlineno__r,   r2   r8   r>   rB   __static_attributes__r<   r4   r+   r   r   &   sT    
J=&18	
&2:EL	r4   r   c                      \ rS rSrSrSS jrSS jrSS jrSS jr      SS jr	        SS jr
      SS	 jrSS
 jr        SS jr        SS jrSrg)RFC2616Policy>   i3c                    UR                  S5      U l        UR                  S5      U l        [	        5       U l        UR                  S5       Vs/ s H  n[        U5      PM     snU l        g s  snf )NHTTPCACHE_ALWAYS_STOREr"   (HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS)getboolalways_storer$   r%   r   
_cc_parsedr   ignore_response_cache_controls)r(   r)   ccs      r+   r,   RFC2616Policy.__init__A   sp    "*"2"23K"L)1)9)9:T)U  	
 &&'QR<
R RLR<
+ <
s   A-c                &   XR                   ;  at  UR                  R                  SS5      nUc   e[        U5      n[	        U[
        5      (       a%  U R                   H  nUR                  US 5        M     X0R                   U'   U R                   U   $ )Ns   Cache-Controlr4   )rX   headersgetparse_cachecontrol
isinstancer   rY   pop)r(   rcchparsedkeys        r+   _parse_cachecontrol!RFC2616Policy._parse_cachecontrolL   s{    OO#))-- 0#6C?"?',F!X&&>>CJJsD) ?!'OOAq!!r4   c                v    [        U5      R                  U R                  ;   a  gU R                  U5      nSU;  $ )NF   no-store)r   r0   r%   rf   )r(   r1   rZ   s      r+   r2   "RFC2616Policy.should_cache_requestW   s9    7#**d.A.AA%%g."$$r4   c                4   U R                  U5      nSU;   a  gUR                  S:X  a  gU R                  (       a  gSU;   d  SUR                  ;   a  gUR                  S;   a  gUR                  S;   a%  S	UR                  ;   =(       d    S
UR                  ;   $ g)Nri   F0  T   max-age   Expiresi,  i-  i4  )      i     Last-Modified   ETag)rf   r6   rW   r]   )r(   r7   r1   rZ   s       r+   r8   #RFC2616Policy.should_cache_response^   s     %%h/"??c!zX-=-==??o-??o-#x'7'77V7hFVFV;VV r4   c                   U R                  U5      nU R                  U5      nSU;   d  SU;   a  g[        5       nU R                  XU5      nU R                  XU5      nU R	                  U5      nUb  [        Xh5      nXv:  a  gSU;   a-  SU;  a'  US   n	U	c  g Xv[        S[        U	5      5      -   :  a  g U R                  X!5        g! [         a     Nf = f)Ns   no-cacheFTs	   max-stale   must-revalidater   )
rf   r   _compute_freshness_lifetime_compute_current_age_get_max_ageminmaxr&   
ValueError_set_conditional_validators)
r(   r=   r1   rZ   ccreqnowfreshnesslifetime
currentage	reqmaxagestaleages
             r+   r>   &RFC2616Policy.is_cached_response_freshz   s    %%n5((1"u 4f <<S
 ..~L
%%e,	  #$5 A)5 %7r%A \*HC3x=4I II J 	((A  s   C 
CCc                p    UR                   S:  a  U R                  U5      nSU;  a  gUR                   S:H  $ )Ni  rv   Trl   )r6   rf   )r(   r=   r7   r1   rZ   s        r+   rB   &RFC2616Policy.is_cached_response_valid   s<    
 ??c!)).9B!+ #%%r4   c                    SUR                   ;   a  UR                   S   UR                   S'   SUR                   ;   a  UR                   S   UR                   S'   g g )Nrr   s   If-Modified-Sincers   s   If-None-Match)r]   )r(   r1   r=   s      r+   r}   )RFC2616Policy._set_conditional_validators   s^     ~5554B4J4J 5GOO01 n,,,0>0F0Fw0OGOO,- -r4   c                `     [        S[        US   5      5      $ ! [        [        4 a     g f = f)Nr   rm   )r{   r&   KeyErrorr|   )r(   rZ   s     r+   ry   RFC2616Policy._get_max_age   s4    	q#bn-..*% 		s    --c                   U R                  U5      nU R                  U5      nUb  U$ [        UR                  R	                  S5      5      =(       d    UnSUR                  ;   a/  [        UR                  S   5      nU(       a  [        SXv-
  5      $ S$ [        UR                  R	                  S5      5      nU(       a  X::  a  Xh-
  S-  $ UR                  S;   a  U R                  $ g)N   Datern   r   rr   
   ro   )rf   ry   rfc1123_to_epochr]   r^   r{   r6   MAXAGE)	r(   r7   r1   r   rZ   maxagedateexpireslastmodifieds	            r+   rw   )RFC2616Policy._compute_freshness_lifetime   s    
 %%h/""2&M   0 0 4 4W =>E# )))&x'7'7
'CDG .53q'.);!; ((8(8(<(<=M(NOL0'2-- ??o-;; r4   c                   Sn[        UR                  R                  S5      5      =(       d    UnX5:  a  X5-
  nSUR                  ;   a&   [        UR                  S   5      n[	        XF5      nU$ U$ ! [
         a     U$ f = f)Nr   r   s   Age)r   r]   r^   r&   r{   r|   )r(   r7   r1   r   r   r   ages          r+   rx   "RFC2616Policy._compute_current_age   s    
 
   0 0 4 4W =>E#:JX%%%(**623 1
 z  s   
#A1 1
A?>A?)rX   rW   rY   r%   NrD   )rb   zRequest | ResponserF   dict[bytes, bytes | None]rE   rH   rI   rJ   )r1   r   r=   r   rF   None)rZ   r   rF   
int | None)r7   r   r1   r   r   floatrF   r   )rK   rL   rM   rN   r   r,   rf   r2   r8   r>   rB   r}   ry   rw   rx   rO   r<   r4   r+   rQ   rQ   >   s    F	
	"%8)&)18)	)V&&&2:&EL&	&	P	P08	P		P +29>	B +29>	r4   rQ   c                  `    \ rS rSrS
S jrSS jrSS jrSS jr        SS jrSS jr	Sr
g	)DbmCacheStorage   c                    [        US   SS9U l        UR                  S5      U l        [	        US   5      U l        S U l        g )NHTTPCACHE_DIRT)	createdirHTTPCACHE_EXPIRATION_SECSHTTPCACHE_DBM_MODULE)r   cachedirgetintexpiration_secsr   dbmoduledbr(   r)   s     r+   r,   DbmCacheStorage.__init__   s@    &x'@DQ$,OO4O$P$1(;Q2R$Sr4   c                @   [        U R                  UR                   S35      nU R                  R	                  [        U5      S5      U l        [        R                  SSU0SU0S9  UR                  R                  (       d   eUR                  R                  U l        g )Nz.dbcz(Using DBM cache storage in %(cachepath)s	cachepathspiderextra)r   r   namer   openstrr   loggerdebugcrawlerrequest_fingerprinter_fingerprinter)r(   r   dbpaths      r+   open_spiderDbmCacheStorage.open_spider  s    dmm}C%89--$$S[#66&!V$ 	 	
 ~~3333NN00 	r4   c                8    U R                   R                  5         g r/   )r   closer(   r   s     r+   close_spiderDbmCacheStorage.close_spider  s    r4   c                    U R                  X5      nUc  g US   nUS   n[        US   5      nUS   n[        R                  " XdUS9nU" XFXWS9$ )Nurlr6   r]   bodyr]   r   r   r   r]   r6   r   )
_read_datar   r   	from_args)	r(   r   r1   datar   r6   r]   r   respclss	            r+   retrieve_response!DbmCacheStorage.retrieve_response  s`    v/<5kh$y/*F|))'N3JJr4   c                V   U R                   R                  U5      R                  5       nUR                  UR                  [        UR                  5      UR                  S.n[        R                  " USS9U R                  U S3'   [        [        5       5      U R                  U S3'   g )N)r6   r   r]   r      protocol_data_time)r   fingerprinthexr6   r   dictr]   r   pickledumpsr   r   r   )r(   r   r1   r7   re   r   s         r+   store_responseDbmCacheStorage.store_response  s     !!--g6::<oo<<H,,-MM	
 "(dQ!?3%u!$TV3%ur4   c                R   U R                   R                  U5      R                  5       nU R                  nU S3nXT;  a  g XE   nSU R                  s=:  a  [        5       [        U5      -
  :  a   g   [        [        [        [        4   [        R                  " XC S3   5      5      $ )Nr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   loads)r(   r   r1   re   r   tkeytss          r+   r   DbmCacheStorage._read_data+  s    !!--g6::<WWe}>Xt##8dfuRy&88 9 DcNFLLe5M1B$CDDr4   )r   r   r   r   r   NrD   r   r   rF   r   r   r   r1   r   rF   zResponse | Noner   r   r1   r   r7   r   rF   r   r   r   r1   r   rF   zdict[str, Any] | None)rK   rL   rM   rN   r,   r   r   r   r   r   rO   r<   r4   r+   r   r      sB    
	K--'.-:B-	-Er4   r   c                  j    \ rS rSrSS jrSS jrSS jrSS jr        SS jrSS jr	SS jr
S	rg
)FilesystemCacheStoragei9  c                    [        US   5      U l        UR                  S5      U l        UR	                  S5      U l        U R
                  (       a  [        R                  U l        g [        U l        g )Nr   r   HTTPCACHE_GZIP)	r   r   r   r   rV   use_gzipgzipr   _openr   s     r+   r,   FilesystemCacheStorage.__init__:  sX    &x'@A$,OO4O$P&../?@ DII 	
,0 	
r4   c                    [         R                  SSU R                  0SU0S9  UR                  R                  (       d   eUR                  R                  U l        g )Nz.Using filesystem cache storage in %(cachedir)sr   r   r   )r   r   r   r   r   r   r   s     r+   r   "FilesystemCacheStorage.open_spiderC  sQ    <'V$ 	 	
 ~~3333$nnBBr4   c                    g r/   r<   r   s     r+   r   #FilesystemCacheStorage.close_spiderM  s    r4   c                   U R                  X5      nUc  g[        U R                  X5      5      nU R                  US-  S5       nUR	                  5       nSSS5        U R                  US-  S5       nUR	                  5       nSSS5        US   nUS   n	[        [        W5      5      n
[        R                  " XWS9nU" XXS9$ ! , (       d  f       Nv= f! , (       d  f       NY= f)	z7Return response if present in cache, or None otherwise.Nresponse_bodyrbresponse_headersresponse_urlr6   r   r   )	
_read_metar   _get_request_pathr   readr   r   r   r   )r(   r   r1   metadatarpathfr   
rawheadersr   r6   r]   r   s               r+   r   (FilesystemCacheStorage.retrieve_responseP  s    ??63T++F<=ZZ/6!668D 7ZZ 22D9QJ :~&(#-j9:))'N3JJ 7699s   C3C
C
C%c                   [        U R                  X5      5      nUR                  5       (       d  UR                  SS9  UR                  UR
                  UR                  UR                  [        5       S.nU R                  US-  S5       nUR                  [        [        U5      5      5        SSS5        U R                  US-  S5       n[        R                  " XVSS	9  SSS5        U R                  US
-  S5       nUR                  [        UR                  5      5        SSS5        U R                  US-  S5       nUR                  UR                   5        SSS5        U R                  US-  S5       nUR                  [        UR                  5      5        SSS5        U R                  US-  S5       nUR                  UR                   5        SSS5        g! , (       d  f       GN9= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g= f)z&Store the given response in the cache.T)parents)r   methodr6   r   	timestampmetawbNpickled_metar   r   r   r   request_headersrequest_body)r   r   existsmkdirr   r   r6   r   r   writer   reprr   dumpr   r]   r   )r(   r   r1   r7   r   r   r   s          r+   r   %FilesystemCacheStorage.store_response`  s    T++F<=||~~KKK%;;nnoo$LL
 ZZ-GGHT(^,- .ZZ.5KKa0 6ZZ 22D9QGG'(8(89: :ZZ/6!GGHMM" 7ZZ 1148AGG'89 9ZZ.5GGGLL! 65 .-5599668855sH   $G"G4 %HH;%H(=H9"
G14
H
H
H%(
H69
Ic           	         U R                   R                  U5      R                  5       n[        [	        U R
                  UR                  USS U5      5      $ )Nr      )r   r   r   r   r   r   r   )r(   r   r1   re   s       r+   r   (FilesystemCacheStorage._get_request_path{  sE    !!--g6::<4v{{C!HcBCCr4   c                   [        U R                  X5      5      nUS-  nUR                  5       (       d  g UR                  5       R                  nSU R
                  s=:  a  [        5       U-
  :  a   g   U R                  US5       n[        [        [        [        4   [        R                  " U5      5      sS S S 5        $ ! , (       d  f       g = f)Nr  r   r   )r   r   r  statst_mtimer   r   r   r   r   r   r   r   load)r(   r   r1   r   metapathmtimer   s          r+   r   !FilesystemCacheStorage._read_meta  s    T++F<=>)  ((t##4dfun4 5ZZ$'1S#XA7 (''s   1C
C)r   r   r   r   r   NrD   r   r   r   )r   r   r1   r   rF   r   r   )rK   rL   rM   rN   r,   r   r   r   r   r   r   rO   r<   r4   r+   r   r   9  sH    
CK ""'.":B"	"6D	8r4   r   c                    0 nU R                  S5       HI  nUR                  5       R                  S5      u  p4nU(       d  M.  U(       a  UOSXR                  5       '   MK     U$ )a  Parse Cache-Control header

https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9

>>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None,
...                                                 b'max-age': b'3600'}
True
>>> parse_cachecontrol(b'') == {}
True

   ,   =N)splitstrip	partitionlower)header
directives	directivere   sepvals         r+   r_   r_     sV     J\\$'	!)33D9#3-0cdJyy{# ( r4   c                `     [        U SS9n [        [        U 5      5      $ ! [         a     g f = f)Nascii)encoding)r   r   r   	Exception)date_strs    r+   r   r     s5    h9h/00 s     
--)r  bytesrF   r   )r%  zstr | bytes | NonerF   r   );
__future__r   r   loggingr   email.utilsr   r   	importlibr   pathlibr   r   typingr	   r
   r   r   weakrefr   
w3lib.httpr   r   scrapy.httpr   r   scrapy.responsetypesr   scrapy.utils.httpobjr   scrapy.utils.projectr   scrapy.utils.pythonr   r   oscollections.abcr   typesr   typing_extensionsr   scrapy.http.requestr   scrapy.settingsr   scrapy.spidersr   scrapy.utils.requestr   	getLoggerrK   r   r   rQ   r   r   r_   r   r<   r4   r+   <module>r=     s    "    / #   / / % ? ) . 0 * 4(  .+,%A 
		8	$ 0y yx<E <E~O8 O8d(r4   