
    !hi                    v   S r SSKJr  SSKrSSKrSSKrSSKrSSKrSSKJ	r	  SSK
J
r
Jr  SSKJrJr  SSKJr  SSKJrJrJrJrJrJrJr  SS	KJrJr  SS
KJrJrJr  SSK J!r!  SSK"J#r#  SSK$J%r%J&r&  SSK'J(r(J)r)  SSK*J+r+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK7J8r8J9r9  SSK:J;r;  \(       a,  SSKJ<r<  SSK=J>r>  SSK?J@r@  SSKAJBrB  SSKCJDrD  SSKEJFrF  SSKGJHrHJIrI  \R                  " \K5      rL\	\M\N\4   \(/\\M\N\4      4   rO\" SSS 9rPSS!S".             S9S# jjrQ " S$ S%5      rR " S& S'\%5      rS " S( S\5      rT\&" \S5       " S) S*5      5       rU\&" \S5       " S+ S,5      5       rV\&" \S5       " S- S.5      5       rW " S/ S0\U5      rX " S1 S2\U5      rY " S3 S4\U5      rZ " S5 S65      r[ " S7 S85      r\g):zK
Feed Exports extension

See documentation in docs/topics/feed-exports.rst
    )annotationsN)Callable)datetimetimezone)PathPureWindowsPath)NamedTemporaryFile)IOTYPE_CHECKINGAnyOptionalProtocolTypeVarcast)unquoteurlparse)DeferredDeferredListmaybeDeferred)deferToThread)file_uri_to_path)	Interfaceimplementer)Spidersignals)NotConfiguredScrapyDeprecationWarning)PostProcessingManager)*feed_complete_default_values_from_settings)maybe_deferred_to_future)ftp_store_file)failure_to_exc_info)build_from_crawlerload_object)without_none_values)Iterable)OpenBinaryMode)Failure)Self)Crawler)BaseItemExporter)BaseSettingsSettings	_StorageTFeedStorageProtocol)bound )feed_optionspreargsc               \    [         R                  " S[        SS9  X%S'   U " / UQUPUQ70 UD6$ )NzVscrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.   category
stacklevelr2   )warningswarnr   )builderurir2   r3   argskwargss         c/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/extensions/feedexport.pybuild_storager@   9   s@     MM`)
 *>2G2S242622    c                  B    \ rS rSr% SrS\S'   S\S'   SS jrSS jrS	rg
)
ItemFilterJ   z
This will be used by FeedExporter to decide if an item should be allowed
to be exported to a particular feed.

:param feed_options: feed specific options passed from FeedExporter
:type feed_options: dict
dict[str, Any] | Noner2   ztuple[type, ...]item_classesc                    Xl         Ub0  [        S UR                  S5      =(       d    S 5       5      U l        g SU l        g )Nc              3  8   #    U  H  n[        U5      v   M     g 7fN)r$   ).0
item_classs     r?   	<genexpr>&ItemFilter.__init__.<locals>.<genexpr>Y   s      &"HJ J''"Hs   rF   r1   )r2   tuplegetrF   )selfr2   s     r?   __init__ItemFilter.__init__V   sE    (# % &"."2"2>"B"Hb"H& !D
 !#DrA   c                P    U R                   (       a  [        XR                   5      $ g)z
Return ``True`` if `item` should be exported or ``False`` otherwise.

:param item: scraped item which user wants to check if is acceptable
:type item: :ref:`Scrapy items <topics-items>`
:return: `True` if accepted, `False` otherwise
:rtype: bool
T)rF   
isinstance)rP   items     r?   acceptsItemFilter.accepts`   s!     d$5$566rA   )r2   rF   N)r2   rE   returnNone)rU   r   rX   bool)	__name__
__module____qualname____firstlineno____doc____annotations__rQ   rV   __static_attributes__r1   rA   r?   rC   rC   J   s     ('""#rA   rC   c                  2    \ rS rSrSrSS.S jrS rS rSrg)	IFeedStoragen   z/Interface that all Feed Storages must implementNr2   c                   gzpInitialize the storage with the parameters given in the URI and the
feed-specific options (see :setting:`FEEDS`)Nr1   )r<   r2   s     r?   rQ   IFeedStorage.__init__s       rA   c                    gzlOpen the storage for the given spider. It must return a file-like
object that will be used for the exportersNr1   )spiders    r?   openIFeedStorage.openw   ri   rA   c                    gzStore the given file streamNr1   )files    r?   storeIFeedStorage.store{   ri   rA   r1   	r[   r\   r]   r^   r_   rQ   rm   rr   ra   r1   rA   r?   rc   rc   n   s    9 '+ 86*rA   rc   c                  >    \ rS rSrSrSS.S	S jjrS
S jrSS jrSrg)r/      zDReimplementation of ``IFeedStorage`` that can be used in type hints.Nre   c                   grg   r1   rP   r<   r2   s      r?   rQ   FeedStorageProtocol.__init__   ri   rA   c                    grk   r1   rP   rl   s     r?   rm   FeedStorageProtocol.open   ri   rA   c                    grp   r1   rP   rq   s     r?   rr   FeedStorageProtocol.store   ri   rA   r1   r<   strr2   rE   rl   r   rX   	IO[bytes]rq   r   rX   Deferred[None] | Nonert   r1   rA   r?   r/   r/      s    NJN 86*rA   c                  2    \ rS rSrSS jrSS jrS	S jrSrg)
BlockingFeedStorage   c                    UR                   R                  S   nU(       a5  [        U5      R                  5       (       d  [	        S[        U5      -   5      e[        SUS9$ )NFEED_TEMPDIRzNot a Directory: zfeed-)prefixdir)crawlersettingsr   is_dirOSErrorr   r	   )rP   rl   paths      r?   rm   BlockingFeedStorage.open   sM    ~~&&~6T
))++-D	9::!d;;rA   c                .    [        U R                  U5      $ rI   )r   _store_in_threadr~   s     r?   rr   BlockingFeedStorage.store   s    T22D99rA   c                    [         erI   )NotImplementedErrorr~   s     r?   r   $BlockingFeedStorage._store_in_thread   s    !!rA   r1   Nr   r   rq   r   rX   rY   )r[   r\   r]   r^   rm   rr   r   ra   r1   rA   r?   r   r      s    <:"rA   r   c                  J    \ rS rSr SSS.     S	S jjjrS
S jrSS jrSrg)StdoutFeedStorage   Nre   c                   U(       d  [         R                  R                  nX l        U(       a,  UR	                  SS5      SL a  [
        R                  S5        g g g )N	overwriteFTzStandard output (stdout) storage does not support overwriting. To suppress this warning, remove the overwrite option from your FEEDS setting, or set it to False.)sysstdoutbuffer_stdoutrO   loggerwarning)rP   r<   r   r2   s       r?   rQ   StdoutFeedStorage.__init__   sK     jj''G")L,,[%@DHNN I<rA   c                    U R                   $ rI   r   r{   s     r?   rm   StdoutFeedStorage.open   s    ||rA   c                    g rI   r1   r~   s     r?   rr   StdoutFeedStorage.store   s    rA   r   rI   )r<   r   r   zIO[bytes] | Noner2   rE   r   r   r[   r\   r]   r^   rQ   rm   rr   ra   r1   rA   r?   r   r      s<    
 %)
 /3 "
 ,$rA   r   c                  :    \ rS rSrSS.SS jjrS	S jrS
S jrSrg)FileFeedStorage   Nre   c                   UR                  S5      (       a  [        U5      OUU l        U=(       d    0 nUR                  SS5      (       a  SU l        g SU l        g )Nzfile://r   Fwbab)
startswithr   r   rO   
write_moderx   s      r?   rQ   FileFeedStorage.__init__   sP    25..2K2K)#.QT	#)r $$[%88D 	>B 	rA   c                    [        U R                  5      R                  nU(       a$  UR                  5       (       d  UR	                  SS9  [        U R                  5      R                  U R                  5      $ )NT)parents)r   r   parentexistsmkdirrm   r   )rP   rl   dirnames      r?   rm   FileFeedStorage.open   sO    tyy/((7>>++MM$M'DII##DOO44rA   c                $    UR                  5         g rI   )closer~   s     r?   rr   FileFeedStorage.store   s    

rA   )r   r   r   r   r   r   r1   rA   r?   r   r      s    JN 
5rA   r   c                      \ rS rSr    S	SSSS.               S
S jjjr\SS.       SS jj5       rSS jrSrg)S3FeedStorage   N)r2   session_tokenregion_namec          	         SS K n	[        U5      n
U
R                  (       d   eU
R                  U l        U
R                  =(       d    UU l        U
R                  =(       d    UU l	        Xpl
        U
R                  SS  U l        X@l        XPl        Xl        U	R                   R#                  5       nUR%                  SU R                  U R                  U R                  U R                  U R                  S9U l        U(       a,  UR)                  SS5      SL a  [*        R-                  S	5        g g g ! [         a    [        S5      ef = f)
Nr   zmissing boto3 library   s3)aws_access_key_idaws_secret_access_keyaws_session_tokenendpoint_urlr   r   TFzS3 does not support appending to files. To suppress this warning, remove the overwrite option from your FEEDS setting or set it to True.)boto3.sessionImportErrorr   r   hostname
bucketnameusername
access_keypassword
secret_keyr   r   keynameaclr   r   sessionSessionclient	s3_clientrO   r   r   )rP   r<   r   r   r   r   r2   r   r   boto3uboto3_sessions               r?   rQ   S3FeedStorage.__init__   s   	9  SMzzz zz&'jj&>J&'jj&>J)6FF12J"(4'2--/&--"oo"&//"00**(( . 
 L,,[$?5HNND I</  	9 788	9s   D. .Ere   c                   U " UUR                   S   UR                   S   UR                   S   UR                   S   =(       d    S UR                   S   =(       d    S UR                   S   =(       d    S US9$ )NAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYAWS_SESSION_TOKENFEED_STORAGE_S3_ACLAWS_ENDPOINT_URLAWS_REGION_NAME)r   r   r   r   r   r   r2   r   clsr   r<   r2   s       r?   from_crawlerS3FeedStorage.from_crawler   s     ''(;<''(?@!**+>?  !67?4 ))*<=E(():;Ct%	
 		
rA   c                    UR                  S5        U R                  (       a  SSU R                  00O0 nU R                  R                  " SU R                  U R
                  US.UD6  UR                  5         g )Nr   	ExtraArgsACL)BucketKeyFileobjr1   )seekr   r   upload_fileobjr   r   r   )rP   rq   r>   s      r?   r   S3FeedStorage._store_in_thread  sb    		!EIXX+txx/@!ASU%% 	
??d	
FL	
 	

rA   )	r   r   r   r   r   r   r   r   r   )NNNN)r<   r   r   
str | Noner   r   r   r   r   r   r2   rE   r   r   r   r   r   r*   r<   r   r2   rE   rX   r)   r   	r[   r\   r]   r^   rQ   classmethodr   r   ra   r1   rA   r?   r   r      s     "&!%#'* /3$("&** * 	*
 * !* ,* "*  *X  /3

 

 ,
 

 
$rA   r   c                  h    \ rS rSrSS.       SS jjr\SS.       S	S jj5       rS
S jrSrg)GCSFeedStoragei  Nre   c                  X l         X0l        [        U5      nUR                  (       d   eUR                  U l        UR
                  SS  U l        U(       a,  UR                  SS5      SL a  [        R                  S5        g g g )Nr   r   TFzGCS does not support appending to files. To suppress this warning, remove the overwrite option from your FEEDS setting or set it to True.)

project_idr   r   r   bucket_namer   	blob_namerO   r   r   )rP   r<   r   r   r2   r   s         r?   rQ   GCSFeedStorage.__init__  sq     '1"SMzzz !

ffQRjL,,[$?5HNND I<rA   c               Z    U " UUR                   S   UR                   S   =(       d    S US9$ )NGCS_PROJECT_IDFEED_STORAGE_GCS_ACLre   r   r   s       r?   r   GCSFeedStorage.from_crawler-  s:     -.34<%	
 	
rA   c                    UR                  S5        SSKJn  U" U R                  S9nUR	                  U R
                  5      nUR                  U R                  5      nUR                  XR                  S9  g )Nr   )Client)project)predefined_acl)
r   google.cloud.storager  r   
get_bucketr   blobr   upload_from_filer   )rP   rq   r  r   bucketr  s         r?   r   GCSFeedStorage._store_in_thread<  s[    		!/0""4#3#34{{4>>*d88<rA   )r   r   r   r   )r<   r   r   r   r   r   r2   rE   r   r   r   r1   rA   r?   r   r     sv     /3  	 ,,  /3

 

 ,
 

 
=rA   r   c                  j    \ rS rSr SSS.     S	S jjjr\SS.       S
S jj5       rSS jrSrg)FTPFeedStorageiF  Nre   c                  [        U5      nUR                  (       d  [        SU 35      eUR                  U l        [	        UR
                  =(       d    S5      U l        UR                  =(       d    SU l        [        UR                  =(       d    S5      U l        UR                  U l	        X l
        U(       + =(       d    UR                  SS5      U l        g )Nz&Got a storage URI without a hostname: 21 r   T)r   r   
ValueErrorhostintportr   r   r   r   use_active_moderO   r   )rP   r<   r  r2   r   s        r?   rQ   FTPFeedStorage.__init__G  s     SMzzEcUKLL	QVV^t,	ZZ-2$QZZ%526	%4#//V<3C3CKQU3VrA   c               D    U " UUR                   R                  S5      US9$ )NFEED_STORAGE_FTP_ACTIVE)r  r2   )r   getboolr   s       r?   r   FTPFeedStorage.from_crawlerY  s,     #,,445NO%
 	
rA   c                    [        U R                  UU R                  U R                  U R                  U R
                  U R                  U R                  S9  g )N)r   rq   r  r  r   r   r  r   )r!   r   r  r  r   r   r  r   r~   s     r?   r   FTPFeedStorage._store_in_threadg  s@    ]]]] 00nn		
rA   )r  r   r   r   r  r  r   )F)r<   r   r  rZ   r2   rE   r   r   r   r1   rA   r?   r  r  F  s{     !&W
 /3WW W
 ,W$  /3

 

 ,
 

 


rA   r  c                      \ rS rSr                        SS jrS	S jr          S
S jrS	S jrSrg)FeedSlotit  c                    S U l         S U l        Xl        XPl        X0l        X@l        X`l        X l        Xpl        Xl	        Xl
        Xl        Xl        Xl        SU l        SU l        SU l        g )Nr   F)rq   exporterstoragebatch_idformatstore_emptyuri_templater<   filterr2   rl   	exportersr   r   	itemcount
_exporting_fileloaded)rP   r   r<   r"  r#  r!  r$  r%  r2   rl   r&  r   r   s                r?   rQ   FeedSlot.__init__u  sg     '+	15,3%!!,!-"(,8$<E&. ' %!&rA   c           	        U R                   (       d  U R                  R                  U R                  5      U l        SU R
                  ;   aH  [        [        [           [        U R
                  S   U R                  U R
                  5      5      U l        U R                  " S	U R                  U R
                  S   U R
                  S   U R
                  S   U R
                  S   S.U R
                  S   D6U l        SU l         U R                  (       d5  U R                  (       d   eU R                  R                  5         SU l        g g )
Npostprocessingr"  fieldsencodingindent)rq   r"  fields_to_exportr.  r/  item_export_kwargsTr1   )r)  r   rm   rl   rq   r2   r   r
   bytesr   _get_exporterr  r(  start_exportingrP   s    r?   r4  FeedSlot.start_exporting  s   ))$++6DI4#4#44 uI)))*:;		))	 !.. YY((2!%!2!28!<**:6((2 ##$89DM  $D== =MM))+"DO rA   c                R    [        U R                  U   U R                  U/UQ70 UD6$ rI   )r#   r&  r   )rP   rq   r"  r=   r>   s        r?   r3  FeedSlot._get_exporter  s5     "NN6"DLL$
9=
AG
 	
rA   c                    U R                   (       a5  U R                  (       d   eU R                  R                  5         SU l         g g )NF)r(  r  finish_exportingr5  s    r?   r:  FeedSlot.finish_exporting  s1    ??== =MM**,#DO rA   )r(  r)  r!  r   r  r&  r2   rq   r%  r"  r'  r   rl   r   r#  r<   r$  N)r   r/   r<   r   r"  r   r#  rZ   r!  r  r$  r   r%  rC   r2   dict[str, Any]rl   r   r&  z!dict[str, type[BaseItemExporter]]r   r,   r   r*   )rX   rY   )
rq   r   r"  r   r=   r   r>   r   rX   r+   )	r[   r\   r]   r^   rQ   r4  r3  r:  ra   r1   rA   r?   r  r  t  s    "'$"' "' 	"'
 "' "' "' "' %"' "' 5"' "' "'H#6

'*
36
BE
	
$rA   r  c                  .   \ rS rSr% / rS\S'   \SS j5       rSS jrSS jr	SS jr
SS jr          SS	 jr          SS
 jr            SS jrSS jrSS jrSS jrS S jrS!S jr      S"S jr S#       S$S jjrS%S jrSrg)&FeedExporteri  zlist[Deferred[None]]_pending_deferredsc                N   U " U5      nUR                   R                  UR                  [         R                  5        UR                   R                  UR                  [         R
                  5        UR                   R                  UR                  [         R                  5        U$ rI   )r   connectopen_spiderspider_openedclose_spiderspider_closeditem_scraped)r   r   r  s      r?   r   FeedExporter.from_crawler  sp    w< 4 4g6K6KL 5 5w7L7LM 5 5w7K7KLrA   c                   Xl         UR                  U l        0 U l        / U l        0 U l        U R                  S   (       d  U R                  S   (       d  [
        eU R                  S   (       a  [        R                  " S[        SS9  U R                  S   n[        U[        5      (       d  [        U5      OUR                  5       R                  5       nSU R                  R                  SS5      0n[        X0R                  5      U R                  U'   U R!                  U5      U R                  U'   U R                  R#                  S5      R%                  5        H  u  p#[        U[        5      (       d  [        U5      OUR                  5       R                  5       n[        X0R                  5      U R                  U'   U R!                  U5      U R                  U'   M     U R'                  S	5      U l        U R'                  S
5      U l        U R                  R%                  5        H[  u  p#U R-                  X#5      (       d  [
        eU R/                  5       (       d  [
        eU R1                  US   5      (       a  MW  [
        e   g )NFEEDSFEED_URIzThe `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of the `FEEDS` setting. Please see the `FEEDS` setting docs for more detailsr5   r6   r"  FEED_FORMAT	jsonlinesFEED_STORAGESFEED_EXPORTERS)r   r   feedsslotsfiltersr   r9   r:   r   rT   r   r   absoluteas_urirO   r   _load_filtergetdictitems_load_componentsstoragesr&  _storage_supported_settings_are_valid_exporter_supported)rP   r   r<   r2   s       r?   rQ   FeedExporter.__init__  s    '")"2"2
%'
.0}}W%dmmJ.G ==$MM\1	 --
+C",S$"7"7#c(S\\^=R=R=TC$dmm&7&7{&STLHmmDJJsO !% 1 1, ?DLL "&!6!6w!?!E!E!GC",S$"7"7#c(S\\^=R=R=TCHmmDJJsO !% 1 1, ?DLL "H ?C>S>S?
 =A<Q<Q=
 "&!1!1!3C**3==##++--##++L,BCC## "4rA   c                    U R                   R                  5        HG  u  p#U R                  XS   5      nU R                  R	                  U R                  SX$-  UUUS95        MI     g )N
uri_paramsr   r!  r<   r2   rl   r$  )rO  rV  _get_uri_paramsrP  append_start_new_batch)rP   rl   r<   r2   r^  s        r?   rB  FeedExporter.open_spider   sh    !%!1!1!3C--f<6PQJJJ%%(!-!!$ &  "4rA   c                R  #    U R                    H  nU R                  X!5        M     U R                  (       a&  [        [	        U R                  5      5      I S h  vN   [        U R
                  R                  R                  [        R                  5      5      I S h  vN   g  NI N7frI   )	rP  _close_slotr?  r    r   r   r   send_catch_log_deferredfeed_exporter_closed)rP   rl   slots      r?   rD  FeedExporter.close_spider  s     JJDT*  ""*<8O8O+PQQQ 'LL  889U9UV
 	
 	
 R	
s%   AB'B#AB'B%B'%B'c                  ^ ^^ SS jnTR                   (       a  TR                  5         OCTR                  (       a1  TR                  S:X  a!  TR	                  5         TR                  5         Og TR
                   STR                    STR                   3n[        TR                  R                  U" T5      5      mTR                  T R                  XB[        TR                  5      R                  5        TR                  T R                  XB[        TR                  5      R                  5        T R                   R#                  T5        TR                  U U4S j5        TR%                  UU 4S j5        T$ )Nc                    U R                   (       d   e[        U R                   [        5      (       a0  U R                   R                  5         U R                   R                   $ U R                   $ rI   )rq   rT   r   r   )slot_s    r?   get_file*FeedExporter._close_slot.<locals>.get_file  sH    :::%**&;<<

  "zz&::rA   r   z feed (z items) in: c                h   > TR                   R                  R                  [        R                  TS9$ )N)rh  )r   r   rf  feed_slot_closed)_rP   rh  s    r?   <lambda>*FeedExporter._close_slot.<locals>.<lambda>8  s+    dll**BB((t C rA   c                :   > TR                   R                  T5      $ rI   )r?  remove)rq  drP   s    r?   rr  rs  <  s    D33::1=rA   )rl  r  rX   r   )r'  r:  r#  r!  r4  r"  r<   r   r   rr   addCallback_handle_store_successtyper[   
addErrback_handle_store_errorr?  ra  addBoth)rP   rh  rl   rm  logmsgrv  s   ``   @r?   re  FeedExporter._close_slot  s   	 >>!!#$--1"4  "!!# KK='7|DHH:N)$,,*<*<htnM	&&T\\8J8S8S	
 	
$$fd4<<6H6Q6Q	
 	&&q)		

 	
		=>rA   c                    [         R                  SU[        U5      SU0S9  U R                  R                  (       d   eU R                  R                  R                  SU 35        g )NzError storing %srl   )exc_infoextrazfeedexport/failed_count/)r   errorr"   r   stats	inc_value)rP   fr}  rl   	slot_types        r?   r{   FeedExporter._handle_store_error@  s`     	(+V$	 	 	
 ||!!!!$$'?	{%KLrA   c                    [         R                  SUSU0S9  U R                  R                  (       d   eU R                  R                  R	                  SU 35        g )Nz	Stored %srl   )r  zfeedexport/success_count/)r   infor   r  r  )rP   resultr}  rl   r  s        r?   rx  "FeedExporter._handle_store_successL  sN     	K&/AB||!!!!$$'@%LMrA   c                    U R                  X#5      n[        UUUS   US   UUU R                  U   UUU R                  U R                  U R
                  S9$ )a  
Redirect the output data stream to a new file.
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed_options: dict with parameters of feed
:param spider: user spider
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
r"  r#  )r   r<   r"  r#  r!  r$  r%  r2   rl   r&  r   r   )_get_storager  rQ  r&  r   r   )rP   r!  r<   r2   rl   r$  r   s          r?   rb  FeedExporter._start_new_batchS  sd    " ##C6)$]3%<<-%nn]]LL
 	
rA   c                <   / nU R                    GH  nUR                  R                  U5      (       d  UR                  U5        M7  UR	                  5         UR
                  (       d   eUR
                  R                  U5        U=R                  S-  sl        U R                  UR                     S   (       a  UR                  U R                  UR                     S   :  a  U R                  X R                  UR                     S   U5      nU R                  XB5        UR                  U R                  UR                  S-   UR                  U-  U R                  UR                     UUR                  S95        GMr  UR                  U5        GM     X0l         g )Nr   batch_item_countr^  r_  )rP  r%  rV   ra  r4  r  export_itemr'  rO  r$  r`  re  rb  r!  )rP   rU   rl   rP  rh  r^  s         r?   rF  FeedExporter.item_scrapedt  sV   JJD;;&&t,,   "== =MM%%d+NNaN 

4,,-.@ANNdjj1B1B&CDV&WW!11JJt'8'89,G
   .))!%!2 --
:%)ZZ0A0A%B%%)%6%6 *  T"= > 
rA   c                >   [        [        [        [        [        4   U R                  R                  U5      5      5      n0 nUR                  5        H5  u  pE[        R                  " [        5         [        U5      X4'   S S S 5        M7     U$ ! , (       d  f       MJ  = frI   )r%   r   dictr   r   getwithbaserV  
contextlibsuppressr   r$   )rP   setting_prefixconfrv  kvs         r?   rW  FeedExporter._load_components  sw    "c3h!:!:>!JK
 JJLDA$$]3"1~ 43 !  43s   1B
B	c                T    XR                   ;   a  g[        R                  SSU05        g)NTzUnknown feed format: %(format)sr"  F)r&  r   r  )rP   r"  s     r?   r[   FeedExporter._exporter_supported  s&    ^^#668JKrA   c                    U R                   R                  5        HE  u  pUS   (       d  M  [        R                  " SU5      (       a  M/  [        R                  SU5          g   g)z
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
r  z%\(batch_time\)s|%\(batch_id\)a  %%(batch_time)s or %%(batch_id)d must be in the feed URI (%s) if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-countFT)rO  rV  researchr   r  )rP   r$  valuess      r?   rZ   FeedExporter._settings_are_valid  sb    
 %)JJ$4$4$6 L())"))1<3 3 n !	  %7 rA   c                H   [        U5      R                  nX0R                  ;   d  [        U5      R                  (       a   U R                  X5        g[        R                  SSU05        g! [         a,  n[        R                  SU[        U5      S.5         S nAgS nAff = f)NTz<Disabled feed storage scheme: %(scheme)s. Reason: %(reason)s)schemereasonz'Unknown feed storage scheme: %(scheme)sr  F)
r   r  rX  r   driver  r   r   r  r   )rP   r<   r2   r  es        r?   rY  FeedExporter._storage_supported  s    #%%]]"oc&:&@&@!!#4 LLBXvDVW ! R%Q8  s    A+ +
B!5"BB!c                    U R                   R                  [        U5      R                  U R                   S   5      n[	        X0R
                  XS9$ )zQBuild a storage object for the specified *uri* with the specified
*feed_options*.rq   re   )rX  rO   r   r  r#   r   )rP   r<   r2   r   s       r?   r  FeedExporter._get_storage  s>    
 mm 4 4dmmF6KL!#||STTrA   Nc                   0 n[        U5       H  n[        X5      XE'   M     [        R                  " [        R
                  S9nUR                  SS9R                  5       R                  SS5      US'   UR                  5       R                  SS5      US'   Ub  UR                  S-   OSUS	'   U(       a  [        U5      OS
 nU" XA5      nUb  U$ U$ )N)tzr   )microsecond:-time
batch_timer   r!  c                    U $ rI   r1   )paramsrq  s     r?   rr  .FeedExporter._get_uri_params.<locals>.<lambda>  s    6rA   )
r   getattrr   nowr   utcreplace	isoformatr!  r$   )	rP   rl   uri_params_functionrh  r  r  utc_nowuripar_function
new_paramss	            r?   r`  FeedExporter._get_uri_params  s     VA*FI ,,(,,/ Q7AACKKCQTUv&002::3D|262BT]]Q.z # +,) 	
 %V4
'3z??rA   c                P    [        UR                  S[        5      5      nU" U5      $ )Nitem_filter)r$   rO   rC   )rP   r2   item_filter_classs      r?   rT  FeedExporter._load_filter  s*    .9]J7/
 !..rA   )r   r&  rO  rQ  r   rP  rX  )r   r*   rX   r)   )r   r*   )rl   r   rX   rY   )rh  r  rl   r   rX   r   )
r  r(   r}  r   rl   r   r  r   rX   rY   )
r  r   r}  r   rl   r   r  r   rX   rY   )r!  r  r<   r   r2   r<  rl   r   r$  r   rX   r  )rU   r   rl   r   rX   rY   )r  r   rX   r<  )r"  r   rX   rZ   )rX   rZ   )r<   r   r2   r<  rX   rZ   )r<   r   r2   r<  rX   r/   rI   )rl   r   r  zstr | UriParamsCallableT | Nonerh  zFeedSlot | NonerX   r<  )r2   r<  rX   rC   )r[   r\   r]   r^   r?  r`   r   r   rQ   rB  rD  re  r{  rx  rb  rF  rW  r[  rZ  rY  r  r`  rT  ra   r1   rA   r?   r>  r>    sQ   /1,1 1$f
$L
M
M"%
M/5
MBE
M	
MNN#&N06NCFN	N

 
 %	

 
 
 

B!F$UU&4U	U !%	@@ =@ 	@
 
@*/rA   r>  )r;   zCallable[..., _StorageT]r<   r   r=   r   r2   rE   r3   zIterable[Any]r>   r   rX   r.   )]r_   
__future__r   r  loggingr  r   r9   collections.abcr   r   r   pathlibr   r   tempfiler	   typingr
   r   r   r   r   r   r   urllib.parser   r   twisted.internet.deferr   r   r   twisted.internet.threadsr   	w3lib.urlr   zope.interfacer   r   scrapyr   r   scrapy.exceptionsr   r    scrapy.extensions.postprocessingr   scrapy.utils.confr   scrapy.utils.deferr    scrapy.utils.ftpr!   scrapy.utils.logr"   scrapy.utils.miscr#   r$   scrapy.utils.pythonr%   r&   	_typeshedr'   twisted.python.failurer(   typing_extensionsr)   scrapy.crawlerr*   scrapy.exportersr+   scrapy.settingsr,   r-   	getLoggerr[   r   r  r   UriParamsCallableTr.   r@   rC   rc   r/   r   r   r   r   r   r  r  r>  r1   rA   r?   <module>r     s   #   	 
  $ ' ) ' L L L * H H 2 & 1 " E B H 7 + 0 = 3((. '&16 
		8	$tCH~v6c3h8PPQ K'<=	 +/3%3	3 3 (	3
 3 3 3"! !H*9 *"*( * \" " " \  4 \  &F' FR-=( -=`+
( +
\K$ K$\h/ h/rA   