
    !h                        S r SSKJr  SSKrSSKJr  SSKJr  SSKJ	r	  \(       a  SSK
Jr  SSKJr  SS	KJrJr  \R"                  " \5      r " S
 S\	5      rg)zV
Url Length Spider Middleware

See documentation in docs/topics/spider-middleware.rst
    )annotationsN)TYPE_CHECKING)NotConfigured)BaseSpiderMiddleware)Self)Crawler)RequestResponsec                  T    \ rS rSr% S\S'   S	S jr\S
S j5       r      SS jrSr	g)UrlLengthMiddleware   r   crawlerc                    Xl         g )N	maxlength)selfr   s     i/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/spidermiddlewares/urllength.py__init__UrlLengthMiddleware.__init__   s    '    c                r    UR                   R                  S5      nU(       d  [        eU " U5      nXl        U$ )NURLLENGTH_LIMIT)settingsgetintr   r   )clsr   r   os       r   from_crawler UrlLengthMiddleware.from_crawler    s5    $$++,=>		N	r   c                |   [        UR                  5      U R                  ::  a  U$ [        R	                  SU R                  UR                  S.SU R
                  R                  0S9  U R
                  R                  (       d   eU R
                  R                  R                  SU R
                  R                  S9  g )Nz4Ignoring link (url length > %(maxlength)d): %(url)s )r   urlspider)extrazurllength/request_ignored_count)r!   )	lenr    r   loggerinfor   r!   stats	inc_value)r   requestresponses      r   get_processed_request)UrlLengthMiddleware.get_processed_request)   s     w{{t~~-NB..=T\\001 	 	

 ||!!!!$$-dll6I6I 	% 	
 r   r   N)r   int)r   r   returnr   )r(   r	   r)   zResponse | Noner-   zRequest | None)
__name__
__module____qualname____firstlineno____annotations__r   classmethodr   r*   __static_attributes__ r   r   r   r      s>    (  *9	r   r   )__doc__
__future__r   loggingtypingr   scrapy.exceptionsr   scrapy.spidermiddlewares.baser   typing_extensionsr   scrapy.crawlerr   scrapy.httpr	   r
   	getLoggerr.   r$   r   r5   r   r   <module>r@      sF    #    + >&&- 
		8	$. r   