
    h-                     z    S SK r S SKJr  S SKrS SKrS SKJr  S SKrS SKrS SK	J
r
   " S S\ R                  5      rg)    N)Request)	Extractor)Pathc                      ^  \ rS rSrSrS/rS/r\R                  " \	R                  R                  \	R                  R                  \5      S5      5      rSrU 4S jrS rS	 rS
rU =r$ )AlibabaCrawlerSpider   alibaba_crawlerz1688.comzhttps://www.1688.com/z../resources/search_results.yml   c                    > [         [        U ]
  " U0 UD6  [        S5      U l        [        S5      U l        U R                  R                  SSS9  U R
                  R                  SSS9  SU l        g )Nzdata/html_rawzdata/json_structuredT)parentsexist_okr   )superr   __init__r   html_dirjson_dirmkdirpage_counter)selfargskwargs	__class__s      K/root/1688_scrapy/alibaba-scraper/scrapy_alibaba/spiders/alibaba_crawler.pyr   AlibabaCrawlerSpider.__init__   sg    "D2DCFC_-34D48D48    c           	   #     #    [        [        R                  R                  [        R                  R	                  [
        5      S5      5       n[        R                  " U5       H=  nUS   nSR                  U5      n[        R                  " X@R                  USS.S9v   M?     SSS5        g! , (       d  f       g= f7f)zBRead keywords from keywords file and construct the 1688 search URLz../resources/keywords.csvkeywordz:https://s.1688.com/selloffer/offer_search.htm?keywords={0}   search_textpage_numcallbackmetaN)openospathjoindirname__file__csv
DictReaderformatscrapyr   parse)r   search_keywordsr   r   urls        r   start_requests#AlibabaCrawlerSpider.start_requests   s      "'',,rwwx8:UVW[j>>/:#I.PWW! nnSZZXcqrHstt ; XWWs   AB?AB.%	B?.
B<8B?c              #   v  #    U =R                   S-  sl         UR                  R                  SS5      nUR                  R                  SS5      n[        R                  " SSU5      R                  5       R                  SS5      nU R                  U S	U SU R                    S
3-  n[        USSS9 nUR                  UR                  5        S S S 5        U R                  R                  SU 35        / nUR                  S5      nU H  n	U	R                  S5      R                  5       U	R                  S5      R                  5       U	R                  S5      R                  5       UR                  U	R                  S5      R                  5       5      U	R                  S5      R                  5       UU[        U5      S.n
UR!                  U
5        U
v   M     U R"                  U S	U SU R                    S3-  n[        USSS9 n[$        R&                  " XvSSS9  S S S 5        U R                  R                  S[)        U5       SU 35        [)        U5      S:  a6  X R*                  :  a&  US-   nSU SU 3n[-        XR.                  X<S.S 9v   g g g ! , (       d  f       GN= f! , (       d  f       N= f7f)!Nr   r    r   unknownz[^\w\s-]  __pagez.htmlwzutf-8)encodingzSaved HTML to z*.sw-offer-item, .sw-dpl-offer, .offer-itemz*.offer-item-title::text, .title-text::textz.price-text::text, .price::textz0.company-name::text, .sw-dpl-offer-company::textza::attr(href)zimg::attr(src))namepriceseller_namelinkimagesearch_keywordpage_number	html_filez.jsonF   )ensure_asciiindentzSaved z products to r   z7https://s.1688.com/selloffer/offer_search.htm?keywords=z&beginPage=r   r!   )r   r#   getresubstripreplacer   r$   writetextloggerinfocssurljoinstrappendr   jsondumplen	max_pagesr   r.   )r   responser    r   clean_searchhtml_filenamefproductsproduct_cardscardproductjson_filename	next_pagenext_urls                 r   r.   AlibabaCrawlerSpider.parse(   sn    Q==$$Z3mm''yA vvk2{;AACKKCQTU<.hZqIZIZH[[`(aa -w71GGHMM" 8 	>-9:  %QR!D!MNRRT"CDHHJ#xx(Z[__a ((/)B)F)F)HI"23779"-' /	G OOG$M " <.hZqIZIZH[[`(aa-w71IIha@ 8 	6#h-m_MN x=1NN!: 1IPQ\P]]hirhstH(ZZk>qrr	 ";A 874 87s8   B1J93JD8J9J(A8J9
J% J9(
J62J9)r   r   r   )__name__
__module____qualname____firstlineno__r;   allowed_domains
start_urlsr   from_yaml_filer%   r&   r'   r(   r)   	extractorrV   r   r1   r.   __static_attributes____classcell__)r   s   @r   r   r      s`    D!lO)*J((bggooh6OQr)stII
u/s /sr   r   )r-   scrapy.httpr   r*   r%   selectorlibr   rG   rS   pathlibr   Spiderr    r   r   <module>rr      s1      
 	 ! 	  Ls6== Lsr   