
    !h                        S SK Jr  S SKrS SKrS SKrS SKJr  S SKJr  S SK	J
r
JrJr  S SKJr  S SKrS SKJr  S SKJr  S S	KJrJr  \
(       a  S SKrSS
 jrSS jrSS jr " S S\5      rg)    )annotationsN)import_module)Path)TYPE_CHECKINGAnycast)urlparse)ScrapyCommand)
UsageError)render_templatefilestring_camelcasec                    U R                  SS5      R                  SS5      n U S   [        R                  ;  a  SU -   n U $ )zSanitize the given module name, by replacing dashes and points
with underscores and prefixing it with a letter if it doesn't start
with one
-_.r   a)replacestringascii_letters)module_names    `/root/1688_scrapy/alibaba-scraper/venv/lib/python3.13/site-packages/scrapy/commands/genspider.pysanitize_module_namer      sE    
 %%c3/77SAK1~V111K'    c                    [        U 5      nUR                  S:X  a-  UR                  S:X  a  [        SU R                  S5      -   5      nUR                  $ )z#Extract domain name from URL string ///)r	   schemenetloclstrip)urlos     r   extract_domainr#      s@    Axx2~!((b.TCJJsO+,88Or   c                    [        U 5      nUR                  S:X  a+  UR                  S:X  a  [        SU -   5      R                  SS9nUR	                  5       $ )z4Check url for scheme and insert https if none found.r   r   https)r   )r	   r   r   _replacegeturl)r!   parseds     r   verify_url_schemer)   '   sJ    c]F}}v}}2$*%..g.>==?r   c                     ^  \ rS rSrSrSS0rSS jrSS jrSU 4S jjrSS jr	          SS jr
            SS	 jrSS
 jrSS jrSS jr\SS j5       rSrU =r$ )Command/   FLOG_ENABLEDc                    g)Nz[options] <name> <domain> selfs    r   syntaxCommand.syntax3   s    *r   c                    g)Nz/Generate new spider using pre-defined templatesr/   r0   s    r   
short_descCommand.short_desc6   s    @r   c                   > [         TU ]  U5        UR                  SSSSSS9  UR                  SSS	SS
S9  UR                  SSSSSS9  UR                  SSSSSS9  UR                  SSSSS9  g )Nz-lz--listlist
store_truezList available templates)destactionhelpz-ez--editeditzEdit spider after creating itz-dz--dumpdumpTEMPLATEz Dump template to standard output)r:   metavarr<   z-tz
--templatetemplatebasiczUses a custom template.)r:   defaultr<   z--forceforcez<If the spider already exists, overwrite it with the template)superadd_optionsadd_argument)r1   parser	__class__s     r   rF   Command.add_options9   s    F#+ 	 	
 	0 	 	
 	3 	 	
 	* 	 	
 	O	 	 	
r   c                   UR                   (       a  U R                  5         g UR                  (       a;  U R                  UR                  5      nU(       a  [	        UR                  SS95        g [        U5      S:w  a  [        eUSS u  pE[        U5      n[        U5      nU R                  R                  S5      U:X  a  [	        S5        g UR                  (       d  U R                  U5      (       a  g U R                  UR                  5      nU(       aO  U R                  XdXRR                  U5        UR                   (       a   ["        R$                  " SU S35      U l        g g g )	Nzutf-8)encoding   r   BOT_NAMEz9Cannot create a spider with the same name as your projectzscrapy edit "")r8   _list_templatesr>   _find_templateprint	read_textlenr   r)   r   settingsgetrD   _spider_existsrA   
_genspiderr=   ossystemexitcode)r1   argsoptstemplate_filenamer!   modules          r   runCommand.run^   s   99  "99 //		:Mm--w-?@t9>1I	$%d+==Z(F2MNzzd11$77++DMM:OOF#}}mLyy "		M$q*A B  r   c                    SR                  S UR                  S5       5       5      nU R                  R                  S5      [	        U R                  R                  S5      5      UUU[        U5      U S3S.$ )Nr   c              3  @   #    U  H  oR                  5       v   M     g 7f)N)
capitalize).0ss     r   	<genexpr>7Command._generate_template_variables.<locals>.<genexpr>   s     $O=N\\^^=Ns   r   rN   Spider)project_nameProjectNamer`   r_   r!   domain	classname)joinsplitrU   rV   r   r#   )r1   r`   r_   r!   template_namecapitalized_modules         r   _generate_template_variables$Command._generate_template_variables{   sq      WW$OV\\#=N$OO MM--j9+DMM,=,=j,IJ$S)./v6
 	
r   c                   U R                  XX45      nU R                  R                  S5      (       aY  [        U R                  S   5      nUR                  (       d   e[        UR                  5      R                  R                  5       nOSn[        5       nX-   S3n	[        R                  " XY5        [        U	40 UD6  [        SU< SU< S3U(       a  SOSS	9  U(       a  [        S
UR                   SU 35        gg)z7Generate the spider module, based on the given templateNEWSPIDER_MODULEN.pyzCreated spider z using template  r   
)endzin module:
  r   )rs   rU   rV   r   __file__r   parentresolveshutilcopyfiler   rR   __name__)
r1   r`   r_   r!   rq   r^   tvarsspiders_modulespiders_dirspider_files
             r   rX   Command._genspider   s     11&S==/00*4==9K+LMN!****~667>>FFHK!N&K$-.c23K151dX%5m5FaH%4	
 N>#:#:";1VHEF r   c                    [        U R                  U S35      nUR                  5       (       a  U$ [        SU S35        [        S5        g )N.tmplzUnable to find template: ry   z=Use "scrapy genspider --list" to see all available templates.)r   templates_direxistsrR   )r1   rA   r^   s      r   rQ   Command._find_template   sN    T//H:U1CD!!  )(267MNr   c                    [        S5        [        [        U R                  5      R	                  5       5       H-  nUR
                  S:X  d  M  [        SUR                   35        M/     g )NzAvailable templates:r     )rR   sortedr   r   iterdirsuffixstem)r1   files     r   rP   Command._list_templates   sM    $%4 2 23;;=>D{{g%499+&' ?r   c                   U R                   R                  S5      (       dA  [        US-   5      nUR                  5       (       a  [	        UR                  5        S35        ggU R                  c   S5       e U R                  R                  R                  U5      n[	        SU< S35        [	        S	UR                   35        g! [         a     Of = f[        U R                   S   5      n[        [        [        UR                  5      5      R                  nUR                  5       nXaS-   -  nUR                  5       (       a  [	        U S35        gg)
Nrv   rw   z already existsTFz.crawler_process must be set before calling runzSpider z already exists in module:r   )rU   rV   r   r   rR   r}   crawler_processspider_loaderload
__module__KeyErrorr   r   strr{   r|   )r1   r_   path	spiderclsr   r   spiders_dir_abss          r   rW   Command._spider_exists   s8   }}  !344u%D{{}}(89##/ 	
<	
/	,,::??EI
 GD8#=>?By++,-.  		 't}}5G'HI4^%<%<=>EE%--/,/;;==TF/*+s   7%C 
CCc                    [        [        U R                  S   =(       d    [        [        R                  S   S5      S5      5      $ )NTEMPLATES_DIRr   	templatesspiders)r   r   rU   scrapy__path__r0   s    r   r   Command.templates_dir   s;    o.W$vq7I;2W
 	
r   )r[   )returnr   )rH   zargparse.ArgumentParserr   None)r\   z	list[str]r]   zargparse.Namespacer   r   )
r`   r   r_   r   r!   r   rq   r   r   zdict[str, Any])r`   r   r_   r   r!   r   rq   r   r^   zstr | os.PathLiker   r   )rA   r   r   zPath | None)r   r   )r_   r   r   bool)r   r   __qualname____firstlineno__requires_projectdefault_settingsr2   r5   rF   ra   rs   rX   rQ   rP   rW   propertyr   __static_attributes____classcell__)rI   s   @r   r+   r+   /   s    %u-+A#
JC:

 
 	

 
 

$GG G 	G
 G )G 
G6( D 
 
r   r+   )r   r   r   r   )r!   r   r   r   )
__future__r   rY   r~   r   	importlibr   pathlibr   typingr   r   r   urllib.parser	   r   scrapy.commandsr
   scrapy.exceptionsr   scrapy.utils.templater   r   argparser   r#   r)   r+   r/   r   r   <module>r      sO    " 	   #  + + !  ) ( Gp
m p
r   