*4ddlZddlmZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZdd lmZmZdd lmZdd lmZmZmZGd d ZGddZeeefZ ee Z!GddZ"y)N)Counter)aliases)sha256)dumps)sub)AnyDictIteratorListOptionalTupleUnion)NOT_PRINTABLE_PATTERNTOO_BIG_SEQUENCE) mess_ratio) iana_nameis_multi_byte_encoding unicode_rangecneZdZ d)dededededddeef d Zd e d efd Z d e d efd Z e d efdZ e d efdZe d efdZe d efdZd efdZd efdZd*dZe d efdZe d eefdZe d efdZe d efdZe d eefdZe d efdZe d efdZe d efdZe d efdZe d efdZe d efdZ e d edfd Z!e d efd!Z"e d eefd"Z#e d eefd#Z$d+d$Z%d+d%Z&d,d&ed efd'Z'e d efd(Z(y)- CharsetMatchNpayloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesCoherenceMatchesdecoded_payloadc||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ y)N) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leaves_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrs 4platform/bq/third_party/charset_normalizer/models.py__init__zCharsetMatch.__init__sW  ) /#-# %("# $& otherreturnc t|tsAtdjt |j t |j |j |j k(xr|j|jk(S)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprintr,r0s r-__eq__zCharsetMatch.__eq__)si%.8??(#dnn*=  }}.X43C3CuGXGX3XXr/ct|tstt|j|jz }t|j |j z }|dkrU|dkDrP|dk(r2|j |j k(r|j |j kDS|j |j kDS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?r )r3r ValueErrorabschaos coherencemulti_byte_usage)r,r0chaos_differencecoherence_differences r-__lt__zCharsetMatch.__lt__2s%. tzzEKK78"4>>EOO#CD d "';d'B3&4>>U__+L,,u/E/EEE>>EOO3 3zzEKK''r/c\dtt|t|jz z S)N?)lenr6rawr,s r-rAzCharsetMatch.multi_byte_usageEs"ST^c$((m333r/c`tjdttt |dS)z Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 z=chaos_secondary_pass is deprecated and will be removed in 3.0rF)warningswarnDeprecationWarningrr6rIs r-chaos_secondary_passz!CharsetMatch.chaos_secondary_passIs)  K  #d)S))r/c8tjdty)zy Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 z)r5r8r9rIs r-__repr__zCharsetMatch.__repr__vs.55dmmTEUEUVVr/ct|tr||k(r$tdj|jd|_|j j|y)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r3rr=r5r7r+r'appendr:s r- add_submatchzCharsetMatch.add_submatchysO%.%4-MTTOO    E"r/c|jSN)r"rIs r-r8zCharsetMatch.encodings ~~r/cg}tjD]G\}}|j|k(r|j|'|j|k(s7|j|I|S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr8r])r, also_known_asups r-encoding_aliaseszCharsetMatch.encoding_aliasessY  MMODAq}}!$$Q'!#$$Q' $ r/c|jSr`r%rIs r-bomzCharsetMatch.bom###r/c|jSr`rhrIs r-byte_order_markzCharsetMatch.byte_order_markrjr/cF|jDcgc]}|d c}Scc}w)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. rr$)r,es r-rzCharsetMatch.languagess$ #oo.o!o...s c|jshd|jvryddlm}m}t |j r||j n||j }t|dk(sd|vry|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)r$could_be_from_charsetcharset_normalizer.cdrsrtrr8rG)r,rsrtrs r-languagezCharsetMatch.languages $444  X*$--8&dmm4' 6  9~"my&@ Q< q!!$$r/c|jSr`)r#rIs r-r?zCharsetMatch.chaoss$$$r/c@|jsy|jddS)Nr rrrnrIs r-r@zCharsetMatch.coherences q!!$$r/c6t|jdzdSNd)ndigits)roundr?rIs r- percent_chaoszCharsetMatch.percent_chaossTZZ#%q11r/c6t|jdzdSr|)rr@rIs r-percent_coherencezCharsetMatch.percent_coherencesT^^c)155r/c|jS)z+ Original untouched bytes. )r!rIs r-rHzCharsetMatch.raws }}r/c|jSr`)r'rIs r-submatchzCharsetMatch.submatchs ||r/c2t|jdkDSNr)rGr'rIs r- has_submatchzCharsetMatch.has_submatchs4<< 1$$r/c|j |jSt|Dcgc] }t|}}tt |Dchc]}|s| c}|_|jScc}wcc}wr`)r&r6rsortedlist)r,chardetected_rangesrs r- alphabetszCharsetMatch.alphabetss}    +'' '-0I ,5DM$ I   &d+L!!A+L&MN###  ,MsA0A5 A5 cp|jg|jDcgc]}|jc}zScc}w)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. )r"r'r8)r,ms r-rvz"CharsetMatch.could_be_from_charsets0t||"D|!1::|"DDD"Ds3c|Sz> Kept for BC reasons. Will be removed in 3.0. rIs r-firstzCharsetMatch.first  r/c|SrrrIs r-bestzCharsetMatch.bestrr/r8c|j|j|k7r'||_t|j|d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. replace)r*r6encoder))r,r8s r-outputzCharsetMatch.outputsJ  (D,A,AX,M$,D !#&t9#3#3Hi#HD ###r/cPt|jjS)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rr hexdigestrIs r-r9zCharsetMatch.fingerprints dkkm$..00r/r`)r0rr1N)r1r)utf_8))__name__ __module__ __qualname__bytesr6floatboolr r.objectr;rDpropertyrArNrPrrVrYr[r^r8r rfrirlrrxr?r@rrrHrrrrvrrrr9rr/r-rrs*.''' '  ' & '"#'2YFYtY(F(t(&4%44 *e * * U   67 6 6 W#W ## $s)  $T$$$$$/49//%#%%6%u%%%5%% 2u226566U $~.%d%% $49 $ $EtCyEE  $s $ $1S11r/rceZdZdZddeefdZdeefdZde e e fdefdZ de fd Z defd Zdeddfd Zded fd Zded fdZy)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. Nresultsc8|rt||_yg|_yr`)r_results)r,rs r-r.zCharsetMatches.__init__s+2w  r/r1c#6K|jD]}|ywr`r)r,results r-__iter__zCharsetMatches.__iter__smmFL$sitemct|tr|j|St|tr/t |d}|jD]}||j vs|cSt )z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. F)r3intrr6rrvKeyError)r,rrs r- __getitem__zCharsetMatches.__getitem__#s[ dC ==& & dC T5)D--6777!M(r/c,t|jSr`rGrrIs r-__len__zCharsetMatches.__len__1s4==!!r/c2t|jdkDSrrrIs r-__bool__zCharsetMatches.__bool__4s4==!A%%r/ct|ts-tdjt |j t |jtkrW|jD]H}|j|jk(s|j|jk(s7|j|y|jj|t|j|_ y)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r3rr=r5r6r7rGrHrrr9r?r^r]r)r,rmatchs r-r]zCharsetMatches.append7s $ -?FF'  txx=, ,$$(8(88U[[DJJ=V&&t,' T"t}}- r/rc:|jsy|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. NrrrIs r-rzCharsetMatches.bestKs}}}}Qr/c"|jS)zP Redundant method, call the method best(). Kept for BC reasons. )rrIs r-rzCharsetMatches.firstSsyy{r/r`)rrr__doc__r rr.r rrrr6rrrrr]r rrrr/r-rrs ;\ 2;(<0 c3h L ""&$&.<.D.( h~. x/r/rceZdZdedeedeedeededeededed ed eed efd Ze d e ee ffdZ d efdZ y)CliDetectionResultpathr8rfalternative_encodingsrxrrr?r@ unicode_path is_preferredc ||_| |_||_||_||_||_||_||_||_| |_ | |_ yr`) rrr8rfrrxrrr?r@r) r,rr8rfrrxrrr?r@rrs r-r.zCliDetectionResult.__init___sV (  0%:"  ", "(r/r1c |j|j|j|j|j|j |j |j|j|j|jd S)N rr8rfrrxrrr?r@rrrrIs r-__dict__zCliDetectionResult.__dict__yseII $ 5 5%)%?%? "11ZZ -- --  r/c2t|jddS)NT) ensure_asciiindent)rrrIs r-to_jsonzCliDetectionResult.to_jsonsT]]a@@r/N)rrrr6r r rrr.rr rrrrr/r-rr^s))3-)s) ) $Cy )  )9))))sm))4  $sCx.    AAr/r)#rK collectionsrencodings.aliasesrhashlibrjsonrrertypingrr r r r r rconstantrrmdrutilsrrrrrr6rCoherenceMatchrrrr/r-rsm%DDD=CCD1D1NAAHsEz"',A,Ar/