a oe0@slddlZddlZddlZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZddlmZdd lmZmZmZmZmZmZeed eed d d Zeed eed ddZeed eeed ddZeed eed ddZ eed eed ddZ!eed eed ddZ"eed eed ddZ#eed eed ddZ$eed eed ddZ%eed eed ddZ&eed eed d d!Z'eed eed d"d#Z(eed eed d$d%Z)eed eed d&d'Z*eed eed d(d)Z+eed eed d*d+Z,ee-ed eed,d-d.Z.eed eed d/d0Z/dSe0e1eed2d3d4Z2ed5d eed6d7d8Z3e0eeee0fd9d:d;Z4eed<d=d>Z5dTeeed@dAdBZ6ee edCdDdEZ7eee8dFdGdHZ9eeedFdIdJZ:dKej;dLfee1eddMdNdOZdS)VN)IncrementalDecoder)aliases) lru_cache)findall) GeneratorListOptionalSetTupleUnion)MultibyteIncrementalDecoder)ENCODING_MARKSIANA_SUPPORTED_SIMILARRE_POSSIBLE_ENCODING_INDICATIONUNICODE_RANGES_COMBINEDUNICODE_SECONDARY_RANGE_KEYWORDUTF8_MAXIMAL_ALLOCATION)maxsize) characterreturncCsdzt|}Wnty"YdS0d|vpbd|vpbd|vpbd|vpbd|vpbd|vpbd|vpbd |vS) NFz WITH GRAVEz WITH ACUTEz WITH CEDILLAzWITH DIAERESISzWITH CIRCUMFLEXz WITH TILDEz WITH MACRONzWITH RING ABOVE unicodedataname ValueErrorr descriptionr9C:\Program Files\Certbot\pkgs\charset_normalizer\utils.pyis_accentuateds& rcCs.t|}|s|S|d}tt|ddS)N r)rZ decompositionsplitchrint)rZ decomposedZcodesrrr remove_accent(s   r%cCs.t|}tD]\}}||vr|SqdS)zK Retrieve the Unicode range official name from a single character. N)ordritems)rZ character_ord range_nameZ ord_rangerrr unicode_range3s  r)cCs,zt|}Wnty"YdS0d|vS)NFZLATINrrrrris_latinAs  r*cCs2t|}d|vrdSt|}|dur*dSd|vS)NPTFZ Punctuationrcategoryr)rcharacter_categorycharacter_rangerrris_punctuationJs r1cCsBt|}d|vsd|vrdSt|}|dur2dSd|vo@|dkS)NSNTFZFormsZLor,r.rrr is_symbolYs r4cCs$t|}|durdSd|vp"d|vS)NFZ EmoticonsZ Pictographs)r))rr0rrr is_emoticonhsr5cCs.|s|dvrdSt|}d|vp,|dvS)N>+TZ>ZPdZPoZPc)isspacerr-)rr/rrr is_separatorrs r;cCs||kSN)islowerisupperrrrris_case_variable|sr@cCs,zt|}Wnty"YdS0d|vS)NFZCJKrrZcharacter_namerrris_cjks  rBcCs,zt|}Wnty"YdS0d|vS)NFZHIRAGANArrArrr is_hiraganas  rCcCs,zt|}Wnty"YdS0d|vS)NFZKATAKANArrArrr is_katakanas  rDcCs,zt|}Wnty"YdS0d|vS)NFZHANGULrrArrr is_hanguls  rEcCs,zt|}Wnty"YdS0d|vS)NFZTHAIrrArrris_thais  rFcCs,zt|}Wnty"YdS0d|vS)NFARABICrrArrr is_arabics  rHcCs4zt|}Wnty"YdS0d|vo2d|vS)NFrGz ISOLATED FORMrrArrris_arabic_isolated_forms  rI)r(rcstfddtDS)Nc3s|]}|vVqdSr<r).0keywordr(rr z-is_unicode_range_secondary..)anyrrLrrLris_unicode_range_secondarysrPcCs(|duo&|duo&|dko&|dkS)NFu)r: isprintabler?rrris_unprintables  rS )sequence search_zonercCst|tstt|}tt|dt||jddd}t|dkrHdS|D]N}| dd}t D]0\}}||kr|S||krh|SqhqLdS)zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nasciiignoreerrorsr-_) isinstancebytes TypeErrorlenrrmindecodelowerreplacerr')rUrVZseq_lenresultsZspecified_encodingencoding_alias encoding_ianarrrany_specified_encodings"   rh)rrcCs |dvpttd|jtS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_32_beutf_8 utf_32_leZ utf_8_sigutf_16utf_32 utf_16_be utf_16_leutf_7 encodings.{}) issubclass importlib import_moduleformatrr )rrrris_multi_byte_encodings  rw)rUrcCsJtD]@}t|}t|tr |g}|D]}||r$||fSq$qdS)z9 Identify and extract SIG/BOM in given sequence. )NrN)rr]r^ startswith)rU iana_encodingZmarksZmarkrrridentify_sig_or_bom s  rz)ryrcCs|dvS)N>rmrnr)ryrrrshould_strip_sig_or_bomsr{T)cp_namestrictrcCsL|dd}tD]\}}|||fvr|Sq|rHtd||S)Nr[r\z Unable to retrieve IANA for '{}')rcrdrr'rrv)r|r}rfrgrrr iana_name"s  r~)decoded_sequencercCs4t}|D] }t|}|dur q ||q t|Sr<)setr)addlist)rZrangesrr0rrr range_scan2s r) iana_name_a iana_name_brc Cst|st|rdStd|j}td|j}|dd}|dd}d}tdD]*}t|g}||||krX|d7}qX|dS) NgrrrXrYrr )rwrtrurvrranger^rb) rrZ decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedrrr cp_similarity@s      rcCs|tvo|t|vS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r)rrrrr is_cp_similarXs rZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)s)rlevel format_stringrcCs:t|}||t}|t|||dSr<)loggingZ getLoggerZsetLevelZ StreamHandlerZ setFormatterZ FormatterZ addHandler)rrrloggerZhandlerrrrset_logging_handlercs   r) sequencesrgoffsets chunk_sizebom_or_sig_availablestrip_sig_or_bom sig_payloadis_multi_byte_decoderdecoded_payloadrc cs*|r6|dur6|D]"} || | |} | s,q4| Vqn|D]} | |} | t|dkrXq:|| | |} |r||dur||| } | j||rdndd} |r| dkrt|d} |r| d| |vrt| | dd D]H}||| } |r|dur|| } | j|dd} | d| |vrqq| Vq:dS) NFrXr}rYrr!)r`rbrar)rrgrrrrrrrrchunkZ chunk_endZ cut_sequenceZchunk_partial_size_chkjrrrcut_sequence_chunksps>       r)rT)T)N)?rtrrcodecsrZencodings.aliasesr functoolsrrertypingrrrr r r Z_multibytecodecr Zconstantrrrrrrstrboolrr%r)r*r1r4r5r;r@rBrCrDrErFrHrIr`rPrSr^r$rhrwrzr{r~rfloatrrINFOrrrrrrrs