Gi,dZddlZddlZddlZddlmZddlmZddlm Z ddl m Z ddl m ZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZdd lmZm Z m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2ddl3m4Z4m5Z5m6Z6e6jne8Z9dZ:dZ;dZe,dz Z,ee e!e"dZ?e:e=dZ@e5e,Gdde/ZAeAZBy)z Tokenization classes for fast tokenizers (provided by HuggingFace's tokenizers library). For slow (python) tokenizers see tokenization_utils.py N) defaultdict)Iterable)copyfile)Any)is_offline_mode) AddedToken processors)Encoding) Tokenizer)Decoder)BPEUnigram) BpeTrainerUnigramTrainerWordLevelTrainerWordPieceTrainer cached_file) SpmConverter)convert_gguf_tokenizer)load_gguf_checkpoint)INIT_TOKENIZER_DOCSTRING BatchEncodingPreTokenizedInputPreTrainedTokenizerBase TextInputTruncationStrategygenerate_merges)PaddingStrategyadd_end_docstringsloggingztokenizer.jsonzspecial_tokens_map.jsonztokenizer_config.jsonztokenizer.modelzadded_tokens.jsonu tokenizer_object ([`tokenizers.Tokenizer`]): A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗 tokenizers](../fast_tokenizers) for more information. tokenizer_file ([`str`]): A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗 tokenizers. )r r WordLevel WordPiece)tokenizer_file vocab_filec) eZdZdZeZdZdZedLdZ fdZ e de fdZ e de fdZdMd ed edzdeefd Zd Ze d Ze dZej*dZej*dZdZe defdZdeeeffdZe deeeffdZe deeeffdZe deeeffdZeZeZ deeeffdZ!de fdZ"defdZ#e de$fdZ%e de&fdZ' dNde(de dzde dzd e d!e d"e d#e d$e deeee)fe*e(ffd%Z+d&edefd'Z,d(ededzfd)Z-dLd*e*eezdefd+Z.dLd,e defd-Z/dLd.ee*ezd/e dee*ezfd0Z0dOd1ed,edzd2e de*efd3Z1d4e2d5e3d6ed7ed8edzd9edzf d:Z4dde2jje3jldd;ddddddddddddfd1e7e8ze*e7ze*e8zde dzde dzde dzd e d!e d"e d#e d$e d?e dzde9f(d@Z:dAe*edefdBZ; dPdCee*ezd/e dDe dzdefdEZ< dQd ee=j|zdFeedGfdHe dzd edzdeedGff dIZ? dRdJZ@e dSdKZAxZBS)TTokenizersBackendaQ Base class for all fast tokenizers (wrapping HuggingFace tokenizers library). Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`]. Handles all the shared methods for tokenization and special tokens, as well as methods for downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary. This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...). NFc ^(t|}|jdd}|Qtjj |r2|t usd|j vs|rtj||d<|S|tjj |rt|d5}tj|}dddjdijd}|d vr[t|}t|d} i| d <|d k(rg| d <| |d<g|d <tjtj|} ntj|} | j|d<| j |d<| j"|d<| j"| j"|d<| j | j |d<|jd} | rk| jdddk(r| d} nt%| t&s| g} | D]8} | jddk(sd| vsddl} | j+| d|d<n|jdijd d}|j,,t%|t&r%t't/t0|}n |j,j2dk(rDt%|t&r|rt%|dt&t0fr|Dcgc] }t1|}}n|j,j2dk(rt5|Dcic]\}}|| }}}nu|j,j2d k(s|j,j2dk(rCt%|t&r3t5|Dcic]\}}t%|t&r|dn||}}}||d <t7|dd}d |jdivr`|r^|j2d k(rO|dd }|Dcgc]7}t%|t8rt1|j;dn t1|9}}||d <|S|jd}|jd}|jd }|jd }t%|t8rY|j=d rHtjj |r)d!d"lm }||#jC|\|d <|d <|St%|t8rntjj |rN|j=d$r< d!d%lm"}||}|jF|j,fi|} d!d&lm$}|j|j2}|tK|d'r|jLd?i|}tK|d+r|jTd?i|}d|vr|t usd|j vr|jd d}|jd d}|jd,xsi}||r|jWDcic]\}}|| }}}|jWD]S\}}tY|}t9|}|j|} | s0| |k7s6||vs;|j| ||<|||<Ut[j\|j^||-}!|!|!|d<|j^j`}"|"jbdk\r |jed.|"jfxsd/|"jhdk\r |jed0|"jjxsd1|"jldk\r |jed2|"jnxsd3|S|9t%|t8r)tjj |r ||d <|d }|9t%|t8r)tjj |r ||d <|d }||j,|j,j2d k(rwt%|trgd:tttvd;t&t8f(fd< (gd=}%ty}&|%D]"}'|'|vs|&j{(||'g$t}||&>}||d <|S#1swYxYwcc}wcc}}wcc}}wcc}w#tN$r1}tPjSd(|j2d)|d*Yd}~ eos_token unk_tokenzz+Could not extract SentencePiece model from z$ using sentencepiece library due to z%. Falling back to TikToken extractor.)TikTokenConverterextra_special_tokens)r&rOvaluesreturncg}|D]M}|t|ttfr|j|4|j t |O|SN) isinstancelisttupleextendappendstr)rP collectedval_iter_special_tokenss l/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/tokenization_utils_tokenizers.pyr\zHTokenizersBackend.convert_to_native_format.._iter_special_tokens)s[') !3C{ !#e}5!(()=c)BC!((S2 3! ) pad_tokenrMrIrK sep_token cls_token mask_tokenadditional_special_tokensrO) skip_tokens)?dictpopospathisfiler(__dict__ TokenizerFast from_fileopenjsonloadgetfrom_strdumpsr4padding truncationrTrUbase64 b64decoder/maprV__name__ enumerategetattrrYsplitendswithconvert_slow_tokenizerrAextract_vocab_merges_from_modelrCextractrDhasattrrE ExceptionloggerwarningrFitemsintrbuild_tokenizer_from_spm_protorH trainer_specbos_id setdefault bos_pieceeos_id eos_pieceunk_id unk_piecerN convertedrrsetupdater))clstrust_remote_codekwargs local_kwargsfast_tokenizer_filetokenizer_handletokenizer_json model_typeminimal_tokenizer_json minimal_model tok_from_filenormalizer_configr9rvr1itemitokenr2merger&r@rArC extractorrDconverter_classerGtoken_id id_to_token new_token current_tokenr+ proto_specrN converterspecial_tokens_keysrdkeyr\s) @r]convert_to_native_formatz*TokenizersBackend.convert_to_native_formates F| *../?F +23))Zs||-KO`/CY]gCg%DJDTDT&'=>E %@A#&&w377FEyy eT* UE!23E##y0eT*uE!HtUZm9\5:;TU4[;E;##{22;E2BChaCC##u, 0B0Bk0QeT*_hin_opS[STV[E4)@U1XeQNpEp$)L ! gt4J>--gr:: zObObfkOk'0:kqrbgZs5K% C 01QVW\Q]]rr)/ X& !%%l3 "&&}5   )!!(+ j# &:+>+>}+MRTRYRYR`R`akRl @>*+E*J]J]^fJgF IJ3:> 0y00KlK  O&=&A&A#,,&OO&2wPb7c'G'G'G'W,'W 3 89#=3#=#=#M #ML &\9,, #,,0N(,,Wd;E)--h=F,8+;+;jc:rww~~k?Z%0L "!(+F >cii3 8J8Je8SXbchjnXo !Xc] !tCy ! # %(EK* R,&&&';\#=Ntt+| |jdd} n|j t-d|2|0|j$|j/dd|j/dd| | |_|j t-d|jddxs|jj0xs|}|q|jj2d8i||j/d|d|j/d |d!|j/d"|d"|j/d#|d$n|jj5|jd%dxs|jj6xs|}||jj8d8i||j/d&|d&|j/d'|d(|j/d)|d!|j/d|d*|j/d+|d+d,|vrd-|d,<d.|vxsd/|v}|jd.d |_|jd/d |_|jd0dx}r||j_|xs|jj>du|_ tC&|d8i|| | |_#| |_$|jJ|j_&|jNDchc]}tQtS|}}tU|jWd12Dcgc]\}}tQtS||vr|}}}t'|jXj[|Dcgc] }t]|c}z} |j^jaD])}!|!t]|!| vs|!|vs|jc|!+|jdD]&}t]|| vs||vs|jc|(t|dkDrg}"|j^jaD#cgc]}#|#st]|#}$}#|D]a}t|t\rtg|d3}n0t|tfr |jhst]||$vrd|_4|"jc|c|"r|jk|" |jjm}%|%d4kDrtq|jd5dl|jdd|jr|j|jtjd df|jt|jd6dd7||_|j@xs|jj>du|_ |j@r|jwyycc}}}wcc}wcc}}wcc}wcc}#w#tn$rd}%YwxYw)9Nr7r8r>r+ gguf_filer%rGadd_prefix_spaceFr&r1r2 name_or_pathconfigr tokenizertokenizer_configrT)r1r2fuse_unkdropoutr)r1ra9Couldn't instantiate the backend tokenizer from one of: (1) a `tokenizers` library serialization file, (2) a slow tokenizer instance to convert or (3) an equivalent slow tokenizer class to instantiate and convert. You need to have sentencepiece or tiktoken installed to convert a slow tokenizer to a fast one.rIrJrKrLz3The backend tokenizer is not correctly initialized.r6 max_lengthtruncation_side directionstridetruncation_strategystrategyr5r_pad_token_type_id pad_type_id padding_sidelengthpad_to_multiple_ofbackend tokenizers add_bos_token add_eos_tokenr4c |dSNrre)xs r]z,TokenizersBackend.__init__..s STUVSWr^r)speciali pre_tokenizerfix_mistral_regex) init_kwargsrre)p$//B\B\p`p  " -DOO - - < <   lK ,E F   / J   h H(= >   3[5L M OO ) ) +::148dDOO ?   2H=Q4R S F " ,F9 %4%>%[/U[B["$jj%@$jj%@#ZZ(8$? ?> ?-;DOO *-G-q4??KiKimqKq* "6"  !(DO 0040I0I-DHD]D]$^5T$u+%6$^!$^!'';'A'A'C X uDK (AA   t005578Ta;b5CJ;bb$(#;#;#B#B#D : "*&'w6;NVc;c$$%89  :// ,E5z(U--G$$U+ , }  !F040H0H0O0O0QW1UVAW W& %eS)&ud;Ez2 ==SZ;K-K(,  e$ %' 779J  74??OT#R#^ JJ{D )7d77  $$^T:!,,"(**-@$"G   DO  . . X$//2P2PTX2X *  - -  & & ( .oDl~%_ .j dhijdkr^rrrGrcontentrvks r]rz&TokenizersBackend.added_tokens_encoderds; *00I0I0O0O0QWk)lmA 1 mmmAc6|jjS)z Returns the added tokens in the vocabulary as a dictionary of index to AddedToken. Returns: `dict[str, int]`: The added tokens. )rget_added_tokens_decoderrs r]rGz&TokenizersBackend.added_tokens_decoderls7799r^ct|jjdDcic]\}}|j|c}}Scc}}w)z Returns the added tokens in the vocabulary as a dictionary of token to index. Returns: `dict[str, int]`: The added tokens. c |dSrrer6s r]rz3TokenizersBackend.get_added_vocab..r7r^rr8r:s r]get_added_vocabz!TokenizersBackend.get_added_vocab{s;*00I0I0O0O0QWk)lmA 1 mmmr=cy)zN Returns True, to avoid expensive `assert tokenizer` gotchas. Trers r]__bool__zTokenizersBackend.__bool__sr^c:|jjdS)zD Size of the full vocabulary with the added tokens. Tr.r0rs r]__len__zTokenizersBackend.__len__s---EEr^c|jS)zc `tokenizers.implementations.BaseTokenizer`: The Rust tokenizer used as a backend. )rrs r]backend_tokenizerz#TokenizersBackend.backend_tokenizers r^c.|jjS)zU `tokenizers.decoders.Decoder`: The Rust decoder for this tokenizer. )rdecoderrs r]rJzTokenizersBackend.decoders &&&r^Tr.return_token_type_idsreturn_attention_maskreturn_overflowing_tokensreturn_special_tokens_maskreturn_offsets_mapping return_lengthverbosec J|d|jv}|d|jv}|r|j|g|jz} n|g} tt} | D]} | dj | j |r| dj | j |r| dj | j|r| dj | j|r| dj | j|s| dj t| j | | fS)a Convert the encoding representation (from low-level HuggingFace tokenizer output) to a python Dict and a list of encodings, take care of building a batch from overflowing tokens. Overflowing tokens are converted to additional examples (like batches) so the output values of the dict are lists (overflows) of lists (tokens). Output shape: (overflows, sequence length) token_type_idsattention_mask input_idsspecial_tokens_maskoffset_mappingr) model_input_names overflowingrrUrXidstype_idsrTrVoffsetsr) rr.rKrLrMrNrOrPrQ encodings encoding_dictrs r]_convert_encodingz#TokenizersBackend._convert_encodings$( ! ($48N8N$N ! ($48N8N$N ! $)=)=)I! X%9%99I! I#D)  ;A + & - -aee 4$./66qzzB$./66q7G7GH)34;;A This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put this inside your training loop. Args: pair (`bool`, *optional*, defaults to `False`): Whether the number of added tokens should be computed in the case of a sequence pair or a single sequence. Returns: `int`: Number of special tokens added to sequences. )rnum_special_tokens_to_add)rrs r]rkz+TokenizersBackend.num_special_tokens_to_adds&88>>r^rZskip_special_tokensc$t|tr|jj|Sg}|rt |j n t }|D]<}t|}||vr|j |jj|>|S)a Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and added tokens. Args: ids (`int` or `list[int]`): The token id (or token ids) to convert to tokens. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Returns: `str` or `list[str]`: The decoded token(s). )rTrrrrall_special_idsrX)rrZrlr ids_to_skiprs r]convert_ids_to_tokensz'TokenizersBackend.convert_ids_to_tokenss c3 ??..s3 33Fc$../CE  >EJE # MM$//55e< =  >  r^textrhc J|jd|||d|jS)N)rq text_pairrhre) _encode_plusr)rrqrrhrs r]tokenizezTokenizersBackend.tokenize s, t  lddOaleklssuur^padding_strategyrrrrrc|jj}|jj}|tjk(r|||jj na|||j |jd} |d} n | D cic]} | |j| d} } | | k7r|jjdi| |tjk(r||jjyy|tjk(r|nd} | ||n |j|j|j |j"|d} || k7r|jj$di| yycc} w)a Define the truncation and the padding strategies for fast tokenizers (provided by HuggingFace tokenizers library) and restore the tokenizer settings afterwards. The provided tokenizer has no padding / truncation strategy before the managed section. If your tokenizer set a padding / truncation strategy before, then it will be reset to no padding / truncation when exiting the managed section. Args: padding_strategy ([`~utils.PaddingStrategy`]): The kind of padding that will be applied to the input truncation_strategy ([`~tokenization_utils_base.TruncationStrategy`]): The kind of truncation that will be applied to the input max_length (`int`): The maximum size of a sequence. stride (`int`): The stride to use when handling overflow. pad_to_multiple_of (`int`, *optional*): If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta). padding_side (`str`, *optional*): The side on which the model should have padding applied. Should be selected between ['right', 'left']. Default value is picked from the class attribute of the same name. N)rrrr)rrpad_idr_rrre)rrurtrDO_NOT_TRUNCATErr&rrqrr DO_NOT_PAD no_padding MAX_LENGTHr pad_token_idr_rr) rrvrrrrrrrtargetcurrentr<rs r]set_truncation_and_paddingz,TokenizersBackend.set_truncation_and_paddingsZBoo00 ??** "4"D"D D&--/) /55!11 F"@FG11kooa66GG& 111;F; 99 9#**,$$47Q7Q#QZW[F -9-E\4K\K\++!^^#55&8 F6!...88"%HsErrsis_split_into_wordsreturn_tensorsrc d}||s td|||s td|r6t|ttfxr|xrt|dttf}nt|ttf}|rrt|tr t d|;t |t |k7r$tdt |dt |d|tt||n|}n |r||fgn|g}t|ttfst dt|d |j||||| | | |j}|jj|k7r||j_ |jj||| }|Dcgc]}|j|| | ||||| }}i}|ddD]'}|Dcgc]\}}||D]}|} }}}| ||<)|Dcgc]\}}|D]}|}!}}}|r2g}"t|D]\}#\}$}|"|#gt |$d zz }"|"|d<|d D]}%|j!|%||t#||!| }&|sb| `|s^t#|&j%D'cic].\}}'|t |'dkDrt|'dtr|'dn|'0c}'}|&j&}&|&Scc}wcc}}}wcc}}}wcc}'}w)Nct|tryt|ttfrt |dk(ryt|dtryt|dttfrtt |ddk(st|ddtryt|ddttfr/t |dddk(xst|dddtSyyy)NTrF)rTrYrUrVr)rs r]_is_valid_text_inputz._is_valid_text_inputqs!S!Ae}-q6Q;!c*!tUm41Q4yA~AaDGS)A##AaDGdE];"1Q47|q0OJqtAwqz34OO$ r^ztext input must be of type `str` (single example), `list[str]` (batch or single pretokenized example) or `list[list[str]]` (batch of pretokenized examples) or `list[tuple[list[str], list[str]]]` (batch of pretokenized sequence pairs).rzdwhen tokenizing batches of text, `text_pair` must be a list or tuple with the same length as `text`.zbatch length of `text`: z- does not match batch length of `text_pair`: .z:batch_text_or_text_pairs has to be a list or a tuple (got ))rvrrrrr)rhis_pretokenized)r.rKrLrMrNrOrPrQrUoverflow_to_sample_mapping) tensor_type)rrTrUrVrY TypeErrorrzipr0rrrr encode_batchr_rz&_eventual_warn_about_too_long_sequencerrr])(rrqrsrhrvrrrrrrrrKrLrMrNrOrPrQrrr is_batchedbatch_text_or_text_pairsr]r.tokens_and_encodingssanitized_tokensrrrrstacksanitized_encodingsrrtoksrUbatched_outputr&s( r]rtzTokenizersBackend._encode_plusYs0 ($D)W   )=i)HW  #D4-8hThjQUVWQX[_afZgFhJ#D4-8J )S)$Tc)n)D .s4yk:I'q*FOEZtCi,@'A`d $?Hy(9':dV $2UDMBLTRjMkLllmn  ''- 3!1% (  '#'#<#< ?? 0 04H H4HDOO 1OO00 $1/1 $&   " "!&;&;*C+E'=+ #   '*1- *C&:NN74DINqQNQNEN$) S ! *1ESSWQdSqSqSS %)+ & )*> ? K 9D!*qcC[8I4J.JJ* K=W 9 :)+6 XI  7 7 :w W X''79LZhin4=V*'5&:&:&<"Uc%j1nE!Hd9S%(Y^^(( NW "OS"s K K 3K3K" rc|jj%|jjj|Sdj|S)Nr?)rHrJdecoder)rrs r]convert_tokens_to_stringz*TokenizersBackend.convert_tokens_to_stringsJ%%--9  " " * * 1 1& 9 &! r^ token_idsclean_up_tokenization_spacesc |jddt|tr|g}t|tr|d}|jj ||}||n |j }|rxt|jjjdk(r;|js/tjd|jjd|S|j|}|S)Nuse_source_tokenizerrU)rlr z=Ignoring clean_up_tokenization_spaces=True for BPE tokenizer aE. The clean_up_tokenization post-processing step is designed for WordPiece tokenizers and is destructive for BPE (it strips spaces before punctuation). Set clean_up_tokenization_spaces=False to suppress this warning, or set clean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_output=True to force cleanup anyway.)rgrTrrfrrrr0rHr/ryGclean_up_tokenization_spaces_for_bpe_even_though_it_will_corrupt_outputr warning_oncerclean_up_tokenization)rrrlrrrqs r]_decodezTokenizersBackend._decodes  )40 i %" I i &!+.I%%iEX%Y,7 )22 % ( T++112;;uDdd##//01-- 11$7 r^ file_names. legacy_formatct|}tjj||r|dzndtz}|j j |||fz}|S)Nr r)rYrhrirTOKENIZER_FILErHsave)rr rrr r%s r]_save_pretrainedz"TokenizersBackend._save_pretrained%s]^, o_s22Q__  ##N3>"33 r^c ` tj|jj}|j d}|j d} d} |dddk(ri|dd<g|dd<np|ddd k(r=|dd ]|dd } |dd| d } | | |vr|| } d |dd <| d gg|dd<n(|ddd vr i|dd<nt d|ddd|"d|dvr|dd|vr||dd|dd<t jtj|} g} |D]b}|j dd}|j dd}|ddd k7r|s5||d|vr ||d|d<| jtd(i|d|| j||dddk(rd|vr|dd |dd|d<|dddk(rd|vr|dd |dd|d<|ddd k(r| | |d<|dV|dddk(s*|dddk(r@d|dvr9td|ddDr!tjj|d<t |dd}|d(|| d|}| j#|||| &tj| j}d| vr| dD]}| d|d}||Dcgc]}|j%||}}|| d|d<|D] }| j'|}|t d |Dcgc]}| j'|c}| d|d!<d"D]?}|| vs| |\}}| ||vr||}| j'|}| t d ||g| |<A| |d<t jtj|} |j(j+}t,j.D]}t1||t1||}| ||vr||}|j2j%|d}t5|tr=t||j6|j8|j:|j<d#$||<|||<|j>r|j>j+ng}||j|tA|d kDr||d%<| |d&< |jBd(i|Scc}wcc}w#tD$rE}d'tG|vr2|j d&d|jBd(i|}| |_|cYd}~Sd}~wwxYw))uf Trains a tokenizer on a new corpus with the same defaults (in terms of special tokens or tokenization pipeline) as the current one. Args: text_iterator (generator of `list[str]`): The training corpus. Should be a generator of batches of texts, for instance a list of lists of texts if you have everything in memory. vocab_size (`int`): The size of the vocabulary you want for your tokenizer. length (`int`, *optional*): The total number of sequences in the iterator. This is used to provide meaningful progress tracking new_special_tokens (list of `str` or `AddedToken`, *optional*): A list of new special tokens to add to the tokenizer you are training. special_tokens_map (`dict[str, str]`, *optional*): If you want to rename some of the special tokens this tokenizer uses, pass along a mapping old special token name to new special token name in this argument. kwargs (`dict[str, Any]`, *optional*): Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library. Returns: [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on `text_iterator`. r3r4Nr/r0r r1r2rrrg)r#r$z;This method does not support this type of tokenizer (found z-) only BPE, Unigram, WordLevel and WordPiece.rMridr9continuing_subword_prefixend_of_word_suffixr ByteLevelr: pretokenizersc3,K|] }|ddk(yw)r0rNre).0 pretokenizers r] z.s"$!(K7sinitial_alphabet)rr)rtrainerrrzQAttempted to set a token in the post processor that does not exist in the mappingrZ)rsepT) single_wordlstriprstripr*rrOr+z7multiple values for keyword argument 'tokenizer_object're)$roloadsrto_strrgrrlrrrsrXrrWanypre_tokenizers_fastralphabetMODEL_TO_TRAINER_MAPPINGtrain_from_iteratorrqrarrrSPECIAL_TOKENS_ATTRIBUTESr{rrTrrrr*rOrrrrY)r text_iteratorrrnew_special_tokensspecial_tokens_maprrr3r4rMrrr added_tokenrr trainer_classrtrained_tokenizer_jsonrrrr special_tokenspecial_token_fullrOr new_tokenizers r]train_new_from_iteratorz)TokenizersBackend.train_new_from_iterator6sDDOO$:$:$<=%)).9 '++,<= ' "6 *e 3/1N7 #G ,02N7 #H - G $V , 9g&x0<'0:*73G4D3Ew'0 G $V ,0J J/1N7 #G ,Mn]dNeflNmMno>>   *~g66w' 48JJ3EnU\F]^iFj3kN7 #K 0!**4::n+EF ' =K!ooi6Gd+Ag&v.);G!-+i2HL^2^);K 1)*:; vC+,<=cB8LF)5TZ![5"4"8"8"F![![FLN#34S9(C!'#,#8#8#?#+", s#ouCuejIDYDYZ_D`CuN#34S9%@ v"0 F  N2-m>+F+ +{"\Dvj HCPQFR -t4 . 8 8 +4 ($$ s0SSS T-(9T(!T-'T((T-c ddlddlm} ddlmddlm} ddlm} | dd td tffd }|s trd }| |s |s||r| |d |||dd|}d}|t|d5}tj|}dddjd}|jd}|r-| j!|| j!dkr |r/|-|dvr)|S|r%| j!|| j!dk\r|Sd }|s |s@||r7|rd|vrt#|d|d| 5t%|dds(t#|ddt&j)d|d|S| d us t%|ddrt#|dd ddl}|j,j/|j1dd}|j2}t5||j,j6r||j2d<|St5||j,j8r|j,j;dd}|j,j7||g|_|S#1swYxYw)af Patches mistral related tokenizers with incorrect regex if detected 1) Local file with an associated config saved next to it >> Model type one of the mistral models (on older versions) 2) Remote models on the hub from official mistral models >> Tags including `base_model:.*mistralai` rN) lru_cache) model_info)versionr)maxsizemodel_idrQc |}|j,jddj|jryy#t$rYywxYw)NFzbase_model:.*mistralairT)rtagssearchr)rr/rres r]is_base_mistralz?TokenizersBackend._patch_mistral_regex..is_base_mistralsT "8,zz%995rwwuzz7JK   sA AATz config.jsonF) cache_dirrlocal_files_only%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errors _commit_hashr,r-transformers_versionrz5.0.0)mistralmistral3voxtral ministralpixtralrz$The tokenizer you are loading from 'a' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue.z[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+isolated)patternbehavior)r use_regex)r functoolsrhuggingface_hubr packagingrtransformers.utils.hubrrYboolrrnrorprqparsesetattrr{rrrpre_tokenizersSplitRegexrrTr: Metaspacer)rrpretrained_model_name_or_pathrrrris_localrrrrrrr _config_filemistral_config_detectedf_configrtransformers_model_typersplit_pretokenizercurrent_pretokenizerrrs @@r]rz&TokenizersBackend._patch_mistral_regexs* '.%6 3  c d    0H ( 4 X/:W*X&-#!16;8=) L', #',9+Q"iilG+'.{{3I'J$*1++l*C' (GMM:N,ORYR_R_`gRh,h 3?3  )()gmm?\>]^eeH?'$.')EXZ_2`I':DA%)3)B)B)H)H * 0 0s!", *I*& ,5+B+B(!"6 8Q8Q8Z8Z[5G //2"&&:JII')FrS)NNFFFFT)NF)FN)NN)NNN)NNFNFNN)Cry __module__ __qualname____doc__rrr/r classmethodrr*propertyrrrrYrVrrrrsetterr,rrrfr2r1rrrG_added_tokens_encoder_added_tokens_decoderrBrDrFrlrH DecoderFastrJ EncodingFastrrUr_rcrerirkrprur rrrzryrrrrtrrrhPathLikerrr __classcell__)rs@r]r(r(Ss * EJ``Da)F    !c !C$J !Z_`cZd ! 46666%%%%)@GCGG A4S>A tCH~  nd38nnn:d3 ?&;::10nc3hn$ FF = '''.2-1*/+0',#-(-( $d{-( $d{ -( $( -( %) -(!%-(-(-( tCH~tL11 2-(^ 7#7#*76d3+;&<6WZ6 ?d?s?*tCyt`cfjknfo`o4vSvd vtvjnorjsvI9)I90I9 I9  I9  $J I9DjI9\gk#',;,F,F2D2T2T!%$))-#'&*-1-1*/+0',#,0)X++d9o=EV@WWX004 ?BTJ[E\\_ccX! X * X 0 X$JXX"X $JXDjXt X $d{X $d{X$(X %)!X"!%#X$%X&'X(#Tk)X, -Xt tCy S %*48 )c?)")'+Tk ) )^&*&* bkk)#s(Od{  t  sCx * CJ CCr^r()Crrrorh collectionsrcollections.abcrshutilrtypingrtokenizers.pre_tokenizersrrrrrrr r rr rltokenizers.decodersr rtokenizers.modelsr rtokenizers.trainersrrrrrrr~rintegrations.ggmlrmodeling_gguf_pytorch_utilsrtokenization_utils_baserrrrrrrutilsr r!r" get_loggerryrrSPECIAL_TOKENS_MAP_FILETOKENIZER_CONFIG_FILETIKTOKEN_VOCAB_FILEADDED_TOKENS_FILErrr(PreTrainedTokenizerFastrer^r]rs  #$7+-/16*^^.05=@?   H %"3/'( !! (6EXY,-k/k.k^),r^