Ë
    Gêñij  ã                   ód   — d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ  G d„ d«      Zdefd„Zy	)
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)Úbytes_to_unicode)ÚPreTrainedTokenizerFastc                   ó>   — e Zd ZdZ	 	 	 	 d	d„Zdefd„Zd„ Zdefd„Z	y)
ÚMistralConverterz'
    A general tiktoken converter.
    Nc                 ó<   — || _         || _        || _        || _        y )N)ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   Úkwargss         úc/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/mistral.pyÚ__init__zMistralConverter.__init__   s"   € ð ˆŒ
ØˆŒØ 0ˆÔØ)BˆÕ&ó    r   c                 ó&  ‡‡— |Št        «       Šˆfd„}g }i }t        ‰j                  «       «      D ]¤  \  }\  }}|| j                  vr‰|| ||«      <   t	        |«      dk(  rŒ1g }t        dt	        |«      «      D ]2  }|d | ||d  }
}	|	‰v sŒ|
‰v sŒ|	|
z   ‰v sŒ|j                  |	|
|f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ |||<   Œ¦ t        |d„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS c c}w )Nc           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w )NÚ zlatin-1)ÚjoinÚdecodeÚord)ÚbÚcharÚbyte_encoders     €r   Útoken_bytes_to_stringzOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string   s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTs    <é   c                 ó$   •— ‰| d      ‰| d      fS )Nr   r!   © )ÚxÚ	bpe_rankss    €r   ú<lambda>zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>-   s   ø€ °Y¸qÀ¹t±_ÀiÐPQÐRSÑPTÁoÐ4V€ r   F)ÚkeyÚreversec                 ó   — | d   S )Né   r#   )Úvals    r   r&   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>1   s
   € °°A±€ r   r   )	r	   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   r    ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr+   r%   r   s               @@r   Úextract_vocab_merges_from_modelz0MistralConverter.extract_vocab_merges_from_model   sK  ù€ Øˆ	Ü'Ó)ˆô	Vð ˆØˆÜ"+¨I¯O©OÓ,=Ó">ò 	#ÑˆC‘�%˜Ø˜D×:Ñ:Ñ:Ø69�Ñ+¨EÓ2Ñ3Ü�u“: ’?ØØ�Ü" 1¤c¨%£jÓ1ò ?�EØ',¨V¨e }°e¸E¸F°m˜W�GØ )Ò+°¸9Ò0DÈ'ÐT[ÑJ[Ð`iÒIiØŸ™ g¨w¸Ð%=Õ>ð?ô ˜uÓ*VÐ`eÔf�Ø—‘˜eÕ$à"��e’ð	#ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðùò ds   Ã+Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S )NF)Úfuse_unkÚignore_mergesT)r;   r   r   r   ÚhasattrÚmodelr>   )r   Úvocab_scoresr3   Ú	tokenizers       r   rB   zMistralConverter.tokenizer5   sN   € Ø#×CÑCÀDÇJÁJÓOÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐr   Úreturnc                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )NÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rB   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rB   s     r   Ú	convertedzMistralConverter.converted<   s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	ÔØ×$Ñ$ T×%CÑ%CÔDä#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr   )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr;   rB   r   rQ   r#   r   r   r   r      s;   „ ñð ð KØØ"&óCð°Só ò6ð˜9ô r   r   Útokenizer_filec                 óH  — ddl m} ddlm} |j	                  | «      }|j
                  j                  j                  }t        |j
                  j                  j                  d„ ¬«      }|D �cg c]  }|d   ‘Œ	 }}t        |«      D ��ci c]  \  }}||“Œ
 }	}}|	j                  |«       |	}|j
                  j                  j                  j                  }
t        t        |||
¬«      j!                  «       ¬«      }|j#                  d	|i«       |j$                  j&                  |j(                  j&                  |j*                  j&                  |j,                  j&                  d
œ}|j/                  «       D ]  \  }}||v sŒ|j#                  ||i«       Œ |S c c}w c c}}w )z1Convert a "tekken" tokenizer to a fast Tokenizer.r   )ÚSpecialTokens)ÚMistralTokenizerc                 ó   — | d   S )Nr6   r#   )r$   s    r   r&   z*convert_tekken_tokenizer.<locals>.<lambda>X   s   € ÐmnÐouÑmv€ r   )r'   Ú	token_str)r   r   r   )Útokenizer_objectr   )Ú	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ	unk_token)Ú%mistral_common.tokens.tokenizers.baserY   Ú(mistral_common.tokens.tokenizers.mistralrZ   Ú	from_fileÚinstruct_tokenizerrB   Ú_tekken_token2id_nospecialr1   Ú_all_special_tokensr,   ÚupdateÚ_modelÚ_pat_strr
   r   rQ   rO   ÚbosÚvalueÚeosÚpadÚunkr-   )rW   rY   rZ   Úmistral_tokenizerr   Úsorted_tokensr5   Úall_specialr4   Úspecials_tokensr   rB   Ú
MAP_SPECALÚspecial_keyÚspecial_tokens                  r   Úconvert_tekken_tokenizerrw   L   sŒ  € õ DÝIð )×2Ñ2°>ÓBÐð ×0Ñ0×:Ñ:×UÑU€EÜÐ,×?Ñ?×IÑI×]Ñ]ÑcvÔw€MØ3@ÖA¨%�5˜Ó%ÐA€KÐAä4=¸kÓ4J×K¡j c¨5�u˜c‘zÐK€OÑKà×Ñ˜5Ô!Ø€Eð  ×2Ñ2×<Ñ<×CÑC×LÑL€Gô (Ü)Ø°;Èô
ç
‰)‹+ô€Ið × Ñ Ð"=¸{Ð!KÔLð #×&Ñ&×,Ñ,Ø"×&Ñ&×,Ñ,Ø"×&Ñ&×,Ñ,Ø"×&Ñ&×,Ñ,ñ	€Jð '1×&6Ñ&6Ó&8ò GÑ"ˆ�]Ø˜KÒ'Ø×(Ñ(¨+°}Ð)EÕFðGð ÐùòA BùãKs   Á.FÂ
FN)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Ú#transformers.convert_slow_tokenizerr	   Ú*transformers.tokenization_utils_tokenizersr
   r   rV   rw   r#   r   r   ú<module>r|      s-   ðß MÕ MÝ !å @Ý N÷Añ AðH-¨Sô -r   