Ë
    Gêñix  ã                   óØ  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej2                  e«      Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!ddddddddddd"dd#d$œ“d%ddddddddddd&d'œ“d(dddddddddddœ
“d)dddddddddddddd*œ“d+dddddddddddœ
“d,d-d.d/d0d1œ“d2dddddddddddœ
“d3d4dd5dd6d7œ“d8d9d:d;d<d=d>dd6d?d@ddAœ“dBddddddddCddDœ	“dEd4dFdGdHd5d6dIœ“dJdddddddKdLœ“dddd6ddMdNdOddPœ	dddddddddQddœ
ddddddddddd#ddRœddddddddddd#ddRœd9d:d;d<d=d>dd6d?d@ddAœdddddddddddœ
i dSd“dTd“dHd“dUd“dVdW“dXd“dYd“dZd“d[d“d\d“d]d“d^d"“d_d“d`d“dd“dadb“dcœ¥Zdddedfdgdhd-d.d/d0didjœ
dkdd-d.d/d0dlœdmœZdndoidpdoidqœZdr„ Z G ds„ dt«      Z  G du„ dve«      Z! G dw„ dxe«      Z" G dy„ dze«      Z# G d{„ d|e«      Z$ G d}„ d~e«      Z% G d„ d€e«      Z&i de!“de"“de"“d(e"“d)e"“d2e#“d3e$“d+e$“dBe$“dEe$“dJe$“d8e%“d�e$“d‚e$“dƒe&“d„e&“d…e%“e!e!e"d†œ¥Z'd‡e(dˆe)ee*f   fd‰„Z+y)Šz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   )Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr#   ÚmistralÚqwen2Ú	qwen2_moeÚnum_expertsÚnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   Úexpert_countÚexpert_used_countÚgpt_ossÚnum_local_expertsÚsliding_window)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   r2   r3   r6   Úlfm2Úconv_L_cache)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   zshortconv.l_cacheÚqwen3Ú	qwen3_moe)r$   r%   r&   r'   r(   r)   úattention.key_lengthr*   r+   r,   r#   r2   r3   ÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r%   r'   r*   r#   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r$   r%   r&   r'   r;   r*   r+   rK   z attention.relative_buckets_countrT   r#   ÚstablelmÚlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rK   r#   Úgpt2Ún_ctxÚn_embdr&   )r%   r$   r'   r&   r*   rK   Ú
starcoder2Únorm_epsilon)r%   r$   r'   r&   r*   r+   rK   Úconv_kernelÚ
state_sizeÚtime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnorm_eps)r$   r%   r&   r'   r(   r)   r;   r*   r+   r,   zattention.sliding_windowr#   r$   r%   r'   r(   Ú
rotary_dimr)   r*   r+   r;   zattention.value_lengthr,   r2   r3   Úexpert_feed_forward_lengthÚexpert_gating_funcÚscoring_func)ÚmambaÚnemotronÚgemma2Úgemma3Úumt5ÚdeciÚ
minimax_m2Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesrB   rC   rD   rE   zggml.add_space_prefixÚchat_template)rr   rq   rB   rC   rD   rE   )r=   Útokenizer_configÚnorm_topk_probTÚuse_routing_bias)r:   rj   c                 ó”  — t        |t        «      s|g}t        |«      dk(  r|d   }d }n|d   dk7  rt        d«      ‚|\  }}|dv rt	        | d   «      } | S |dv rt        | d   «      } | S |dk(  rt        | d   «      } | S |dk(  r3t        d	t        | «      «      j                  «       j                  «       } | S |dk(  rt        | |«      } | S )
Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rw   r   é   é   é   é
   é   )é   é   é   é   ÚB)Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      ú`/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyrŒ   rŒ   m  sô   € Ü�i¤Ô&Ø�Kˆ	Ü
ˆ9ƒ~˜ÒØ˜a‘Lˆ	Ø‰à�Q‰<˜1ÒÜÐoÓpÐpØ%.Ñ"ˆ	�?àÐ.Ñ.Ü�V˜A‘Y“ˆð €Mð 
�gÑ	Ü�v˜a‘yÓ!ˆð €Mð 
�aŠÜ�f˜Q‘i“ˆð
 €Mð	 
�aŠÜ�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�aŠÜ" 6¨?Ó;ˆØ€Mó    c                   ó   — e Zd Zd„ Zy)ÚGGUFTokenizerSkeletonc                 óp  ‡— |j                  «       D ]  \  }}t        | ||«       Œ t        | d«      �s)t        | d«      rt        | d«      st        d«      ‚| j                  }| j
                  }t        |«      D ��ci c]  \  }}|||   “Œ c}}Št        j                  d«       g }t        ‰j                  «       «      D ]k  \  }	}
g }t        dt        |	«      «      D ]*  }|	d | |	|d  }}||v sŒ||v sŒ|j                  |||
f«       Œ, t        |ˆfd„d¬	«      }|j                  |«       Œm t        |d
„ d¬	«      }|D �cg c]  }|d   |d   f‘Œ }}|| _        nt| j                  D �	cg c]  }	t!        |	j#                  d«      «      ‘Œ c}	| _        t        | d«      s1t        t        | j                  «      «      D �cg c]  }d ‘Œ c}| _        t        | d«      sg | _        t        | d«      sd | _        t        | d«      r| j&                  €| j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nro   rl   rm   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rw   c                 ó$   •— ‰| d      ‰| d      fS )Nr   rw   © )ÚxÚvocabs    €r�   ú<lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>›  s   ø€ °U¸1¸Q¹4±[À%ÈÈ!ÉÁ+Ð4N€ r‘   T)ÚkeyÚreversec                 ó   — | d   S )Nr   r–   )Úvals    r�   r™   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>�  s
   € °C¸±F€ r‘   r   ú Úadded_tokensr@   Úunknown_token_id)ÚitemsÚsetattrÚhasattrr†   rl   rm   Ú	enumerateÚloggerÚwarningr   Úranger…   ÚappendÚsortedÚextendro   ÚtupleÚsplitrŸ   r@   r    )ÚselfÚdict_ÚkÚvrl   rm   ÚiÚtro   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rr�   Ú_r˜   s                    @r�   Ú__init__zGGUFTokenizerSkeleton.__init__†  s   ø€ Ø—K‘K“Mò 	 ‰DˆAˆqÜ�D˜!˜QÕð	 ô �t˜XÕ&Ü˜4 Ô*´'¸$ÀÔ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ó.?×@¡d a¨�Q˜˜q™	‘\Ó@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ó&9ò %Ñ"��{Ø�Ü" 1¤c¨%£jÓ1ò F�EØ',¨V¨e }°e¸E¸F°m˜W�GØ &Ò(¨W¸Ò->ØŸ™ g¨w¸Ð%DÕEðFô ˜uÓ*NÐX\Ô]�Ø—‘˜eÕ$ð%ô ˜FÑ(:ÀDÔIˆFØ28Ö9¨3�s˜1‘v˜s 1™vÒ&Ð9ˆFÐ9Ø ˆD�Kà@DÇÁÖL°uœ5 §¡¨SÓ!1Õ2ÒLˆDŒKÜ˜4 Ô*Ü-2´3°t·{±{Ó3CÓ-DÖE¨štÒE�”ä�t˜^Ô,Ø "ˆDÔä�t˜^Ô,Ø $ˆDÔô �4Ð+Ô,°×1BÑ1BÐ1JØ $× 5Ñ 5ˆDÕð 2KÐ,ùó7 Aùò :ùò MùâEs   Á=H#ÅH)Å+!H.Ç 	H3N)Ú__name__Ú
__module__Ú__qualname__rº   r–   r‘   r�   r“   r“   …  s   „ ó'6r‘   r“   c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zy)ÚGGUFLlamaConverterc                 ó’   — t        |«      | _        | j                  | _        i | _        t	        | j                  dd«      dk7  | _        y )Nrk   r   )r“   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r­   Útokenizer_dicts     r�   rº   zGGUFLlamaConverter.__init__±  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÕ!r‘   c                 óT   — t        t        |j                  |j                  «      «      S ©N©r„   Úziprl   rm   ©r­   rÁ   s     r�   r˜   zGGUFLlamaConverter.vocab·  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4r‘   c                 ó   — |j                   S rÉ   ©ro   rÌ   s     r�   ro   zGGUFLlamaConverter.mergesº  ó   € Ø�|‰|Ðr‘   c           
      óÄ  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}|j                  �|j
                  |j                     nd }t        |dd «      �|j
                  |j                     nd }	t        |dd «      �|j
                  |j                     nd }
t        t        |||dd¬«      «      }g }t        | j                  d«      s^|�|j                  t        |dd¬«      «       |	�|j                  t        |	dd¬«      «       |
�›|j                  t        |
dd¬«      «       n}t        j                  t        j                  | j                  j                   «      dk(  «      d	   }|D ]6  }|j                  t        | j                  j
                  |   dd¬«      «       Œ8 t#        |«      d	k7  r|j%                  |«       t#        | j                  j&                  «      d	k7  r?|j)                  | j                  j&                  D �cg c]  }t        |dd¬«      ‘Œ c}«       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nr>   r?   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrn   F©Ú
normalizedÚspecialry   r   rÒ   Ú	eos_tokenÚ	bos_tokenrp   Úclean_up_tokenization_spacesÚlegacy)r˜   rÁ   ro   r¤   r@   rl   rÄ   r>   r   r	   r£   r¨   r   ÚnpÚwherer   rn   r…   Úadd_special_tokensrŸ   Ú
add_tokensrÃ   rÅ   rÂ   rÛ   )r­   rÁ   Úvocab_scoresro   r±   ÚwordÚ_scoreÚ	bpe_vocabrÒ   rÙ   rØ   r=   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens                   r�   r=   zGGUFLlamaConverter.tokenizer½  s¨  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNà8=×8JÑ8JÐ8V�E—L‘L ×!3Ñ!3Ò4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	äÜØØØ#ØØ"ôó
ˆ	ð ˆä�t—z‘z <Ô0ØÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Õ\ô "$§¡¬"¯(©(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÑ!RÐà)ò j�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Ñ1GÐTYÐcgÔ&hÕiðjô ˆ~Ó !Ò#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ò,Ø× Ñ Ø]a×]gÑ]g×]tÑ]tÖuÈk”˜K°EÀ5ÖIÒuôð /8ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+à×$Ò$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁKÉKc                 ó0  — t        j                  «       t        j                  «       t        j                  dd«      g}| j                  r|t        j
                  ddd¬«      gz  }|r|t        j                  dd¬«      gz  }t        j                  |«      S )Nõ   â–�rž   FT©rp   Útrim_offsetsÚ	use_regexrw   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacerÅ   Ú	ByteLevelÚStripÚSequence©r­   Úreplacementrp   Úsequences       r�   ÚdecoderzGGUFLlamaConverter.decoder÷  sƒ   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜U CÓ(ð
ˆð ×$Ò$Øœ×+Ñ+¸UÐQVÐbfÔgÐhÑhˆHáØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r‘   c                 ó   — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        | j                  r7t        j                  ddd¬«      |_        t        j                  g «      |_        |S )Nré   Trp   Frê   )r=   rÁ   Ú
normalizerr£   rÂ   rp   Úpre_tokenizerrù   Úpost_processorrÅ   r   ró   r   rõ   )r­   r=   rû   r÷   rp   rü   rý   s          r�   Ú	convertedzGGUFLlamaConverter.converted  sò   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$ð ×$Ò$Ü&4×&>Ñ&>Ø!&°UÀdô'ˆIÔ#ô
 $/×#7Ñ#7¸Ó#;ˆIÔ àÐr‘   N)	r»   r¼   r½   rº   r˜   ro   r=   rù   rþ   r–   r‘   r�   r¿   r¿   °  s"   „ ò^ò5òò8òt+ó!r‘   r¿   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFQwen2Converterc                 ó2   — t        |«      | _        i | _        y rÉ   ©r“   rÂ   rÃ   rÆ   s     r�   rº   zGGUFQwen2Converter.__init__*  ó   € Ü"7¸Ó"GˆÔØ!#ˆÕr‘   Úreturnc           	      ó4  •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|j                  t        ddd¬«      t        ddd¬«      t        ddd¬«      g«       |S c c}}w )Nú<|endoftext|>FTrÕ   z<|im_start|>z
<|im_end|>)r¤   rÂ   rl   ro   Úsuperrþ   rÞ   r   ©r­   r±   rá   r˜   ro   r=   Ú	__class__s         €r�   rþ   zGGUFQwen2Converter.converted.  s–   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	à×$Ñ$ä˜?°uÀdÔKÜ˜>°eÀTÔJÜ˜<°EÀ4ÔHðô	
ð Ðùó Ss   ¤B©r»   r¼   r½   rº   r   rþ   Ú__classcell__©r	  s   @r�   r   r   )  s   ø„ ò$ð˜9÷ ñ r‘   r   c                   ó6   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zdefd„Z	y)	ÚGGUFPhi3Converterc                 óT   — t        |«      | _        | j                  | _        i | _        y rÉ   ©r“   rÁ   rÂ   rÃ   rÆ   s     r�   rº   zGGUFPhi3Converter.__init__>  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr‘   c                 óT   — t        t        |j                  |j                  «      «      S rÉ   rÊ   rÌ   s     r�   r˜   zGGUFPhi3Converter.vocabC  rÍ   r‘   c                 ó   — |j                   S rÉ   rÏ   rÌ   s     r�   ro   zGGUFPhi3Converter.mergesF  rÐ   r‘   c                 ó  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}t	        t        ||«      «      }|j                  t        ddddd¬«      t        ddd¬«      t        dddd¬«      t        d	ddd¬«      t        d
ddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      g«       |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |S c c}}}w )Nú</s>TF)ÚrstripÚlstriprÖ   r×   r  rÕ   z<|assistant|>)r  rÖ   r×   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÒ   rØ   rÙ   Ú	pad_token)r˜   rÁ   ro   r¤   r   r	   rÞ   r   r@   rl   rÃ   r?   r>   rA   )	r­   rÁ   rà   ro   r±   rá   râ   rã   r=   s	            r�   r=   zGGUFPhi3Converter.tokenizerI  s   € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNäœc )¨VÓ4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ô]Ü˜?°uÀdÔKÜ˜?°4ÀEÐSWÔXÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜<°ÀÐPTÔUÜ˜9¨T¸eÈTÔRÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜:¨d¸uÈdÔSðô	
ð$ 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁG:c                 óà   — t        j                  «       t        j                  «       t        j                  |d«      g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nrž   rw   rí   )r   rð   rñ   rò   rô   rõ   rö   s       r�   rù   zGGUFPhi3Converter.decoderp  s\   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜[¨#Ó.ð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r‘   r  c                 óÊ   — | j                  | j                  «      }d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_        |S )Nré   Trp   )r=   rÁ   r£   rÂ   rp   rù   )r­   r=   r÷   rp   s       r�   rþ   zGGUFPhi3Converter.converted{  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔàÐr‘   N)
r»   r¼   r½   rº   r˜   ro   r=   rù   r   rþ   r–   r‘   r�   r  r  =  s(   „ ò$ò
5òò%òN	+ð
˜9ô 
r‘   r  c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFGPTConverterc                 ó2   — t        |«      | _        i | _        y rÉ   r  rÆ   s     r�   rº   zGGUFGPTConverter.__init__‰  r  r‘   r  c                 óÄ   •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|S c c}}w rÉ   )r¤   rÂ   rl   ro   r  rþ   r  s         €r�   rþ   zGGUFGPTConverter.converted�  s]   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	ØÐùó Ss   ¤Ar
  r  s   @r�   r  r  ˆ  s   ø„ ò$ð˜9÷ ñ r‘   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFT5Converterc                 óÜ   — dg|d<   t        |«      | _        t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 c}}| _        | j                  | _        i | _        y c c}}w ©Nz
dummy textro   )r“   rÁ   r¤   rl   Útoken2idrÂ   rÃ   )r­   rÇ   r°   r¯   s       r�   rº   zGGUFT5Converter.__init__•  s]   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÓ*F×G¡$ ! Q˜˜A™ÓGˆŒØ"&§*¡*ˆÔØ!#ˆÕùó Hs   ¹A(c                 óT   — t        t        |j                  |j                  «      «      S rÉ   rÊ   rÌ   s     r�   r˜   zGGUFT5Converter.vocabž  rÍ   r‘   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )NrÛ   Trp   ré   )Úprependrž   )Úpatternrî   )rÄ   rÂ   r   ÚPrependrò   rõ   )r­   rÁ   rø   s      r�   rû   zGGUFT5Converter.normalizer¡  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ør‘   c                 óZ   — t        j                  ddgg d¢d| j                  d   fg¬«      S )Nú$Ar  )r)  r  z$Br  )ÚsingleÚpairrä   )r   ÚTemplateProcessingr"  )r­   s    r�   rý   zGGUFT5Converter.post_processorª  s5   € Ü×,Ñ,Ø˜&�>Ú-à˜Ÿ™ vÑ.Ð/ðô
ð 	
r‘   r  c                 óÐ  — | j                  | j                  «      }t        t        || j                  j                  d¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      }|�||_	        | j                  ||«      |_
        | j                  «       }|r||_        |S )NF©Úunk_idrÔ   ré   Trp   )r˜   rÁ   r   r
   r@   rû   r£   rÂ   rp   rü   rù   rý   )r­   rà   r=   rû   r÷   rp   rü   rý   s           r�   rþ   zGGUFT5Converter.converted³  sÝ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐr‘   N)	r»   r¼   r½   rº   r˜   rû   rý   r   rþ   r–   r‘   r�   r  r  ”  s"   „ ò$ò5òò
ð˜9ô r‘   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFGemmaConverterc                 ó`   — dg|d<   t        |«      | _        | j                  | _        i | _        y r!  r  rÆ   s     r�   rº   zGGUFGemmaConverter.__init__Ô  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr‘   c                 óL  — t        t        |j                  |j                  «      «      }g }|D ]s  \  }}|dk(  r|j	                  d|f«       Œd|v r>t        |j                  «       «      dk(  r"dt        |«      z  }|j	                  ||f«       Œa|j	                  ||f«       Œu |S )Nz<0x09>ú	rž   r   ré   )r„   rË   rl   rm   r¨   r…   Ústrip)r­   rÁ   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r�   r˜   zGGUFGemmaConverter.vocabÜ  s¡   € Üœc %§,¡,°·±Ó=Ó>ˆØˆà*ò 	5‰LˆE�5Ø˜Ò Ø×$Ñ$ d¨E ]Õ3Ø˜‘¤# e§k¡k£mÓ"4¸Ò"9Ø#¤c¨%£jÑ0�Ø×$Ñ$ k°5Ð%9Õ:à×$Ñ$ e¨U ^Õ4ð	5ð Ðr‘   c                 ó.   — t        j                  dd«      S )Nrž   ré   )r   rò   rÌ   s     r�   rû   zGGUFGemmaConverter.normalizerë  s   € Ü×"Ñ" 3¨Ó.Ð.r‘   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nré   rž   rw   rí   )r   rò   rð   rñ   rô   rõ   rö   s       r�   rù   zGGUFGemmaConverter.decoderî  s\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r‘   r  c                 ó²  — | j                  | j                  «      }t        t        || j                  j                  | j
                  ¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_
        | j                  ||«      }|�||_        |S )Nr.  ré   Trp   )r˜   rÁ   r   r
   r@   Úhandle_byte_fallbackrû   r£   rÂ   rp   rù   rü   )r­   rà   r=   rû   r÷   rp   rü   s          r�   rþ   zGGUFGemmaConverter.convertedù  sÊ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔØ×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#àÐr‘   N)	r»   r¼   r½   rº   r˜   rû   rù   r   rþ   r–   r‘   r�   r1  r1  Ó  s"   „ ò$òò/ò	+ð˜9ô r‘   r1  rd   re   rf   Úgemma3_textrh   )ri   Údecilmrj   r   r  c                 ó^   — | }t        |   |«      }|j                  «       }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrþ   rÃ   )r   rÇ   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r�   Úconvert_gguf_tokenizerrF  ,  s9   € ð (ÐÜ'Ð(<Ñ=¸nÓM€IØ×(Ñ(Ó*€NØ˜9×6Ñ6Ð6Ð6r‘   ),Ú__doc__r   ÚnumpyrÜ   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr»   r¥   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGÚGGUF_CONFIG_DEFAULTS_MAPPINGrŒ   r“   r¿   r   r  r  r  r1  rB  Ústrr«   ÚdictrF  r–   r‘   r�   ú<module>rU     sË  ðñõ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
ˆ×	Ñ	˜HÓ	%€ð`ØØ$Ø%ñð`ð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð`ð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð%`ð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñð?`ðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðW`ðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+Ø2Ø*ñðs`ðP Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+ñðQ`ðj Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðk`ðB Ø3Ø*Ø2Ø)Ø $Ø&Ø *Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðC`ð` Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñða`ðx Ø+Ø+Ø!/Ø!/ñ	ðy`ðD Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðE`ð\ Ø Ø)Ø (Ø"Ø(<ñð]`ðj 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñðk`ðD Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ñ
ðE`ðZ Ø Ø!Ø$Ø4Ø (Ø(<ñð[`ðj Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ñðk`ð~ #Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ñ
ð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ñð 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" (Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðØÐ3ðàÐ*ðð 	Ð2ðð 	˜Mð	ð
 	 ðð 	˜,ðð 	Ð 5ðð 	"Ð#8ðð 	 
ðð 	! $ðð 	+¨Nðð 	Ð+ðð 	Ð2ðð 	% dðð 	�lðð  	˜nð!ò]`Ð ðH	 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ñð )Ø"Ø+Ø+Ø!/Ø!/ññÐ ð: 	˜$ð	ð 	˜Dð	ñ Ð ò ÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" ˆOð#ð$ Ø Ø$ò)Ð ð07¨ð 7ÀÀyÐRVÀÑAWô 7r‘   