Ë
    Gêñib1 ã                   óN  — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlmZmZmZ ddlmZ d	d
lmZmZmZmZ d	dlmZ  ej8                  e«      Zg d¢Zeg d¢z   Z d¦d„Z!de"de#fd„Z$d§dee#   dz  fd„Z% G d„ d«      Z& G d„ de&«      Z'de#de"fd„Z( G d„ d«      Z) G d„ de)«      Z* G d„ de)«      Z+ G d „ d!e)«      Z, G d"„ d#e)«      Z- G d$„ d%e)«      Z. G d&„ d'e)«      Z/ G d(„ d)e)«      Z0 G d*„ d+e)«      Z1 G d,„ d-e)«      Z2 G d.„ d/e)«      Z3 G d0„ d1e)«      Z4 G d2„ d3e)«      Z5 G d4„ d5e5«      Z6 G d6„ d7e5«      Z7 G d8„ d9e5«      Z8 G d:„ d;e5«      Z9 G d<„ d=e5«      Z: G d>„ d?e5«      Z; G d@„ dAe5«      Z< G dB„ dCe5«      Z= G dD„ dEe5«      Z> G dF„ dGe5«      Z? G dH„ dIe5«      Z@ G dJ„ dKe5«      ZA G dL„ dMe5«      ZB G dN„ dOe5«      ZC G dP„ dQe5«      ZD G dR„ dSe5«      ZE G dT„ dUe)«      ZF G dV„ dWe5«      ZG G dX„ dYe)«      ZH G dZ„ d[e)«      ZI G d\„ d]e)«      ZJ G d^„ d_e5«      ZK G d`„ dae5«      ZL G db„ dce5«      ZM G dd„ dee)«      ZN G df„ dge5«      ZO G dh„ die5«      ZP G dj„ dke5«      ZQdl„ ZR G dm„ dn«      ZS G do„ dp«      ZTi dqe6“dre2“dse7“dte*“dueG“dveJ“dwe8“dxeH“dye/“dze*“d{e4“d|e9“d}e*“d~e*“de*“d€e*“d�e*“i d‚e6“dƒe,“d„e/“d…e0“d†e*“d‡e*“dˆe2“d‰e>“dŠe2“d‹e2“dŒe*“d�eN“dŽe:“d�e;“d�e-“d‘e*“d’e2“¥i d“e<“d”e.“d•eC“d–e1“d—e@“d˜eA“d™e2“dše3“d›e=“dœe*“d�eD“džeE“dŸeF“d e>“d¡e?“d¢e+“d£eK“¥eMeMeLeMd¤œ¥ZUd¨defd¥„ZVy)©zù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Ú
Collection)Ú	lru_cache)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Útqdmé   )Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERROR)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CN)Úaf_ZAÚaz_AZÚbn_INÚfa_IRÚhe_ILÚhr_HRÚid_IDÚka_GEÚkm_KHÚmk_MKÚml_INÚmn_MNÚmr_INÚpl_PLÚps_AFÚpt_XXÚsv_SEÚsw_KEÚta_INÚte_INÚth_THÚtl_XXÚuk_UAÚur_PKÚxh_ZAÚgl_ESÚsl_SIc                 ó  — t        «       rddlm} |S t        «       rSdd l}t        j                  |j                  j                  «      t        j                  d«      k  rddl	m} |S ddl	m
} |S t        t        j                  | «      «      ‚)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   ÚsentencepiecerL   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsrM   ÚImportErrorr   Úformat)Úerror_messagerL   Úgoogles      úe/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufrY   _   sl   € Ü!Ô#Ý9à&Ð&ÜÔÛä�=‰=˜Ÿ™×4Ñ4Ó5¼¿¹ÀgÓ8NÒNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ñ6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 ó4   — | rd}t        |dd«      sd}|S d}|S )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r[   Úoriginal_tokenizerÚprepend_schemes      rX   Ú_get_prepend_schemere   p   s1   € ÙØ!ˆÜÐ)¨8°TÔ:Ø$ˆNð Ðð !ˆØÐrZ   Úskip_tokensc                 óÌ  ‡ — |�t        |«      n	t        «       }|d u}|rt        |«      n‰ }g }|j                  «       D ]x  \  }}||v rŒg }t        dt	        |«      «      D ]3  }|d | ||d  }
}	|	|v s|
|v rŒ|	‰ v sŒ|
‰ v sŒ |j                  |	|
|f«       Œ5 t        |ˆ fd„¬«      }|j                  |«       Œz t        |d„ |¬«      }|D �cg c]  }|d   |d   f‘Œ }}|S c c}w )Nr   c                 ó$   •— ‰| d      ‰| d      fS ©Nr   r   © )ÚxÚvocabs    €rX   ú<lambda>z!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ð,F€ rZ   ©Úkeyc                 óB   — | d   t        | d   «      t        | d   «      fS )Né   r   r   )Úlen©Úvals    rX   rm   z!generate_merges.<locals>.<lambda>�   s!   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ð,N€ rZ   ©ro   Úreverser   )ÚsetÚdictÚitemsÚrangerr   ÚappendÚsortedÚextend)rl   Úvocab_scoresrf   rv   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrt   s   `           rX   Úgenerate_mergesr†   z   s  ø€ Ø&1Ð&=”#�kÔ"Ä3Ã5€KØ $Ð&€GÙ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ó2ò Ñˆˆ{Ø�KÑØØˆÜ˜1œc %›jÓ)ò 	>ˆEØ$ V e˜}¨e°E°F¨m�WˆGØ˜+Ñ%¨°KÑ)?ØØ˜%Ò G¨uÒ$4Ø—‘˜g w°Ð<Õ=ð	>ô �uÓ"FÔGˆØ�‰�eÕðô �FÑ NÐX_Ô`€FØ*0Ö1 3ˆs�1‰v�s˜1‘vÒÐ1€FÐ1Ø€Mùò 2s   ÃC!c                   óB   — e Zd ZdZdefd„Zdeeeef   e	e   f   fd„Z
y)ÚSentencePieceExtractorzl
    Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
    Úmodelc                 ó  — t        | d«       t        | d«       t        «       }|j                  «       }t        |d«      5 }|j	                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w)NrN   rQ   Úrb)r   rY   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)Úselfr‰   Ú	model_pb2ÚmÚfs        rX   Ú__init__zSentencePieceExtractor.__init__—   sl   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜Óð 	( !Ø×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
ús   ¿ A/Á/A?r\   c           
      ó”  — | j                   j                  j                   |€/ddlm}m} | j                   j                  j                  dk(  r|n|}| j                   j                  D �cg c]  }|j                  |j                  f‘Œ }}|j                  dk7  r)| j                   j                  j                  |d<   ||d<   n;ddlm} t        |«      D ��	�
ci c]  \  }\  }	}
|	|“Œ }}	}}
 ||«      }||d<   ||d	<   t        | j                   j                  «      D ��cg c]0  \  }}|j                  d
v sŒ||j                  |j                  dk(  f‘Œ2 }}}t        |d„ ¬«      D ���cg c]  \  }}}t!        |d|¬«      ‘Œ c}}}|d<   t#        | j                   j$                  dd«      |d<   |S c c}w c c}
}	}w c c}}w c c}}}w )úÂ
        By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
        order the merges with respect to the piece scores instead.
        Nr   )r   r   r   r   Úunk_idrl   )r†   r   ©é   é   rš   c                 ó   — | d   S ©Nr   rj   ©rk   s    rX   rm   z0SentencePieceExtractor.extract.<locals>.<lambda>Á   s
   € ÈQÈqÉT€ rZ   rn   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap)r�   Útrainer_specr˜   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baser†   Ú	enumerateÚtyper|   r   rb   Únormalizer_spec)r‘   r§   Úkwargsr   r   r©   rl   r†   ÚiÚwordrª   r   ÚidÚpÚspm_added_tokensÚtokenr¡   s                    rX   ÚextractzSentencePieceExtractor.extract£   s©  € ð
 	�
‰
×Ñ×&Ò&ØÐß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ò$K™ÐQTˆJØ9=¿¹×9JÑ9JÖK°�%—+‘+˜uŸ{™{Ò+ÐKˆÐKà×Ñ %Ò'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#ˆF�7ŠOå@ä5>¸uÓ5E×FÐFÑ!1 ¡M T¨5�T˜1‘WÐFˆEÒFÙ$ UÓ+ˆFØ#ˆF�7‰OØ%ˆF�8Ñô ENÈdÏjÉj×N_ÑN_ÓD`×u¹5¸2¸qÐde×djÑdjÐntÒdt˜R §¡¨!¯&©&°A©+Ò6ÐuÐÑuô '-Ð-=Á>Ô&R÷/
ð /
á"��E˜7ô �u¨¸Ö@ô/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á*F1ÃF6Ä F=Ä7F=Å*GN)r«   Ú
__module__Ú__qualname__Ú__doc__Ústrr•   Útuplerx   ÚintÚlistr·   rj   rZ   rX   rˆ   rˆ   ’   s5   „ ñð
˜có 
ð!¨u°T¸#¸s¸(±^ÀTÈ%Á[Ð5PÑ/Qô !rZ   rˆ   c                   ó4   — e Zd Zddeeeef   ee   f   fd„Zy)ÚGemmaSentencePieceExtractorNr\   c                 óà   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}d|vr|j	                  d«      |d<   t        ||«      }||fS c c}w )r—   ú	ú<0x09>)Úsprz   ÚGetPieceSizeÚid_to_pieceÚgetr†   )r‘   r~   rÄ   rƒ   rl   r   s         rX   r·   z#GemmaSentencePieceExtractor.extractÈ   sr   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTð �uÑØŸ)™) HÓ-ˆE�$‰KÜ  ¨Ó5ˆØ�fˆ}Ðùò Us   ¨A+©N)	r«   r¸   r¹   r¼   rx   r»   r½   r¾   r·   rj   rZ   rX   rÀ   rÀ   Ç   s$   „ ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô rZ   rÀ   r©   c                 ó^   — t        | «      dk  xs | d   dk7  xs | d   j                  «        S )Nrq   éÿÿÿÿú,éþÿÿÿ)rr   Úisdigit)r©   s    rX   Úcheck_number_commarÎ   Ø   s3   € Üˆu‹:˜‰>ÒH˜U 2™Y¨#Ñ-ÒH°U¸2±Y×5FÑ5FÓ5HÐ1HÐHrZ   c                   ó   — e Zd Zd„ Zdefd„Zy)Ú	Converterc                 ó   — || _         y rÈ   )rc   )r‘   rc   s     rX   r•   zConverter.__init__Ý   s
   € Ø"4ˆÕrZ   r\   c                 ó   — t        «       ‚rÈ   )ÚNotImplementedError©r‘   s    rX   Ú	convertedzConverter.convertedà   s   € Ü!Ó#Ð#rZ   N)r«   r¸   r¹   r•   r   rÕ   rj   rZ   rX   rÐ   rÐ   Ü   s   „ ò5ð$˜9ô $rZ   rÐ   c                   ó   — e Zd Zdefd„Zy)ÚBertConverterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©rc   rl   r   r   r»   rÚ   ÚhasattrrÛ   Útokenize_chinese_charsrß   Údo_lower_caser
   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr	   Údecoder©
r‘   rl   Ú	tokenizerrî   rß   rï   ÚclsÚseprö   r÷   s
             rX   rÕ   zBertConverter.convertedå   ó…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   N©r«   r¸   r¹   r   rÕ   rj   rZ   rX   r×   r×   ä   ó   „ ð#˜9ô #rZ   r×   c                   ó   — e Zd Zdefd„Zy)ÚSplinterConverterr\   c           
      ó‚  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }t	        | j                   j&                  «      }d}	| j                   j(                  }
| j                   j*                  }| j                   j,                  }| j                   j/                  d«      }| j                   j0                  dk(  r|› d|› d	|	› d	|› d
|› d�
}n|› d|› d
|› d	|	› d	|› d�
}t3        j4                  |› d|› d�|||
f||f||f|	|fg¬«      |_        t9        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   ú.Úrightrá   ú rã   rä   râ   rå   ré   rê   )rc   rl   r   r   r»   rÚ   rí   rÛ   rî   rß   rï   r
   rð   rñ   r   rò   ró   rô   rõ   Úquestion_tokenrö   r÷   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   rø   rù   r	   rú   )r‘   rl   rü   rî   rß   rï   rý   rþ   ÚquestionÚdotrö   r÷   r	  Údot_token_idrç   s                  rX   rÕ   zSplinterConverter.converted  s"  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ò:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ô	$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r    s   „ ð.˜9ô .rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚFunnelConverterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   z:2 $A:0 râ   rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zFunnelConverter.converted>  rÿ   rZ   Nr   rj   rZ   rX   r  r  =  r  rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚMPNetConverterr\   c                 ór  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	|› d
�||f||	fg¬«      |_        t1        j                  d¬«      |_        |S )NrÙ   FrÛ   TrÜ   rá   râ   z:0 rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zMPNetConverter.convertede  s‹  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r  d  r  rZ   r  c                   ó   — e Zd Zdefd„Zy)ÚOpenAIGPTConverterr\   c           
      ó  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d t        |«      dd¬«      «      }|j                  t        |«      «      �|j                  t        |«      g«       t        j                  d¬«      |_        t        j                  «       |_        t#        j$                  d¬«      |_        |S )Nú</w>F)rl   r   ÚdropoutrÚ   Úend_of_word_suffixÚfuse_unkT)rà   ©Úsuffix)rc   Úencoderr¾   Ú	bpe_ranksÚkeysrÚ   r   r   r»   Útoken_to_idÚadd_special_tokensr
   rð   rñ   r   rò   ró   r	   Ú
BPEDecoderrú   ©r‘   rl   r   rÚ   rü   s        rX   rÕ   zOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ñ9ÀDÔIˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°vÔ>ˆ	ÔàÐrZ   Nr   rj   rZ   rX   r  r  ‹  s   „ ð˜9ô rZ   r  c                   óH   — e Zd Zddeeef   dz  deeeef      dz  defd„Z	y)ÚGPT2ConverterNrl   r   r\   c           
      óN  — |s| j                   j                  }|st        | j                   j                  «      }t	        t        ||d ddd¬«      «      }t        | j                   dd«      }t        j                  |¬«      |_	        t        j                  «       |_        t        | j                   dd«      rT| j                   j                  }| j                   j                  }t        j                  |› d�|› d�||fg¬	«      |_        |S t        j                  d¬
«      |_        |S )NÚ F©rl   r   r  Úcontinuing_subword_prefixr  r  r[   ©r[   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rå   ©Útrim_offsets)rc   r  r¾   r  r   r   rb   r   Ú	ByteLevelró   r	   rú   Ú	bos_tokenÚbos_token_idr   rø   rù   )r‘   rl   r   rü   r[   Úbosr1  s          rX   rÕ   zGPT2Converter.converted§  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ñ":ÐL\Ô"]ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸UÔCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÑ'DØ˜˜g�Ø�u˜LÐ)à˜,Ð'ð ô(ˆIÔ$ð Ðô (2×';Ñ';ÈÔ'OˆIÔ$ØÐrZ   ©NN©
r«   r¸   r¹   rx   r»   r½   r¾   r¼   r   rÕ   rj   rZ   rX   r&  r&  ¦  s@   „ ñ"˜t C¨ H™~°Ñ4ð "ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað "Ðmvô "rZ   r&  c                   ó   — e Zd Zdefd„Zy)ÚHerbertConverterr\   c           	      óŒ  — d}d}| j                   j                  }t        | j                   j                  j	                  «       «      }||d   d   v r|dd  }t        t        ||d | j                   j                  |¬«      «      }t        j                  dd¬«      |_
        t        j                  «       |_        t        j                  |¬«      |_        t#        j$                  | j                   j&                  | j                   j(                  f| j                   j*                  | j                   j,                  f¬	«      |_        |S )
Nz	#version:r  r   r   )r  rÚ   r  F)rà   rß   r  )rþ   rý   )rc   r  r¾   r  r   r   r   rÚ   r
   rð   rñ   r   rò   ró   r	   r#  rú   r   ÚBertProcessingrõ   r÷   rô   rö   rù   )r‘   Útokenizer_info_strÚtoken_suffixrl   r   rü   s         rX   rÕ   zHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ ¨¡¨1¡Ñ-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ñ9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°|ÔDˆ	ÔÜ#-×#<Ñ#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYô$
ˆ	Ô ð
 ÐrZ   Nr   rj   rZ   rX   r6  r6  Ì  ó   „ ð˜9ô rZ   r6  c                   óH   — e Zd Zddeeef   dz  deeeef      dz  defd„Z	y)ÚQwen2ConverterNrl   r   r\   c                 ó0  — |s| j                   j                  }|s-t        | j                   j                  j	                  «       «      }t        t        ||d d dddd¬«      «      }t        j                  «       |_	        t        j                  t        j                  t        d«      dd¬«      t        j                  t        | j                   dd«      d¬«      g«      |_        t#        j                  «       |_        t'        j                  d¬	«      |_        |S )
Nr(  F)rl   r   r  rÚ   r*  r  r  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr[   ©r[   Ú	use_regexr-  )rc   r  r¾   r  r   r   r   r
   ÚNFCrñ   r   ÚSequenceÚSplitr   r/  rb   ró   r	   rú   r   rù   )r‘   rl   r   rü   s       rX   rÕ   zQwen2Converter.convertedì  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +Ÿ™Ó0ˆ	Ôä"0×"9Ñ"9ä×$Ñ$Üð Nóð (Ø ôô ×(Ñ(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ñ.Ó0ˆ	ÔÜ#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐrZ   r3  r4  rj   rZ   rX   r=  r=  ë  s@   „ ñ(˜t C¨ H™~°Ñ4ð (ÀTÈ%ÐPSÐUXÐPXÉ/ÑEZÐ]aÑEað (Ðmvô (rZ   r=  c                   ó   — e Zd Zdefd„Zy)ÚRobertaConverterr\   c           
      óÔ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  |j                  |j                   f|j"                  |j$                  f|j                  d¬«      |_        |S )Nr(  Fr)  r+  T©rþ   rý   r[   r.  )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   ÚRobertaProcessingrõ   r÷   rô   rö   rù   ©r‘   Úotrl   r   rü   s        rX   rÕ   zRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#?Ñ#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øô	$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rJ  rJ    ó   „ ð˜9ô rZ   rJ  c                   ó   — e Zd Zdefd„Zy)ÚRoFormerConverterr\   c           	      óV  — ddl m} | j                  j                  }t	        t        |t        | j                  j                  «      ¬«      «      }d}d}t        | j                  d«      r@| j                  j                  j                  }| j                  j                  j                  }t        j                  dd||¬«      |_        t        j                   j#                   ||«      «      |_        t        | j                  j&                  «      }t        | j                  j(                  «      }| j                  j*                  }| j                  j,                  }	t/        j0                  |› d|› d	�|› d|› d
|› d�||f||	fg¬«      |_        t5        j
                  d¬«      |_        |S )Nr   )ÚJiebaPreTokenizerrÙ   FrÛ   TrÜ   rá   râ   rã   rä   rå   ré   rê   )Ú"models.roformer.tokenization_utilsrT  rc   rl   r   r   r»   rÚ   rí   rÛ   rß   rï   r
   rð   rñ   r   ÚPreTokenizerÚcustomró   rô   rõ   rö   r÷   r   rø   rù   r	   rú   )
r‘   rT  rl   rü   rß   rï   rý   rþ   rö   r÷   s
             rX   rÕ   zRoFormerConverter.converted5  sy  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=Ô>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   rR  rR  4  r  rZ   rR  c                   ó   — e Zd Zdefd„Zy)ÚDebertaConverterr\   c           
      óÖ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  ddd| j                   j                  d«      fd| j                   j                  d«      fg¬	«      |_        |S )
Nr(  Fr)  r+  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rå   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   rø   r
  rù   rN  s        rX   rÕ   zDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rY  rY  [  r;  rZ   rY  c                   ó„   ‡ — e Zd ZdZeZi Zedd„«       Ze	dd„«       Z
ˆ fd„Zd„ Zd„ Zd„ Zd„ Zd	„ Zd
„ Zd„ Zdefd„Zˆ xZS )ÚSpmConverterFc           
      óZ  — | j                   j                  }| j                   j                  }| j                  j                  }t        |t        «      rt        t        ||xs g |d|d¬«      «      }nYt        |t        «      rH|rFt        |d   t        t        z  «      r,t        t        || j                   j                  |¬«      «      }nyt        j                  dd«      g}|r%|j                  dt        j                   |«      «       t        j"                  |«      |_        |rWt'        j"                  t'        j                  dd«      t'        j(                  «       t'        j*                  «       g«      |_        |S t'        j"                  t'        j                  dd«      g«      |_        |S )z÷
        Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
        TN)rl   r   rÚ   r  r?  r  r   )rl   r˜   r?  r  õ   â–�)r¥   r?  Ú	unk_piecer¯   r£   Ú
isinstancerx   r   r   r¾   r¼   r   r˜   r
   ÚReplaceÚinsertÚPrecompiledrG  rñ   r	   ÚByteFallbackÚFuserú   )r�   rl   r   r?  rc  r£   rü   Ú_normalizerss           rX   Úbuild_tokenizer_from_spm_protoz+SpmConverter.build_tokenizer_from_spm_proto  sr  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœTÔ"Ü!ÜØØ!š< RØ'Ø!Ø"/Ø ôó	‰Iô ˜œtÔ$©´:¸eÀA¹hÌÔPTÉÔ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ôó‰Ið ô $×+Ñ+¨C°Ó7Ð8ˆÙØ×Ñ ¤;×#:Ñ#:Ð;OÓ#PÔQÜ*×3Ñ3°LÓAˆ	Ôñ Ü (× 1Ñ 1Ü×!Ñ! %¨Ó-¬x×/DÑ/DÓ/FÌÏÉËÐXó!ˆIÔð Ðô !)× 1Ñ 1´8×3CÑ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐrZ   c                 ó   — |�||d<   |S )zŸ
        Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
        By default, return kwargs unchanged.
        rl   rj   )rý   rl   r°   s      rX   Úconvert_from_spmzSpmConverter.convert_from_spm¯  s   € ð ÐØ#ˆF�7‰OØˆrZ   c                 ó¦  •— t        | d«       t        ‰| �  |Ž  t        «       }|j	                  «       }t        | j                  j                  d«      5 }|j                  |j                  «       «       d d d «       || _
        | j                  j                  j                  r#| j                  st        j                  d«       y y y # 1 sw Y   ŒTxY w)NrQ   r‹   a  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperr•   rY   rŒ   r�   rc   Ú
vocab_filerŽ   r�   r�   r¥   r?  Úhandle_byte_fallbackÚwarningsÚwarn)r‘   Úargsr’   r“   r”   Ú	__class__s        €rX   r•   zSpmConverter.__init__¹  s°   ø€ Ü˜$ 
Ô+ä‰Ñ˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÓ;ð 	(¸qØ×Ñ˜aŸf™f›hÔ'÷	(àˆŒ
à�:‰:×"Ñ"×0Ò0¸×9RÒ9RÜ�M‰Mðeõð :SÐ0÷		(ð 	(ús   Á CÃCc                 ól   — |j                   D �cg c]  }|j                  |j                  f‘Œ c}S c c}w rÈ   ©r¨   r©   rª   ©r‘   r�   r©   s      rX   rl   zSpmConverter.vocabÎ  s'   € Ø8=¿¹ÖE¨u�—‘˜eŸk™kÒ*ÒEÐEùÒEs   �1c                 ó.   — |j                   j                  S rÈ   )r¥   r˜   ©r‘   r�   s     rX   r˜   zSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(rZ   c                 ó~  — |j                   j                  }| j                  |«      }|dk(  r1t        t	        || j                  |«      | j                  ¬«      «      }n£|dk(  r“| j                  | j                  j                  «      j                  |«      \  }}t        |«      D ���	ci c]  \  }\  }}	||“Œ }
}}}	t        t        |
||j                   j                  d| j                  d ¬«      «      }nt        d«      ‚t        |j                  «      D ��cg c]I  \  }}|j                   dv r6||j"                  |j                   dk(  xs |j"                  | j$                  v f‘ŒK }}}|j'                  t)        |d	„ ¬
«      D ���cg c]  \  }}}t+        |d|¬«      ‘Œ c}}}«       |S c c}	}}w c c}}w c c}}}w )Nr   ©r˜   r?  rq   T©rÚ   r  r?  r  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmr™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÑQU€ rZ   rn   FrŸ   )r¥   r§   rl   r   r   r˜   rq  ÚSpmExtractorrc   rp  r·   r­   r   rc  Ú	Exceptionr¨   r®   r©   rè   Ú
add_tokensr|   r   )r‘   r�   r§   r~   rü   Ú_r   r±   r²   rª   Ú	bpe_vocabr³   r´   rµ   r¶   r¡   s                   rX   rü   zSpmConverter.tokenizerÔ  s´  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Š?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Š_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAˆvÜ9BÀ<Ó9P×QÐQÑ%5 Q©¨¨u˜˜q™ÐQˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð ÐùôC Rùó*
ùôs   Â)F+ÄAF2ÆF8c                 ó   — |j                   j                  }t        j                  dd¬«      t        j                  t        d«      d«      g}|st        j                  |«      S t        j                  t        j                  |«      g|z   «      S )NFT)Úleftr  ú {2,}rb  )r¯   r£   r
   ÚStripre  r   rG  rg  ©r‘   r�   r£   rj  s       rX   rñ   zSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ñ 5°Ô5Ü×Ñ¤ g£°Ó6ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S ©N©Úreplacementrd   )re   rc   r   Ú	Metaspace©r‘   r�  r[   rd   s       rX   ró   zSpmConverter.pre_tokenizer  s)   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×'Ñ'°KÐP^Ô_Ð_rZ   c                  ó   — y rÈ   rj   rÔ   s    rX   rù   zSpmConverter.post_processor  s   € ØrZ   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S r‹  )re   rc   r	   rŽ  r�  s       rX   rú   zSpmConverter.decoder  s(   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×!Ñ!¨kÈ.ÔYÐYrZ   r\   c                 óz  — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        |S )Nrb  Tr[   )	rü   r�   rñ   rí   rc   r[   ró   rú   rù   )r‘   rü   rñ   r�  r[   ró   rù   s          rX   rÕ   zSpmConverter.converted  s¸   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐrZ   rÈ   )r«   r¸   r¹   rq  rˆ   r€  rè   Ústaticmethodrk  Úclassmethodrm  r•   rl   r˜   rü   rñ   ró   rù   rú   r   rÕ   Ú__classcell__)ru  s   @rX   r`  r`  z  ss   ø„ Ø ÐØ)€LØ€Nàò-ó ð-ð^ òó ðôò*Fò)ò0òd	hò`òòZð˜9÷ rZ   r`  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚAlbertConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w ©Néd   ©r¨   rÎ   r©   rª   rx  s      rX   rl   zAlbertConverter.vocab6  ó^   € ð Ÿ™ö
àô +=¸U¿[¹[Ô*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÑoò
ð 	
ùò 
ó   �AA!c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S ©Nz``ú"z''r‡  r  ©r
   re  rc   Úkeep_accentsr{   ÚNFKDÚStripAccentsrï   Ú	Lowercaser¯   r£   rg  r   rG  ©r‘   r�   Úlist_normalizersr£   s       rX   rñ   zAlbertConverter.normalizer<  óï   € ä×Ñ  cÓ*Ü×Ñ  cÓ*ð
Ðð ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRà×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIÜ×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nr[  r\  r]  r^  rå   ©r   rø   rc   r
  rÔ   s    rX   rù   zAlbertConverter.post_processorO  óR   € Ü×,Ñ,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
rZ   N©r«   r¸   r¹   rl   rñ   rù   rj   rZ   rX   r—  r—  5  ó   „ ò
ò6ó&
rZ   r—  c                   ó   — e Zd Zd„ Zd„ Zy)ÚBarthezConverterc                 ó
   — d}|S ©Nrš   rj   ©r‘   r�   r˜   s      rX   r˜   zBarthezConverter.unk_id[  ó   € ØˆØˆrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rå   r«  rÔ   s    rX   rù   zBarthezConverter.post_processor_  óR   € Ü×,Ñ,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   N)r«   r¸   r¹   r˜   rù   rj   rZ   rX   r°  r°  Z  s   „ òó
rZ   r°  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚCamembertConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )N)©z
<s>NOTUSEDç        ©ú<pad>r¾  ©z</s>NOTUSEDr¾  ©ú<unk>r¾  )ú<unk>NOTUSEDiœÿÿÿr   ©ú<mask>r¾  rw  ©r‘   r�   rl   r©   s       rX   rl   zCamembertConverter.vocabk  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   —Ac                  ó   — yr²  rj   rz  s     rX   r˜   zCamembertConverter.unk_idx  s   € àrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r¶  r«  rÔ   s    rX   rù   z!CamembertConverter.post_processor|  r¹  rZ   Nc                 ó,  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }d|dfd	|dfd
g}|�|j                  t        |«      dd  «       |j	                  |df«       ||d<   |S )NÚ	pad_tokenrÀ  rÚ   rÃ  Ú
mask_tokenrÆ  r½  r¾  rÁ  )rÄ  ç      YÀr   rl   ©r»   rÇ   r}   r¾   r{   )rý   rl   r°   rÌ  rÚ   rÍ  Ú
vocab_lists          rX   rm  z#CamembertConverter.convert_from_spm†  s¤   € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   ©r«   r¸   r¹   rl   r˜   rù   r”  rm  rj   rZ   rX   r»  r»  j  s%   „ òòò
ð òó ñrZ   r»  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚDebertaV2Converterc                 ó  — g }| j                   j                  r%|j                  t        j                  d¬«      «       t        || j                   «      }|j                  t        j                  ||¬«      «       t        j                  |«      S )Nr@  )rB  rŒ  )rc   Úsplit_by_punctr{   r   ÚPunctuationre   rŽ  rG  )r‘   r�  r[   Úlist_pretokenizersrd   s        rX   ró   z DebertaV2Converter.pre_tokenizer›  sq   € ØÐØ×"Ñ"×1Ò1Ø×%Ñ%¤n×&@Ñ&@È*Ô&UÔVÜ,Ð-=¸t×?VÑ?VÓWˆØ×!Ñ!¤.×":Ñ":À{ÐcqÔ"rÔsÜ×&Ñ&Ð'9Ó:Ð:rZ   c                 ó¼  — g }| j                   j                  r#|j                  t        j                  «       «       |j                  t        j
                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       |j                  t        j                  t        d«      d«      «       t        j                  |«      S )Nr‡  r  )rc   rï   r{   r
   r¥  rˆ  r¯   r£   rg  re  r   rG  r¦  s       rX   rñ   zDebertaV2Converter.normalizer£  s¥   € ØÐØ×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<Ø×Ñ¤× 1Ñ 1Ó 3Ô4à$×4Ñ4×IÑIÐÙØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRØ×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIä×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rª  r«  rÔ   s    rX   rù   z!DebertaV2Converter.post_processor°  r¬  rZ   N)r«   r¸   r¹   ró   rñ   rù   rj   rZ   rX   rÓ  rÓ  š  s   „ ò;ò6ó
rZ   rÓ  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚMBartConverterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )N©©r·  r¾  r¿  ©r¸  r¾  rÂ  rš   )©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r   r¾  ©r    r¾  ©r!   r¾  ©r"   r¾  ©r#   r¾  ©r$   r¾  ©r%   r¾  ©r&   r¾  ©r'   r¾  ©r(   r¾  ©r)   r¾  ©r*   r¾  ©r+   r¾  ©r,   r¾  ©r-   r¾  ©r.   r¾  ©r/   r¾  rÅ  rw  rÇ  s       rX   rl   zMBartConverter.vocab¼  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   —A
c                  ó   — yr²  rj   rz  s     rX   r˜   zMBartConverter.unk_idâ  ó   € ØrZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A </s> en_XXz$A $B </s> en_XXr   r¸  rå   r«  rÔ   s    rX   rù   zMBartConverter.post_processorå  óR   € Ü×,Ñ,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óÖ  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j                  d„ t        D «       «       |j                  |df«       ||d<   |S )Nr0  r·  rÌ  rÀ  Ú	eos_tokenr¸  rÚ   rÃ  rÍ  rÆ  r¾  rš   c              3   ó$   K  — | ]  }|d f–— Œ
 y­w©r¾  Nrj   ©Ú.0Ú	lang_codes     rX   ú	<genexpr>z2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   è ø€ ÒL¨y˜9 cÔ*ÑLùó   ‚rl   )r»   rÇ   r}   r¾   ÚMBART_LANGUAGESr{   ©	rý   rl   r°   r0  rÌ  rÿ  rÚ   rÍ  rÐ  s	            rX   rm  zMBartConverter.convert_from_spmï  sé   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑL¼OÔLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÑ  rj   rZ   rX   rÛ  rÛ  »  s&   „ ò$òLò
ð òó ñrZ   rÛ  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚMBart50Converterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )NrÝ  rš   )4rà  rá  râ  rã  rä  rå  ræ  rç  rè  ré  rê  rë  rì  rí  rî  rï  rð  rñ  rò  ró  rô  rõ  rö  r÷  rø  )r0   r¾  )r1   r¾  )r2   r¾  )r3   r¾  )r4   r¾  )r5   r¾  )r6   r¾  )r7   r¾  )r8   r¾  )r9   r¾  )r:   r¾  )r;   r¾  )r<   r¾  )r=   r¾  )r>   r¾  )r?   r¾  )r@   r¾  )rA   r¾  )rB   r¾  )rC   r¾  )rD   r¾  )rE   r¾  )rF   r¾  )rG   r¾  )rH   r¾  )rI   r¾  )rJ   r¾  rÅ  rw  rÇ  s       rX   rl   zMBart50Converter.vocab  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò Lrù  c                  ó   — yr²  rj   rz  s     rX   r˜   zMBart50Converter.unk_id  rû  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzen_XX $A </s>zen_XX $A $B </s>r   r¸  rå   r«  rÔ   s    rX   rù   zMBart50Converter.post_processor  rý  rZ   Nc                 óÖ  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j                  d„ t        D «       «       |j                  |df«       ||d<   |S )Nrô   r·  rÌ  rÀ  rÿ  r¸  rÚ   rÃ  rÍ  rÆ  r¾  rš   c              3   ó$   K  — | ]  }|d f–— Œ
 y­wr  rj   r  s     rX   r  z4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   è ø€ ÒN¨y˜9 cÔ*ÑNùr  rl   )r»   rÇ   r}   r¾   ÚMBART50_LANGUAGESr{   )	rý   rl   r°   rô   rÌ  rÿ  rÚ   rÍ  rÐ  s	            rX   rm  z!MBart50Converter.convert_from_spm  sê   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×ÑÑNÔ<MÔNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÑ  rj   rZ   rX   r
  r
    s%   „ ò
òò
ð òó ñrZ   r
  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚNllbConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )NrÝ  rš   rw  rÇ  s       rX   rl   zNllbConverter.vocab6  óC   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ló   —=c                  ó   — yr²  rj   rz  s     rX   r˜   zNllbConverter.unk_id@  rû  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnr¸  rå   r«  rÔ   s    rX   rù   zNllbConverter.post_processorC  sR   € Ü×,Ñ,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óª  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }|d	|d
|d|di}|�Ot        |t        «      r|j	                  «       n|D ��	cg c]  \  }}	|‘Œ	 c}	}}
|
D ]  }||v rŒt        |«      ||<   Œ ||d<   |S c c}	}w )Nr0  r·  rÌ  rÀ  rÿ  r¸  rÚ   rÃ  r   r   rq   rš   rl   )r»   rÇ   rd  rx   r   rr   )rý   rl   r°   r0  rÌ  rÿ  rÚ   Úreordered_vocabÚtokrƒ  Útokensr¶   s               rX   rm  zNllbConverter.convert_from_spmM  sâ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÐÜ%/°´tÔ%<�U—Z‘Z”\ÐUZ×B[É6È3ÐPQÂ3ÓB[ˆFØò >�Ø˜OÑ+ØÜ),¨_Ó)=� Ò&ð>ð *ˆˆw‰Øˆùó C\s   ÂCrÈ   rÑ  rj   rZ   rX   r  r  5  s%   „ òòò
ð òó ñrZ   r  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚSeamlessM4TConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )N)r¿  rÂ  rÞ  rß  rš   rw  rÇ  s       rX   rl   zSeamlessM4TConverter.vocabe  r  r  c                 ó.   — | j                   j                  S rÈ   )rc   Úunk_token_idrz  s     rX   r˜   zSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__r¸  rå   r«  rÔ   s    rX   rù   z#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ñ,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   N©r«   r¸   r¹   rl   r˜   rù   rj   rZ   rX   r  r  d  s   „ òò4ó
rZ   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zedd„«       Zy)ÚXLMRobertaConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )NrÝ  rš   rÅ  rw  rÇ  s       rX   rl   zXLMRobertaConverter.vocab~  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò LrÈ  c                 ó
   — d}|S r²  rj   r³  s      rX   r˜   zXLMRobertaConverter.unk_id‰  r´  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S r¶  r«  rÔ   s    rX   rù   z"XLMRobertaConverter.post_processor�  r¹  rZ   Nc                 óž  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  d	d
«      «      }|df|df|df|dfg}|�|j                  t        |«      dd  «       |j	                  |df«       ||d<   |S )Nr0  r·  rÌ  rÀ  rÿ  r¸  rÚ   rÃ  rÍ  rÆ  r¾  rš   rl   rÏ  r  s	            rX   rm  z$XLMRobertaConverter.convert_from_spm—  sÕ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆˆw‰ØˆrZ   rÈ   rÑ  rj   rZ   rX   r&  r&  }  s%   „ ò	òò
ð òó ñrZ   r&  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLNetConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w r™  r›  rx  s      rX   rl   zXLNetConverter.vocab­  rœ  r�  c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S rŸ  r¡  r¦  s       rX   rñ   zXLNetConverter.normalizer³  r¨  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rå   r«  rÔ   s    rX   rù   zXLNetConverter.post_processorÆ  r¬  rZ   Nr­  rj   rZ   rX   r,  r,  ¬  r®  rZ   r,  c                   ó   — e Zd Zy)ÚReformerConverterN©r«   r¸   r¹   rj   rZ   rX   r1  r1  Ñ  ó   „ ØrZ   r1  c                   ó   — e Zd Zd„ Zd„ Zy)ÚRemBertConverterc                 ób  — t        j                  dd«      t        j                  dd«      t        j                  t        d«      d«      g}| j                  j                  sF|j                  t        j                  «       «       |j                  t        j                  «       «       | j                  j                  r#|j                  t        j                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       t        j                  |«      S rŸ  )r
   re  r   rc   r¢  r{   r£  r¤  rï   r¥  r¯   r£   rg  rG  r¦  s       rX   rñ   zRemBertConverter.normalizer×  sä   € ä×Ñ  cÓ*Ü×Ñ  cÓ*Ü×Ñ¤ g£°Ó4ð
Ðð
 ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRä×#Ñ#Ð$4Ó5Ð5rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rª  r«  rÔ   s    rX   rù   zRemBertConverter.post_processorê  r¬  rZ   N)r«   r¸   r¹   rñ   rù   rj   rZ   rX   r5  r5  Õ  s   „ ò6ó&
rZ   r5  c                   ó   — e Zd Zy)ÚBertGenerationConverterNr2  rj   rZ   rX   r9  r9  õ  r3  rZ   r9  c                   ó6   — e Zd Zd„ Zedd„«       Zd„ Zd„ Zd„ Zy)ÚPegasusConverterc                 óv  — | j                   j                  df| j                   j                  dfg}| j                   j                  �|| j                   j                  dfgz  }| j                   j                  �I| j                   j
                  | j                   j                  k  r|| j                   j                  dfgz  }|t        d| j                   j                  «      D �cg c]
  }d|› d�df‘Œ c}z  }||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w c c}w )Nr¾  rq   ú<unk_ú>rÎ  )rc   rÌ  rÿ  Úmask_token_sentrÍ  Úmask_token_idÚoffsetrz   r¨   r©   rª   )r‘   r�   rl   r±   r©   s        rX   rl   zPegasusConverter.vocabú  s%  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ð>Ø�t×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ð:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÒVà�t×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÓ2ZÖ[¨Q�U˜1˜#˜Q�< Ò(Ò[Ñ[ˆØ¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò \ùÚKs   Ã%D1ÄD6Nc           	      ó(  — t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }t        |j                  dd«      «      }|d	f|d	fg}|d
k7  r|j                  |d	f«       |d
k7  r||k7  r|j                  |d	f«       |j                  t	        d|j                  dd«      «      D �cg c]
  }d|› d�df‘Œ c}«       |�|j                  t        |«      dd  «       ||d<   |S c c}w )NrÌ  rÀ  rÿ  r¸  rÍ  z<mask_1>r?  z<mask_2>r¾  ÚNonerq   rA  ég   r=  r>  rÎ  rl   )r»   rÇ   r{   r}   rz   r¾   )	rý   rl   r°   rÌ  rÿ  rÍ  r?  rÐ  r±   s	            rX   rm  z!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÒØ×Ñ˜z¨3Ð/Ô0Ø˜fÒ$¨¸JÒ)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ó;^Ö_°a˜e A 3 a˜L¨&Ò1Ò_Ô`ØÐØ×Ñœd 5›k¨!¨"˜oÔ.Ø$ˆˆw‰Øˆùò	 `s   ÃDc                 ó\   — |j                   j                  | j                  j                  z   S rÈ   )r¥   r˜   rc   rA  rz  s     rX   r˜   zPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIrZ   c                 óª   — t        || j                  «      }t        j                  t        j                  «       t        j
                  ||¬«      g«      S r‹  )re   rc   r   rG  ÚWhitespaceSplitrŽ  r�  s       rX   ró   zPegasusConverter.pre_tokenizer&  sJ   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×&Ñ&ä×.Ñ.Ó0Ü×(Ñ(°[ÐQ_Ô`ðó
ð 	
rZ   c                 óš   — | j                   j                  }|| j                   j                  fg}t        j                  d|gdd|g|¬«      S )Nú$Aú$Brå   )rc   rÿ  Úeos_token_idr   rø   )r‘   Úeosrè   s      rX   rù   zPegasusConverter.post_processor/  sR   € Ø×%Ñ%×/Ñ/ˆà�$×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ñ,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwrZ   rÈ   )	r«   r¸   r¹   rl   r”  rm  r˜   ró   rù   rj   rZ   rX   r;  r;  ù  s,   „ òð& òó ðò*Jò
óxrZ   r;  c                   ó*   — e Zd Zd„ Zd„ Zedd„«       Zy)ÚT5Converterc                 óò   — | j                   j                  }|j                  D �cg c]  }|j                  |j                  f‘Œ }}|t        |dz
  dd«      D �cg c]
  }d|› d�df‘Œ c}z  }|S c c}w c c}w )Nr   rÊ   ú
<extra_id_r>  r¾  )rc   Ú
_extra_idsr¨   r©   rª   rz   )r‘   r�   Únum_extra_idsr©   rl   r±   s         rX   rl   zT5Converter.vocab8  sw   € Ø×/Ñ/×:Ñ:ˆØ9>¿¹ÖF°�%—+‘+˜uŸ{™{Ò+ÐFˆÐFØ´E¸-È!Ñ:KÈRÐQSÓ4TÖU¨q�Z ˜s !Ð$ cÒ*ÒUÑUˆØˆùò GùÚUs   ¥A/ÁA4c                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S ©NrI  r¸  )rI  r¸  rJ  r¸  rå   r«  rÔ   s    rX   rù   zT5Converter.post_processor>  ó=   € Ü×,Ñ,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nc                 óò   — |j                  dd«      }t        |dz
  dd«      D �cg c]  }d|› d�‘Œ
 }}|�t        |«      ng }|j                  d„ |D «       «       |j	                  d|«       ||d	<   |S c c}w )
NÚ	extra_idsrš  r   rÊ   rP  r>  c              3   ó$   K  — | ]  }|d f–— Œ
 y­wr  rj   )r  r¶   s     rX   r  z/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   è ø€ ÒA¨5˜5 #œ,ÑAùr  r¢   rl   )rÇ   rz   r¾   r}   Ú
setdefault)rý   rl   r°   rW  r±   Úextra_tokensrÐ  s          rX   rm  zT5Converter.convert_from_spmG  s„   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ¹ÈÈBÓ3OÖP¨a˜* Q C qÒ)ÐPˆÐPØ$)Ð$5”T˜%”[¸2ˆ
Ø×ÑÑA°LÔAÔAà×ÑÐ5°|ÔDØ$ˆˆw‰Øˆùò Qs   ¥A4rÈ   )r«   r¸   r¹   rl   rù   r”  rm  rj   rZ   rX   rN  rN  7  s    „ òò
ð òó ñrZ   rN  c                   ó   — e Zd Zd„ Zy)ÚUdopConverterc                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S rT  r«  rÔ   s    rX   rù   zUdopConverter.post_processorT  rU  rZ   N©r«   r¸   r¹   rù   rj   rZ   rX   r\  r\  S  s   „ ó
rZ   r\  c                   ó   — e Zd Zdefd„Zy)ÚWhisperConverterr\   c           
      óÈ  — | j                   j                  }t        | j                   j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  | j                   j                  ¬«      |_
        t        j                  «       |_        | j                   j                  }| j                   j                  |«      }| j                   j                  }| j                   j                   }dj#                  |D �cg c]  }|› d�‘Œ	 c}«      }	t%        j&                  |	› d|› d�|	› d|› d	�||fgt)        ||«      ¢¬
«      |_        |S c c}w )Nr(  Fr)  r+  r  râ   z $A:0 z $A:0 $B:1 rä   rå   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   Úprefix_tokensÚconvert_ids_to_tokensrÿ  rK  Újoinr   rø   Úziprù   )
r‘   rl   r   rü   Úprefix_token_idsÚprefixesrL  rK  r¶   Úprefix_templates
             rX   rÕ   zWhisperConverter.converted_  sR  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¸hÖ#G°U u g¨R¢LÒ#GÓHˆÜ#-×#@Ñ#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7à�lÐ#ðä�XÐ/Ó0ðô$
ˆ	Ô ð Ðùò $Hs   ÄENr   rj   rZ   rX   r`  r`  ^  s   „ ð ˜9ô  rZ   r`  c                   ó   — e Zd Zd„ Zy)ÚBigBirdConverterc           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rª  r«  rÔ   s    rX   rù   zBigBirdConverter.post_processorƒ  r¬  rZ   Nr^  rj   rZ   rX   rj  rj  ‚  s   „ ó
rZ   rj  c                   ó   — e Zd Zdefd„Zy)ÚCLIPConverterr\   c                 óp  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d dddt        |«      ¬«      «      }t        j                  t        j                  «       t        j                  t        d«      d«      t        j                  «       g«      |_        t!        j                  t!        j"                  t        d«      dd	¬
«      t!        j$                  d¬«      g«      |_        t)        j$                  «       |_        t-        j.                  | j                   j0                  | j                   j2                  f| j                   j4                  | j                   j6                  fdd¬«      |_        |S )Nr(  r  F©rl   r   r  r*  r  r  rÚ   z\s+r  z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrA  r+  rL  )rc   r  r¾   r  r   rÚ   r   r   r»   r
   rG  rF  re  r   r¥  rñ   r   rH  r/  ró   r	   rú   r   rM  rÿ  rK  r0  r1  rù   r$  s        rX   rÕ   zCLIPConverter.converted�  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ñ3Ü�_‰_Ó¤× 3Ñ 3´E¸&³MÀ3Ó GÌ×I^ÑI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ñ"9ä×$Ñ$ÜÐZÓ[Ø&Øôô
 ×(Ñ(¸%Ô@ðó	#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôô $.×#?Ñ#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øô	$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rm  rm  Ž  s   „ ð'˜9ô 'rZ   rm  c                   ó   — e Zd Zdefd„Zy)ÚLayoutLMv2Converterr\   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )NrÙ   FTrÛ   rÜ   rá   râ   rã   rä   rå   ré   rê   rì   rû   s
             rX   rÕ   zLayoutLMv2Converter.convertedº  s…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐrZ   Nr   rj   rZ   rX   rr  rr  ¹  r  rZ   rr  c                   ó   — e Zd Zdefd„Zy)ÚBlenderbotConverterr\   c           
      ó®  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  d|j                  › d�|j                  |j                   fg¬«      |_        |S )Nr(  Fr)  r+  z$A:0 râ   )ræ   rè   )rc   r  r¾   r  r   r   r   r   r/  r[   ró   r	   rú   r   rø   rÿ  rK  rù   rN  s        rX   rÕ   zBlenderbotConverter.convertedá  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   ru  ru  à  rP  rZ   ru  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXGLMConverterc                 ó’   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|S c c}w )NrÝ  rš   ))z<madeupword0>r¾  )z<madeupword1>r¾  )z<madeupword2>r¾  )z<madeupword3>r¾  )z<madeupword4>r¾  )z<madeupword5>r¾  )z<madeupword6>r¾  rw  rÇ  s       rX   rl   zXGLMConverter.vocabþ  sT   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  zñ  	zˆØˆùò Ls   —Ac                 ó
   — d}|S r²  rj   r³  s      rX   r˜   zXGLMConverter.unk_id	  r´  rZ   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz</s> $Az</s> $A </s> </s> $Br·  r¸  rå   r«  rÔ   s    rX   rù   zXGLMConverter.post_processor  sR   € Ü×,Ñ,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
rZ   Nr$  rj   rZ   rX   rx  rx  ý  s   „ ò	òó
rZ   rx  c                   ó<   — e Zd ZdZeZddhZ	 d„ Zd„ Zd„ Z	d„ Z
d„ Zy	)
ÚGemmaConverterTz<start_of_turn>z<end_of_turn>c                 ó.   — t        j                  dd«      S ©Nr  rb  )r
   re  rz  s     rX   rñ   zGemmaConverter.normalizer(  s   € Ü×"Ñ" 3¨Ó.Ð.rZ   c                 ót  — | j                   j                  df| j                   j                  df| j                   j                  dfg}||j                  dd  D �cg c]  }|j
                  |j                  f‘Œ c}z  }t        d„ |D «       «      s#t        d„ t        |«      D «       d «      }|�d||<   |S c c}w )Nr¾  rš   c              3   ó,   K  — | ]  }|d    dk(  –— Œ y­w)r   rÂ   Nrj   )r  rk   s     rX   r  z'GemmaConverter.vocab.<locals>.<genexpr>4  s   è ø€ Ò/ A�1�Q‘4˜4•<Ñ/ùs   ‚c              3   ó8   K  — | ]  \  }}|d    dk(  sŒ|–— Œ y­w)r   rÃ   Nrj   )r  r±   rk   s      rX   r  z'GemmaConverter.vocab.<locals>.<genexpr>5  s   è ø€ Ò"V©¨¨AÀQÀqÁTÈXÓEU¤1Ñ"Vùs   ‚“)rÂ   r¾  )
rc   rÌ  rÿ  r0  r¨   r©   rª   ÚanyÚnextr­   )r‘   r�   rl   r©   Úoverride_indexs        rX   rl   zGemmaConverter.vocab+  s·   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆô Ñ/¨Ô/Ô/Ü!Ñ"V´¸5Ó1AÔ"VÐX\Ó]ˆNØÐ)Ø(3��nÑ%àˆùò Ls   ÁB5c                 ó.   — t        j                  dd«      S )Nr  Úmerged_with_previous)r   rH  ©r‘   r�  r[   s      rX   ró   zGemmaConverter.pre_tokenizer;  s   € Ü×#Ñ# CÐ)?Ó@Ð@rZ   c                 ó
   — d}|S r²  rj   r³  s      rX   r˜   zGemmaConverter.unk_id>  r´  rZ   c                 ó¢   — t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      S )Nrb  r  )r	   rG  re  rh  ri  rˆ  s      rX   rú   zGemmaConverter.decoderB  s?   € Ü× Ñ ä× Ñ  ¨Ó,Ü×%Ñ%Ó'Ü—‘“ðó
ð 	
rZ   N)r«   r¸   r¹   rq  rÀ   r€  rè   rñ   rl   ró   r˜   rú   rj   rZ   rX   r}  r}    s6   „ ØÐØ.€Là'¨Ð9€Nðò/òò Aòó
rZ   r}  c                   ó4   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	y)	ÚLlamaConverterTc                 ó(  — | j                   j                  d«      df| j                   j                  d«      df| j                   j                  d«      dfg}||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )Nr   r¾  r   rq   rš   )rc   rc  r¨   r©   rª   rÇ  s       rX   rl   zLlamaConverter.vocabO  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ls   Á)Bc                 ó
   — d}|S r�   rj   r³  s      rX   r˜   zLlamaConverter.unk_idX  r´  rZ   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S ©Nrb  r  r   )Úcontentr†  ©r	   re  rh  ri  rˆ  rG  ©r‘   r�  r[   Úsequences       rX   rú   zLlamaConverter.decoder\  ó\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ
 ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*rZ   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )Nr_   Tr[   rb  )Úprependr  )Úpatternr‘  )rb   rc   r
   ÚPrependre  rG  )r‘   r�   r”  s      rX   rñ   zLlamaConverter.normalizerf  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1ØrZ   c                 óŽ   — t        | j                  dd«      s.t        || j                  «      }t        j                  ||d¬«      S y )Nr_   TF©r�  rd   Úsplit)rb   rc   re   r   rŽ  r�  s       rX   ró   zLlamaConverter.pre_tokenizero  sA   € Ü�t×.Ñ.°¸$Ô?Ü0Ð1AÀ4×CZÑCZÓ[ˆNÜ!×+Ñ+¸ÐTbÐjoÔpÐpØrZ   c                  ó   — y rÈ   rj   rÔ   s    rX   rù   zLlamaConverter.post_processoru  s   € àrZ   N)
r«   r¸   r¹   rq  rl   r˜   rú   rñ   ró   rù   rj   rZ   rX   rŒ  rŒ  L  s&   „ ØÐòòò+òòórZ   rŒ  c                   ó   — e Zd Zdefd„Zy)ÚMarkupLMConverterr\   c                 óŠ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd| j                   j                  ¬«      «      }t        j                  |j                  ¬«      |_        t        j                  «       |_        t        | j                   j                  «      }t        | j                   j                   «      }| j                   j"                  }| j                   j$                  }t'        j(                  |› d|› �|› d|› d|› �||f||fg¬«      |_        |S )Nr(  Fro  r+  z $A z $B rå   )rc   r  r¾   r  r   r   r   rÚ   r   r/  r[   ró   r	   rú   r»   rô   rõ   rö   r÷   r   rø   rù   )	r‘   rO  rl   r   rü   rý   rþ   rö   r÷   s	            rX   rÕ   zMarkupLMConverter.converted{  s,  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ð ÐrZ   Nr   rj   rZ   rX   rŸ  rŸ  z  s   „ ð"˜9ô "rZ   rŸ  c                   ó(   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zy)ÚMoshiConverterTc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w©NrQ   r‹   ©	r   rÐ   r•   rY   rŒ   r�   rŽ   r�   r�   ©r‘   rp  r°   r’   r“   r”   s         rX   r•   zMoshiConverter.__init__£  sr   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úó   Á	 A9Á9B	c                 óâ   — |j                   j                  }t        j                  dd«      g}|st        j                  |«      S t        j                  t        j
                  |«      g|z   «      S r  )r¯   r£   r
   re  rG  rg  r‰  s       rX   rñ   zMoshiConverter.normalizer°  sg   € Ø$×4Ñ4×IÑIÐä×Ñ  UÓ+ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgrZ   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S r�  r’  r“  s       rX   rú   zMoshiConverter.decoderº  r•  rZ   c                 ó6   — d}t        j                  ||d¬«      S )Nr`   Fr›  )r   rŽ  r�  s       rX   ró   zMoshiConverter.pre_tokenizerÄ  s   € Ø ˆÜ×'Ñ'°KÐP^ÐfkÔlÐlrZ   N)r«   r¸   r¹   rq  r•   rñ   rú   ró   rj   rZ   rX   r¢  r¢     s   „ ØÐòòhò+ómrZ   r¢  c                   óB   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zy)ÚHeliumConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY wr¤  r¥  r¦  s         rX   r•   zHeliumConverter.__init__Ì  sp   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úr§  c                 óV  — | j                  |«      }t        t        || j                  |«      | j                  ¬«      «      }t        |j                  «      D ��cg c]I  \  }}|j                  dv r6||j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}}|j                  t        |d„ ¬«      D ���cg c]  \  }}}t        |d|d¬«      ‘Œ c}}}«       |j                  t        d	dd¬
«      g«       |j                  dd¬«       |S c c}}w c c}}}w )Nr|  r™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z+HeliumConverter.tokenizer.<locals>.<lambda>í  r  rZ   rn   FT)r    r¡   Úsingle_wordú
rŸ   rÀ  )rÌ  Úpad_id)rl   r   r   r˜   rq  r­   r¨   r®   r©   rè   r‚  r|   r   Úenable_padding)	r‘   r�   r~   rü   r³   r´   rµ   r¶   r¡   s	            rX   rü   zHeliumConverter.tokenizerØ  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÐQUÖVôô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁADÃD$c                 ó¨   — g }|j                   D ]@  }|j                  dk(  r|d|j                  fgz  }Œ%||j                  |j                  fgz  }ŒB |S )Nz<0x0A>r±  rw  rÇ  s       rX   rl   zHeliumConverter.vocabô  s]   € ØˆØ—\‘\ò 	6ˆEØ�{‰{˜hÒ&Ø˜4 §¡Ð-Ð.Ñ.‘à˜5Ÿ;™;¨¯©Ð4Ð5Ñ5‘ð		6ð
 ˆrZ   c                 ó
   — d}|S r�   rj   r³  s      rX   r˜   zHeliumConverter.unk_idý  r´  rZ   c                 óÜ   — t        j                  dd«      t        j                  «       t        j                  «       g}|t        j                  dd¬«      gz  }t        j
                  |«      S r�  r’  r“  s       rX   rú   zHeliumConverter.decoder  sY   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆð
 	”X—^‘^¨C°aÔ8Ð9Ñ9ˆÜ× Ñ  Ó*Ð*rZ   c                 ó~   — t        j                  t        j                  d«      t        j                  dd«      g«      S r  )r
   rG  r™  re  rz  s     rX   rñ   zHeliumConverter.normalizer
  s2   € Ü×#Ñ#¤[×%8Ñ%8¸Ó%=¼{×?RÑ?RÐSWÐY^Ó?_Ð$`ÓaÐarZ   c                 óV   — t        j                  t        j                  dd«      g«      S )Nr±  Ú
contiguous)r   rG  rH  rˆ  s      rX   ró   zHeliumConverter.pre_tokenizer  s#   € Ü×&Ñ&¬×(<Ñ(<¸TÀ<Ó(PÐ'QÓRÐRrZ   c                 ó<   — t        j                  ddgg d¢dg¬«      S )Nr·  rI  )r·  rI  r·  rJ  )r·  r   rå   )r   rø   rÔ   s    rX   rù   zHeliumConverter.post_processor  s/   € Ü×,Ñ,àØðòð ðô
ð 	
rZ   rÈ   )r«   r¸   r¹   rq  r•   rü   rl   r˜   rú   rñ   ró   rù   rj   rZ   rX   r¬  r¬  É  s2   „ ØÐó
òò8òò+òbòSó
rZ   r¬  c                   ó   — e Zd ZdZdd„Zd„ Zy)ÚParakeetConverterTNc                 ó&  — || _         t        | d«       t        j                  | |«       t	        «       }|j                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _	        y # 1 sw Y   || _	        y xY wr¤  )
rp  r   rÐ   r•   rY   rŒ   r�   rŽ   r�   r�   )r‘   rp  rt  r’   r“   r”   s         rX   r•   zParakeetConverter.__init__%  sw   € Ø$ˆŒä˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
ús   Á B Â Bc                 ó°  — | j                  |«      }| j                  | j                  «      j                  |«      \  }}t	        |«      D ���ci c]  \  }\  }}||“Œ }}}}t        t        |||j                  j                  d| j                  d ¬«      «      }	t	        |j                  «      D �
�cg c]I  \  }
}|j                  dv r6|
|j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}
}|	j                  t        |d„ ¬«      D �
��cg c]  \  }
}}t!        |d|¬«      ‘Œ c}}}
«       |	S c c}}}w c c}}
w c c}}}
w )	NTr}  r™   rš   c                 ó   — | d   S r�   rj   rž   s    rX   rm   z-ParakeetConverter.tokenizer.<locals>.<lambda>K  r  rZ   rn   FrŸ   )rl   r€  rp  r·   r­   r   r   r¥   rc  rq  r¨   r®   r©   rè   r‚  r|   r   )r‘   r�   r~   rƒ  r   r±   r²   rª   r„  rü   r³   r´   rµ   r¶   r¡   s                  rX   rü   zParakeetConverter.tokenizer2  sM  € Ø—z‘z %Ó(ˆà×%Ñ% d§o¡oÓ6×>Ñ>¸|ÓL‰	ˆˆ6Ü5>¸|Ó5L×MÐMÑ!1 ¡M T¨5�T˜1‘WÐMˆ	ÒMÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øôó	
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð Ðùô3 Nùó
ùôs   ÁEÂ2AEÄ ErÈ   )r«   r¸   r¹   rq  r•   rü   rj   rZ   rX   r¼  r¼  "  s   „ ØÐóórZ   r¼  c            	      óò  — t        t        t        d«      t        d«      dz   «      «      t        t        t        d«      t        d«      dz   «      «      z   t        t        t        d«      t        d«      dz   «      «      z   } | dd }d	}t        d
«      D ]1  }|| vsŒ| j                  |«       |j                  d
|z   «       |dz  }Œ3 |D �cg c]  }t	        |«      ‘Œ }}t        t        | |«      «      S c c}w )a8  
    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
    characters the bpe code barfs on.

    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
    tables between utf-8 bytes and unicode strings.
    ú!ú~r   õ   Â¡õ   Â¬õ   Â®õ   Ã¿Nr   é   )r¾   rz   Úordr{   Úchrrx   re  )ÚbsÚcsÚnÚbs       rX   Úbytes_to_unicoderÎ  R  sæ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4‹[ò ˆØ�BŠ;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰F‰Að	ð
 Ö	�QŒ#ˆa�&Ð	€BÐ	Ü”�B˜“ÓÐùò 
s   ÃC4c                   ó>   — e Zd ZdZ	 	 	 	 d	d„Zdefd„Zd„ Zdefd„Z	y)
ÚTikTokenConverterz'
    A general tiktoken converter.
    Nc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y rÈ   )rp  r˜  r[   rd  rx   r   Úextra_special_tokens)r‘   rp  r˜  r[   rÒ  r°   s         rX   r•   zTikTokenConverter.__init__o  sD   € ð %ˆŒØˆŒØ 0ˆÔä+5Ð6JÌDÔ+QÐ ×%Ñ%Ó'ð 	Õ!ØWkð 	Õ!rZ   Útiktoken_urlc                 ó0  ‡‡— 	 ddl m}  ||«      Št	        «       Šˆfd„}g }i }‰j                  «       D ]�  \  }}|| ||«      <   t        |«      dk(  rŒ g }t        dt        |«      «      D ]2  }	|d |	 ||	d  }}
|
‰v sŒ|‰v sŒ|
|z   ‰v sŒ|j                  |
||f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ� t        |d	„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS # t        $ r t        d«      ‚w xY wc c}w )
Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w ©Nr(  zlatin-1©rd  ÚdecoderÈ  ©rÍ  ÚcharÚbyte_encoders     €rX   Útoken_bytes_to_stringzPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string‰  s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTó    <r   c                 ó$   •— ‰| d      ‰| d      fS ri   rj   )rk   r  s    €rX   rm   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>—  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ð0R€ rZ   Fru   c                 ó   — | d   S ©Nrq   rj   rs   s    rX   rm   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>™  ó
   € °°A±€ rZ   )Útiktoken.loadrÕ  r�  Ú
ValueErrorrÎ  ry   rr   rz   r{   r|   r}   )r‘   rÓ  rÕ  rÝ  r   rl   r¶   Úrankr‚   rƒ   r„   r…   rt   r  rÜ  s                @@rX   Úextract_vocab_merges_from_modelz1TikTokenConverter.extract_vocab_merges_from_model~  sY  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆô	Vð ˆØˆØ$Ÿ?™?Ó,ò 
	!‰KˆE�4Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜iÒ'¨G°yÒ,@ÀgÐPWÑFWÐ\eÒEeØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&RÐ\aÔbˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðøô5 ò 	ÜØkóð ð	üò2 ds   „C; ÃDÃ;Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S ©NF)r  Úignore_mergesT©ræ  rp  r   r   rí   r‰   ré  ©r‘   r~   r   rü   s       rX   rü   zTikTokenConverter.tokenizer�  óN   € Ø#×CÑCÀDÇOÁOÓTÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐrZ   r\   c           
      óâ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        | j                  �5|j                  | j                  D �cg c]  }t        |dd¬«      ‘Œ c}«       t        j                  d¬«      |_        |S c c}w )Nr@  FrA  rD  TrŸ   r-  )rü   r   rG  rH  r   r˜  r/  r[   ró   r	   rú   rÒ  r"  r   r   rù   )r‘   rü   r¶   s      rX   rÕ   zTikTokenConverter.converted¤  sÄ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×$Ñ$Ð0Ø×(Ñ(ØPT×PiÑPiÖjÀu”˜E¨e¸TÖBÒjôô $.×#7Ñ#7ÀUÔ#Kˆ	Ô àÐùò ks   Â3C,©Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)
r«   r¸   r¹   rº   r•   r»   ræ  rü   r   rÕ   rj   rZ   rX   rÐ  rÐ  j  s:   „ ñð ð KØØ!ó
ð¸Có ò>ð˜9ô rZ   rÐ  c                   ó:   — e Zd Z	 	 	 	 dd„Zdefd„Zd„ Zdefd„Zy)	ÚMistralConverterNc                 óˆ   — || _         || _        || _        t        |t        «      r|j                  «       | _        y || _        y rÈ   )rp  r˜  r[   rd  rx   r   r¢   )r‘   rp  r˜  r[   r¢   r°   s         rX   r•   zMistralConverter.__init__¹  sJ   € ð %ˆŒØˆŒØ 0ˆÔô Ð3´TÔ:ð &×*Ñ*Ó,ð 	Õ&ð +ð 	Õ&rZ   rÓ  c                 ó  ‡‡— dd l }dd l}t        | j                  dd¬«      5 }|j	                  |«      }d d d «       d   d   | _        |d   D �cg c]  }t        |d   |d	   ¬
«      ‘Œ c}| _        |d   }t        «       Št        ˆfd„«       }g }	i }
t        | j                  «      D ]  \  }}||
|j                  <   Œ |D �cg c]  }|j                  |d   «      ‘Œ }}t        |«      }t        |«      D ��ci c]  \  }}||“Œ
 c}}Št        t        |d¬«      «      D ]�  \  }}||
 ||«      <   t        |«      dk(  rŒ g }t!        dt        |«      «      D ]2  }|d | ||d  }}||v sŒ||v sŒ||z   |v sŒ|j#                  |||f«       Œ4 t%        |ˆfd„d¬«      }|	j'                  |«       Œ� t%        |	d„ d¬«      }	|	D �cg c]  } ||d   «       ||d   «      f‘Œ }	}|
|	fS # 1 sw Y   �Œ´xY wc c}w c c}w c c}}w c c}w )Nr   Úrzutf-8)ÚencodingÚconfigr˜  rè   Ú	token_strÚ
is_control)r¡   rl   c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w r×  rØ  rÚ  s     €rX   rÝ  zOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string×  s2   ø€ à—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTrÞ  Útoken_bytesz(Converting tekken.json to tokenizer.json)Údescr   c                 ó$   •— ‰| d      ‰| d      fS ri   rj   )rk   Útoken_to_ranks    €rX   rm   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>ë  s!   ø€ °¸qÀ¹tÑ1DÀmÐTUÐVWÑTXÑFYÐ0Z€ rZ   Fru   c                 ó   — | d   S rá  rj   rs   s    rX   rm   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>í  râ  rZ   )Úbase64Újsonr�   rp  Úloadr˜  r   r¢   rÎ  r   r­   r‘  Ú	b64decoderw   r   rr   rz   r{   r|   r}   )r‘   rÓ  rþ  rÿ  r”   ÚuntypedÚkr  rÝ  r   rl   Úidxr¶   Úrank_setrå  r‚   rƒ   r„   r…   rt   rÜ  rü  s                       @@rX   ræ  z0MistralConverter.extract_vocab_merges_from_modelÊ  s?  ù€ ÛÛä�$—/‘/ 3°Ô9ð 	#¸QØ—i‘i “lˆG÷	#à˜xÑ(¨Ñ3ˆŒàIPÐQaÑIbö*
ØDEŒJ�q˜‘~¨q°©Ö?ò*
ˆÔ&ð ˜GÑ$ˆ	Ü'Ó)ˆä	ó	Vó 
ð	Vð ˆØˆÜ# D×$BÑ$BÓCò 	'‰JˆC�Ø#&ˆE�%—-‘-Ò ð	'àAJÖK¸A�V×%Ñ% a¨Ñ&6Õ7ÐKˆ	ÐKÜ�y“>ˆÜ8AÀ)Ó8L×M©¨¨u˜ ™ÓMˆÜ$¤T¨)Ð:dÔ%eÓfò 
	!‰KˆD�%Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜hÒ&¨7°hÒ+>ÀGÈgÑDUÐZbÒCbØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&ZÐdiÔjˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ð÷C	#ñ 	#üò*
ùò LùãMùò ds#   £G'ÁG4ÃG9Ã:G>ÇHÇ'G1c                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S rè  rê  rë  s       rX   rü   zMistralConverter.tokenizerñ  rì  rZ   r\   c                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )Nr@  FrA  rD  r-  )rü   r   rG  rH  r   r˜  r/  r[   ró   r	   rú   r‚  r¢   r   rù   )r‘   rü   s     rX   rÕ   zMistralConverter.convertedø  s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×Ñ˜T×;Ñ;Ô<Ü#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐrZ   rî  )	r«   r¸   r¹   r•   r»   ræ  rü   r   rÕ   rj   rZ   rX   rð  rð  ¸  s6   „ ð ð KØØ"&ó
ð"%¸Có %òNð˜9ô rZ   rð  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizerÚXGLMTokenizer)ÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 ó*  — | j                   j                  }|t        v r!|st        |   } || «      j                  «       S | j                  j                  d«      r?| | _        t        j                  d«       t        | j                  «      j                  «       S 	 t        j                  d«       t        | j                  | j                  ¬«      j                  «       S # t        $ r* t        dt        t        j                  «       «      › �«      ‚w xY w)aˆ  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].
       from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
            Defaults to False.

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    ztekken.jsonz#Converting from Mistral tekken.jsonzConverting from Tiktoken)rp  rÒ  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )ru  r«   ÚSLOW_TO_FAST_CONVERTERSrÕ   rp  Úendswithrc   ÚloggerÚinforð  rÐ  rÒ  r�  rä  r¾   r   )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs       rX   Úconvert_slow_tokenizerrH  C  sþ   € ð  1×:Ñ:×CÑCÐØÔ6Ñ6¹}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAØ	×	)Ñ	)×	2Ñ	2°=Ô	AØ3HÐÔ0Ü�‰Ð9Ô:ÜÐ 5× @Ñ @ÓA×KÑKÓMÐMð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oô÷ ‰i‹kðøô ò 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÂAC Ã3D)r(  rÈ   )F)Wrº   rr  Úcollections.abcr   Ú	functoolsr   Ú	packagingr   Ú
tokenizersr   r   r   r	   r
   r   r   r¦   r   r   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr«   rB  r  r  rY   Úboolr»   re   r†   rˆ   rÀ   rÎ   rÐ   r×   r  r  r  r  r&  r6  r=  rJ  rR  rY  r`  r—  r°  r»  rÓ  rÛ  r
  r  r  r&  r,  r1  r5  r9  r;  rN  r\  r`  rj  rm  rr  ru  rx  r}  rŒ  rŸ  r¢  r¬  r¼  rÎ  rÐ  rð  r@  rH  rj   rZ   rX   ú<module>rQ     sµ  ðñó Ý &Ý å ß f× fÑ fß 5Ñ 5Ý ç `Ó `Ý 5ð 
ˆ×	Ñ	˜HÓ	%€ò€ð8 $ò 'ñ Ð ó>Gð"¨$ð Àsó ñ°jÀ±oÈÑ6Ló ÷02ñ 2ôjÐ"8ô ð"I˜cð I dó I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ò`÷0Kñ K÷\Mñ Mð`8Ø�ð8àÐ%ð8ð Ð(ð8ð �]ð	8ð
 Ð(ð8ð Ð.ð8ð Ð,ð8ð �]ð8ð ˜ð8ð ˜ð8ð Ð(ð8ð Ð,ð8ð ˜=ð8ð ˜-ð8ð " =ð8ð  ! -ð!8ð" ˜ñ#8ð$ �_ð%8ð& �ð'8ð( �]ð)8ð* Ð(ð+8ð, ˜ð-8ð. ˜=ð/8ð0 Ð+ð18ð2 Ð-ð38ð4 Ð+ð58ð6 Ð$ð78ð8 �}ð98ð: Ð*ð;8ð< �nð=8ð> Ð(ð?8ð@ �nðA8ðB ˜=ðC8ðD Ð$òE8ðF �]ðG8ðH Ð,ðI8ðJ Ð(ðK8ðL �nðM8ðN Ð*ðO8ðP Ð(ðQ8ðR Ð(ðS8ðT Ð*ðU8ðV Ð0ðW8ðX ˜MðY8ðZ �;ð[8ð\ �]ð]8ð^ Ð(ð_8ð` Ð.ða8ðb �nðc8ðd Ð*ðe8ðf �]ñg8ðh %Ø(Ø$Ø#òo8Ð ñv$È)ô $rZ   