Ë
    Gêñi¹#  ã                   óJ  — d dl Z d dlZd dlmZmZ d dlmZ d dlZd dlm	Z	 d dl
mZ ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZmZ  ej0                  e«      Z e ej8                  «       «      Z ed„ eD «       «      Ze G d„ d«      «       Z  G d„ de«      Z! G d„ de«      Z"y)é    N)Ú	dataclassÚfield)ÚEnum)ÚFileLock)ÚDataseté   )Ú$MODEL_FOR_QUESTION_ANSWERING_MAPPING)ÚPreTrainedTokenizer)Úcheck_torch_load_is_safeÚloggingé   )ÚSquadFeaturesÚSquadV1ProcessorÚSquadV2ProcessorÚ"squad_convert_examples_to_featuresc              #   ó4   K  — | ]  }|j                   –— Œ y ­w©N)Ú
model_type)Ú.0Úconfs     úb/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/data/datasets/squad.pyú	<genexpr>r   !   s   è ø€ ÒE¨�D—O•OÑEùs   ‚c                   óÎ  — e Zd ZU dZ eddddj                  e«      z   i¬«      Zee	d<    edddi¬«      Z
ee	d	<    ed
ddi¬«      Zee	d<    ed
ddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    eddd i¬«      Zee	d!<    ed"dd#i¬«      Zee	d$<   y)%ÚSquadDataTrainingArgumentszb
    Arguments pertaining to what data we are going to input our model for training and eval.
    NÚhelpz!Model type selected in the list: z, )ÚdefaultÚmetadatar   zFThe input data dir. Should contain the .json files for the SQuAD task.Údata_diré€   z‹The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.Úmax_seq_lengthzVWhen splitting up a long document into chunks, how much stride to take between chunks.Ú
doc_strideé@   zkThe maximum number of tokens for the question. Questions longer than this will be truncated to this length.Úmax_query_lengthé   z�The maximum length of an answer that can be generated. This is needed because the start and end predictions are not conditioned on one another.Úmax_answer_lengthFz1Overwrite the cached training and evaluation setsÚoverwrite_cachezDIf true, the SQuAD examples contain some that do not have an answer.Úversion_2_with_negativeg        zIIf null_score - best_non_null is greater than the threshold predict null.Únull_score_diff_thresholdé   Ún_best_sizer   zjlanguage id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)Úlang_idé   z3multiple threads for converting example to featuresÚthreads)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚjoinÚMODEL_TYPESr   ÚstrÚ__annotations__r   r    Úintr!   r#   r%   r&   Úboolr'   r(   Úfloatr*   r+   r-   © ó    r   r   r   $   s•  … ññ Ø Ð(KÈdÏiÉiÐXcÓNdÑ(dÐeô€J�ó ñ Ø Ð(pÐqô€Hˆcó ñ  ØàðQð
ô€N�Có ñ ØØÐrÐsô€J�ó ñ "Øàð/ð
ôÐ�có ñ #ØàðJð
ôÐ�só ñ "Ø Ð)\Ð ]ô€O�Tó ñ %*Ø Ð)oÐ pô%Ð˜Tó ñ (-Ø˜vÐ'rÐsô(Ð˜uó ñ Ø˜fÐ&qÐrô€K�ó ñ ØàðCð
ô€GˆSó ñ  ¨fÐ6kÐ-lÔm€GˆSÔmr:   r   c                   ó   — e Zd ZdZdZy)ÚSplitÚtrainÚdevN)r.   r/   r0   r=   r>   r9   r:   r   r<   r<   g   s   „ Ø€EØ
�Cr:   r<   c                   óÂ   — e Zd ZU eed<   ee   ed<   eed<   eed<   dej                  dddfdede
d	edz  deez  ded
edz  defd„Zd„ Zdeeej"                  f   fd„Zy)ÚSquadDatasetÚargsÚfeaturesÚmodeÚis_language_sensitiveNFÚptÚ	tokenizerÚlimit_lengthÚ	cache_dirÚdataset_formatc                 óØ  — || _         || _        |j                  r
t        «       n	t	        «       | _        t        |t        «      r
	 t        |   }|| _
        |j                  rdnd}t        j                  j                  |�|n|j                  d|j                  › d|j                   j"                  › d|j$                  › d|› �«      }	|	dz   }
t'        |
«      5  t        j                  j)                  |	«      rÿ|j*                  sót-        j,                  «       }t/        «        t1        j2                  |	d¬«      | _        | j4                  d	   | _        | j4                  j9                  d
d «      | _        | j4                  j9                  dd «      | _        t>        jA                  d|	› d�t-        j,                  «       |z
  «       | j:                  �| j<                  �€dt>        jC                  d|	› d�«       �nI|t        jD                  k(  r+| j
                  jG                  |j                  «      | _        n*| j
                  jI                  |j                  «      | _        tK        | j<                  ||j$                  |jL                  |jN                  |t        jP                  k(  |jR                  |¬«      \  | _        | _        t-        j,                  «       }t1        jT                  | j6                  | j:                  | j<                  dœ|	«       t>        jA                  d|	› dt-        j,                  «       |z
  d›d�«       d d d «       y # t        $ r t        d«      ‚w xY w# 1 sw Y   y xY w)Nzmode is not a valid split nameÚv2Úv1Úcached_Ú_z.lockT)Úweights_onlyrB   ÚdatasetÚexamplesz"Loading features from cached file z [took %.3f s]zDeleting cached file z; will allow dataset and examples to be cached in future run)rQ   rF   r    r!   r#   Úis_trainingr-   Úreturn_dataset)rB   rP   rQ   z!Saving features into cached file z [took z.3fz s])+rA   rD   r'   r   r   Ú	processorÚ
isinstancer4   r<   ÚKeyErrorrC   ÚosÚpathr2   r   ÚvalueÚ	__class__r.   r    r   Úexistsr&   Útimer   ÚtorchÚloadÚold_featuresrB   ÚgetrP   rQ   ÚloggerÚinfoÚwarningr>   Úget_dev_examplesÚget_train_examplesr   r!   r#   r=   r-   Úsave)ÚselfrA   rF   rG   rC   rD   rH   rI   Úversion_tagÚcached_features_fileÚ	lock_pathÚstarts               r   Ú__init__zSquadDataset.__init__r   så  € ð ˆŒ	Ø%:ˆÔ"Ø/3×/KÒ/KÔ)Ô+ÔQaÓQcˆŒÜ�dœCÔ ðAÜ˜T‘{�ð ˆŒ	à"×:Ò:‘dÀˆÜ!Ÿw™wŸ|™|Ø"Ð.‰I°D·M±MØ�d—j‘j�\  9×#6Ñ#6×#?Ñ#?Ð"@ÀÀ$×BUÑBUÐAVÐVWÐXcÐWdÐeó 
Ðð )¨7Ñ2ˆ	Ü�iÓ ñ -	Ü�w‰w�~‰~Ð2Ô3¸D×<PÒ<PÜŸ	™	›�Ü(Ô*Ü$)§J¡JÐ/CÐRVÔ$W�Ô!ð !%× 1Ñ 1°*Ñ =�”Ø#×0Ñ0×4Ñ4°YÀÓE�”Ø $× 1Ñ 1× 5Ñ 5°jÀ$Ó G�”Ü—‘Ø8Ð9MÐ8NÈnÐ]Ô_c×_hÑ_hÓ_jÐmrÑ_rôð —<‘<Ð'¨4¯=©=Ñ+@Ü—N‘NØ/Ð0DÐ/Eð F&ð &öð
 œ5Ÿ9™9Ò$Ø$(§N¡N×$CÑ$CÀDÇMÁMÓ$R�D•Mà$(§N¡N×$EÑ$EÀdÇmÁmÓ$T�D”Mä.PØ!Ÿ]™]Ø'Ø#'×#6Ñ#6Ø#Ÿ™Ø%)×%:Ñ%:Ø $¬¯©Ñ 3Ø ŸL™LØ#1ô	/Ñ+�”˜tœ|ô Ÿ	™	›�Ü—
‘
Ø!%§¡¸4¿<¹<ÐUY×UbÑUbÑcØ(ôô
 —‘Ø7Ð8LÐ7MÈWÔUY×U^ÑU^ÓU`ÐchÑUhÐilÐTmÐmpÐqô÷W-	ð -	øô ò AÜÐ?Ó@Ð@ðAú÷-	ð -	ús   Á	M ÃI(M ÍMÍ M)c                 ó,   — t        | j                  «      S r   )ÚlenrB   )rg   s    r   Ú__len__zSquadDataset.__len__¾   s   € Ü�4—=‘=Ó!Ð!r:   Úreturnc                 ó(  — | j                   |   }t        j                  |j                  t        j                  ¬«      }t        j                  |j
                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }|||dœ}	| j                  j                  dv r|	d= | j                  j                  dv r |	j                  ||dœ«       | j                  j                  r|	j                  d|i«       | j                  rW|	j                  dt        j                   |j"                  t        j$                  ¬«      | j                  j&                  z  i«       | j(                  t*        j,                  k(  rrt        j                  |j.                  t        j                  ¬«      }
t        j                  |j0                  t        j                  ¬«      }|	j                  |
|d	œ«       |	S )
N)Údtype)Ú	input_idsÚattention_maskÚtoken_type_ids)ÚxlmÚrobertaÚ
distilbertÚ	camembertru   )Úxlnetrv   )Ú	cls_indexÚp_maskÚis_impossibleÚlangs)Ústart_positionsÚend_positions)rB   r]   Útensorrs   Úlongrt   ru   r{   r|   r8   r}   rA   r   Úupdater'   rD   ÚonesÚshapeÚint64r+   rC   r<   r=   Ústart_positionÚend_position)rg   ÚiÚfeaturers   rt   ru   r{   r|   r}   Úinputsr   r€   s               r   Ú__getitem__zSquadDataset.__getitem__Á   s¶  € à—-‘- Ñ"ˆä—L‘L ×!2Ñ!2¼%¿*¹*ÔEˆ	ÜŸ™ g×&<Ñ&<ÄEÇJÁJÔOˆÜŸ™ g×&<Ñ&<ÄEÇJÁJÔOˆÜ—L‘L ×!2Ñ!2¼%¿*¹*ÔEˆ	Ü—‘˜gŸn™n´E·K±KÔ@ˆÜŸ™ W×%:Ñ%:Ä%Ç+Á+ÔNˆð #Ø,Ø,ñ
ˆð �9‰9×ÑÐ#PÑPØÐ'Ð(à�9‰9×ÑÐ#3Ñ3Ø�M‰M¨	¸VÑDÔEØ�y‰y×0Ò0Ø—‘˜°Ð>Ô?Ø×)Ò)Ø—‘˜w¬¯©°I·O±OÌ5Ï;É;Ô)WÐZ^×ZcÑZc×ZkÑZkÑ)kÐmÔnà�9‰9œŸ™Ò#Ü#Ÿl™l¨7×+AÑ+AÌÏÉÔTˆOÜ!ŸL™L¨×)=Ñ)=ÄUÇZÁZÔPˆMØ�M‰M¨oÐP]Ñ^Ô_àˆr:   )r.   r/   r0   r   r5   Úlistr   r<   r7   r=   r
   r6   r4   rl   ro   Údictr]   ÚTensorrŒ   r9   r:   r   r@   r@   l   s°   … Ø
$Ó$Ø�=Ñ!Ó!Ø
ƒKØÓð $(Ø!ŸK™KØ&+Ø $Ø"ñJà(ðJð 'ðJð ˜D‘jð	Jð
 �E‰kðJð  $ðJð ˜‘:ðJð óJòX"ð   S¨%¯,©,Ð%6Ñ 7ô  r:   r@   )#rW   r\   Údataclassesr   r   Úenumr   r]   Úfilelockr   Útorch.utils.datar   Úmodels.auto.modeling_autor	   Útokenization_pythonr
   Úutilsr   r   Úprocessors.squadr   r   r   r   Ú
get_loggerr.   ra   r�   ÚkeysÚMODEL_CONFIG_CLASSESÚtupler3   r   r<   r@   r9   r:   r   ú<module>rœ      sŸ   ðó 
Û ß (Ý ã Ý Ý $å MÝ 6ß 6ß tÓ tð 
ˆ×	Ñ	˜HÓ	%€áÐEÐ@×EÑEÓGÓHÐ ÙÑEÐ0DÔEÓE€ð ÷?nð ?nó ð?nôDˆDô ô
u�7õ ur:   