Ë
    Gêñió  ã                   óî  — d dl Z ddlmZmZmZ ddlmZ  ej                  e«      Z	 edd¬«      Z
 edd¬«      Z e«       Z e«       Zd	e j                  d
ede j                  fd„Zde j                  dz  de j                  defd„Z	 	 	 dde j&                  j(                  de j                  de j                  de j                  de j                  dz  dededz  dedz  dee j                  df   fd„Zy)é    Né   )Úis_torch_npu_availableÚis_torch_xpu_availableÚlogging)Úis_torch_greater_or_equalz2.5T)Ú
accept_devz2.8Úhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r	   r
   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         új/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/sdpa_attention.pyÚ	repeat_kvr      so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTó    Úattention_maskÚkeyc                 ó2   — t         rt        S t        xr | d u S )N)Ú_is_torch_xpu_availableÚ#_is_torch_greater_or_equal_than_2_8Ú#_is_torch_greater_or_equal_than_2_5)r   r   s     r   Úuse_gqa_in_sdpar      s   € õ Ü2Ð2Ü.ÒI°>ÀTÐ3IÐIr   ÚmoduleÚqueryÚvalueÚdropoutÚscalingÚ	is_causalc                 ó<  — |j                  dd«      rt        j                  d«       i }	t        | d«      r=t	        ||«      s-t        || j                  «      }t        || j                  «      }nddi}	|�|nt        | dd«      }|j                  d   d	kD  xr |d u xr |}t        j                  j                  «       r*t        |t        j                  «      r|j                  «       }t        r[|�Y|j                   t        j"                  k7  r<t        j$                  |j#                  «       «      j'                  |j(                  «      }t        j*                  j,                  j.                  |||f||||d
œ|	¤Ž}
|
j1                  d	d«      j3                  «       }
|
d fS )NÚoutput_attentionsFzƒ`sdpa` attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.Únum_key_value_groupsÚ
enable_gqaTr$   r   r   )Ú	attn_maskÚ	dropout_pÚscaler$   )ÚgetÚloggerÚwarning_onceÚhasattrr   r   r'   Úgetattrr   ÚtorchÚjitÚ
is_tracingÚ
isinstanceÚTensorÚitemÚ_is_torch_npu_availableÚdtypeÚboolÚlogical_notÚtoÚdeviceÚnnÚ
functionalÚscaled_dot_product_attentionÚ	transposeÚ
contiguous)r   r    r   r!   r   r"   r#   r$   ÚkwargsÚsdpa_kwargsÚattn_outputs              r   Úsdpa_attention_forwardrE   (   sƒ  € ð ‡z�zÐ% uÔ-Ü×ÑðWô	
ð €KÜˆvÐ-Ô.Ü˜~¨sÔ3Ü˜C ×!<Ñ!<Ó=ˆCÜ˜e V×%@Ñ%@ÓA‰Eà'¨Ð.ˆKð 'Ð2‘	¼ÀÈÐUYÓ8Z€Ið —‘˜A‘ Ñ"ÒK ~¸Ð'=ÒKÀ)€Iô ‡y�y×ÑÔ¤*¨Y¼¿¹Ô"EØ—N‘NÓ$ˆ	õ
 ØÐ%¨.×*>Ñ*>Ä%Ç*Á*Ò*Lä"×.Ñ.¨~×/BÑ/BÓ/DÓE×HÑHÈÏÉÓVˆNä—(‘(×%Ñ%×BÑBØØØð	ð !ØØØñ	ð ñ	€Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        NN)r1   Úutilsr   r   r   Úutils.import_utilsr   Ú
get_loggerÚ__name__r-   r   r   r   r7   r5   Úintr   r9   r   r=   ÚModuleÚfloatÚtuplerE   © r   r   ú<module>rO      sJ  ðÛ ç KÑ KÝ :ð 
ˆ×	Ñ	˜HÓ	%€ñ '@ÀÐRVÔ&WÐ #Ù&?ÀÐRVÔ&WÐ #Ù0Ó2Ð Ù0Ó2Ð ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð	J E§L¡L°4Ñ$7ð 	J¸e¿l¹lð 	JÈtó 	Jð$ Ø Ø!ñ@Ø�H‰H�O‰Oð@à�<‰<ð@ð 
�‰ð@ð �<‰<ð	@ð
 —L‘L 4Ñ'ð@ð ð@ð �T‰\ð@ð �d‰{ð@ð ˆ5�<‰<˜ÐÑô@r   