Ë
    Gêñi§  ã                   óv  — d dl Z ddlmZ ddlmZ de j
                  j                  de j                  de j                  de j                  d	e j                  dz  d
ede j                  de j                  ee	e j                  f   z  de
de
ee	e
f   z  de j                  dz  dee j                  df   fd„Ze j                  j                  de j
                  j                  de j                  de j                  de j                  d
ede j                  dee
e
f   de j                  de j                  fd„«       Zy)é    Né   )ÚPagedAttentionCache)Ú!lazy_import_paged_flash_attentionÚmoduleÚqÚkÚvÚattention_maskÚcacheÚcu_seq_lens_qÚcu_seq_lens_kÚmax_seqlen_qÚmax_seqlen_kÚblock_tableÚreturnc                 ó.  — t        | j                  j                  «      \  }}t        | dd«      sdn| j                  dz
  df}|dk(  rdnd}t        |t        «      r
||   }|	|   }	|
�€	|j                  ||| j                  |d	   |d
   ¬«      \  }}d|v rd|j                  d«      ini } ||j                  dd«      j                  d«      j                  «       |j                  «       |j                  «       |j                  t        j                  «      |j                  t        j                  «      j!                  «       ||	f| j"                  d|dœ|¤Ž}t        |t$        «      r|d   }|dfS d|v rd|d   ini }t'        | ||||||||
f	i |¤Ž}|dfS )ap  Performs the forward pass of attention with paged key-value cache. This function handles the cache updates and
    performs the attention computation. For decode-only batches (when block_table is provided), uses
    `flash_attn_with_kvcache` for fused attention + cache update. Otherwise uses `flash_attn_varlen_func`.
    See the [paged attention guide](https://huggingface.co/docs/transformers/en/paged_attention) for more details.

    Args:
        q: (1, nheads, total_q, headdim), where total_q = total number of query tokens in the batch.
        k: (1, nheads_k, total_k, headdim), where total_k = total number of key tokens in the batch.
        v: (1, nheads_k, total_k, headdim), where total_k = total number of key tokens in the batch.
        cu_seq_lens_q: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths
           of the sequences in the batch, used to index into q.
        cu_seq_lens_k: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths
           of the sequences in the batch, used to index into kv.
        max_seqlen_q: int. Maximum query sequence length in the batch.
        max_seqlen_k: int. Maximum key sequence length in the batch.
        block_table: (num_groups, batch_size, max_blocks_per_seq), dtype int32. Block table for paged KV cache.
            If provided, uses flash_attn_with_kvcache for fused attention + cache update. For each request, the block
            table is a vector of size (max_blocks_per_seq,) with indices indicating the physical location of the cache
            to read from and write to. The kernel, using the cache_seqlens for that request, knows how much cache to
            read and dispatches the read using the block table. Same for the write. If a request has fewer than
            max_blocks_per_seq blocks, the block table is padded with -1s to indicate that the block is not allocated.
    Úsliding_windowF)éÿÿÿÿr   é   r   Úfull_attentionÚsliding_attentionNÚ
read_indexÚwrite_index)Ú
key_statesÚvalue_statesÚ	layer_idxr   r   Ús_auxr   T)Úsoftmax_scaleÚcausalÚwindow_size)r   ÚconfigÚ_attn_implementationÚgetattrr   Ú
isinstanceÚdictÚupdater   ÚgetÚ	transposeÚsqueezeÚ
contiguousÚtoÚtorchÚint32ÚcloneÚscalingÚtupleÚ_paged_decode_forward)r   r   r   r	   r
   r   r   r   r   r   r   ÚkwargsÚflash_attn_varlen_funcÚflash_attn_with_kvcacher   Ú
layer_typeÚcustom_kwargsÚattn_outputÚflash_kwargss                      úg/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/flash_paged.pyÚpaged_attention_forwardr:      sÖ  € ôJ 7XØ�‰×*Ñ*ó7Ñ3ÐÐ3ô
 &-¨VÐ5EÀuÔ%M‘XÐTZ×TiÑTiÐlmÑTmÐopÐSq€NØ%3°xÒ%?Ñ!ÐEX€JÜ�-¤Ô&Ø% jÑ1ˆØ# JÑ/ˆð Ñà�|‰|ØØØ×&Ñ&Ø˜lÑ+Ø˜}Ñ-ð ó 
‰ˆˆ1ð ;BÀVÑ:K˜ &§*¡*¨WÓ"5Ñ6ÐQSˆÙ,Ø�K‰K˜˜1Ó×%Ñ% aÓ(×3Ñ3Ó5Ø�L‰L‹NØ�L‰L‹NØ×ÑœUŸ[™[Ó)Ø×ÑœUŸ[™[Ó)×/Ñ/Ó1ØØð
ð !Ÿ.™.ØØ&ñ
ð ñ
ˆô �k¤5Ô)Ø% a™.ˆKð ˜ÐÐð	 6=ÀÑ5F˜ ¨¡Ñ1ÈBˆÜ+Ø�A�q˜!˜U M°>ÐCZÐ\gñ
Økwñ
ˆð ˜ÐÐó    r   c	                 ó2  — |j                   | j                     \  }
}|j                  |   j                  d|j                  |j
                  |j                  «      }|j                  |   j                  d|j                  |j
                  |j                  «      }|j                  dddd«      j                  «       }|j                  dddd«      j                  «       }|j                  dddd«      j                  «       }|j                  d«      }|d|dz    |d| z
  dz
  j                  t        j                  «      }||
   |	|j                  |«      <    |d	||||||| j                  d|dœ	|	¤Ž}t!        |t"        «      r|d   }|j%                  d«      S )
zaDecode fast path using flash_attn_with_kvcache. Disabled because FA3 has issue with tracing this.r   r   r   r   é   NT)	r   Úk_cacheÚv_cacher   r	   Úcache_seqlensr   r   r    © )Úlayer_index_to_group_indicesr   Ú	key_cacheÚviewÚ
block_sizeÚnum_key_value_headsÚhead_dimÚvalue_cacheÚpermuter*   Úsizer+   r,   r-   Úget_block_table_keyr/   r$   r0   r)   )r   r   r   r	   r   r   r   r4   r   r8   Ú	group_idxÚlayer_idx_in_groupr>   r?   Ú
batch_sizer@   r7   s                    r9   r1   r1   [   sž  € ð %*×$FÑ$FÀv×GWÑGWÑ$XÑ!€IÐ!à�o‰oÐ0Ñ1×6Ñ6°r¸5×;KÑ;KÈU×MfÑMfÐhm×hvÑhvÓw€GØ×ÑÐ 2Ñ3×8Ñ8Ø
ˆE×Ñ˜e×7Ñ7¸¿¹ó€Gð 	
�	‰	�!�Q˜˜1Ó×(Ñ(Ó*€AØ	�	‰	�!�Q˜˜1Ó×(Ñ(Ó*€AØ	�	‰	�!�Q˜˜1Ó×(Ñ(Ó*€Að —‘˜“€JØ" 1 z°A¡~Ð6¸À{È
Ð9SÑSÐVWÑW×[Ñ[Ô\a×\gÑ\gÓh€MàGRÐS\ÑG]€L�×*Ñ*Ð+BÓCÑDá)ð Ø
ØØØ
Ø
Ø#Ø—n‘nØØ"ñð ñ€Kô �+œuÔ%Ø! !‘nˆà×Ñ˜qÓ!Ð!r;   )r,   Úgeneration.continuous_batchingr   Úmodeling_flash_attention_utilsr   ÚnnÚModuleÚTensorr%   ÚstrÚintr0   r:   ÚcompilerÚdisabler1   rA   r;   r9   ú<module>rX      s�  ðÛ å @Ý NðQØ�H‰H�O‰OðQà‡|�|ðQð ‡|�|ðQð ‡|�|ð	Qð
 —L‘L 4Ñ'ðQð ðQð —<‘<ðQð —<‘< $ s¨E¯L©LÐ'8Ñ"9Ñ9ðQð ðQð ˜˜S #˜X™Ñ&ðQð —‘ Ñ$ðQð ˆ5�<‰<˜ÐÑóQðh ‡�×Ñð."Ø�H‰H�O‰Oð."à‡|�|ð."ð ‡|�|ð."ð ‡|�|ð	."ð
 ð."ð —<‘<ð."ð ˜#˜s˜(‘Oð."ð —‘ð."ð ‡\�\ò."ó ñ."r;   