Ë
    Gêñi®—  ã                   óº  — d dl Z d dlmZ d dlmZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZmZ d	d
lmZ d	dlmZmZ  ej,                  e«      Ze j2                  Z e j6                  e«      j8                  Z e j6                  e«      j<                  ZdZ da!da"da#da$da%da&da'da(da)d„ Z*d„ Z+d„ Z,de-de-de-fd„Z.e j^                  fde j`                  de j`                  de j`                  de j`                  de1e-   de jd                  de j`                  fd„Z3 G d„ dejh                  «      Z5de j                  jl                  de j`                  de j`                  d e j`                  de j`                  f
d!„Z7de j                  jl                  de j`                  de j`                  d e j`                  de j`                  f
d"„Z8d#e j`                  d$e-d%e-de9fd&„Z:de j`                  d'e j`                  d(e j`                  d)e-de9e j`                  e j`                  f   f
d*„Z;d+e j`                  d(e j`                  de j`                  fd,„Z<de j                  jl                  de j`                  de j`                  d e j`                  de j`                  f
d-„Z= G d.„ d/ejl                  «      Z> G d0„ d1e«      Z? e?«       Z@	 d8d2e1eA   dz  fd3„ZB G d4„ d5e«      ZC G d6„ d7e«      ZDy)9é    N)Ú
functionalé   )ÚACT2FN)ÚConversionOpsÚ_IdentityOp)Úshould_convert_module)Úlogging)Úget_cuda_runtime_versionÚresolve_internal_importé   )Úlazy_load_kernel)ÚExpertsInterfaceÚuse_experts_implementationé€   c                 óŠ   — |D ]  }t        | |«      sŒt        | |«      c S  t        t        | «      j                  › d|› �«      ‚)Nz has none of: )ÚhasattrÚgetattrÚAttributeErrorÚtypeÚ__name__)ÚobjÚnamesÚnames      úk/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/finegrained_fp8.pyÚ_first_attrr   7   sK   € Øò &ˆÜ�3˜ÕÜ˜3 Ó%Ò%ð&ô œD ›I×.Ñ.Ð/¨~¸e¸WÐEÓ
FÐFó    c                  ój  — t         �t         st        d«      ‚yda t        d«      } t        | dd«      at        | dd«      at        | dd«      at        | dd«      adt        fdt
        fdt        fdt        ffD ��cg c]	  \  }}|€|‘Œ }}}|rt        d	d
j                  |«      › d�«      ‚da yc c}}w )zëLazily load the finegrained-fp8 Triton kernel and extract functions.

    Uses the hub kernels lazy loading pattern. Raises an error if the kernel
    cannot be loaded or required functions are missing. Only attempts loading once.
    NzGfinegrained-fp8 kernel is not available (previous load attempt failed).Fzfinegrained-fp8Úw8a8_fp8_matmulÚfp8_act_quantÚw8a8_fp8_matmul_batchedÚw8a8_fp8_matmul_groupedz6finegrained-fp8 kernel is missing required functions: ú, úA. Please update the `kernels` package (`pip install -U kernels`).T)	Ú_triton_availableÚImportErrorr   r   Útriton_fp8_matmulÚtriton_fp8_act_quantÚtriton_batched_fp8_matmulÚtriton_grouped_fp8_matmulÚjoin)Úkernelr   ÚattrÚmissings       r   Ú_load_triton_kernelr.   >   sù   € ô Ð$Ý ÜÐgÓhÐhØàÐäÐ/Ó0€FÜ Ð(9¸4Ó@ÐÜ" 6¨?¸DÓAÐÜ '¨Ð0IÈ4Ó PÐÜ '¨Ð0IÈ4Ó PÐð
 Ô 1Ð2ØÔ2Ð3Ø&Ô(AÐBØ&Ô(AÐBð	
÷	áˆD�$ð ˆ<ò 	ð	€Gñ 	ñ ÜØDÀTÇYÁYÈwÓEWÐDXð YNð Nó
ð 	
ð
 Ñùó!	s   Á;B/c                  óZ  — t         �t         st        d«      ‚yda t        j                  j	                  «       st        d«      ‚t        j                  j                  «       d   } | dk  rt        d| › d�«      ‚t        «       \  }}|d	k  s
|d	k(  r|d
k  rt        d|› d|› d�«      ‚t        d«      }t        |dd«      a	t        |dd«      a
t        |d¬«      adt        fdt        fdt        ffD ��cg c]	  \  }}|€|‘Œ }}}|rt        ddj                  |«      › d�«      ‚da yc c}}w )a  Lazily load the DeepGEMM kernel and extract functions with proper names.

    Uses the hub kernels lazy loading pattern. Raises an error if the kernel
    cannot be loaded, required functions are missing, or the hardware is insufficient.
    Only attempts loading once.
    Nz@DeepGEMM kernel is not available (previous load attempt failed).FzcDeepGEMM kernel requires CUDA, but CUDA is not available. Use a different `experts_implementation`.r   é	   z_DeepGEMM requires a Hopper (SM90+) or newer GPU, but the current device has compute capability z-.x. Use a different `experts_implementation`.é   é   z0DeepGEMM requires CUDA runtime 12.3+, but found ú.zO. Please upgrade your CUDA toolkit or use a different `experts_implementation`.z	deep-gemmÚfp8_gemm_ntÚ m_grouped_fp8_gemm_nt_contiguouszutils.per_token_cast_to_fp8)Úchained_pathz/DeepGEMM kernel is missing required functions: r"   r#   T)Ú_deepgemm_availabler%   ÚtorchÚcudaÚis_availableÚget_device_capabilityr
   r   r   Údeepgemm_fp8_matmulÚdeepgemm_grouped_fp8_matmulr   Údeepgemm_per_token_cast_to_fp8r*   )ÚmajorÚ
cuda_majorÚ
cuda_minorr+   r   r,   r-   s          r   Ú_load_deepgemm_kernelrB   k   sŒ  € ô Ð&Ý"ÜÐ`ÓaÐaØàÐô �:‰:×"Ñ"Ô$ÜØqó
ð 	
ô
 �J‰J×,Ñ,Ó.¨qÑ1€EØˆq‚yÜð&Ø&+ WÐ,Yð[ó
ð 	
ô 6Ó7Ñ€J�
Ø�B‚˜:¨Ò+°
¸Q²ÜØ>¸z¸lÈ!ÈJÈ<ð X\ð \ó
ð 	
ô
 ˜kÓ*€FÜ! &¨-¸Ó>ÐÜ")¨&Ð2TÐVZÓ"[ÐÜ%<¸VÐRoÔ%pÐ"ð
 Ô/Ð0Ø/Ô1LÐMØ*Ô,JÐKð
÷áˆD�$ð
 ˆ<ò 	ð€Gñ ñ ÜØ=¸d¿i¹iÈÓ>PÐ=Qð RNð Nó
ð 	
ð
 Ñùós   Ã3D'ÚaÚbÚreturnc                 ó   — | |z   dz
  |z  S )zCeiling division.r   © )rC   rD   s     r   Ú_cdivrH   ¨   s   € à�‰E�A‰I˜!ÑÐr   ÚAÚBÚAsÚBsÚ
block_sizeÚoutput_dtypec                 ón  — |�ö|d   |d   cxk(  rdk(  rån nâ	 t        «        | j                  d| j                  d   «      }|j                  d|j                  d   «      }t        j                  |j                  d   |j                  d   | j
                  |¬«      }t        ||j                  «       f||j                  «       f|«       |j                  | j                  dd |j                  d   fz   «      S t        «        t        | |||||«      S # t        $ r t        j                  d«       Y Œ:w xY w)u„  FP8 matmul: C = dequant(A, As) @ dequant(B, Bs)^T.

    Supports both per-tensor and block-wise quantization:
      - block_size=None or block_size=[N, K]: per-tensor mode (As is scalar/per-row, Bs is scalar)
      - block_size=[block_n, block_k]: block-wise mode (As and Bs are per-block scale grids)

    Dispatch order:
      1. DeepGEMM (Hopper+, block_size 128x128) if available
      2. Triton finegrained-fp8 kernel (universal fallback)

    Args:
        A:  (M, K) float8_e4m3fn â€” quantized activations
        B:  (N, K) float8_e4m3fn â€” quantized weights
        As: block-wise: (M, K//block_k) float32; per-tensor: (M,) per-row scales
        Bs: block-wise: (N//block_n, K//block_k) float32; per-tensor: scalar or (1,) single weight scale
        block_size: [block_n, block_k] for block-wise quantization, or None/[N, K] for per-tensor
        output_dtype: desired output dtype
    Nr   r   r   éÿÿÿÿ©ÚdeviceÚdtypea  DeepGEMM kernel is not available or compatible, falling back to Triton finegrained-fp8 kernel. To use DeepGEMM FP8 matmul, ensure you have a Hopper (SM90+) or newer GPU with CUDA runtime 12.3+, and that the `kernels` package is installed and up to date (`pip install -U kernels`).)rB   ÚviewÚshaper8   ÚemptyrR   r<   Úfloatr%   ÚloggerÚwarning_oncer.   r&   )	rI   rJ   rK   rL   rM   rN   ÚA_2dÚAs_2dÚoutputs	            r   r   r   ­   s  € ð4 Ð *¨Q¡-°:¸a±=Ô"GÀCÕ"Gð	=Ü!Ô#ð —6‘6˜"˜aŸg™g b™kÓ*ˆDØ—G‘G˜B §¡¨¡Ó-ˆEÜ—[‘[ §¡¨A¡°·±¸±
À1Ç8Á8ÐS_Ô`ˆFÜ  u§{¡{£}Ð 5¸¸2¿8¹8»:°ÈÔOØ—;‘;˜qŸw™w s¨˜|¨q¯w©w°q©z¨mÑ;Ó<Ð<äÔä˜Q  2 r¨:°|ÓDÐDøô! ò 	Ü×Ñðiöð	ús   ˜
D ÄD4Ä3D4c                   ó†   ‡ — e Zd Zdddefdededeeef   dz  dedef
ˆ fd	„Zd
e	j                  de	j                  fd„Zˆ xZS )Ú	FP8LinearNÚdynamicFÚin_featuresÚout_featuresrM   Úactivation_schemeÚhas_biasc                 óÒ  •— t         ‰	| �  ||«       || _        || _        || _        t
        j                  j                  t        j                  |||¬«      «      | _	        | j                  €>t        j                  t        j                  dt
        j                  ¬«      «      | _        nˆ|| j                  d   z   dz
  | j                  d   z  }|| j                  d   z   dz
  | j                  d   z  }t        j                  t        j                  ||t
        j                  ¬«      «      | _        | j                  dk(  r>t        j                  t        j                  dt
        j                  ¬«      «      | _        n| j                  dd «       | j                  r8t        j                  t        j                  | j                  «      «      | _        y | j                  dd «       y )N©rS   ç      ð?r   r   ÚstaticÚactivation_scaleÚbias)ÚsuperÚ__init__rc   rM   rb   r8   ÚnnÚ	ParameterrV   ÚweightÚtensorÚfloat32Úweight_scale_invrh   Úregister_parameterra   ri   )
Úselfr`   ra   rM   rb   rc   rS   Úscale_out_featuresÚscale_in_featuresÚ	__class__s
            €r   rk   zFP8Linear.__init__ß   sf  ø€ ô 	‰Ñ˜ lÔ3à ˆŒØ$ˆŒØ!2ˆÔÜ—h‘h×(Ñ(¬¯©°\À;ÐV[Ô)\Ó]ˆŒà�?‰?Ð"ä$&§L¡L´·±¸cÌÏÉÔ1WÓ$XˆDÕ!à".°·±ÀÑ1CÑ"CÀaÑ"GÈDÏOÉOÐ\]ÑL^Ñ!^ÐØ!,¨t¯©¸qÑ/AÑ!AÀAÑ!EÈ$Ï/É/ÐZ[ÑJ\Ñ \ÐÜ$&§L¡LÜ—‘Ð.Ð0AÌÏÉÔWó%ˆDÔ!ð ×!Ñ! XÒ-Ü$&§L¡L´·±¸cÌÏÉÔ1WÓ$XˆDÕ!à×#Ñ#Ð$6¸Ô=à�=Š=ÜŸ™¤U§[¡[°×1BÑ1BÓ%CÓDˆD�Ià×#Ñ# F¨DÕ1r   ÚinputrE   c                 óZ  — | j                   j                  «       dkD  r+t        j                  || j                   | j                  «      S t        | j                   t        j                  j                  j                  «      rI| j                   j                  j                  «       }| j                  j                  j                  «       }n4| j                   j                  «       }| j                  j                  «       }| j                  dk(  rBt        «        t        || j                   �| j                   d   n|j"                  d   «      \  }}n‚| j                  dk(  r[| j$                  j'                  t        j(                  «      }||z  j+                  t,        t.        ¬«      j'                  t0        «      }nt3        d| j                  › �«      ‚t5        ||||| j                   |j6                  ¬«      }| j                  �|| j                  z   }|j'                  |j6                  ¬«      S )	Nr   r_   rP   rg   ©ÚminÚmaxzUnsupported activation scheme: ©rN   re   )rn   Úelement_sizeÚFÚlinearri   Ú
isinstancer8   Údistributedro   ÚDTensorÚ_local_tensorÚ
contiguousrq   rb   r.   r'   rM   rU   rh   Útorp   ÚclampÚ_FP8_MINÚ_FP8_MAXÚ
_FP8_DTYPEÚNotImplementedErrorr   rS   )rs   rw   rn   Ú	scale_invÚqinputÚscaler\   s          r   ÚforwardzFP8Linear.forward  s«  € Ø�;‰;×#Ñ#Ó%¨Ò)Ü—8‘8˜E 4§;¡;°·	±	Ó:Ð:ä�d—k‘k¤5×#4Ñ#4×#;Ñ#;×#CÑ#CÔDØ—[‘[×.Ñ.×9Ñ9Ó;ˆFØ×-Ñ-×;Ñ;×FÑFÓH‰Ið —[‘[×+Ñ+Ó-ˆFØ×-Ñ-×8Ñ8Ó:ˆIà×!Ñ! YÒ.ÜÔ!ä0Ø¨T¯_©_Ð-H�t—‘ qÒ)ÈeÏkÉkÐZ\Éoó‰MˆF‘Eð ×#Ñ# xÒ/Ø×)Ñ)×,Ñ,¬U¯]©]Ó;ˆEØ˜e‘m×*Ñ*¬x¼XÐ*ÓF×IÑIÌ*ÓU‰Fä%Ð(GÈ×H^ÑH^ÐG_Ð&`ÓaÐaä ØØØØØ�O‰OØŸ™ô
ˆð �9‰9Ð Ø˜dŸi™iÑ'ˆFà�y‰y˜uŸ{™{ˆyÓ+Ð+r   )r   Ú
__module__Ú__qualname__r‰   ÚintÚtupleÚstrÚboolrk   r8   ÚTensorrŽ   Ú__classcell__©rv   s   @r   r^   r^   Þ   so   ø„ ð
 .2Ø!*ØØñ"2àð"2ð ð"2ð ˜#˜s˜(‘O dÑ*ð	"2ð
 ð"2ð õ"2ðH$,˜UŸ\™\ð $,¨e¯l©l÷ $,r   r^   rs   Úhidden_statesÚtop_k_indexÚtop_k_weightsc                 óì  — | j                   dk(  rt        d«      ‚t        «        |j                  }|j	                  d«      }|j	                  d«      }|j	                  d«      }t        j                  ||¬«      j                  d«      j                  d|«      j                  d«      }|j                  d«      }	|j                  d«      }
||   }t        || j                  r| j                  n| j                  | j                  r| j                  n| j                  | j                   |
¬«      }| j                  r| j#                  |«      }n| j%                  |«      }t        || j&                  | j(                  | j                   |
¬«      }||	j+                  |j,                  «      j                  d«      z  }|j/                  |||«      j1                  d¬«      }|j+                  |j,                  «      S )	Nrg   z‘batched_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rP   r   ©rR   r   )rM   Ú
expert_ids©Údim)rb   rŠ   r.   rR   Úsizer8   ÚarangeÚ	unsqueezeÚexpandÚreshaper(   Úhas_gateÚgate_up_projÚup_projÚgate_up_proj_scale_invÚup_proj_scale_invrM   Ú_apply_gateÚact_fnÚ	down_projÚdown_proj_scale_invr…   rS   rT   Úsum)rs   r˜   r™   rš   rR   Ú	num_top_kÚ
num_tokensÚ
hidden_dimÚ	token_idxÚsample_weightsr�   Úselected_hidden_statesÚproj_outÚweighted_outÚfinal_hidden_statess                  r   Úfp8_batched_mm_experts_forwardr¸   *  sÈ  € ð ×Ñ Ò)Ü!ðWó
ð 	
ô
 Ôð ×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jô —‘˜Z°Ô7×AÑAÀ!ÓD×KÑKÈBÐPYÓZ×bÑbÐceÓf€IØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð +¨9Ñ5Ðô )ØØ!Ÿ]š]ˆ×Ò°·±Ø'+§}¢}ˆ×#Ò#¸$×:PÑ:PØ—?‘?Øô€Hð ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆô )ØØ�‰Ø× Ñ Ø—?‘?Øô€Hð ˜n×/Ñ/°·±Ó?×IÑIÈ"ÓMÑM€Lð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c                 óÄ  — | j                   dk(  rt        d«      ‚t        «        |j                  }|j	                  d«      }|j	                  d«      }|j	                  d«      }t        j                  ||¬«      j                  d«      j                  d|«      j                  d«      }|j                  d«      }	|j                  d«      }
t        j                  |
«      }t        j                  |«      }t        j                  |j	                  d«      |¬«      ||<   |
|   }|	|   }|||      }|j                  dk(  r|j                  «       n|j                  «       }t        j                  || j                   d| j                   dz
  ¬«      }t        j"                  |dt
        j$                  ¬	«      }t'        || j(                  r| j*                  n| j,                  | j(                  r| j.                  n| j0                  || j2                  |¬
«      }| j(                  r| j5                  |«      }n| j7                  |«      }t'        || j8                  | j:                  || j2                  |¬
«      }||j=                  |j>                  «      j                  d«      z  }||   }|jA                  |||«      jC                  d¬«      }|j=                  |j>                  «      S )Nrg   z‘grouped_mm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.rP   r   rœ   r   Úcpu©Úbinsrz   r{   )rŸ   rS   )Útokens_per_expertrM   Úoffsetsrž   )"rb   rŠ   r.   rR   r    r8   r¡   r¢   r£   r¤   ÚargsortÚ
empty_liker   rW   r‘   ÚhistcÚnum_expertsÚcumsumÚint32r)   r¥   r¦   r§   r¨   r©   rM   rª   r«   r¬   r­   r…   rS   rT   r®   )rs   r˜   r™   rš   rR   r¯   r°   r±   r²   r³   r�   ÚpermÚinv_permÚexpert_ids_gÚsample_weights_gÚselected_hidden_states_gÚhistc_inputr½   r¾   rµ   r¶   r·   s                         r   Úfp8_grouped_mm_experts_forwardrË   j  s“  € ð ×Ñ Ò)Ü!ðWó
ð 	
ô
 Ôð ×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jô —‘˜Z°Ô7×AÑAÀ!ÓD×KÑKÈBÐPYÓZ×bÑbÐceÓf€IØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jô �=‰=˜Ó$€DÜ×Ñ Ó%€HÜ—\‘\ $§)¡)¨A£,°vÔ>€HˆT�Nà˜dÑ#€LØ% dÑ+ÐØ,¨Y°t©_Ñ=Ðð
 +1¯+©+¸Ò*>�,×$Ñ$Ô&ÀL×DTÑDTÓDV€KÜŸ™ K°d×6FÑ6FÈAÐSW×ScÑScÐfgÑSgÔhÐÜ�l‰lÐ,°!¼5¿;¹;ÔG€Gô )Ø Ø!Ÿ]š]ˆ×Ò°·±Ø'+§}¢}ˆ×#Ò#¸$×:PÑ:PØ+Ø—?‘?Øô€Hð ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆô )ØØ�‰Ø× Ñ Ø+Ø—?‘?Øô€Hð Ð.×1Ñ1°(·.±.ÓA×KÑKÈBÓOÑO€Lð   Ñ)€Lð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   Úexpert_ids_sortedrÂ   Ú	alignmentc                 óh  — | j                   }| j                  d«      }t        j                  | j	                  «       |d|dz
  ¬«      j                  «       }||z   dz
  |z  |z  }|t        ||«      |dz
  z  z   }||z
  }|j                  d«      |z
  }	t        j                  ||¬«      |	|    z   }
t        j                  j                  |«      d   dk\  r |j                  d«      j	                  «       }n;t        j                  |fd|t        j                  ¬«      }| j	                  «       ||
<   |
||fS )a&  Build a TMA-aligned contiguous layout for DeepGEMM grouped GEMM.

    DeepGEMM requires M-dimension alignment per expert for TMA. This computes
    the mapping from sorted token positions to padded row positions, and the
    layout tensor that DeepGEMM uses to identify expert boundaries.

    Returns:
        sorted_to_padded: (num_tokens,) index map from sorted position to padded row
        grouped_layout: expert layout tensor (format depends on GPU architecture)
        total_padded_rows: total number of rows including alignment padding
    r   r   r»   rœ   é
   rP   rQ   )rR   r    r8   rÁ   r‘   Úlongrz   rÃ   r¡   r9   r;   ÚfullrÄ   )rÌ   rÂ   rÍ   rR   r°   r½   Úaligned_tokens_per_expertÚtotal_padded_rowsÚpadding_per_expertÚcumulative_paddingÚsorted_to_paddedÚgrouped_layouts               r   Ú!_build_deepgemm_contiguous_layoutrØ   ¼  s>  € ð ×%Ñ%€FØ"×'Ñ'¨Ó*€JÜŸ™Ð$5×$9Ñ$9Ó$;À+ÐSTÐZeÐhiÑZiÔj×oÑoÓqÐØ"3°iÑ"?À!Ñ"CÈ	Ñ!QÐU^Ñ ^Ðà"¤S¨°[Ó%AÀYÐQRÁ]Ñ%SÑSÐà2Ð5FÑFÐØ+×2Ñ2°1Ó5Ð8JÑJÐÜ—|‘| J°vÔ>ÐASÐTeÑAfÑfÐä‡z�z×'Ñ'¨Ó/°Ñ2°bÒ8Ø*×1Ñ1°!Ó4×8Ñ8Ó:‰ô Ÿ™Ð%6Ð$8¸"ÀVÔSX×S^ÑS^Ô_ˆØ+<×+@Ñ+@Ó+BˆÐ'Ñ(à˜^Ð->Ð>Ð>r   ÚscalesrÖ   rÓ   c                 ó  — t        j                  || j                  d   | j                  | j                  ¬«      }| ||<   t        j                  ||j                  d   | j                  t         j
                  ¬«      }|||<   ||fS )zKPad sorted hidden states and scales into the TMA-aligned contiguous layout.r   rQ   )r8   ÚzerosrU   rR   rS   rp   )r˜   rÙ   rÖ   rÓ   Úhidden_paddedÚscales_paddeds         r   Ú"_pad_to_deepgemm_contiguous_layoutrÞ   Ý  s€   € ô —K‘KØ˜=×.Ñ.¨qÑ1¸-×:NÑ:NÐVc×ViÑViô€Mð '4€MÐ"Ñ#Ü—K‘KÐ 1°6·<±<À±?È=×K_ÑK_Ôgl×gtÑgtÔu€MØ&,€MÐ"Ñ#Ø˜-Ð'Ð'r   Úhidden_states_paddedc                 ó   — | |   S )z;Remove padding rows from the TMA-aligned contiguous layout.rG   )rß   rÖ   s     r   Ú&_unpad_from_deepgemm_contiguous_layoutrá   í  s   € ð  Ð 0Ñ1Ð1r   c                 óF  — | j                   dk(  rt        d«      ‚| j                  €t        d«      ‚| j                  d   dk7  s| j                  d   dk7  rt        d| j                  › �«      ‚t	        «        |j
                  }|j                  d«      }|j                  d«      }|j                  d«      }t        j                  ||¬	«      j                  d«      j                  d|«      j                  d«      }|j                  d«      }	|j                  d«      }
t        j                  |
«      }t        j                  |«      }t        j                  |j                  d«      |¬	«      ||<   |
|   }|	|   }|||      }t        || j                  t         ¬
«      \  }}}| j"                  r| j$                  n| j&                  }| j"                  r| j(                  n| j*                  }t-        |d¬«      \  }}t/        ||||«      \  }}t        j0                  ||j2                  d   |t        j4                  ¬«      }t        j6                  j9                  |«      d   dk\  }t;        ||f||j=                  «       f|||¬«       | j"                  r| j?                  |«      }n| jA                  |«      }| jB                  }| jD                  }t-        |d¬«      \  }}t        j0                  |||t        j4                  ¬«      }t;        ||f||j=                  «       f|||¬«       tG        ||«      }||jI                  |jJ                  «      j                  d«      z  }||   }|jM                  |||«      jO                  d¬«      }|jI                  |jJ                  «      S )Nrg   z�deepgemm experts dispatch does not support activation_scheme='static'. Use the default eager dispatch or switch to activation_scheme='dynamic'.zuDeepGEMM requires block-wise quantization (block_size=[128, 128]), but got per-tensor quantization (block_size=None).r   r   r   z-DeepGEMM requires block_size=(128, 128), got rP   rœ   )rÍ   F)Ú	use_ue8m0rQ   rÏ   )Úuse_psum_layoutrž   )(rb   rŠ   rM   Ú
ValueErrorrB   rR   r    r8   r¡   r¢   r£   r¤   r¿   rÀ   rØ   rÂ   Ú_DEEPGEMM_M_ALIGNMENTr¥   r¦   r§   r¨   r©   r>   rÞ   rÛ   rU   Úbfloat16r9   r;   r=   rW   rª   r«   r¬   r­   rá   r…   rS   rT   r®   )rs   r˜   r™   rš   rR   r¯   r°   r±   r²   r³   r�   rÅ   rÆ   rÇ   rÈ   rÉ   rÖ   r×   rÓ   Úw_upÚws_upÚact_fp8Ú
act_scalesrµ   rä   Úw_downÚws_downÚproj_fp8Úproj_scalesr¶   r·   s                                  r   Úfp8_deepgemm_experts_forwardrð   ô  si  € ð ×Ñ Ò)Ü!ðWó
ð 	
ð ‡�ÐÜðAó
ð 	
ð ‡��qÑ˜SÒ  D§O¡O°AÑ$6¸#Ò$=ÜÐHÈÏÉÐHYÐZÓ[Ð[äÔð ×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jô —‘˜Z°Ô7×AÑAÀ!ÓD×KÑKÈBÐPYÓZ×bÑbÐceÓf€IØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jô �=‰=˜Ó$€DÜ×Ñ Ó%€HÜ—\‘\ $§)¡)¨A£,°vÔ>€HˆT�Nà˜dÑ#€LØ% dÑ+ÐØ,¨Y°t©_Ñ=Ðô ;\Ø�d×&Ñ&Ô2Gô;Ñ7Ð�nÐ&7ð
 !%§¢ˆ4×Ò°4·<±<€DØ+/¯=ª=ˆD×'Ò'¸d×>TÑ>T€EÜ8Ð9QÐ]bÔcÑ€GˆZÜ<¸WÀjÐRbÐduÓvÑ€GˆZÜ�{‰{Ð,¨d¯j©j¸©mÀFÔRW×R`ÑR`Ôa€HÜ—j‘j×6Ñ6°vÓ>¸qÑAÀRÑG€OÜØ	�*Ð  e§k¡k£mÐ4°hÀÐ`oõð
 ‡}‚}Ø×#Ñ# HÓ-‰à—;‘;˜xÓ(ˆð �^‰^€FØ×&Ñ&€GÜ:¸8ÈuÔUÑ€HˆkÜ�{‰{Ð,¨jÀÌuÏ~É~Ô^€HÜØ	�;Ð &¨'¯-©-«/Ð!:¸HÀnÐfuõô
 6°hÐ@PÓQ€Hð Ð.×1Ñ1°(·.±.ÓA×KÑKÈBÓOÑO€Lð   Ñ)€Lð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c                   óv  ‡ — e Zd Zddddefdeeef   dz  dededefˆ fd	„Zd
e	j                  de	j                  fd„Zde	j                  de	j                  de	j                  de	j                  fd„Z	 dde	j                  de	j                  de	j                  de	j                  dz  de	j                  f
d„Zˆ xZS )Ú
FP8ExpertsNr_   FTrM   rb   rc   r¥   c                 ó`  •— t         ‰| �  «        |du sJ d«       ‚|| _        || _        || _        || _        |j                  | _        || _        t        |dd«      | _
        t        |dd«      | _        t        t        |dd«         | _        | j                  �r d	| j                  z  | j                  }}t        j                  t!        j"                  | j                  |||¬
«      «      | _        | j
                  �t'        || j
                  d   «      nd}	| j
                  �t'        || j
                  d   «      nd}
t        j                  t!        j"                  | j                  |	|
t         j(                  ¬
«      «      | _        | j-                  dd «       nü| j                  | j                  }}t        j                  t!        j"                  | j                  |||¬
«      «      | _        | j
                  �t'        || j
                  d   «      nd}| j
                  �t'        || j
                  d   «      nd}t        j                  t!        j"                  | j                  ||t         j(                  ¬
«      «      | _        | j-                  dd «       | j                  | j                  }}t        j                  t!        j"                  | j                  |||¬
«      «      | _        | j
                  �t'        || j
                  d   «      nd}| j
                  �t'        || j
                  d   «      nd}t        j                  t!        j"                  | j                  ||t         j(                  ¬
«      «      | _        | j-                  dd «       | j                  dk(  r�t        j                  t!        j6                  | j                  t         j(                  ¬
«      «      | _        t        j                  t!        j6                  | j                  t         j(                  ¬
«      «      | _        y y )NFzWFP8Experts does not support bias for now, please open an issue if you want this featureÚnum_local_expertsrÂ   Úmoe_intermediate_sizeÚintermediate_sizeÚhidden_activationÚ
hidden_actr   re   r   r   Úgate_up_proj_biasÚup_proj_biasÚdown_proj_biasrg   )rj   rk   Úconfigrc   r¥   rM   Úhidden_sizer±   rb   r   rÂ   Úintermediate_dimr   r«   rl   rm   r8   rV   r¦   rH   rp   r¨   rr   r§   r©   r¬   r­   ÚonesÚgate_up_proj_activation_scaleÚdown_proj_activation_scale)rs   rü   rM   rb   rc   r¥   rS   Úgu_proj_outÚ
gu_proj_inÚgu_scale_outÚgu_scale_inÚ
u_proj_outÚ	u_proj_inÚu_scale_outÚ
u_scale_inÚ
d_proj_outÚ	d_proj_inÚd_scale_outÚ
d_scale_inrv   s                      €r   rk   zFP8Experts.__init__M  s;  ø€ ô 	‰ÑÔà˜5Ñ ð 	
Øeó	
Ð ð ˆŒØ ˆŒØ ˆŒØ$ˆŒØ ×,Ñ,ˆŒØ!2ˆÔÜ& vÐ/BÀMÓRˆÔÜ +¨FÐ4KÐM`Ó aˆÔÜœ[¨Ð1DÀlÓSÑTˆŒà�=‹=Ø&'¨$×*?Ñ*?Ñ&?ÀÇÁ˜ˆKÜ "§¡¬U¯[©[¸×9IÑ9IÈ;ÐXbÐjoÔ-pÓ qˆDÔØEIÇ_Á_ÐE`œ5 ¨d¯o©o¸aÑ.@ÔAÐfgˆLØCGÇ?Á?ÐC^œ% 
¨D¯O©O¸AÑ,>Ô?ÐdeˆKÜ*,¯,©,Ü—‘˜D×,Ñ,¨l¸KÌuÏ}É}Ô]ó+ˆDÔ'ð ×#Ñ#Ð$7¸Õ>à$(×$9Ñ$9¸4¿?¹?˜	ˆJÜŸ<™<¬¯©°D×4DÑ4DÀjÐR[ÐchÔ(iÓjˆDŒLØCGÇ?Á?ÐC^œ% 
¨D¯O©O¸AÑ,>Ô?ÐdeˆKØAEÇÁÐA\œ˜y¨$¯/©/¸!Ñ*<Ô=ÐbcˆJÜ%'§\¡\Ü—‘˜D×,Ñ,¨k¸:ÌUÏ]É]Ô[ó&ˆDÔ"ð ×#Ñ# N°DÔ9à $§¡°×1FÑ1F�Iˆ
ÜŸ™¤e§k¡k°$×2BÑ2BÀJÐPYÐafÔ&gÓhˆŒØ?C¿¹Ð?Z”e˜J¨¯©¸Ñ(:Ô;Ð`aˆØ=A¿_¹_Ð=X”U˜9 d§o¡o°aÑ&8Ô9Ð^_ˆ
Ü#%§<¡<Ü�K‰K˜×(Ñ(¨+°zÌÏÉÔWó$
ˆÔ ð 	×ÑÐ 0°$Ô7à×!Ñ! XÒ-Ü13·±¼e¿j¹jÈ×IYÑIYÔaf×anÑanÔ>oÓ1pˆDÔ.Ü.0¯l©l¼5¿:¹:Àd×FVÑFVÔ^c×^kÑ^kÔ;lÓ.mˆDÕ+ð .r   Úgate_uprE   c                 óV   — |j                  dd¬«      \  }}| j                  |«      |z  S )Nr   rP   rž   )Úchunkr«   )rs   r  ÚgateÚups       r   rª   zFP8Experts._apply_gate†  s,   € Ø—=‘= ¨�=Ó+‰ˆˆbØ�{‰{˜4Ó  2Ñ%Ð%r   r˜   r™   rš   c                 ó
  — t        j                  |t         j                  ¬«      }t        j                  «       5  t         j                  j
                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  d¬	«      j                  d
«      }d d d «       D �]‡  }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
| j                  dk(  r| j                  |   nd }| j!                  |
| j"                  r| j$                  |   n| j&                  |   | j"                  r| j(                  |   n| j*                  |   |¬«      }| j"                  r| j-                  |«      n| j/                  |«      }| j                  dk(  r| j0                  |   nd }| j!                  || j2                  |   | j4                  |   |¬«      }||	|d f   }||j7                  |j8                  «      z  }|j;                  d|	|j7                  |j8                  «      «       �ŒŠ |j7                  |j8                  «      S # 1 sw Y   �Œ²xY w)Nre   )Únum_classesr   r   r   )rP   éþÿÿÿrž   F)Úas_tuplerP   rg   )rh   )r8   Ú
zeros_likerp   Úno_gradrl   r   Úone_hotrÂ   ÚpermuteÚgreaterr®   ÚnonzerorT   Úwhererb   r   r   r¥   r¦   r§   r¨   r©   rª   r«   r  r¬   r­   r…   rS   Ú
index_add_)rs   r˜   r™   rš   r·   Úexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posr²   Úcurrent_stateÚgate_up_act_scalerµ   Údown_act_scaleÚrouting_weightsr¶   s                   r   rŽ   zFP8Experts.forwardŠ  sX  € ô
 $×.Ñ.¨}ÄEÇMÁMÔRÐä�]‰]‹_ñ 	jÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÐZ_ÐPÓ`×eÑeÐfhÓiˆJ÷	jð
 %ó 	eˆJØ˜T×-Ñ-Ò-Øä#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMàBF×BXÑBXÐ\dÒBd�×2Ñ2°:Ò>Ðjnð ð —{‘{ØØ15·²�×!Ñ! *Ò-ÀDÇLÁLÐQ[ÑD\Ø;?¿=º=�×+Ñ+¨JÒ7Èd×NdÑNdÐeoÑNpØ!2ð	 #ó ˆHð 6:·]²]�t×'Ñ'¨Ô1ÈÏÉÐT\ÓH]ˆHà?C×?UÑ?UÐYaÒ?a�×/Ñ/°
Ò;Ðgkð ð —{‘{ØØ—‘˜zÑ*Ø×(Ñ(¨Ñ4Ø!/ð	 #ó ˆHð ,¨I°yÀ$Ð,FÑGˆOØ# o×&8Ñ&8¸¿¹Ó&HÑHˆLØ×*Ñ*¨1¨i¸¿¹ÐI\×IbÑIbÓ9cÖdð7	eð8 #×%Ñ% m×&9Ñ&9Ó:Ð:÷C	jñ 	jús   ºBI8É8Jrw   rn   rq   rh   c                 ó  — |j                  «       dkD  rt        j                  ||d «      S | j                  dk(  rS|�Q|j	                  t
        j                  «      }||z  j                  t        t        ¬«      j	                  t        «      }nAt        «        t        || j                  �| j                  d   n|j                  d   «      \  }}t        ||||| j                  |j                   ¬«      }|j	                  |j                   ¬«      S )Nr   rg   ry   rP   r|   re   )r}   r~   r   rb   r…   r8   rp   r†   r‡   rˆ   r‰   r.   r'   rM   rU   r   rS   )rs   rw   rn   rq   rh   r�   rŒ   r\   s           r   r   zFP8Experts.linear´  sç   € ð ×ÑÓ  1Ò$Ü—8‘8˜E 6¨4Ó0Ð0à×!Ñ! XÒ-Ð2BÐ2NØ$×'Ñ'¬¯©Ó6ˆEØ˜e‘m×*Ñ*¬x¼XÐ*ÓF×IÑIÌ*ÓU‰FäÔ!ä0Ø¨T¯_©_Ð-H�t—‘ qÒ)ÈeÏkÉkÐZ\Éoó‰MˆF�Eô !ØØØØØ�O‰OØŸ™ô
ˆð �y‰y˜uŸ{™{ˆyÓ+Ð+r   ©N)r   r�   r�   r‰   r’   r‘   r“   r”   rk   r8   r•   rª   rŽ   r   r–   r—   s   @r   rò   rò   L  sú   ø„ ð .2Ø!*ØØØñ7nð ˜#˜s˜(‘O dÑ*ð7nð ð	7nð
 ð7nð õ7nðr& 5§<¡<ð &°E·L±Ló &ð(;Ø"Ÿ\™\ð(;Ø8=¿¹ð(;ØUZ×UaÑUað(;à	�‰ó(;ð^ 15ñ,à�|‰|ð,ð —‘ð,ð  Ÿ,™,ð	,ð
  Ÿ,™,¨Ñ-ð,ð 
�‰÷,r   rò   c                   ó   — e Zd ZdZeeedœZy)ÚFP8ExpertsInterfacez?Interface for registering custom FP8 experts forward functions.)Ú
batched_mmÚ
grouped_mmÚdeepgemmN)r   r�   r�   Ú__doc__r¸   rË   rð   Ú_global_mappingrG   r   r   r*  r*  Ó  s   „ ÙIð 5Ø4Ø0ñ�Or   r*  Úmodules_to_not_convertc                 ó
  — |j                   r| S d}| j                  «       D �]:  \  }}t        ||«      sŒ|ri nddi}d}t        j                  d«      5  |j                  d«      ryt        |dd«      }	t        |dd«      }
t        |d	| j                  j                  «       «      }t        t        t        |
|	¬
«      } |d||j                  |j                  |
|	dœ|¤Ž}n_t        |t        j                   «      rEt#        d|j$                  |j&                  |j                  |j                  |j(                  dudœ|¤Ž}|�| j+                  ||«       d}ddd«       �Œ= |st,        j/                  d«       | S # 1 sw Y   �ŒaxY w)a©  
    A helper function to replace all `torch.nn.Linear` modules by `FP8Linear` modules.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `None`):
            Names of the modules to not convert. In practice we keep the `lm_head` in full precision for numerical stability reasons.
        quantization_config (`FbgemmFp8Config`):
            The quantization config object that contains the quantization parameters.
        pre_quantized (`book`, defaults to `False`):
            Whether the model is pre-quantized or not
    FrS   NÚmetaz.expertsr¥   Trc   rü   )Úexperts_classÚexperts_interfacerc   r¥   )rü   rM   rb   rc   r¥   )r`   ra   rM   rb   rc   z�You are loading your model using fp8 but no linear modules were found in your model. Please double check your model architecture.rG   )Ú
dequantizeÚnamed_modulesr   r8   rR   Úendswithr   rü   Úget_text_configr   rò   ÚALL_FP8_EXPERTS_FUNCTIONSÚweight_block_sizerb   r€   rl   ÚLinearr^   r`   ra   ri   Úset_submodulerX   Úwarning)Úmodelr0  Úquantization_configÚpre_quantizedÚhas_been_replacedÚmodule_nameÚmoduleÚmodule_kwargsÚ
new_moduler¥   rc   rü   Ú	new_classs                r   Úreplace_with_fp8_linearrG  à  s¤  € ð" ×%Ò%ØˆàÐØ$×2Ñ2Ó4ó %)Ñˆ�VÜ$ [Ð2HÔIØñ ,™°'¸4°ˆØˆ
Ü�\‰\˜&Ó!ñ 	)Ø×#Ñ# JÔ/Ü" 6¨:°tÓ<�Ü" 6¨:°uÓ=�Ü  ¨°5·<±<×3OÑ3OÓ3QÓR�Ü6Ü",Ü&?Ø%Ø%ô	�	ñ 'ð Ø!Ø2×DÑDØ&9×&KÑ&KØ%Ø%ñð $ñ‘
ô ˜F¤B§I¡IÔ.Ü&ð Ø &× 2Ñ 2Ø!'×!4Ñ!4Ø2×DÑDØ&9×&KÑ&KØ#Ÿ[™[°Ð4ñð $ñ�
ð Ð%Ø×#Ñ# K°Ô<Ø$(Ð!÷=	)ñ 	)ð%)ñN Ü�‰ð<ô	
ð €L÷K	)ñ 	)ús   ÁD E8Å8F	c                   óX   — e Zd ZdZd„ Zdej                  deeej                  f   fd„Z	y)ÚFp8Quantizez^
    A quantization operation that creates two tensors, weight and scale out of a weight.
    c                 ó   — || _         y r(  ©Úhf_quantizer©rs   rL  s     r   rk   zFp8Quantize.__init__)  ó
   € Ø(ˆÕr   Ú
input_dictrE   c                 ól  — t        |j                  «       «      d   \  }}|d   }d }| j                  j                  �kt	        | j                  j                  t
        «      r&| j                  j                  j                  d«      }n!t        | j                  j                  dd «      }|€|j                  d   |j                  d   f}|\  }}|j                  d   |j                  d   }	}||z  dk7  s|	|z  dk7  rt        d|› d|	› d|› d|› d|› �
«      ‚|j                  d d }
||z  }|	|z  }|j                  }|j                  t        j                  «      } |j                  g |
¢|‘|‘|‘|‘­Ž }|j                  «       j                  d	¬
«      }t        j                   |dkD  |t        j"                  |«      «      }t$        |z  }t        j                   |dkD  |t        j"                  |«      «      }|j'                  d«      j'                  d«      }||z  }t        j(                  |t*        t$        ¬«      j                  t,        «      }|j                  |«      }d|z  j                  t        j                  «      }|j/                  d«      r|j1                  dd«      d   dz   }n|dz   }||||iS )Nr   r:  r  rP   úMatrix dimensions (r"   ú$) must be divisible by block sizes (z). for )éýÿÿÿrP   rž   rS  ry   rf   rn   r3   r   z.weight_scale_invÚ
_scale_inv)r’   ÚitemsrL  r?  r€   ÚdictÚgetr   rU   rå   r…   r8   rp   r¤   ÚabsÚamaxr  Ú	ones_likerˆ   r¢   r†   r‡   r‰   r7  Úrsplit)rs   rO  ÚkwargsÚtarget_keysÚvaluerM   Úblock_mÚblock_nÚrowsÚcolsÚleading_shapeÚ
rows_tilesÚ
cols_tilesÚoriginal_shapeÚ
value_fp32ÚreshapedÚmax_absÚsafe_max_absrÙ   Úscales_broadcastÚscaledÚ	quantizedÚ
inv_scalesÚ	scale_keys                           r   ÚconvertzFp8Quantize.convert,  s°  € ä" :×#3Ñ#3Ó#5Ó6°qÑ9Ñˆ�UØ�a‘ˆð ˆ
Ø×Ñ×0Ñ0Ð<Ü˜$×+Ñ+×?Ñ?ÄÔFØ!×.Ñ.×BÑB×FÑFÐGZÓ[‘
ä$ T×%6Ñ%6×%JÑ%JÐL_ÐaeÓf�
ØÐØŸ+™+ b™/¨5¯;©;°r©?Ð;ˆJà%Ñˆ�Ø—[‘[ ‘_ e§k¡k°"¡oˆdˆð �'‰>˜QÒ $¨¡.°AÒ"5ÜØ% d V¨2¨d¨VÐ3WÐX_ÐW`Ð`bÐcjÐbkÐkrÐs~Ðrð  Aóð ð
 Ÿ™ C RÐ(ˆØ˜W‘_ˆ
Ø˜W‘_ˆ
àŸ™ˆØ—X‘XœeŸm™mÓ,ˆ
ð &�:×%Ñ%Ð_ }Ð_°jÐ_À'Ð_È:Ð_ÐW^Ò_ˆð —,‘,“.×%Ñ%¨(Ð%Ó3ˆÜ—{‘{ 7¨Q¡;°¼¿¹ÈÓ9QÓRˆô ˜LÑ(ˆÜ—‘˜W q™[¨&´%·/±/À&Ó2IÓJˆð "×+Ñ+¨BÓ/×9Ñ9¸"Ó=ÐØÐ,Ñ,ˆä—K‘K ¬H¼(ÔC×FÑFÄzÓRˆ	à×%Ñ% nÓ5ˆ	à˜F‘l×&Ñ&¤u§}¡}Ó5ˆ
Ø×Ñ Ô)Ø#×*Ñ*¨3°Ó2°1Ñ5Ð8KÑK‰Ià# lÑ2ˆIð ˜Ø�zð
ð 	
r   N)
r   r�   r�   r.  rk   r8   r•   rV  r“   rp  rG   r   r   rI  rI  $  s1   „ ñò)ð?
 %§,¡,ð ?
¸TÀ#ÀuÇ|Á|ÐBSÑ=Tô ?
r   rI  c            	       ó‚   — e Zd ZdZd„ Z	 d	deeej                  f   dedz  deeej                  f   fd„Z	e
d
d„«       Zy)ÚFp8DequantizeziInverse operation of :class:`Fp8Quantize`. Takes a pair (weight, scale) and reconstructs the fp32 tensor.c                 ó   — || _         y r(  rK  rM  s     r   rk   zFp8Dequantize.__init__q  rN  r   NrO  Úfull_layer_namerE   c                 ól  — t        |«      dk  r||d   iS |d   d   }|d   d   }|j                  dd  \  }}| j                  j                  j                  }|€|j                  d   |j                  d   f}|\  }	}
||	z  dk7  s||
z  dk7  rt        d|› d|› d	|	› d|
› d
�	«      ‚|j                  |j                  «      }|j                  d||	z  |	||
z  |
«      }|j                  d||	z  ||
z  «      }|j                  d«      j                  d«      }||z  }||j                  |j                  «      iS )Nr   zweight$r   rq   r  rP   rQ  r"   rR  z).)
ÚlenrU   rL  r?  r:  rå   r…   rS   r¤   r¢   )rs   rO  rt  r\  rm  rÙ   ra  rb  rM   r_  r`  rh  Úexpanded_scalesÚdequantizeds                 r   rp  zFp8Dequantize.convertt  ss  € ô ˆz‹?˜QÒà# Z°	Ñ%:Ð;Ð;à˜yÑ)¨!Ñ,ˆ	ØÐ.Ñ/°Ñ2ˆà—_‘_ R SÐ)‰
ˆˆdØ×&Ñ&×:Ñ:×LÑLˆ
ØÐØ#Ÿ/™/¨"Ñ-¨y¯©¸rÑ/BÐCˆJà%Ñˆ�à�'‰>˜QÒ $¨¡.°AÒ"5ÜØ% d V¨2¨d¨VÐ3WÐX_ÐW`Ð`bÐcjÐbkÐkmÐnóð ð —L‘L §¡Ó.ˆ	Ø×$Ñ$ R¨°©¸'À4È7Á?ÐT[Ó\ˆØ Ÿ.™.¨¨T°W©_¸dÀg¹oÓNˆØ)×3Ñ3°BÓ7×AÑAÀ!ÓDˆØ Ñ0ˆð ˜[×0Ñ0°·±ÓAð
ð 	
r   c                 ó   — t        «       S r(  )r   )rs   s    r   Ú
reverse_opzFp8Dequantize.reverse_op–  s
   € ä‹}Ðr   r(  )rE   r   )r   r�   r�   r.  rk   rV  r“   r8   r•   rp  Úpropertyrz  rG   r   r   rr  rr  n  se   „ Ùsò)ð '+ñ 
à˜˜eŸl™lÐ*Ñ+ð 
ð ˜t™ð 
ð
 
ˆc�5—<‘<ÐÑ	 ó 
ðD òó ñr   rr  )NNF)Er8   Útorch.nnrl   r   r~   Úactivationsr   Úcore_model_loadingr   r   Úquantizers.quantizers_utilsr   Úutilsr	   Úutils.import_utilsr
   r   Úhub_kernelsr   Úmoer   r   Ú
get_loggerr   rX   Úfloat8_e4m3fnr‰   Úfinforz   r‡   r{   rˆ   ræ   r&   r'   r(   r)   r$   r<   r=   r>   r7   r   r.   rB   r‘   rH   rp   r•   ÚlistrS   r   r;  r^   ÚModuler¸   rË   r’   rØ   rÞ   rá   rð   rò   r*  r9  r“   rG  rI  rr  rG   r   r   ú<module>r‰     sa  ðó Ý Ý $å  ß ;Ý ?Ý ß RÝ )ß =ð 
ˆ×	Ñ	˜HÓ	%€ð × Ñ €
Øˆ5�;‰;�zÓ"×&Ñ&€Øˆ5�;‰;�zÓ"×&Ñ&€ð
 Ð ð Ð ØÐ Ø Ð Ø Ð àÐ ð Ð Ø"Ð Ø!%Ð àÐ òGò*òZ:ðzˆSð �Sð ˜Só ð !&§¡ñ.EØ‡|�|ð.Eà‡|�|ð.Eð 	�‰ð.Eð 	�‰ð	.Eð
 �S‘	ð.Eð —+‘+ð.Eð ‡\�\ó.EôbI,�—	‘	ô I,ðX=7Ø
�(‰(�/‰/ð=7à—<‘<ð=7ð —‘ð=7ð —<‘<ð	=7ð
 ‡\�\ó=7ð@O7Ø
�(‰(�/‰/ðO7à—<‘<ðO7ð —‘ðO7ð —<‘<ð	O7ð
 ‡\�\óO7ðd?¸¿¹ð ?ÐTWð ?Ðdgð ?Ðlqó ?ðB(Ø—<‘<ð(à�L‰Lð(ð —l‘lð(ð ð	(ð
 ˆ5�<‰<˜Ÿ™Ð%Ñ&ó(ð 2ØŸ,™,ð2Ø:?¿,¹,ð2à
‡\�\ó2ðU7Ø
�(‰(�/‰/ðU7à—<‘<ðU7ð —‘ðU7ð —<‘<ð	U7ð
 ‡\�\óU7ôpD,�—‘ô D,ôNÐ*ô ñ 0Ó1Ð ð ejñAØ#'¨¡9¨tÑ#3óAôHG
�-ô G
ôT*�Mõ *r   