Ë
    Gêñi­n  ã                   ó  — d dl mZmZ  e«       r
ddlZddlmZ ddlmZ d dlmZm	Z	 d dl
mZmZ  ej                  e«      Zg d¢Zed	„ «       Z G d
„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Zd„ Zd„ Zej0                  ddœdej2                  dedej6                  fd„Zej0                  ddœdej2                  dedej6                  fd„Z G d„ dej<                  «      Zd„ Z d„ Z!d„ Z"d „ Z#d!„ Z$d"„ Z%d%d#e&e'   dz  fd$„Z(y)&é   )Úis_torch_availableÚloggingé    N)Únn)Úcontextmanager)ÚConversionOpsÚ_IdentityOp)Úget_module_from_nameÚshould_convert_module)g        g      à?g      ð?g      ø?g       @g      @g      @g      @g       €g      à¿g      ð¿g      ø¿g       Àg      Àg      Àg      Àc              #   óØ  K  — t        «       rÀdd l}t        | |j                  «      r| j                  } n"t        | t
        «      r |j                  | «      } t        | dd «      }|dk(  r*|j                  j	                  | «      5  d –— 	 d d d «       y |dk(  r6t        |d«      r*|j                  j	                  | «      5  d –— 	 d d d «       y d –— y # 1 sw Y   ŒIxY w# 1 sw Y   ŒxY w­w)Nr   ÚtypeÚcudaÚxpu)
r   ÚtorchÚ
isinstanceÚTensorÚdeviceÚstrÚgetattrr   Úhasattrr   )Údevr   Údev_types      úa/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/mxfp4.pyÚ	on_devicer   1   sÑ   è ø€ äÔÛä�c˜5Ÿ<™<Ô(Ø—*‘*‰CÜ˜œSÔ!Ø�%—,‘,˜sÓ#ˆCÜ˜3 ¨Ó-ˆØ�vÒØ—‘×"Ñ" 3Ó'ñ ÛØ÷ð ð �uÒ¤¨°Ô!6Ø—‘×!Ñ! #Ó&ñ ÛØ÷ð ô 
÷ð ú÷ð üs6   ‚BC*ÂCÂ	5C*Â>CÃC*ÃCÃC*ÃC'Ã#C*c                   ó²   — e Zd Zd„ Z	 	 	 d	deeej                  f   dej                  j                  dz  de
e   dz  dedz  deeej                  f   f
d„Zy)
ÚMxfp4Quantizec                 ó   — || _         y ©N©Úhf_quantizer©Úselfr    s     r   Ú__init__zMxfp4Quantize.__init__H   ó
   € Ø(ˆÕó    NÚ
input_dictÚmodelÚmissing_keysÚfull_layer_nameÚreturnc                 ó  — t        |j                  «       «      d   \  }}t        |t        «      r|d   n|}t	        ||«      \  }}t        j                  |j                  «      5  t        |t        «      rýt        |j                  dd«      t        «      \  }	}
t        j                  j                  t        j                  j                  t        j                  j                  }}}t        |	|
t        «      \  }	}
d|v rdnd}||j                   v r|j                   |= t#        |||	«       t#        ||› d� ||
 | |«       ¬«      ¬«      «       |j%                  |› «       d	|_        i cd d d «       S 	 d d d «       y # 1 sw Y   y xY w)
Nr   éÿÿÿÿéþÿÿÿÚgate_up_projÚ	down_projÚ_precision_config©Úrhs_data©Úweight_scaleÚflex_ctxT)ÚtupleÚitemsr   Úlistr
   r   r   ÚMxfp4GptOssExpertsÚquantize_to_mxfp4Ú	transposeÚtriton_kernels_hubÚ
matmul_ogsÚPrecisionConfigÚFlexCtxÚ
InFlexDataÚswizzle_mxfp4Ú_parametersÚsetattrÚdiscardÚ_is_hf_initialized)r"   r&   r'   r(   r)   ÚkwargsÚ_ÚvalueÚmoduleÚtriton_weight_tensorr4   r>   r?   r@   Úprojs                  r   ÚconvertzMxfp4Quantize.convertK   ss  € ô ˜×)Ñ)Ó+Ó,¨QÑ/‰ˆˆ5Ü& u¬dÔ3��a’¸ˆä(¨°Ó@‰	ˆ�ä�\‰\˜%Ÿ,™,Ó'ñ 	Ü˜&Ô"4Ô5Ü5FÀuÇÁÐWYÐ[]ÓG^Ô`rÓ5sÑ2Ð$ lä&×1Ñ1×AÑAÜ&×1Ñ1×9Ñ9Ü&×1Ñ1×<Ñ<ð +5 �ô
 6CØ(¨,Ô8Jó6Ñ2Ð$ lð *8¸?Ñ)J‘~ÐP[�à˜6×-Ñ-Ñ-à×*Ñ*¨4Ð0ä˜ Ð&:Ô;ÜØØ�fÐ-Ð.Ù#°ÉÑYcÓYeÔHfÔgôð ×$Ñ$¨Ð'8Ô:Ø,0�Ô)à÷9	ñ 	Ø5÷	÷ 	ñ 	ús   Á%DE=Å=F©NNN©Ú__name__Ú
__module__Ú__qualname__r#   Údictr   r   r   r   ÚModuler8   rL   © r%   r   r   r   G   s}   „ ò)ð )-Ø)-Ø&*ñ)à˜˜eŸl™lÐ*Ñ+ð)ð �x‰x�‰ Ñ%ð)ð ˜3‘i $Ñ&ð	)ð
 ˜t™ð)ð 
ˆc�5—<‘<ÐÑ	 ô)r%   r   c                   óÄ   — e Zd Zd„ Z	 	 	 d
deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
d„Zedd	„«       Zy)ÚMxfp4Dequantizec                 ó   — || _         y r   r   r!   s     r   r#   zMxfp4Dequantize.__init__x   r$   r%   Nr&   r'   r)   r(   r*   c                 óz  — i }d|v rdnd}|› d�|j                  «       v r6t        ||› d�   t        «      r||› d�   d   ||› d�<   n||› d�   ||› d�<   |› d�|j                  «       v r6t        ||› d�   t        «      r||› d�   d   ||› d�<   n||› d�   ||› d�<   t        ||› d�   ||› d�   «      }||iS )Nr.   r/   Ú_blocksr   Ú_scales)Úkeysr   r8   Údequantize_convertops)	r"   r&   r'   r)   r(   rF   Ú
param_datarK   Údequantizeds	            r   rL   zMxfp4Dequantize.convert{   s!  € ð ˆ
Ø!/°?Ñ!B‰~ÈˆØˆV�7Ð˜zŸ™Ó0Ñ0Ü˜*¨ v¨WÐ%5Ñ6¼Ô=Ø/9¸T¸FÀ'Ð:JÑ/KÈAÑ/N�
˜d˜V 7Ð+Ò,à/9¸T¸FÀ'Ð:JÑ/K�
˜d˜V 7Ð+Ñ,ØˆV�7Ð˜zŸ™Ó0Ñ0Ü˜*¨ v¨WÐ%5Ñ6¼Ô=Ø/9¸T¸FÀ'Ð:JÑ/KÈAÑ/N�
˜d˜V 7Ð+Ò,à/9¸T¸FÀ'Ð:JÑ/K�
˜d˜V 7Ð+Ñ,ô ,¨J¸$¸¸wÐ7GÑ,HÈ*ÐX\ÐW]Ð]dÐUeÑJfÓgˆØ Ð-Ð-r%   c                 ó   — t        «       S r   )r	   ©r"   s    r   Ú
reverse_opzMxfp4Dequantize.reverse_op”   s
   € ä‹}Ðr%   rM   )r*   r   )rO   rP   rQ   r#   rR   r   r   r   r   rS   r8   rL   Úpropertyra   rT   r%   r   rV   rV   w   s‘   „ ò)ð )-Ø&*Ø)-ñ.à˜˜eŸl™lÐ*Ñ+ð.ð �x‰x�‰ Ñ%ð.ð ˜t™ð	.ð
 ˜3‘i $Ñ&ð.ð 
ˆc�5—<‘<ÐÑ	 ó.ð2 òó ñr%   rV   c                   óÈ   — e Zd Zd„ Z	 	 	 d
deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
d„Zedefd	„«       Zy)ÚMxfp4Deserializec                 ó   — || _         y r   r   r!   s     r   r#   zMxfp4Deserialize.__init__š   r$   r%   Nr&   r'   r)   r(   r*   c           	      óö  — i }d|v rdnd}|› d�|j                  «       v r6t        ||› d�   t        «      r||› d�   d   ||› d�<   n||› d�   ||› d�<   |› d�|j                  «       v r6t        ||› d�   t        «      r||› d�   d   ||› d�<   n||› d�   ||› d�<   t        ||«      \  }}	t	        ||› d�   ||› d�   ||||› d�   j
                  t        «       |j                  |› «       d|_        i S )Nr.   r/   rY   r   rZ   T)	r[   r   r8   r
   Úswizzle_mxfp4_convertopsr   r<   rD   rE   )
r"   r&   r'   r)   r(   rF   r]   rK   rI   rG   s
             r   rL   zMxfp4Deserialize.convert�   se  € ð ˆ
Ø!/°?Ñ!B‰~ÈˆàˆV�7Ð˜zŸ™Ó0Ñ0Ü˜*¨ v¨WÐ%5Ñ6¼Ô=Ø/9¸T¸FÀ'Ð:JÑ/KÈAÑ/N�
˜d˜V 7Ð+Ò,à/9¸T¸FÀ'Ð:JÑ/K�
˜d˜V 7Ð+Ñ,ØˆV�7Ð˜zŸ™Ó0Ñ0Ü˜*¨ v¨WÐ%5Ñ6¼Ô=Ø/9¸T¸FÀ'Ð:JÑ/KÈAÑ/N�
˜d˜V 7Ð+Ò,à/9¸T¸FÀ'Ð:JÑ/K�
˜d˜V 7Ð+Ñ,ô )¨°Ó@‰	ˆ�Ü Ø˜$˜˜wÐ'Ñ(Ø˜$˜˜wÐ'Ñ(ØØØ˜$˜˜wÐ'Ñ(×/Ñ/Üô	
ð 	×Ñ Ð0Ô2Ø$(ˆÔ!ð ˆ	r%   c                 ó,   — t        | j                  «      S r   )ÚMxfp4ReverseDeserializer    r`   s    r   ra   zMxfp4Deserialize.reverse_opÄ   s   € ä& t×'8Ñ'8Ó9Ð9r%   rM   )rO   rP   rQ   r#   rR   r   r   r   r   rS   r8   rL   rb   r   ra   rT   r%   r   rd   rd   ™   s™   „ ò)ð )-Ø&*Ø)-ñ%à˜˜eŸl™lÐ*Ñ+ð%ð �x‰x�‰ Ñ%ð%ð ˜t™ð	%ð
 ˜3‘i $Ñ&ð%ð 
ˆc�5—<‘<ÐÑ	 ó%ðN ð:˜Mò :ó ñ:r%   rd   c                   ó²   — e Zd Zd„ Z	 	 	 d	deeej                  f   dej                  j                  dz  dedz  de
e   dz  deeej                  f   f
d„Zy)
ri   c                 ó   — || _         y r   r   r!   s     r   r#   z Mxfp4ReverseDeserialize.__init__Ê   r$   r%   Nr&   r'   r)   r(   r*   c                 óð  — t        |j                  dd«      }t        |j                  dd«      }d|v rdnd}|j                  dd«      d	   }	t        ||«      \  }
}i }t	        |
t
        «      �rd
|v r&|j                  dd«      }	t        |
|dz   «      ||	<   |S d|v rî|
j                  j                  j                  j                  |
j                  j                  j                  «      j                  dd«      j                  |ddd«      ||	› d�<   |
j                  j                  j                  j                  j                  |
j                  j                  j                  j                  «      j                  dd«      ||	› d�<   |S |
j                   j                  j                  j                  |
j                   j                  j                  «      j                  dd«      j                  ||dd«      ||	› d�<   |
j"                  j                  j                  j                  j                  |
j"                  j                  j                  j                  «      j                  dd«      ||	› d�<   |S )NÚnum_local_expertsé    Úhidden_sizei@  r.   r/   rG   é   r   ÚbiasrY   Ú Ú_biasr,   r-   éZ   é   rZ   )r   ÚconfigÚrsplitr
   r   r9   Úreplacer.   ÚstorageÚlayoutÚunswizzle_dataÚdatar;   ÚreshapeÚgate_up_proj_precision_configr4   r/   Údown_proj_precision_config)r"   r&   r'   r)   r(   rF   rm   ro   rK   ÚnamerI   rG   Ú
state_dicts                r   rL   zMxfp4ReverseDeserialize.convertÍ   s>  € ô $ E§L¡LÐ2EÀrÓJÐÜ˜eŸl™l¨M¸4Ó@ˆà!/°?Ñ!B‰~Èˆà×%Ñ% c¨1Ó-¨aÑ0ˆÜ(¨°Ó@‰	ˆ�Øˆ
Ü�fÔ0Õ1Ø˜Ñ(Ø&×.Ñ.¨y¸"Ó=�Ü#*¨6°4¸'±>Ó#B�
˜4Ñ Ø!Ð!Ø Ñ0à×'Ñ'×/Ñ/×6Ñ6×EÑEÀf×FYÑFY×FaÑFa×FfÑFfÓgß‘Y˜r 2Ó&ß‘WÐ.°°B¸Ó;ð ˜d˜V 7Ð+Ñ,ð ×8Ñ8×EÑE×MÑM×TÑT×cÑcØ×<Ñ<×IÑI×QÑQ×VÑVóç‘i  BÓ'ð ˜d˜V 7Ð+Ñ,ð" Ðð ×$Ñ$×,Ñ,×3Ñ3×BÑBÀ6×CSÑCS×C[ÑC[×C`ÑC`Óaß‘Y˜r 2Ó&ß‘WÐ.°¸RÀÓDð ˜d˜V 7Ð+Ñ,ð ×5Ñ5×BÑB×JÑJ×QÑQ×`Ñ`Ø×9Ñ9×FÑF×NÑN×SÑSóç‘i  BÓ'ð ˜d˜V 7Ð+Ñ,ð Ðr%   rM   rN   rT   r%   r   ri   ri   É   s}   „ ò)ð )-Ø&*Ø)-ñ,à˜˜eŸl™lÐ*Ñ+ð,ð �x‰x�‰ Ñ%ð,ð ˜t™ð	,ð
 ˜3‘i $Ñ&ð,ð 
ˆc�5—<‘<ÐÑ	 ô,r%   ri   c                 ó¼   — |j                   j                  j                  } || j                  t        j
                  «      t        j                  d¬«      \  } }| |fS )Nrp   )Úaxis)Únumerics_detailsÚmxfpÚdowncast_to_mxfp_torchÚtor   Úbfloat16Úuint8)Úwr<   r†   Úw_scales       r   r:   r:   ý   sH   € Ø/×@Ñ@×EÑE×\Ñ\ÐÙ'¨¯©¬U¯^©^Ó(<¼e¿k¹kÐPQÔR�J€A€wØˆgˆ:Ðr%   c                 óf  — |j                   j                  |j                   j                  |j                   j                  }}}|j                  j
                  }|j                  j
                  j                  }|j                  d¬«      \  }}	 | || |¬«      |fi |	¤Ž}  | ||«      |«      }| |fS )zE
    Changes the layout of the tensors depending on the hardware
    rp   )Úmx_axis©Údtype)ÚtensorÚFP4Úconvert_layoutÚwrap_torch_tensorÚtensor_detailsrz   ÚStridedLayoutÚ"make_default_matmul_mxfp4_w_layout)
rŠ   r‹   r<   r‘   r’   r“   rz   r•   Úvalue_layoutÚvalue_layout_optss
             r   rA   rA     s´   € ð
 	×!Ñ!×%Ñ%Ø×!Ñ!×0Ñ0Ø×!Ñ!×3Ñ3ð +ˆ€Cð
  ×.Ñ.×5Ñ5€FØ&×5Ñ5×<Ñ<×JÑJ€Mà&,×&OÑ&OÐXYÐ&OÓ&ZÑ#€LÐ#ÙÑ(¨°#Ô6¸ÑZÐHYÑZ€AÙÑ.¨wÓ7¸ÓG€GØˆgˆ:Ðr%   i   ©r�   Úrows_per_chunkr�   rš   r*   c                ó4  — ddl }| j                  t        j                  «      } |j                  t        j                  «      dz
  }| j
                  dd |j
                  k(  s$J d| j
                  dd ›d|j
                  ›�«       ‚t        j                  t        || j                  ¬«      }| j
                  �^ }}}|j                  |«      |z  }	| j                  |	|«      } |j                  |	d«      }t        j                  |	|d	z  || j                  ¬«      }
t        d|	|«      D ]Ÿ  }t        ||z   |	«      }| || }||| }|
|| }|d
z  j                  t        j                  «      }||   |dd…ddd	…f<   ~|dz	  j                  t        j                  «      }||   |dd…ddd	…f<   ~t        j                  |||¬«       ~~~Œ¡   |
j                  g |¢|‘|d	z  ‘­Ž j                   g |¢||z  d	z  ‘­Ž }
|
j#                  dd	«      j%                  «       S )úw
    Convert the mxfp4 weights again, dequantizing and makes them compatible with the forward
    pass of GPT_OSS.
    r   Né   r,   zblocks.shape[:-1]=z does not match scales.shape=)r�   r   rp   r   é   é   )Úout)Úmathr‡   r   r‰   Úint32Úshaper�   Ú
FP4_VALUESr   Úprodr}   ÚemptyÚrangeÚminÚintÚldexpÚviewr;   Ú
contiguous)ÚblocksÚscalesr�   rš   r¡   ÚlutÚprefix_shapeÚGÚBÚ
rows_totalr    Úr0Úr1ÚblkÚexpÚsubÚidx_loÚidx_his                     r   Ú_convert_moe_packed_tensorsr»     s  € ó à�Y‰Y”u—{‘{Ó#€FØ�Y‰Y”u—{‘{Ó# cÑ)€Fà�<‰<˜˜Ð §¡Ò,ÐdÐ1C°·±¸c¸rÐ1BÐ0DÐDbÐU[×UaÑUaÐTcÐ.dÓdÐ,ä
�,‰,”z¨°v·}±}Ô
E€Cà Ÿ,™,Ñ€\�1�aØ—‘˜<Ó(¨1Ñ,€Jà�^‰^˜J¨Ó*€FØ�^‰^˜J¨Ó*€Fä
�+‰+�j ! a¡%¨u¸V¿]¹]Ô
K€Cä�A�z >Ó2ò ˆÜ��nÑ$ jÓ1ˆà�R˜ˆmˆØ�R˜ˆmˆØ�"�Rˆjˆð ˜‘*—‘¤§¡Ó+ˆØ˜6‘{ˆŠAˆqˆt�!ˆtˆG‰Øð ˜‘(—‘œuŸy™yÓ)ˆØ˜6‘{ˆŠAˆqˆt�!ˆtˆG‰Øô 	�‰�C˜ #Õ&Ø�‘cð'ð* 4ˆ+ˆ#�+‰+Ð
.�|Ð
. QÐ
.¨¨A©Ò
.×
3Ñ
3Ð
M°\Ð
MÀ1ÀqÁ5È1Á9Ò
M€Cà�=‰=˜˜AÓ×)Ñ)Ó+Ð+r%   c                ó¶   — 	 t        | |||¬«      S # t        j                  $ r4 | j                  d«      } |j                  d«      }t        | |||¬«      cY S w xY w)rœ   r™   Úcpu)r»   r   ÚOutOfMemoryErrorr‡   )r­   r®   r�   rš   s       r   Úconvert_moe_packed_tensorsr¿   M  sa   € ðgÜ*¨6°6ÀÐWeÔfÐføô ×!Ñ!ò gØ—‘˜5Ó!ˆØ—‘˜5Ó!ˆÜ*¨6°6ÀÐWeÔfÒfðgús   ‚ ‘AAÁAc                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )r9   c           	      óä  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  dz  dt        j                  ¬«      d¬«      | _        t        j                  t        j                  | j                  d| j                  z  t        j                  ¬«      d¬«      | _        t        j                  t        j                  | j                  | j
                  | j                  dz  dft        j                  ¬«      d¬«      | _        t        j                  t        j                  | j                  | j
                  t        j                  ¬«      d¬«      | _        d| _        t#        |dd	«      | _        d | _        d | _        t#        |dd	«      | _        y )
Nr   rn   ru   rŽ   F©Úrequires_gradg¬Zd;û?Úswiglu_limitg      @)Úsuperr#   rm   Únum_expertsÚintermediate_sizero   r   Ú	Parameterr   Úzerosr‰   r.   Úfloat32Úgate_up_proj_biasr/   Údown_proj_biasÚalphar   Úlimitr~   r   )r"   rv   Ú	__class__s     €r   r#   zMxfp4GptOssExperts.__init__g  s{  ø€ Ü‰ÑÔà!×3Ñ3ˆÔØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔäŸL™LÜ�K‰K˜×(Ñ(¨!¨d×.DÑ.DÑ*DÀd×FVÑFVÐZ\ÑF\Ð^`Ôhm×hsÑhsÔtØô
ˆÔô
 "$§¡Ü�K‰K˜×(Ñ(¨!¨d×.DÑ.DÑ*DÌEÏMÉMÔZÐjoô"
ˆÔô Ÿ™Ü�K‰K˜×)Ñ)¨4×+;Ñ+;¸T×=SÑ=SÐWYÑ=YÐ[]Ð^Ôfk×fqÑfqÔrØô
ˆŒô
 !Ÿl™lÜ�K‰K˜×(Ñ(¨$×*:Ñ*:Ä%Ç-Á-ÔPÐ`eô
ˆÔð ˆŒ
Ü˜V ^°SÓ9ˆŒ
Ø-1ˆÔ*Ø*.ˆÔ'Ü˜V ^°SÓ9ˆ�
r%   Úhidden_statesr*   c                 óº  — t         j                  j                  t         j                  j                  t         j                  j                  }}}t         j                  j
                  }t        |j                  «      5   | |d|d«      | j                  | j                  fd«      }	 ||| j                  | j                  j                  t        j                  «      ||| j                  d |	¬«      }
 ||
| j                   | j"                  j                  t        j                  «      ||| j$                  |j&                  ¬«      }d d d «       |S # 1 sw Y   S xY w)NÚswiglu)rÍ   rÎ   r   )Úgather_indxÚprecision_configÚgammasÚfused_activation)Úscatter_indxrÔ   rÕ   )r<   r=   ÚFnSpecsÚFusedActivationrÒ   Ú	swiglu_fnr   r   rÍ   rÎ   r.   rË   r‡   r   rÊ   r~   r/   rÌ   r   Ú	gate_scal)r"   rÐ   Úrouting_dataÚ
gather_idxÚscatter_idxrØ   rÙ   r=   rÚ   ÚactÚintermediate_cache1Úintermediate_cache3s               r   ÚforwardzMxfp4GptOssExperts.forward…  s0  € ä×)Ñ)×1Ñ1Ü×)Ñ)×9Ñ9Ü×)Ñ)×4Ñ4ð #-�ˆô
 '×-Ñ-×7Ñ7ˆ	ä�}×+Ñ+Ó,ñ 	Ù!¡'¨(°IÐ?QÓ"RÐUY×U_ÑU_Ðae×akÑakÐTlÐnoÓpˆCá",ØØ×!Ñ!Ø×&Ñ&×)Ñ)¬%¯-©-Ó8ØØ&Ø!%×!CÑ!CØØ!$ô	#Ðñ #-Ø#Ø—‘Ø×#Ñ#×&Ñ&¤u§}¡}Ó5ØØ(Ø!%×!@Ñ!@Ø#×-Ñ-ô#Ð÷	ð. #Ð"÷/	ð. #Ð"ús   Á>CEÅE)rO   rP   rQ   r#   r   r   râ   Ú__classcell__)rÏ   s   @r   r9   r9   f  s&   ø„ ô:ð<# U§\¡\ð #Ð]b×]iÑ]i÷ #r%   r9   c                 óB  — dd l }t        j                  j                  t        j                  j                  t        j                  j
                  t        j                  j                  f\  }}}}t        | j                  «      5  t        j                  j                  «       }t        |j                  j                  dd«      «      }d}	| j                  d   }
| j                  d   }||z  }||z  }|dz   |z  }|
|z  }d„ } || |«      \  }}t        j                   |d¬«      }t        j"                  |d¬«      \  }}t        j$                  |d|«      }|j'                  d«      }t        j(                  |||dz
  ¬«      || }|j+                  d«      j-                  t        j.                  «      }d	}t        j0                  ||k  ||«      }t        j2                  |d
¬«      j-                  t        j.                  «      }t        j2                  |«      j-                  t        j.                  «      }t        j0                  ||k  ||	«      }t        j0                  ||k  ||	«      }t        j0                  ||	k(  |	|«      }||   }t        j0                  ||   |	k(  |	|«      } ||j                  «       |j                  «       ¬«      } ||j                  «       |j                  «       ¬«      } ||||«      }|}d d d «        |«      fS # 1 sw Y   ŒxY w)Nr   Ú
LOCAL_RANKÚ0r,   rp   c                 óº   — t        j                  |  dd¬«      d d …d |…f   }|j                  «       }t        j                  | |d¬«      }||j	                  «       fS )Nrp   T)ÚdimÚstable©rè   )r   ÚargsortÚlongÚtake_along_dimr©   )ÚvalsÚkÚtk_indxÚtk_vals       r   Útopkz routing_torch_dist.<locals>.topkÄ  sS   € Ü—m‘m T E¨q¸Ô>ºqÀ"À1À"¸uÑEˆGØ—l‘l“nˆGÜ×)Ñ)¨$°¸QÔ?ˆFØ˜7Ÿ;™;›=Ð(Ð(r%   rê   )ÚbinsÚmaxiè  T)ré   )Úsrc_indxÚdst_indx)Úosr<   ÚroutingÚ
GatherIndxÚRoutingDataÚScatterIndxÚcompute_expt_data_torchr   r   r   ÚdistributedÚget_world_sizer©   ÚenvironÚgetr£   ÚsoftmaxÚsortÚgatherr}   Úhistcr«   r‡   r¢   Úwhererë   )ÚlogitsÚn_expts_actr÷   rù   rú   rû   rü   Ú
world_sizeÚrankÚreplace_valueÚn_tokensÚn_expts_totÚn_local_expertsÚlocal_expert_startÚlocal_expert_endÚn_gates_padrò   Ú	expt_scalÚ	expt_indxÚsort_indicesÚhistÚvarÚ	topk_indxÚ	gate_indxrÛ   rÓ   r×   Ú	expt_dataÚhit_expertss                                r   Úrouting_torch_distr  ©  sÀ  € ó ô 	×"Ñ"×-Ñ-Ü×"Ñ"×.Ñ.Ü×"Ñ"×.Ñ.Ü×"Ñ"×:Ñ:ð	EÑA€J�˜[Ð*Aô 
�6—=‘=Ó	!ñ 3"Ü×&Ñ&×5Ñ5Ó7ˆ
Ü�2—:‘:—>‘> ,°Ó4Ó5ˆØˆà—<‘< ‘?ˆØ—l‘l 1‘oˆà%¨Ñ3ˆØ! OÑ3ÐØ  1™H¨Ñ7Ðà Ñ,ˆò	)ñ  $ F¨KÓ8Ñˆ	�9Ü—M‘M )°Ô4ˆ	Ü"'§*¡*¨Y¸AÔ">Ñˆ	�<Ü—L‘L ¨A¨|Ó<ˆ	ð ×%Ñ% bÓ)ˆ	ä�{‰{˜9¨;¸KÈ!¹OÔLÐM_Ð`pÐqˆà—N‘N 2Ó&×)Ñ)¬%¯+©+Ó6ˆ	ð ˆÜ—K‘K 	Ð,>Ñ >ÀÀYÓOˆ	Ü—M‘M )°DÔ9×<Ñ<¼U¿[¹[ÓIˆ	Ü—M‘M )Ó,×/Ñ/´·±Ó<ˆ	Ü—K‘K 	Ð,<Ñ <¸iÈÓWˆ	Ü—K‘KÐ 2°iÑ ?ÀÈMÓZˆ	ä—K‘K 	¨]Ñ :¸MÈ9ÓUˆ	Ø˜iÑ(ˆ	ä—K‘K 	¨)Ñ 4¸Ñ EÀ}ÐV_Ó`ˆ	ñ !¨)¯-©-«/ÀIÇMÁMÃOÔTˆÙ"¨I¯M©M«OÀiÇmÁmÃoÔVˆá+¨D°/À;ÓOˆ	à!ˆ÷g3"ñh �y $¨¸ÀiÓPÐR]Ð_kÐkÐk÷i3"ð 3"ús   ÂI9LÌLc                 óº  — dd l m} |j                  «       r#|j                  «       rt	        | d«      rt
        }nt        j                  j                  }|j                  d   }|j                  d| j                  j                  «      }t        j                  j                  || j                  j                  | j                  j                   «      }t#        |j$                  «      5   ||| j                  j&                  «      \  }}}d d d «       | j)                  |¬«      }	|	j                  |d| j                  j                  «      }	|	|fS # 1 sw Y   ŒIxY w)Nr   Ú
_is_hookedr,   )rÞ   )Útorch.distributedrý   Úis_availableÚis_initializedr   r  r<   rø   r£   r}   ÚrouterÚ
hidden_dimr   Ú
functionalÚlinearÚweightrq   r   r   Útop_kÚexperts)
r"   rÐ   Údistrø   Ú
batch_sizeÚrouter_logitsrÜ   rÝ   rÞ   Ú
routed_outs
             r   Úmlp_forwardr+  í  s  € Ý$à×ÑÔ˜t×2Ñ2Ô4¼ÀÀ|Ô9TÜ$‰ä$×,Ñ,×4Ñ4ˆà×$Ñ$ QÑ'€JØ!×)Ñ)¨"¨d¯k©k×.DÑ.DÓE€MÜ—M‘M×(Ñ(¨¸¿¹×8JÑ8JÈDÏKÉK×L\ÑL\Ó]€Mä	�=×'Ñ'Ó	(ñ ZÙ07¸ÀtÇ{Á{×GXÑGXÓ0YÑ-ˆ�j +÷Zð —‘˜m¨\¸:ÐS^�Ó_€JØ×#Ñ# J°°D·K±K×4JÑ4JÓK€JØ�}Ð$Ð$÷Zð Zús   Ã'"EÅEc                 ó€  — ddl m} |j                  d«      }|j                  d«      }|j                  d«      }	|j                  d«      }
|j                  d«      }|j                  d«      }d	D ]Í  }||v sŒ|� ||||||	|
||«      }|› d
�}|› d�}t        | |j	                  dd«      d   |«       t        | |«      sŒPt        | |«      sŒ]t        t        | |«      t        | |«      «      }t        | |t        j                  j                  |j                  |«      «      «       t        | |«       t        | |«       ŒÏ y )Nr   ©Úshard_and_distribute_moduler'   Úempty_paramÚcasting_dtypeÚto_contiguousr	  Údevice_mesh)r.   r/   rY   rZ   ú.rp   )Úintegrations.tensor_parallelr.  r   rC   rw   r   r¿   r   r   r   rÈ   r‡   Údelattr)rI   Ú
param_nameÚparam_valueÚtarget_deviceÚdq_param_namerF   r.  r'   r/  r0  r1  r	  r2  rK   Úblocks_attrÚscales_attrr^   s                    r   Ú
dequantizer<    s8  € ÝJà�J‰J�wÓ€EØ—*‘*˜]Ó+€KØ—J‘J˜Ó/€MØ—J‘J˜Ó/€MØ�:‰:�fÓ€DØ—*‘*˜]Ó+€Kà-ò -ˆØ�:ÒØÐ&Ù9ØØØØ!Ø!Ø!ØØó	�ð "˜F 'Ð*ˆKØ!˜F 'Ð*ˆKÜ�F˜J×-Ñ-¨c°1Ó5°aÑ8¸+ÔFÜ�v˜{Õ+´¸ÀÕ0LÜ8¼ÀÈÓ9UÔW^Ð_eÐgrÓWsÓt�Ü˜ ¤e§h¡h×&8Ñ&8¸¿¹ÈÓ9VÓ&WÔXÜ˜ Ô,Ü˜ Õ,ñ)-r%   c                 óX   — t        | |«      }t        j                  j                  |«      S r   )r¿   r   r   rÈ   )r­   r®   r^   s      r   r\   r\   "  s#   € Ü,¨V°VÓ<€KÜ�8‰8×Ñ˜kÓ*Ð*r%   c                 óZ  — |j                   j                  |j                   j                  |j                   j                  }}}ddlm}	 |j                  d«      }
|j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      }d	|v r&|j                  d
«      d   j                  d«      d   }d|v r&|j                  d
«      d   j                  d«      d   }|� |	|
|||||||«       n?t        | |j                  d
d«      d   t        j                  j                  |d¬«      «       › d�}|› d�}t        | |«      }t        | |«      }|j                  j                  dk7  �r|j                  j                  dk7  �rõ|j!                  d«      }|dk(  r!|j#                  || j$                  dz  d«      }n |j#                  |d| j$                  dz  «      }t        |d|«      dk(  rVt'        t        d«      rFt        j(                  j+                  «       �(t        j(                  j+                  «       j                  }|j-                  |«      j/                  «       }|j-                  |«      j/                  «       }t1        |«      5  t3        |j5                  dd«      |j5                  dd«      |«      \  }}ddd«       |dk(  r5t        j6                  || j8                  | j$                  dz  g«      _        n1t        j6                  || j$                  | j8                  g«      _        t        | ||«       t        | |› d� | | |«       ¬«      ¬«      «       t=        | |«       t=        | |«       ~yyy# 1 sw Y   ŒÁxY w)úq
    This transforms the weights obtained using `convert_gpt_oss.py` to load them into `Mxfp4GptOssExperts`.
    r   r-  r'   r/  r0  r1  r	  r2  r­   r3  r,   rY   r   r®   rZ   Nrp   FrÂ   Úmetar.   r   r½   Úacceleratorr-   r0   r1   r3   )r=   r>   r?   r@   r4  r.  r   ÚsplitrC   rw   r   r   rÈ   r   r   r   Úsizer}   rÇ   r   rA  Úcurrent_acceleratorr‡   r¬   r   rA   r;   ÚSizero   r£   r5  )rI   r6  r7  r8  r<   rF   r>   r?   r@   r.  r'   r/  r0  r1  r	  r2  rK   r:  r;  r­   r®   Úlocal_expertsrJ   r4   s                           r   Úload_and_swizzle_mxfp4rG  '  sf  € ð
 	×%Ñ%×5Ñ5Ø×%Ñ%×-Ñ-Ø×%Ñ%×0Ñ0ð )�W€Oõ
 Kà�J‰J�wÓ€EØ—*‘*˜]Ó+€KØ—J‘J˜Ó/€MØ—J‘J˜Ó/€MØ�:‰:�fÓ€DØ—*‘*˜]Ó+€KØ�:ÑØ×Ñ Ó$ RÑ(×.Ñ.¨yÓ9¸!Ñ<ˆØ�:ÑØ×Ñ Ó$ RÑ(×.Ñ.¨yÓ9¸!Ñ<ˆØÐÙ#Ø�; ¨Z¸ÈÐW[Ð]hõ	
ô 	�˜
×)Ñ)¨#¨qÓ1°!Ñ4´e·h±h×6HÑ6HÈÐdiÐ6HÓ6jÔkØ�F˜'Ð"€KØ�F˜'Ð"€KÜ�V˜[Ó)€FÜ�V˜[Ó)€Fà‡}�}×Ñ˜VÓ#¨¯©×(:Ñ(:¸fÓ(DØŸ™ A›ˆØ�>Ò!Ø—^‘^ M°6×3KÑ3KÈaÑ3OÐQSÓT‰Fà—^‘^ M°2°v×7OÑ7OÐSTÑ7TÓUˆFä�M 6¨=Ó9¸UÒBÜœ˜}Ô-Ü×!Ñ!×5Ñ5Ó7ÐCä!×-Ñ-×AÑAÓC×HÑHˆMØ—‘˜=Ó)×4Ñ4Ó6ˆØ—‘˜=Ó)×4Ñ4Ó6ˆÜ�}Ó%ñ 	Ü1>Ø× Ñ   RÓ(¨&×*:Ñ*:¸2¸rÓ*BÐDVó2Ñ.Ð  ,÷	ð �>Ò!Ü).¯©°]ÀF×DVÑDVÐX^×XpÑXpÐstÑXtÐ4uÓ)vÐ Õ&ä).¯©°]ÀF×D\ÑD\Ð^d×^pÑ^pÐ4qÓ)rÐ Ô&ô 	�˜Ð2Ô3ÜØØˆfÐ%Ð&Ù¨ÁÑQ[ÓQ]Ô@^Ô_ô	
ô 	�˜Ô$Ü�˜Ô$ÙðI )EÐ#÷	ð 	ús   Ê01N!Î!N*c                 óŒ  — |j                   j                  |j                   j                  |j                   j                  }}}| j	                  d«      }	t        |d|«      dk(  rVt        t        d«      rFt        j                  j                  «       �(t        j                  j                  «       j                  }| j                  |«      j                  «       } |j                  |«      j                  «       }|dk(  r!| j                  |	|j                  dz  d«      } n | j                  |	d|j                  dz  «      } t        |«      5  t!        | j#                  d	d«      |j#                  d	d«      |«      \  }
}ddd«       |dk(  r5t        j$                  |	|j&                  |j                  dz  g«      
_        n1t        j$                  |	|j                  |j&                  g«      
_        ||j*                  v r|j*                  |= t-        |||
«       t-        ||› d
� | | |«       ¬«      ¬«      «       y# 1 sw Y   ŒÁxY w)r?  r   r   r½   rA  Nr.   r   r,   r-   r0   r1   r3   )r=   r>   r?   r@   rC  r   r   r   rA  rD  r   r‡   r¬   r}   rÇ   r   rA   r;   rE  ro   r£   rB   rC   )r­   r®   rI   rK   r8  r<   r>   r?   r@   rF  rJ   r4   s               r   rg   rg   n  s   € ð
 	×%Ñ%×5Ñ5Ø×%Ñ%×-Ñ-Ø×%Ñ%×0Ñ0ð )�W€Oð —K‘K “N€Mä�˜v }Ó5¸Ò>Ü”E˜=Ô)Ü×Ñ×1Ñ1Ó3Ð?ä×)Ñ)×=Ñ=Ó?×DÑDˆà�Y‰Y�}Ó%×0Ñ0Ó2€FØ�Y‰Y�}Ó%×0Ñ0Ó2€Fàˆ~ÒØ—‘ ¨v×/GÑ/GÈ!Ñ/KÈRÓP‰à—‘ ¨r°6×3KÑ3KÈqÑ3PÓQˆä	�=Ó	!ñ 
Ü-:Ø×Ñ˜R Ó$ f×&6Ñ&6°r¸2Ó&>Ð@Ró.
Ñ*Ð˜l÷
ð
 ˆ~ÒÜ%*§Z¡Z°À×@RÑ@RÐTZ×TlÑTlÐopÑTpÐ0qÓ%rÐÕ"ä%*§Z¡Z°À×@XÑ@XÐZ`×ZlÑZlÐ0mÓ%nÐÔ"ð ˆv×!Ñ!Ñ!à×Ñ˜tÐ$ÜˆF�DÐ.Ô/ÜØØˆ&Ð!Ð"Ù \¹GÉZË\Ô<ZÔ[õ÷!
ð 
ús   Å	1H:È:IÚmodules_to_not_convertc                 ó$  — |j                   r| S ddlm}  |d«      ad}| j	                  «       D ]»  \  }}t        ||«      sŒ|j                  j                  dk(  rQ|j                   sEt        j                  d«      5  | j                  |t        | j                  «      «       d}ddd«       |j                  j                  d	k(  sŒ—|j                   rŒ¤d
dlm}  |t        |«      |_        Œ½ |st"        j%                  d«       | S # 1 sw Y   ŒcxY w)aD  
    Public method that replaces the expert layers of the given model with mxfp4 quantized layers.

    Args:
        model (`torch.nn.Module`):
            The model to convert, can be any `torch.nn.Module` instance.
        quantization_config (`Mxfp4Config`, defaults to `None`):
            The quantization config object that contains the quantization parameters.
        modules_to_not_convert (`list`, *optional*, defaults to `None`):
            A list of modules to not convert. If a module name is in the list (e.g. `lm_head`), it will not be
            converted.
    rp   )Ú
get_kernelz(kernels-community/gpt-oss-triton-kernelsFÚGptOssExpertsr@  TNÚ	GptOssMLPr   )Ú
MethodTypez×You are loading your model using mixed-precision FP4 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)r<  Úhub_kernelsrK  r<   Únamed_modulesr   rÏ   rO   r   r   Úset_submoduler9   rv   ÚtypesrN  r+  râ   ÚloggerÚwarning)r'   Úquantization_configrI  rK  Úhas_been_replacedÚmodule_namerI   rN  s           r   Úreplace_with_mxfp4_linearrX  Ÿ  sÿ   € ð ×%Ò%Øˆå'ñ $Ð$NÓOÐàÐØ$×2Ñ2Ó4ò 
=Ñˆ�VÜ$ [Ð2HÔIØØ×Ñ×$Ñ$¨Ò7Ð@S×@^Ò@^Ü—‘˜fÓ%ñ )Ø×#Ñ# KÔ1CÀEÇLÁLÓ1QÔRØ$(Ð!÷)ð ×Ñ×$Ñ$¨Ó3Ð<O×<ZÓ<ZÝ(á'¬°VÓ<ˆF�Nð
=ñ Ü�‰ðô	
ð €L÷)ð )ús   Á<(DÄD	)NN))Úutilsr   r   r   r   Ú
contextlibr   Úcore_model_loadingr   r	   Úquantizers.quantizers_utilsr
   r   Ú
get_loggerrO   rS  r¤   r   r   rV   rd   ri   r:   rA   rˆ   r�   r©   r   r»   r¿   rS   r9   r  r+  r<  r\   rG  rg   r8   r   rX  rT   r%   r   ú<module>r^     sP  ð÷ 0ñ ÔÛÝÝ %ç ;ß Uð 
ˆ×	Ñ	˜HÓ	%€ò€
ð( ñ
ó ð
ô*-�Mô -ô`�mô ôD-:�}ô -:ô`0˜mô 0òhòð0 Ÿ™Ø&ò3,ð �;‰;ð	3,ð
 ð3,ð ‡\�\ó3,ðt Ÿ™Ø&ògð �;‰;ð	gð
 ðgð ‡\�\ógô2>#˜Ÿ™ô >#òFAlòH%ò(-òB+ò
DòN.ñb)ÐW[Ð\_ÑW`ÐcgÑWgô )r%   