Ë
    Gêñi/c  ã                   ó  — d dl mZ d dlmZ ddlmZ ddlmZ ddlm	Z	m
Z
mZmZ ddlmZ  e	«       rd d	lZ ej                   e«      Z	 	 d/dej&                  dej&                  dej&                  d	z  dedej&                  f
d„Zdej,                  j.                  dej&                  dej&                  dej&                  dej&                  f
d„Zdej&                  dej&                  dej&                  dej&                  fd„Zdej&                  dej&                  dej&                  dej&                  fd„Zd„ Zd„ Z e	«       rXej:                  j=                  ded¬«       ej:                  j?                  de«       ej:                  jA                  dee¬«       dej&                  dej&                  dej&                  defd„Z!dej&                  dej&                  dej&                  dej&                  fd „Z"	 	 d/dej&                  dej&                  dej&                  dej&                  d	z  dedej&                  fd!„Z#dej,                  j.                  dej&                  dej&                  dej&                  dej&                  f
d"„Z$ G d#„ d$e«      Z% e%«       Z&d%ej&                  dej&                  fd&„Z'	 d0e&d'd
d
d'd(œd)e(ej,                  j.                     d	z  d*e%d+eded,ed-ede(ej,                  j.                     fd.„Z)y	)1é    )ÚCallable)Úwrapsé   )Úlogging)ÚGeneralInterface)Úis_torch_availableÚis_torch_greater_or_equalÚis_torch_less_or_equalÚis_torchdynamo_compilingé   )Úsonicmoe_experts_forwardNFÚinputÚweightÚbiasÚis_transposedÚreturnc                 óê   — |r5t        j                  | j                  d«      |«      j                  d«      }n4t        j                  || j                  d«      «      j                  d«      }|�||z   }|S )a¶  Batched linear layer supporting optional bias and transposed weights.

    Args:
        input (`torch.Tensor`):
            Input tensor of shape (batch_size, input_dim).
        weight (`torch.Tensor`):
            Weight tensor of shape (batch_size, output_dim, input_dim) if transposed is `False`,
            else of shape (batch_size, input_dim, output_dim).
        bias (`torch.Tensor`, *optional*):
            Bias tensor of shape (batch_size, output_dim). Default is `None`.
        is_transposed (`bool`, *optional*, defaults to `False`):
            Whether the weight tensor is transposed.
    Returns:
        `torch.Tensor`: Output tensor of shape (batch_size, output_dim).
    r   éÿÿÿÿ)ÚtorchÚbmmÚ	unsqueezeÚsqueeze)r   r   r   r   Úouts        ú_/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/integrations/moe.pyÚ_batched_linearr   L   se   € ñ* ä�i‰i˜Ÿ™¨Ó*¨FÓ3×;Ñ;¸AÓ>‰ô �i‰i˜ §¡°Ó 3Ó4×<Ñ<¸RÓ@ˆàÐØ�D‰jˆà€Jó    ÚselfÚhidden_statesÚtop_k_indexÚtop_k_weightsc                 ó~  — |j                   }|j                  d«      }|j                  d«      }|j                  d«      }t        j                  ||¬«      j	                  d«      j                  d|«      j                  d«      }|j                  d«      }	|j                  d«      }
|
| j                  k\  }|
j                  d| j                  dz
  «      }
||   }| j                  r-| j                  |
   }| j                  r| j                  |
   nd }n,| j                  |
   }| j                  r| j                  |
   nd }t        |||| j                   ¬«      }| j                  r| j#                  |«      }n| j%                  |«      }| j&                  |
   }| j                  r| j(                  |
   nd }t        |||| j                   ¬«      }||	j	                  d«      z  }|j+                  |j	                  d«      d«       |j-                  |||«      j/                  d¬«      }|j1                  |j2                  «      S )Nr   r   ©Údevicer   ©r   r   ç        ©Údim)r#   Úsizer   Úaranger   ÚexpandÚreshapeÚnum_expertsÚclampÚhas_gateÚgate_up_projÚhas_biasÚgate_up_proj_biasÚup_projÚup_proj_biasr   r   Ú_apply_gateÚact_fnÚ	down_projÚdown_proj_biasÚmasked_fill_ÚviewÚsumÚtoÚdtype)r   r   r   r    r#   Ú	num_top_kÚ
num_tokensÚ
hidden_dimÚ	token_idxÚsample_weightsÚ
expert_idsÚinvalid_maskÚselected_hidden_statesÚselected_weightsÚselected_biasesÚproj_outÚweighted_outÚfinal_hidden_statess                     r   Úbatched_mm_experts_forwardrJ   n   s&  € ð ×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jô —‘˜Z°Ô7×AÑAÀ!ÓD×KÑKÈBÐPYÓZ×bÑbÐceÓf€IØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð  ×!1Ñ!1Ñ1€LØ×!Ñ! ! T×%5Ñ%5¸Ñ%9Ó:€Jð +¨9Ñ5Ðð ‡}‚}Ø×,Ñ,¨ZÑ8ÐØ@DÇÂ˜$×0Ñ0°Ò<ÐSW‰àŸ<™<¨
Ñ3ÐØ;?¿=º=˜$×+Ñ+¨JÒ7Èdˆô ØÐ 0°ÐVZ×VhÑVhô€Hð
 ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð —~‘~ jÑ1ÐØ9=¿º�d×)Ñ)¨*Ò5ÈD€Oô ØÐ"¨È×HZÑHZô€Hð
 ˜n×6Ñ6°rÓ:Ñ:€LØ×Ñ˜l×4Ñ4°RÓ8¸#Ô>ð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   Úoffsc                 ó4  — t        j                  | j                  d«      |j                  d«      | j                  | j                  ¬«      }d}t        |j                  «       «      D ].  \  }}||k(  rŒt        j                  | || ||   ||| ¬«       |}Œ0 |S )a(  
    Fallback grouped matrix multiplication used when `torch.nn.functional.grouped_mm` and `torch._grouped_mm`
    are unavailable or incompatible with `torch.compile` (e.g. non-bfloat16 weights).

    Args:
        input (`torch.Tensor`): Input of shape (S, input_dim), sorted by expert id.
        weight (`torch.Tensor`): Expert weights of shape (num_experts, input_dim, output_dim).
        offs (`torch.Tensor`): Cumulative token counts per expert of shape (num_experts,).
    Returns:
        `torch.Tensor`: Output of shape (S, output_dim).
    r   r   ©r#   r<   ©r   )r   Úzerosr(   r#   r<   Ú	enumerateÚtolistÚmm)r   r   rK   ÚoutputÚstartÚiÚends          r   Ú_grouped_mm_fallbackrW   µ   s�   € ô �[‰[˜Ÿ™ A›¨¯©°A«¸u¿|¹|ÐSX×S^ÑS^Ô_€Fà€Eô ˜DŸK™K›MÓ*ò ‰ˆˆ3Ø�CŠ<ØÜ�‰��u˜SÐ! 6¨!¡9°&¸¸sÐ2CÕDØ‰ð	ð €Mr   c                 óp  — | j                  «       dk(  sJ dt        | j                  «      › �«       ‚|j                  «       dk(  sJ dt        |j                  «      › �«       ‚|j                  «       dk(  sJ dt        |j                  «      › �«       ‚|j                  d«      |j                  d«      k(  s+J d|j                  d«      › d	|j                  d«      › �«       ‚| j                  d«      |j                  d«      k(  s+J d
| j                  d«      › d|j                  d«      › �«       ‚|j                  t
        j                  t
        j                  fv sJ d|j                  › �«       ‚t        j                  | j                  d«      |j                  d«      | j                  | j                  ¬«      S )zRShape/dtype inference stub for `_grouped_mm_fallback` required by `torch.compile`.r   z+input must be 2D (S, input_dim), got shape é   zBweight must be 3D (num_experts, input_dim, output_dim), got shape r   z*offs must be 1D (num_experts,), got shape r   zoffs length z must match number of experts zinput_dim mismatch: input has z, weight has z$offs must be an integer tensor, got rM   )
r'   ÚtupleÚshaper(   r<   r   Úint32Úint64Úemptyr#   ©r   r   rK   s      r   Ú_grouped_mm_fallback_faker`   Ï   s�  € à�9‰9‹;˜!ÒÐ_ÐJÌ5ÐQV×Q\ÑQ\ÓK]ÐJ^Ð_Ó_ÐØ�:‰:‹<˜1Òð Ø
LÌUÐSY×S_ÑS_ÓM`ÐLaÐbóÐð �8‰8‹:˜Š?Ð\ÐHÌÈtÏzÉzÓIZÐH[Ð\Ó\ˆ?Ø�9‰9�Q‹<˜6Ÿ;™; q›>Ò)Ðv¨\¸$¿)¹)ÀA»,¸ÐGeÐfl×fqÑfqÐrsÓftÐeuÐ+vÓvÐ)Ø�:‰:�a‹=˜FŸK™K¨›NÒ*ð Ø
(¨¯©°A«¨°}ÀVÇ[Á[ÐQRÃ^ÐDTÐUóÐ*ð �:‰:œ%Ÿ+™+¤u§{¡{Ð3Ñ3ÐhÐ7[Ð\`×\fÑ\fÐ[gÐ5hÓhÐ3Ü�;‰;�u—z‘z !“} f§k¡k°!£n¸U¿\¹\ÐQV×Q\ÑQ\Ô]Ð]r   c                 óH   — | j                  |d   |d   «       |d   | _        y)zjSaves input and weight for backward; offs is stored directly as it is a non-differentiable integer tensor.r   r   r   N)Úsave_for_backwardrK   )ÚctxÚinputsrS   s      r   Ú"_grouped_mm_fallback_setup_contextre   Þ   s%   € à×Ñ˜& ™) V¨A¡YÔ/Ø�a‰y€C…Hr   c                 ó”  — | j                   \  }}t        j                  |«      }t        j                  |«      }d}t        | j                  j                  «       «      D ]c  \  }}||k(  rŒt        j                  ||| ||   j                  ||| ¬«       t        j                  ||| j                  ||| ||   ¬«       |}Œe ||dfS )zuBackward pass for `_grouped_mm_fallback`. Computes grad_input and grad_weight per expert group; offs has no gradient.r   rN   N)Úsaved_tensorsr   Ú
zeros_likerP   rK   rQ   rR   ÚT)	rc   Úgrad_outputr   r   Ú
grad_inputÚgrad_weightrT   rU   rV   s	            r   Ú_grouped_mm_fallback_backwardrm   ä   sÇ   € à×%Ñ%�M€Eˆ6Ü×!Ñ! %Ó(€JÜ×"Ñ" 6Ó*€Kà€Eô ˜CŸH™HŸO™OÓ-Ó.ò ‰ˆˆ3Ø�CŠ<ØÜ�‰�˜U 3Ð'¨°©¯©¸*ÀUÈ3Ð:OÕPÜ�‰��u˜SÐ!×#Ñ# [°°sÐ%;ÀÈQÁÕPØ‰ðð �{ DÐ(Ð(r   z!transformers::grouped_mm_fallback© )Úmutates_args)Úsetup_contextc                 ó   — t        «       r|j                  t        j                  k7  sR|j                  j
                  dk(  r:t        dd¬«      r-|j                  «       dz  dk7  s| j                  «       dz  dk7  ry|j                  j
                  dk(  rÆt        t        j                  j                  d	«      r,t        j                  j                  |j                  «      d
k\  S t        t        d«      ret        dd¬«      r,t        j                  j                  |j                  «      d
k\  S t        j                  j                  |j                  «      dk\  S yt        t        j                  j                  d	«      xs t        t        d«      S )a  
    Check if torch.nn.functional.grouped_mm or torch._grouped_mm can be used based on availability and compatibility with torch.compile.

    Args:
        input (`torch.Tensor`):
            Input tensor of shape (S, input_dim).
        weight (`torch.Tensor`):
            Weight tensor of shape (num_experts, input_dim, output_dim).
        offs (`torch.Tensor`):
            Offsets tensor indicating the boundaries of each group in the input tensor.
    Returns:
        `bool`: True if grouped_mm can be used, False otherwise.
    Úcpuz2.10.0T)Ú
accept_devé   r   FÚcudaÚ
grouped_mm)é   r   Ú_grouped_mmz2.9)é	   r   )r   r<   r   Úbfloat16r#   Útyper
   Údata_ptrÚhasattrÚnnÚ
functionalru   Úget_device_capabilityr	   r_   s      r   Ú_can_use_grouped_mmr�     s  € ô 	!Ô" v§|¡|´u·~±~Ò'EØ�‰×Ñ˜eÒ#ä" 8¸Õ=Ø�_‰_Ó Ñ# qÒ(¨E¯N©NÓ,<¸rÑ,AÀQÒ,Fð ð
 ‡}�}×Ñ˜VÒ#Ü”5—8‘8×&Ñ&¨Ô5Ü—:‘:×3Ñ3°F·M±MÓBÀfÑLÐLÜ”5˜-Ô(Ü(¨¸4Õ@Ü—z‘z×7Ñ7¸¿¹ÓFÈ&ÑPÐPä—z‘z×7Ñ7¸¿¹ÓFÈ&ÑPÐPàä”5—8‘8×&Ñ&¨Ó5ÒV¼ÄÈÓ9VÐVr   c                 óÈ  — t        | ||«      rªt        t        j                  j                  d«      rEt        j                  j                  j                  | j                  |j                  «      ||¬«      S t        t        d«      r1t        j                  | j                  |j                  «      ||¬«      S t        j                  j                  j                  | ||¬«      S )a  Grouped matrix multiplication dispatcher that uses torch.nn.functional.grouped_mm if available, else falls back to torch._grouped_mm.

    Args:
        input (`torch.Tensor`):
            Input tensor of shape (S, input_dim).
        weight (`torch.Tensor`):
            Weight tensor of shape (num_experts, input_dim, output_dim).
        offs (`torch.Tensor`):
            Offsets tensor indicating the boundaries of each group in the input tensor.
    Returns:
        `torch.Tensor`: Output tensor of shape (S, output_dim).
    rv   ©rK   rx   )r�   r}   r   r~   r   rv   r;   r<   rx   ÚopsÚtransformersÚgrouped_mm_fallbackr_   s      r   rx   rx   .  s¤   € ô$ ˜5 &¨$Ô/ô
 ”5—8‘8×&Ñ&¨Ô5Ü—8‘8×&Ñ&×1Ñ1°%·(±(¸6¿<¹<Ó2HÈ&ÐW[Ð1Ó\Ð\Ü”U˜MÔ*Ü×$Ñ$ U§X¡X¨f¯l©lÓ%;¸VÈ$ÔOÐOä�9‰9×!Ñ!×5Ñ5°e¸VÈ$Ð5ÓOÐOr   c                 ór   — |rt        | ||¬«      }nt        | |j                  dd«      |¬«      }|�||z   }|S )a  Grouped linear layer supporting optional bias and transposed weights.

    Args:
        input (`torch.Tensor`):
            Input tensor of shape (S, input_dim).
        weight (`torch.Tensor`):
            Weight tensor of shape (num_experts, input_dim, output_dim) if `is_transposed`,
            else of shape (num_experts, output_dim, input_dim).
        offs (`torch.Tensor`):
            Offsets tensor indicating the boundaries of each group in the input tensor.
        bias (`torch.Tensor`, *optional*):
            Bias tensor of shape (num_experts, output_dim). Default is `None`.
        is_transposed (`bool`, *optional*, defaults to `False`):
            Whether the weight tensor is transposed.
    Returns:
        `torch.Tensor`: Output tensor of shape (S, output_dim).
    rƒ   éþÿÿÿr   )rx   Ú	transpose)r   r   rK   r   r   r   s         r   Ú_grouped_linearrŠ   M  sF   € ñ0 ä˜% ¨dÔ3‰ô ˜% ×!1Ñ!1°"°bÓ!9ÀÔEˆàÐà�D‰jˆà€Jr   c                 óN  — |j                   }|j                  d«      }|j                  d«      }|j                  d«      }t        j                  ||¬«      j	                  d«      j                  d|«      j                  d«      }|j                  d«      }	|j                  d«      }
|
| j                  k\  }|
j                  d| j                  dz
  «      }
t        j                  |
«      }t        j                  |«      }t        j                  |j                  d«      |¬«      ||<   |
|   }|	|   }|||      }|j                  dk(  r|j                  «       n|j                  «       }t        j                  || j                  d| j                  dz
  ¬«      }t        j                  |dt        j                   ¬«      }| j"                  r*| j$                  }| j&                  r| j(                  |   nd }n)| j*                  }| j&                  r| j,                  |   nd }t/        ||||| j0                  ¬«      }| j"                  r| j3                  |«      }n| j5                  |«      }| j6                  }| j&                  r| j8                  |   nd }t/        ||||| j0                  ¬«      }||j	                  d«      z  }||   }|j;                  |j	                  d«      d	«       ||   }|j=                  |||«      j?                  d¬
«      }|jA                  |jB                  «      S )Nr   r   r"   r   rr   )ÚbinsÚminÚmax)r'   r<   r$   r%   r&   )"r#   r(   r   r)   r   r*   r+   r,   r-   ÚargsortÚ
empty_liker{   ÚfloatÚintÚhistcÚcumsumr\   r.   r/   r0   r1   r2   r3   rŠ   r   r4   r5   r6   r7   r8   r9   r:   r;   r<   )r   r   r   r    r#   r=   r>   r?   r@   rA   rB   rC   ÚpermÚinv_permÚexpert_ids_gÚsample_weights_gÚselected_hidden_states_gÚhistc_inputÚtokens_per_expertÚoffsetsrE   rF   rG   rH   Úinvalid_mask_grI   s                             r   Úgrouped_mm_experts_forwardrž   s  sì  € ð ×!Ñ!€FØ× Ñ  Ó$€IØ×#Ñ# AÓ&€JØ×#Ñ# BÓ'€Jô —‘˜Z°Ô7×AÑAÀ!ÓD×KÑKÈBÐPYÓZ×bÑbÐceÓf€IØ"×*Ñ*¨2Ó.€NØ×$Ñ$ RÓ(€Jð  ×!1Ñ!1Ñ1€LØ×!Ñ! ! T×%5Ñ%5¸Ñ%9Ó:€Jô �=‰=˜Ó$€DÜ×Ñ Ó%€HÜ—\‘\ $§)¡)¨A£,°vÔ>€HˆT�Nà˜dÑ#€LØ% dÑ+ÐØ,¨Y°t©_Ñ=Ðð
 +1¯+©+¸Ò*>�,×$Ñ$Ô&ÀL×DTÑDTÓDV€KÜŸ™ K°d×6FÑ6FÈAÐSW×ScÑScÐfgÑSgÔhÐÜ�l‰lÐ,°!¼5¿;¹;ÔG€Gð ‡}‚}Ø×,Ñ,ÐØBFÇ-Â-˜$×0Ñ0°Ò>ÐUY‰àŸ<™<ÐØ=A¿]º]˜$×+Ñ+¨LÒ9ÐPTˆô Ø Ð"2°GÀ/Ðae×asÑasô€Hð
 ‡}‚}à×#Ñ# HÓ-‰ð —;‘;˜xÓ(ˆð —~‘~ÐØ;?¿=º=�d×)Ñ)¨,Ò7Èd€Oô ØÐ" G°/ÐQU×QcÑQcô€Hð
 Ð.×8Ñ8¸Ó<Ñ<€LØ! $Ñ'€NØ×Ñ˜n×6Ñ6°rÓ:¸CÔ@ð   Ñ)€Lð '×+Ñ+¨J¸	À:ÓN×RÑRÐWXÐRÓYÐà×!Ñ! -×"5Ñ"5Ó6Ð6r   c                   ó<   ‡ — e Zd ZdZeeedœZdede	de	fˆ fd„Z
ˆ xZS )ÚExpertsInterfacez;Interface for registering custom experts forward functions.)ÚsonicmoeÚ
batched_mmrv   Úexperts_implementationÚdefaultr   c                 ó„   •— |€t         j                  d«       n|dk7  r|| vrt        d|› d�«      ‚t        ‰| �  ||«      S )zfReturn the requested `experts_implementation`. Also strictly check its validity, and raise if invalid.a
  You tried to access the `ExpertsInterface` with a `config._experts_implementation` set to `None`. This is expected if you use an Expert Module as a standalone Module. If this is not the case, something went wrong with the dispatch of `config._experts_implementation`Úeagerú`zL` is not a valid experts implementation registered in the `ExpertsInterface`)ÚloggerÚwarning_onceÚKeyErrorÚsuperÚget)r   r£   r¤   Ú	__class__s      €r   Úget_interfacezExpertsInterface.get_interfaceÔ  s`   ø€ à!Ð)Ü×ÑðNõð
 $ wÒ.Ð3IÐQUÑ3UÜØÐ*Ð+Ð+wÐxóð ô ‰w‰{Ð1°7Ó;Ð;r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   rJ   rž   Ú_global_mappingÚstrr   r®   Ú__classcell__)r­   s   @r   r    r    Ë  s7   ø„ ÙEð -Ø0Ø0ñ€Oð<°Cð <À(ð <Èx÷ <ñ <r   r    Úgate_up_outc                 óV   — |j                  dd¬«      \  }}| j                  |«      |z  S )a›  
    Default gating mechanism: splits the gate_up_out into gate and up parts,
    applies the activation function to the gate part, and multiplies it with the up part.
    Args:
        gate_up_out (`torch.Tensor`):
            The output tensor from the gate and up projection of shape (S, 2 * intermediate_dim).
    Returns:
        `torch.Tensor`: The gated output tensor of shape (S, intermediate_dim).
    r   r   r&   )Úchunkr5   )r   r¶   ÚgateÚups       r   Ú_default_apply_gater»   æ  s1   € ð × Ñ  ¨Ð Ó+�H€Dˆ"Ø�;‰;�tÓ˜rÑ!Ð!r   T)Úexperts_interfaceÚis_concatenatedr   r0   r.   Úexperts_classr¼   r½   r0   r.   c                ó¼   ‡‡‡‡‡— dt         t        j                  j                     dt         t        j                  j                     fˆˆˆˆˆfd„}| � || «      S |S )a¤  Decorator to modify experts class to support different experts implementations.

    Args:
        experts_class (`type[torch.nn.Module]`, *optional*):
            The experts class to modify. If not provided, returns a decorator that can be applied to the class.
        experts_interface (`ExpertsInterface`, *optional*, defaults to `ALL_EXPERTS_FUNCTIONS`):
            The experts interface to use for dispatching the forward method.
        is_concatenated (`bool`, *optional*, defaults to `True`):
            Whether the expert weights are stored in concatenated layout [gate;up]
            or interleaved layout [gate0, up0, gate1, up1, ...].
        is_transposed (`bool`, *optional*, defaults to `False`):
            Whether the expert weights are stored in transposed format.
        has_bias (`bool`, *optional*, defaults to `False`):
            Whether the expert layers include bias terms or not.
        has_gate (`bool`, *optional*, defaults to `True`):
            Whether the experts use a gating mechanism or not.
            Whether it has gate_up_proj weights or just up_proj weights.

    Returns:
        `type[torch.nn.Module]`: The modified experts class.
    r¾   r   c                 óÜ   •‡‡— | j                   Š| j                  Št        ‰«      ˆˆˆˆ	ˆfd„«       }t        ‰«      ˆˆfd„«       }t        | d«      st        | _        || _         || _        | S )Nc                 óf   •—  ‰| |g|¢­i |¤Ž || _         ‰| _        ‰| _        ‰| _        ‰| _        y ©N)Úconfigr.   r0   r   r½   )	r   rÃ   ÚargsÚkwargsr0   r.   r½   r   Úoriginal_inits	       €€€€€r   Ú__init__z=use_experts_implementation.<locals>.wrapper.<locals>.__init__  s<   ø€ á˜$ Ð8¨Ò8°Ò8Ø ˆDŒKØ$ˆDŒMØ$ˆDŒMØ!.ˆDÔØ#2ˆDÕ r   c                 óh   •— ‰j                  | j                  j                  ‰«      } || g|¢­i |¤ŽS rÂ   )r®   rÃ   Ú_experts_implementation)r   rÄ   rÅ   Úexperts_forwardr¼   Úoriginal_forwards       €€r   Úforwardz<use_experts_implementation.<locals>.wrapper.<locals>.forward   s5   ø€ à/×=Ñ=¸d¿k¹k×>aÑ>aÐcsÓtˆOÙ" 4Ð9¨$Ò9°&Ñ9Ð9r   r4   )rÇ   rÌ   r   r}   r»   r4   )
r¾   rÇ   rÌ   rË   rÆ   r¼   r0   r.   r½   r   s
      @@€€€€€r   Úwrapperz+use_experts_implementation.<locals>.wrapper  su   ú€ Ø%×.Ñ.ˆØ(×0Ñ0Ðä	ˆ}Ó	÷	3ó 
ð	3ô 
ÐÓ	 ô	:ó 
!ð	:ô �} mÔ4Ü(;ˆMÔ%à!)ˆÔØ 'ˆÔØÐr   )r{   r   r~   ÚModule)r¾   r¼   r½   r   r0   r.   rÍ   s    ````` r   Úuse_experts_implementationrÏ   ô  sM   ü€ ð>œt¤E§H¡H§O¡OÑ4ð ¼¼e¿h¹h¿o¹oÑ9N÷ ñ ð2 Ð Ù�}Ó%Ð%à€Nr   )NFrÂ   )*Úcollections.abcr   Ú	functoolsr   Úutilsr   Úutils.genericr   Úutils.import_utilsr   r	   r
   r   r¡   r   r   Ú
get_loggerr¯   r¨   ÚTensorÚboolr   r~   rÎ   rJ   rW   r`   re   rm   ÚlibraryÚ	custom_opÚregister_fakeÚregister_autogradr�   rx   rŠ   rž   r    ÚALL_EXPERTS_FUNCTIONSr»   r{   rÏ   rn   r   r   ú<module>rÝ      s‘  ðõ %Ý å Ý ,÷ó õ /ñ ÔÛð 
ˆ×	Ñ	˜HÓ	%€ð\ !%Øñ	Ø�<‰<ðà�L‰Lðð �,‰,˜Ñ
ðð ð	ð
 ‡\�\óðDA7Ø
�(‰(�/‰/ðA7à—<‘<ðA7ð —‘ðA7ð —<‘<ð	A7ð
 ‡\�\óA7ðN §¡ð °e·l±lð È%Ï,É,ð Ð[`×[gÑ[gó ð4^ U§\¡\ð ^¸5¿<¹<ð ^ÈuÏ|É|ð ^Ð`e×`lÑ`ló ^òò)ñ& ÔØ	‡M�M×ÑÐ?ÐAUÐdfÐÔgØ	‡M�M×ÑÐ CÐE^Ô_Ø	‡M�M×#Ñ#Ø+Ø%Ø8ð $ô ð*W˜uŸ|™|ð *W°U·\±\ð *WÈÏÉð *WÐZ^ó *WðZPØ�<‰<ðPà�L‰LðPð �,‰,ðPð ‡\�\ó	PðF !%Øñ#Ø�<‰<ð#à�L‰Lð#ð �,‰,ð#ð �,‰,˜Ñ
ð	#ð
 ð#ð ‡\�\ó#ðLU7Ø
�(‰(�/‰/ðU7à—<‘<ðU7ð —‘ðU7ð —<‘<ð	U7ð
 ‡\�\óU7ôp<Ð'ô <ñ0 )Ó*Ð ð"¨5¯<©<ð "¸E¿L¹Ló "ð 37ð;ð +@Ø ØØØò;Ø˜Ÿ™Ÿ™Ñ(¨4Ñ/ð;ð (ð;ð ð	;ð
 ð;ð ð;ð ð;ð 
ˆ%�(‰(�/‰/Ñô;r   