Ë
    Gêñiž]  ã                  ó   — U d Z ddlmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZmZ ddlZddlmZ ddlmZ d	d
lmZ d	dlmZ d	dlmZ d	dlmZmZ d	dlmZmZmZmZmZm Z m!Z!m"Z"m#Z# erd	dl$m%Z% d	dlm&Z&  ejN                  e(«      Z)e	 G d„ d«      «       Z*d)d„Z+ee*ge,ee-e.ef   f   f   Z/d*d„Z0	 d+	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,d„Z1d-d„Z2d-d„Z3d-d„Z4d-d„Z5d-d„Z6d-d„Z7d-d„Z8d-d„Z9d-d„Z:d-d„Z;d-d „Z<d-d!„Z=d-d"„Z>d-d#„Z?d-d$„Z@d-d%„ZAd-d&„ZBej†                  ejˆ                  ejŠ                  ejŒ                  ejŽ                  ej�                  ej’                  ej”                  ej–                  ej˜                  ejš                  ejœ                  ejž                  ej                   ej¢                  gZRej¦                  ej¨                  ejª                  ej¬                  ej®                  ej°                  gZYej´                  ej¶                  gZ\ejº                  ej¼                  gZ_ejÀ                  ejÂ                  ejÄ                  gZcejÈ                  e2ejÊ                  e3ejÌ                  e3ejÎ                  e4ejÐ                  e5ejÒ                  e6ejÔ                  e8ejÖ                  e9ejØ                  e:ejÚ                  e;ejÜ                  e?ejÞ                  eBejà                  e>ejâ                  e>ie-jå                  eRe7«      ¥e-jå                  eYe<«      ¥e-jå                  e\e=«      ¥e-jå                  e_e@«      ¥e-jå                  eceA«      ¥Zsd'etd(<   y).z,
Optimizer utilities for the Trainer class.
é    )ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únné   )Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óD   — e Zd ZU dZded<   ded<   ded<   ded<   d	ed
<   y)ÚOptimizerContextz0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_argsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    ú`/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/trainer_optimizer.pyr   r   6   s#   … á:à
ÓØ!Ó!Ø$Ó$ØÓØÔr*   r   c                ó�   — | si S i }| j                  dd«      j                  d«      D ]  }|j                  d«      \  }}|||<   Œ |S )z8Parse optimizer arguments from a comma-separated string.ú Ú ú,ú=)ÚreplaceÚsplit)Úoptim_args_strr#   ÚmappingÚkeyÚvalues        r+   Ú_parse_optim_argsr7   A   sY   € áØˆ	Ø€JØ!×)Ñ)¨#¨rÓ2×8Ñ8¸Ó=ò  ˆØ—]‘] 3Ó'‰
ˆˆUØˆ
�3Šð ð Ðr*   c                ón   — t        | t        «      r%t        | t        j                  j
                  «      ryy)a4  
    Check if the returned value from a handler is a factory rather than an Optimizer class.

    Factory callables are used for complex optimizers like Muon or Dion that need to:
    - Split parameters between multiple internal optimizers
    - Handle complex sharding logic
    - Access the full model structure for parameter grouping

    Args:
        optimizer_cls_or_factory: The first element returned by an optimizer handler.

    Returns:
        `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
    FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)Úoptimizer_cls_or_factorys    r+   Úis_optimizer_factoryr@   P   s,   € ô  Ð*¬DÔ1´jÐAYÔ[`×[fÑ[f×[pÑ[pÔ6qØØr*   c                óŠ  ‡— |j                  «       j                  d«      }|r.| j                  t        j                  k(  r|rt        d|› d�«      ‚||   }| j                  €t        d|› d�«      ‚t        | j                  t        t        f«      st        d| j                  › �«      ‚|€t        d|› d�«      ‚t        | j                  t        «      xr | j                  j                  d	d
«      dk(  }	g }
|j                  «       D ]s  \  }}t        | j                  |d¬«      \  }}t        |t        j                   «      s |r|st"        j%                  |› d|› d�«       Œ[|s|	sŒ`|
j'                  |dz   «       Œu t)        |
«      dk(  rt        d|› d| j                  › d�«      ‚|j+                  «       D ��cg c]  \  }}||
v sŒ|‘Œ }}}|j+                  «       D ��cg c]  \  }}||
vsŒ|‘Œ }}}d|id|i|¥g}|r¡| j,                  dk7  rt        d|› d�«      ‚i Š|D ]  } |d|gigfi |¤Ž‰|<   Œ |D ]  } |d|gi|¥gfi |¤Ž‰|<   Œ ˆfd„}|j/                  «       D ]   }|j0                  sŒ|j3                  |«       Œ" t4        }|j7                  d‰i«       |j7                  d|i«       ||fS c c}}w c c}}w )z¶
    Helper function to set up low-rank optimizers like GaLore and Apollo.

    These optimizers apply low-rank projections to specific target modules (typically linear layers).
    Ú	layerwisezLayer-wise z" does not support DDP at this timez1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightr   zNo target modules found for z (z).Úparamsr
   z
Layerwise z( does not support gradient accumulation!c                ól   •— | j                   �'‰|    j                  «        ‰|    j                  «        y y )N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts    €r+   Úoptimizer_hookz1_setup_low_rank_optimizer.<locals>.optimizer_hook®   s4   ø€ Ø�z‰zÐ%Ø˜uÑ%×*Ñ*Ô,Ø˜uÑ%×/Ñ/Õ1ð &r*   rL   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorr9   ÚlistÚstrÚ	TypeErrorr1   Únamed_modulesr   r	   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr   Úupdate)r   r   Úoptimizer_nameÚoptimizer_mappingÚoptim_kwargsr    Úis_layerwise_supportedÚis_layerwiseÚoptimizer_clsÚ
all_linearÚtarget_params_namesÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚnÚpÚtarget_paramsÚnon_target_paramsÚparam_groupsrK   rM   rL   s                         @r+   Ú_setup_low_rank_optimizerru   e   sI  ø€ ð "×'Ñ'Ó)×2Ñ2°;Ó?€LÙ˜×*Ñ*¬l×.FÑ.FÒFÑKaÜ! K°Ð/?Ð?aÐ"bÓcÐcà% nÑ5€Mà× Ñ Ð(ÜÐLÈ^ÐL\Ð\gÐhÓiÐiä�d×/Ñ/´$¼°Ô=ÜðØ×-Ñ-Ð.ð0ó
ð 	
ð
 €}ÜÐBÀ>ÐBRÐR]Ð^Ó_Ð_ô 	�4×,Ñ,¬cÓ2Òr°t×7PÑ7P×7XÑ7XÐY\Ð^aÓ7bÐfrÑ7rð ð ÐØ$×2Ñ2Ó4ò <Ñˆ�VÜ)CØ×%Ñ% {ÀDô*
Ñ&Ð˜hô ˜&¤"§)¡)Ô,Ù#©HÜ—‘ + Ð.DÀ^ÐDTÐTqÐrÔsØá#©JØà×"Ñ" ;°Ñ#:Õ;ð<ô ÐÓ 1Ò$ÜÐ7¸Ð7GÀrÈ$×JcÑJcÐIdÐdfÐgÓhÐhà#(×#9Ñ#9Ó#;×X™4˜1˜a¸qÐDWÒ?W’QÐX€MÑXØ',×'=Ñ'=Ó'?×`™t˜q !À1ÐL_ÒC_šÐ`ÐÑ`ð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€Lñ
 Ø×+Ñ+¨qÒ0Ü˜z¨.Ð)9Ð9aÐbÓcÐcàˆØ&ò 	]ˆEÙ$1°H¸u¸gÐ3FÐ2GÑ$\ÐK[Ñ$\ˆN˜5Ò!ð	]à"ò 	mˆEÙ$1°H¸u¸gÐ3VÈÐ3VÐ2WÑ$lÐ[kÑ$lˆN˜5Ò!ð	mô	2ð
 ×%Ñ%Ó'ò 	IˆEØ×"Ó"Ø×8Ñ8¸ÕHð	Iô 0ˆØ×ÑÐ!1°>Ð BÔCà×Ñ˜X |Ð4Ô5ØÐ*Ð*Ð*ùó? YùÛ`s   Æ:J9ÇJ9Ç!J?Ç.J?c                ób   — | j                   j                  dddœ«       t        | j                   fS )zGet Adafactor optimizer.F©Úscale_parameterÚrelative_step)r    rc   r   )Úctxs    r+   Ú_get_adafactorr{   Ã   s,   € à×Ñ×Ñ°EÈEÑ RÔSÜ�c×*Ñ*Ð*Ð*r*   c                óü   — ddl m} | j                  j                  | j                  «       | j
                  j                  t        j                  k(  r| j                  j                  ddi«       || j                  fS )z/Get PyTorch AdamW optimizer (regular or fused).r   ©ÚAdamWÚfusedT)	Útorch.optimr~   r    rc   r!   r   r=   r   ÚADAMW_TORCH_FUSED©rz   r~   s     r+   Ú_get_adamw_torchrƒ   É   s\   € å!à×Ñ×Ñ §¡Ô0Ø
‡x�x‡~�~œ×9Ñ9Ò9Ø×Ñ×#Ñ# W¨d OÔ4Ø�#×&Ñ&Ð&Ð&r*   c                ó¦   — 	 ddl m} | j                  j                  | j                  «       || j                  fS # t
        $ r t        d«      ‚w xY w)z'Get Torch XLA syncfree AdamW optimizer.r   r}   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreer~   r    rc   r!   ÚImportErrorrT   r‚   s     r+   Ú_get_adamw_torch_xlar‡   Ó   sQ   € ðTÝ0à×Ñ×#Ñ# C§O¡OÔ4Ø�c×*Ñ*Ð*Ð*øÜò TÜÐRÓSÐSðTúó	   ‚8; »Ac                ó¦   — 	 ddl m} | j                  j                  | j                  «       || j                  fS # t
        $ r t        d«      ‚w xY w)zGet NPU Fused AdamW optimizer.r   )ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimrŠ   r    rc   r!   r†   rT   )rz   rŠ   s     r+   Ú_get_adamw_torch_npu_fusedrŒ   Þ   sQ   € ðPÝ1à×Ñ×#Ñ# C§O¡OÔ4Ø˜c×2Ñ2Ð2Ð2øÜò PÜÐNÓOÐOðPúrˆ   c                ó¦   — 	 ddl m} | j                  j                  | j                  «       || j                  fS # t
        $ r t        d«      ‚w xY w)zGet Apex Fused Adam optimizer.r   )Ú	FusedAdamzFTrainer tried to instantiate apex FusedAdam but apex is not installed!)Úapex.optimizersrŽ   r    rc   r!   r†   rT   )rz   rŽ   s     r+   Ú_get_adamw_apex_fusedr�   é   sQ   € ðcÝ-à×Ñ×#Ñ# C§O¡OÔ4Ø˜#×.Ñ.Ð.Ð.øÜò cÜÐaÓbÐbðcúrˆ   c                óŒ  — t        «       st        d«      ‚ddlm}m}m} | j                  j                  }d|v }d|v rdnd}d}| j                  }d	|v r|}�n�d
|v r2|}d| j                  j                  | j                  j                  fi}�ngd|v r|}| j                  }�nSd|v �rNddlm}	 |	}t        | j                  j                  d| j                  j                  «      «      t        | j                  j                  d| j                  j                  «      «      t        | j                  j                  dd«      «      ft        | j                  j                  dd«      «      t        | j                  j                  d| j                  j                  «      «      dœ}d| j                  v rt!        | j                  d   «      |d<   d| j                  v rt!        | j                  d   «      |d<   d|i}
d|vr||
d<   | j"                  j%                  |«       | j"                  j%                  |
«       || j"                  fS )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`r   )r~   ÚLionÚRMSpropÚpagedÚ8bité   é    NÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @Úeps)rš   r¡   r¢   Út_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   r†   Úbitsandbytes.optimr~   r’   r“   r   r=   r!   Ú
adam_beta1Ú
adam_beta2r#   r�   ÚfloatÚgetÚadam_epsilonÚintr    rc   )rz   r~   r’   r“   Ú
optim_namer¦   r¥   ri   Úadditional_optim_kwargsr�   Ú
bnb_kwargss              r+   Ú_get_bitsandbytes_optimizerr±   ô   s  € ä$Ô&ÜØwó
ð 	
÷ 8Ñ7à—‘—‘€JØ˜*Ð$€HØ 
Ñ*‘°€JØ€MØ!Ÿo™oÐà�ÑØŠØ	�:Ñ	ØˆØ#*¨S¯X©X×-@Ñ-@À#Ç(Á(×BUÑBUÐ,VÐ"WÒØ	�jÑ	 ØˆØ"%§.¡.ÒØ	�zÒ	!Ý/à ˆô �c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°&Ó9Ó:ðô
 ˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü˜Ÿ™×+Ñ+¨E°3·8±8×3HÑ3HÓIÓJñ#
Ðð ˜Ÿ™Ñ&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.Ø˜Ÿ™Ñ&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.à 
Ð+€JØ˜
Ñ"Ø!)ˆ
�:Ñà×Ñ×ÑÐ 7Ô8Ø×Ñ×Ñ 
Ô+Ø˜#×.Ñ.Ð.Ð.r*   c                ó  — 	 ddl m} | j                  j                  | j                  «       | j                  j                  t        | j                  j                  dd«      «      t        t        | j                  j                  dd«      «      t        t        | j                  j                  dd«      «      t        t        | j                  j                  dd	«      «      d
œ«       || j                  fS # t        $ r t        d«      ‚w xY w)z!Get AnyPrecision AdamW optimizer.r   )ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16)r´   r¶   r¸   r¹   z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersr³   r    rc   r!   r   r#   r«   Úgetattrr<   r†   rT   )rz   r³   s     r+   Ú_get_adamw_anyprecisionr½   &  sà   € ðQÝ;à×Ñ×#Ñ# C§O¡OÔ4Ø×Ñ×#Ñ#ä'0°·±×1CÑ1CÐDYÐ[bÓ1cÓ'dÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ-4Ü˜3Ÿ>™>×-Ñ-Ð.IÈ:ÓVó.ñ	ô		
ð ! #×"6Ñ"6Ð6Ð6øÜò QÜÐOÓPÐPðQús   ‚C3C6 Ã6Dc                óH  — | j                   j                  «       }| j                  radD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. d| j                  v r"| j                  d   j	                  «       dv |d<   t
        j                  j                  |fS )zGet SGD optimizer.)ÚmomentumÚ	dampeningÚweight_decayÚnesterov©ÚtrueÚ1Úyes)r    Úcopyr#   rª   rN   r<   r=   ÚSGD©rz   Úkwargsr5   s      r+   Ú_get_sgdrË   ;  s’   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø<ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ð ˜Ÿ™Ñ'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ð![ˆF�:ÑÜ�;‰;�?‰?˜FÐ"Ð"r*   c                óè   — | j                   j                  «       }| j                  r1dD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. t        j
                  j                  |fS )zGet Adagrad optimizer.)Úlr_decayrÁ   r¢   )r    rÇ   r#   rª   r<   r=   ÚAdagradrÉ   s      r+   Ú_get_adagradrÏ   G  sf   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø6ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ô �;‰;×Ñ Ð&Ð&r*   c                óH  — | j                   j                  «       }| j                  radD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. d| j                  v r"| j                  d   j	                  «       dv |d<   t
        j                  j                  |fS )zGet RMSprop optimizer.)r¿   r¡   r¢   rÁ   ÚcenteredrÃ   )r    rÇ   r#   rª   rN   r<   r=   r“   rÉ   s      r+   Ú_get_rmsproprÒ   Q  s”   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø?ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ð ˜Ÿ™Ñ'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ð![ˆF�:ÑÜ�;‰;×Ñ Ð&Ð&r*   c                ó   — t        «       st        d«      ‚ddlm}m}m} t        j                  |t        j                  |t        j                  |t        j                  |t        j                  |t        j                  |i}t        | j                  j                  dd«      «      t        | j                  j                  dd«      «      t!        | j                  j                  dd	«      «      | j                  j                  d
d«      dœ}t#        | j$                  | j&                  | j$                  j(                  ||| j*                  «      \  }}| j$                  j(                  t        j                  k(  r|j-                  dddœ«       ||fS )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`r   )ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚranké€   Úupdate_proj_gapéÈ   Úscaleg      Ð?Ú	proj_typeÚstd)r×   rÙ   rÛ   rÜ   Frw   )r   r†   Úgalore_torchrÔ   rÕ   rÖ   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr­   r#   Úpoprª   ru   r   r   r=   r    rc   )rz   rÔ   rÕ   rÖ   re   Úgalore_optim_kwargsri   r    s           r+   Ú_get_galore_optimizerrç   ]  sJ  € ä$Ô&ÜðVó
ð 	
÷ KÑJô 	×#Ñ# [Ü×(Ñ(¨/Ü×'Ñ'¨Ü×-Ñ-¨{Ü×2Ñ2°OÜ×1Ñ1°?ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ü˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°4Ó8Ó9Ø—^‘^×'Ñ'¨°UÓ;ñ	Ðô '@Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->Ð@SÐUX×UiÑUió'Ñ#€MÐ#ð ‡x�x‡~�~œ×8Ñ8Ò8Ø×Ñ°EÈEÑ RÔSØÐ*Ð*Ð*r*   c           
     ó¶  — t        «       st        d«      ‚ddlm} t        j
                  |t        j                  |i}t        | j                  j                  dd«      «      | j                  j                  dd«      | j                  j                  dd	«      t        | j                  j                  d
d«      «      t        | j                  j                  dd«      «      | j                  j                  dd«      dœ}|j                  | j                  «       t        | j                  | j                  | j                  j                   ||| j"                  «      S )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`r   )ÚAPOLLOAdamWr×   rØ   ÚprojÚrandomÚ
scale_typeÚchannelrÙ   rÚ   rÛ   ç      ð?rÜ   rÝ   )r×   rê   rì   rÙ   rÛ   rÜ   )r   r†   Úapollo_torchré   r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISEr­   r#   rå   rª   rc   r!   ru   r   r   r=   r    )rz   ré   re   Úapollo_optim_kwargss       r+   Ú_get_apollo_optimizerró   ~  s  € ä$Ô&ÜðUó
ð 	
õ )ô 	×#Ñ# [Ü×-Ñ-¨{ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ø—‘×"Ñ" 6¨8Ó4Ø—n‘n×(Ñ(¨°yÓAÜ˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°3Ó7Ó8Ø—^‘^×'Ñ'¨°UÓ;ñÐð ×Ñ˜sŸ™Ô/ä$Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->Ð@SÐUX×UiÑUióð r*   c                ó  — t        «       st        d«      ‚| j                  €t        d«      ‚ddlm}m} d| j                  j                  v r|n|}| j                  j                  d| j                  i«       || j                  fS )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`zMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.r   )ÚAdaLomoÚLomoÚadar   )r   r†   r   rT   Ú
lomo_optimrõ   rö   r   r=   r    rc   )rz   rõ   rö   ri   s       r+   Ú_get_lomo_optimizerrù   ›  sy   € äÔÜð7ó
ð 	
ð
 ‡y�yÐÜÐhÓiÐiç(à$¨¯©¯©Ñ6‘G¸D€MØ×Ñ×Ñ ¨#¯)©)Ð 4Ô5Ø˜#×.Ñ.Ð.Ð.r*   c                óô  — t        «       st        d«      ‚ddlm} | j                  j                  t        | j                  j                  dd«      «      t        | j                  j                  dd«      «      t        | j                  j                  dd	«      «      t        | j                  j                  d
d	«      «      t        | j                  j                  dd«      «      dœ«       || j                  fS )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`r   )Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingrî   )rü   rý   rÿ   r   r  )	r   rT   Ú	grokadamwrû   r    rc   rª   r#   r«   )rz   rû   s     r+   Ú_get_grokadamwr  ­  sÃ   € ä!Ô#ÜÐPÓQÐQå#à×Ñ×Ñä §¡× 2Ñ 2°<ÀÓ FÓGÜ˜#Ÿ.™.×,Ñ,¨V°SÓ9Ó:Ü˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü*/°·±×0BÑ0BÐC_ÐadÓ0eÓ*fÜ!& s§~¡~×'9Ñ'9Ð:MÈsÓ'SÓ!Tñ	
ôð �c×*Ñ*Ð*Ð*r*   c           	     óˆ  — t        «       rHt        j                  t        j                  j                  d«      «      t        j                  d«      k  rt        d«      ‚t        j                  t        j                  j                  d«      «      t        j                  d«      k  rt        d«      ‚t        j                  t        j                  j                  d«      «      t        j                  d«      k\  r	dd	lm}m} ndd	l	m}m} | j                  j                  t        j                  k(  r|}n|}| j                  j                  | j                   j#                  d
d«      t%        | j                   j#                  dd«      «      dœ«       | j                  j                  | j&                  «       || j                  fS )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aor<   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0r   )Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundrµ   )r  r
  )r   r   ÚparseÚ	importlibÚmetadatar†   Útorchao.optimr  r  Útorchao.prototype.low_bit_optimr   r=   r   ÚADAMW_TORCH_4BITr    rc   r#   r«   r   r!   )rz   r  r  ri   s       r+   Ú_get_torchao_optimizerr  À  sP  € äÔ!¤W§]¡]´9×3EÑ3E×3MÑ3MÈiÓ3XÓ%YÔ\c×\iÑ\iÐjqÓ\rÒ%rÜð,ó
ð 	
ô
 ‡}�}”Y×'Ñ'×/Ñ/°Ó8Ó9¼W¿]¹]È5Ó=QÒQÜð<ó
ð 	
ô ‡}�}”Y×'Ñ'×/Ñ/°	Ó:Ó;¼w¿}¹}ÈXÓ?VÒVß6Ð6çHà
‡x�x‡~�~œ×8Ñ8Ò8Ø!‰à!ˆà×Ñ×ÑàŸ.™.×,Ñ,¨\¸3Ó?Ü%.¨s¯~©~×/AÑ/AÐBYÐ[bÓ/cÓ%dñ	
ôð ×Ñ×Ñ §¡Ô0Ø˜#×.Ñ.Ð.Ð.r*   c           	     ó0  — t        «       st        d«      ‚ddlm}m} i }d}| j
                  j                  t        j                  k(  r-t        d«      st        d«      ‚ddlm	} |}| j                  }d}nk| j
                  j                  t        j                  k(  r|}| j                  }n5| j
                  j                  t        j                  k(  r|}nt        d	«      ‚| j
                  j                  |d
<   |r| j
                  j                  |d<   |j!                  t#        | j$                  j'                  dd«      «      t#        | j$                  j'                  dd«      «      dœ«       | j(                  j!                  |«       || j(                  fS )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`r   )ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerrÁ   Úwarmup_stepsÚweight_lr_powerrþ   Úrg        )r  r  )r   r†   Úschedulefreer  r  r   r=   r   ÚSCHEDULE_FREE_RADAMr  r!   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDrT   rÁ   r  rc   rª   r#   r«   r    )rz   r  r  r¯   Úrequire_warmupr  ri   s          r+   Ú_get_schedule_free_optimizerr  ã  s[  € ä$Ô&Üð:ó
ð 	
÷ @à ÐØ€Nà
‡x�x‡~�~œ×;Ñ;Ò;Ü(¨Ô1Üð>óð õ 	3à)ˆØ"%§/¡/ÐØ‰Ø	�‰�‰œ>×=Ñ=Ò	=Ø)ˆØ"%§/¡/ÑØ	�‰�‰œ>×;Ñ;Ò	;Ø'‰äÐ9Ó:Ð:à.1¯h©h×.CÑ.CÐ˜NÑ+ÙØ25·(±(×2GÑ2GÐ Ñ/Ø×"Ñ"ä$ S§^¡^×%7Ñ%7Ð8IÈ3Ó%OÓPÜ�s—~‘~×)Ñ)¨#¨sÓ3Ó4ñ	
ôð ×Ñ×ÑÐ 7Ô8Ø˜#×.Ñ.Ð.Ð.r*   c                ó  — t        «       st        d«      ‚ddlm} | j                  j                  dd«      }|�t        |«      }| j                  j                  dd«      }|�t        |«      }| j                  j                  | j                  d<   t        | j                  j                  dd	«      «      ||d
œ}| j                  j                  | j                  «       | j                  j                  |«       || j                  fS )z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.r   )ÚStableAdamWÚmax_lrNÚ	kahan_sumrÁ   Údecouple_lrF)r#  r!  r"  )r   r†   Úoptimir   r#   rå   rª   Úboolr   rÁ   r!   r    rc   )rz   r   r!  r"  Ústable_adamw_kwargss        r+   Ú_get_stable_adamwr'    sè   € ä$Ô&Üð:ó
ð 	
õ #à�^‰^×Ñ ¨$Ó/€FØÐÜ�v“ˆà—‘×"Ñ" ;°Ó5€IØÐÜ˜“Oˆ	à&)§h¡h×&;Ñ&;€C‡O�O�NÑ#ä˜CŸN™N×.Ñ.¨}¸eÓDÓEØØñÐð ×Ñ×Ñ §¡Ô0Ø×Ñ×ÑÐ 3Ô4Ø˜×,Ñ,Ð,Ð,r*   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)r3   z
str | NoneÚreturnr"   )r?   r   r)  r%  )T)r   r   r   r   rd   rV   re   r   rf   r   r    r   rg   r%  r)  útuple[Any, dict[str, Any]])rz   r   r)  r*  )ur'   Ú
__future__r   Úimportlib.metadatar  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   r<   Ú	packagingr   r	   Úoptimizationr   Útrainer_pt_utilsr   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr$   rZ   r   r7   ÚtupleÚdictrV   ÚOptimizerHandlerr@   ru   r{   rƒ   r‡   rŒ   r�   r±   r½   rË   rÏ   rÒ   rç   ró   rù   r  r  r  r'  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSrß   rà   rá   râ   rã   rä   Ú_GALORE_OPTIMIZERSrð   rñ   Ú_APOLLO_OPTIMIZERSr  ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr  r  r  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHr�   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_APEX_FUSEDÚADAMW_ANYPRECISIONrÈ   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysr(  r(   r)   r*   r+   ú<module>r^     s¾  ðòõ #ã Û Ý $Ý !ß %ã Ý Ý å #Ý 5Ý 5ß 7÷
÷ 
õ 
ñ Ý/Ý0à	ˆ×	Ñ	˜8Ó	$€ð ÷ð ó ðóð Ð-Ð.°°c¸4ÀÀSÀ¹>Ð6IÑ0JÐJÑKÐ óð8 $(ðV+Ø
ðV+àðV+ð ðV+ð &ð	V+ð
 !ðV+ð %ðV+ð !ðV+ð  óV+ó|+ó'óTóPócó//ódQó*	#ó'ó	'ó+óBó:/ó$+ó& /óF)/óX-ðD ×ÑØ×ÑØ×ÑØ×#Ñ#Ø×ÑØ× Ñ Ø×!Ñ!Ø×&Ñ&Ø×ÑØ×ÑØ×ÑØ×"Ñ"Ø×ÑØ×ÑØ× Ñ ðÐ ð& ×ÑØ×$Ñ$Ø×#Ñ#Ø×)Ñ)Ø×.Ñ.Ø×-Ñ-ðÐ ð ×ÑØ×)Ñ)ðÐ ð ×#Ñ#Ø×#Ñ#ðÐ ð ×&Ñ&Ø×&Ñ&Ø×$Ñ$ðÐ ð ×Ñ˜nØ×ÑÐ 0Ø×$Ñ$Ð&6Ø×"Ñ"Ð$8Ø×(Ñ(Ð*DØ×#Ñ#Ð%:Ø×%Ñ%Ð'>Ø×Ñ˜Ø×Ñ˜LØ×Ñ˜LØ×Ñ˜nØ×ÑÐ!2Ø×ÑÐ,Ø×ÑÐ/ð4ð ‡m�mÐ,Ð.IÓJð4ð  ‡m�mÐ&Ð(=Ó>ð!4ð" ‡m�mÐ&Ð(=Ó>ð#4ð$ ‡m�mÐ'Ð)?Ó@ð%4ð& ‡m�mÐ-Ð/KÓLð'4Ð Ð0ô r*   