Ë
    Gêñiš  ã                   óŽ   — d dl Z d dlZd dlZddlmZ ddlmZ ddlmZ  ej                  e
«      Z G d„ d«      Z G d„ d	e«      Zy)
é    Né   )ÚTrainerCallback)ÚPREFIX_CHECKPOINT_DIR)Úloggingc                   ó2   — e Zd Zddefd„Zd„ Zd„ Zd„ Zd„ Zy)	ÚCheckpointManagerÚ	kill_waitc                 ó<   — || _         d| _        d| _        || _        y)aD  
        Initialize the CheckpointManager for Just-In-Time checkpoint handling.

        Args:
            trainer: The Trainer instance that will be used to save checkpoints when SIGTERM is received.
            kill_wait (`int`, *optional*, defaults to 3): Grace period to distinguish between SIGTERM and SIGKILL.
        FN)ÚtrainerÚis_checkpoint_requestedÚ_original_sigterm_handlerr	   )Úselfr   r	   s      úe/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/transformers/trainer_jit_checkpoint.pyÚ__init__zCheckpointManager.__init__   s"   € ð ˆŒØ',ˆÔ$Ø)-ˆÔ&Ø"ˆ�ó    c                 ó”   — t        j                   t         j                  | j                  «      | _        t        j                  d«       y )Nz4JIT checkpoint signal handler registered for SIGTERM)ÚsignalÚSIGTERMÚ_sigterm_handlerr   ÚloggerÚinfo©r   s    r   Úsetup_signal_handlerz&CheckpointManager.setup_signal_handler   s,   € Ü)/¯©´v·~±~Àt×G\ÑG\Ó)]ˆÔ&Ü�‰ÐJÕKr   c                 óÔ   — | j                   ry t        j                  d| j                  › d�«       t	        j
                  | j                  | j                  «      j                  «        y )Nz4SIGTERM received, will request JIT checkpoint after Ús)r   r   r   r	   Ú	threadingÚTimerÚ_enable_checkpointÚstart)r   ÚsignumÚframes      r   r   z"CheckpointManager._sigterm_handler   sK   € Ø×'Ò'Øä�‰ÐJÈ4Ï>É>ÐJZÐZ[Ð\Ô]Ü�‰˜Ÿ™¨×(?Ñ(?Ó@×FÑFÕHr   c                 ó<   — t         j                  d«       d| _        y )Nz/Kill wait period elapsed, requesting checkpointT)r   r   r   r   s    r   r   z$CheckpointManager._enable_checkpoint&   s   € Ü�‰ÐEÔFØ'+ˆÕ$r   c                 óº  — 	 d| _         t        j                  d«       | j                  j                  j
                  }t        j                  d|› �«       | j                  j                  d ¬«      }t        › d|› �}t        j                  j                  ||«      }t        j                  |d¬«       t        j                  j                  ||d«      }t        |d	«      5 }|j                  d
|› d�«       d d d «       t        j                  d|› �«       | j                  j                  | j                  j                  d ¬«       t        j                  j!                  |«      r*t        j"                  |«       t        j                  d«       t        j                  d«       y # 1 sw Y   Œ±xY w# t$        $ r}t        j'                  d|› �«       ‚ d }~ww xY w)NFzStarting JIT checkpointing...zSaving JIT checkpoint at step )Útrialú-T)Úexist_okzcheckpoint-is-incomplete.txtÚwzCheckpoint started at step z and in progress...z2Created checkpoint progress sentinel marker file: zSentinel marker file removedz/Immediate JIT checkpoint completed successfullyzFailed to save JIT checkpoint: )r   r   r   r   ÚstateÚglobal_stepÚ_get_output_dirr   ÚosÚpathÚjoinÚmakedirsÚopenÚwriteÚ_save_checkpointÚmodelÚexistsÚremoveÚ	ExceptionÚerror)r   Úcurrent_stepÚ
output_dirÚcheckpoint_folderÚcheckpoint_pathÚsentinel_fileÚfÚes           r   Úexecute_jit_checkpointz(CheckpointManager.execute_jit_checkpoint*   s€  € ð!	à+0ˆDÔ(ä�K‰KÐ7Ô8ØŸ<™<×-Ñ-×9Ñ9ˆLÜ�K‰KÐ8¸¸ÐGÔHàŸ™×5Ñ5¸DÐ5ÓAˆJÜ#8Ð"9¸¸<¸.Ð IÐÜ Ÿg™gŸl™l¨:Ð7HÓIˆOô �K‰K˜°$Õ7ô ŸG™GŸL™L¨Ð5FÐHfÓgˆMÜ�m SÓ)ð Y¨QØ—‘Ð5°l°^ÐCVÐWÔX÷Yä�K‰KÐLÈ]ÈOÐ\Ô]ð �L‰L×)Ñ)¨$¯,©,×*<Ñ*<ÀDÐ)ÔIô �w‰w�~‰~˜mÔ,Ü—	‘	˜-Ô(Ü—‘Ð:Ô;ä�K‰KÐIÕJ÷Yð Yûô ò 	Ü�L‰LÐ:¸1¸#Ð>Ô?Øûð	ús1   ‚CF3 Ã!F'Ã7B/F3 Æ'F0Æ,F3 Æ3	GÆ<GÇGN)é   )	Ú__name__Ú
__module__Ú__qualname__Úintr   r   r   r   r>   © r   r   r   r      s$   „ ñ#¨3ó #òLòIò,ó"r   r   c                   ó:   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
y	)
ÚJITCheckpointCallbackaN  
    Callback for Just-In-Time checkpointing on SIGTERM signals.

    When SIGTERM is received, the checkpoint manager sets `is_checkpoint_requested=True`.
    The callbacks detect this flag and set `control.should_training_stop=True`, which signals
    the Trainer's training loop to exit gracefully after saving the checkpoint.
    c                 ó    — d | _         d | _        y )N)r   Újit_managerr   s    r   r   zJITCheckpointCallback.__init__X   s   € ØˆŒØ59ˆÕr   c                 óÀ   — || _         |j                  j                  rAt        |¬«      | _        | j                  j                  «        t        j                  d«       y y )N)r   zJIT checkpointing enabled)r   ÚargsÚenable_jit_checkpointr   rH   r   r   r   )r   r   s     r   Úset_trainerz!JITCheckpointCallback.set_trainer\   sG   € ØˆŒØ�<‰<×-Ò-Ü0¸ÔAˆDÔØ×Ñ×1Ñ1Ô3Ü�K‰KÐ3Õ4ð .r   c                 óŽ   — | j                   r9| j                   j                  r"d|_        | j                   j                  «        y y y ©NT©rH   r   Úshould_training_stopr>   ©r   rJ   r(   ÚcontrolÚkwargss        r   Úon_pre_optimizer_stepz+JITCheckpointCallback.on_pre_optimizer_stepc   ó<   € Ø×Ò × 0Ñ 0× HÒ HØ+/ˆGÔ(Ø×Ñ×3Ñ3Õ5ð !IÐr   c                 óŽ   — | j                   r9| j                   j                  r"d|_        | j                   j                  «        y y y rN   rO   rQ   s        r   Úon_step_beginz#JITCheckpointCallback.on_step_beginh   rU   r   c                 óœ   — | j                   r@| j                   j                  r)d|_        d|_        | j                   j	                  «        y y y ©NFT©rH   r   Úshould_saverP   r>   rQ   s        r   Úon_step_endz!JITCheckpointCallback.on_step_endm   óD   € Ø×Ò × 0Ñ 0× HÒ HØ"'ˆGÔØ+/ˆGÔ(Ø×Ñ×3Ñ3Õ5ð !IÐr   c                 óœ   — | j                   r@| j                   j                  r)d|_        d|_        | j                   j	                  «        y y y rY   rZ   rQ   s        r   Úon_epoch_endz"JITCheckpointCallback.on_epoch_ends   r]   r   c                 óæ   — | j                   re| j                   j                  �Nt        j                  t        j                  | j                   j                  «       t        j                  d«       y y y )Nz;Restored original SIGTERM handler after training completion)rH   r   r   r   r   r   rQ   s        r   Úon_train_endz"JITCheckpointCallback.on_train_endy   sP   € à×Ò × 0Ñ 0× JÑ JÐ VÜ�M‰Mœ&Ÿ.™.¨$×*:Ñ*:×*TÑ*TÔUÜ�K‰KÐUÕVð !WÐr   N)r@   rA   rB   Ú__doc__r   rL   rT   rW   r\   r_   ra   rD   r   r   rF   rF   O   s+   „ ñò:ò5ò6ò
6ò
6ò6óWr   rF   )r+   r   r   Útrainer_callbackr   Útrainer_utilsr   Úutilsr   Ú
get_loggerr@   r   r   rF   rD   r   r   ú<module>rg      sD   ðÛ 	Û Û å -Ý 0Ý ð 
ˆ×	Ñ	˜HÓ	%€÷?ñ ?ôD.W˜Oõ .Wr   