Ë
    *êñi¾+ ã                   ó–  — d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZ d dlZd dlmZ d d	lmZ d d
lmZm Z  d dl!m"Z" d dl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z- g d¢Z. ej^                  e0«      Z1 G d„ de2e«      Z3e3jh                  Z4e3jj                  Z5e3jl                  Z6e3jn                  Z7e3jp                  Z8e3jr                  Z9e3jt                  Z:e3jv                  Z;e3jx                  Z<e3jz                  Z=e3j|                  Z>e3j~                  Z?e4Z@e5ZAe6ZBe=ZC ejˆ                  d«      ZE G d„ de«      ZFedeFde2fd„«       ZG	 dZdeHeIeJeFdz     f   deIdz  de2fd„ZK G d„ de«      ZLdZdeJejš                     d e2dz  deJejœ                     fd!„ZO	 dZdeJejš                     d e2dz  deHeIeJejœ                     f   fd"„ZPd#eJejœ                     fd$„ZQ G d%„ d&eL«      ZR G d'„ d(eR«      ZS G d)„ d*eR«      ZT G d+„ d,eR«      ZUd-e3deVfd.„ZWd/eJeFdz     d0eIdeJeFdz     fd1„ZX	 d[d2eJeFdz     d3eIdeJeF   fd4„ZYd2eJeFdz     deJeF   fd5„ZZd2eHeIeJeF   f   d6eeIgeIf   d7eIdeHeIeJeF   f   fd8„Z[d/eHeIeJeFdz     f   d9eId7eId:eIdeHeIeIf   f
d;„Z\ G d<„ d=eL«      Z]e G d>„ d?«      «       Z^ G d@„ dAe«      Z_ G dB„ dCe]«      Z` G dD„ dEe`«      Za	 	 d\dF„Zb	 d]dGeIdHeIdIeId9eId0eIdJeIdeIfdK„Zc G dL„ dMe`«      Zd G dN„ dOe`«      Ze G dP„ dQe`«      Zf G dR„ dSe`«      ZgdTe2fdU„Zhd6eeIgeIf   d7eIfdV„ZidW„ ZjdXeJe-   dTe2fdY„Zky)^é    N)ÚABCÚabstractmethod)ÚCounterÚdefaultdict)ÚCallable)Ú	dataclass)ÚEnum)Ú	lru_cache)ÚAnyÚcastÚ
NamedTupleÚProtocol)ÚOptimizedModule)Ú
FSDPModuleÚUnshardHandle)Ú_Loss)Úrecord_functioné   )Úgenerate_rank_to_stage_mappingÚgenerate_stage_to_rank_mapping)Úmerge_chunksÚsplit_args_kwargs_into_chunksÚTensorChunkSpec)Ú_PipelineStageBase)
Úget_schedule_classÚPipelineScheduleSingleÚPipelineScheduleMultiÚSchedule1F1BÚScheduleGPipeÚScheduleInterleaved1F1BÚScheduleLoopedBFSÚScheduleInterleavedZeroBubbleÚScheduleZBVZeroBubbleÚScheduleDualPipeVc                   óV   — e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZededd fd„«       Zy)Ú_ComputationTypeÚFÚIÚWÚUNSHARDÚRESHARDÚSEND_FÚRECV_FÚSEND_BÚRECV_BÚBÚOVERLAP_F_BÚREDUCE_GRADÚactionÚreturnc                 óZ   — 	 t        | «      S # t        $ r}t        d| › �«      |‚d }~ww xY w)NúInvalid computation type )r&   Ú
ValueErrorÚRuntimeError)r3   Úexcs     úh/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/torch/distributed/pipelining/schedules.pyÚfrom_strz_ComputationType.from_str<   s:   € ð	NÜ# FÓ+Ð+øÜò 	NÜÐ!:¸6¸(ÐCÓDÈ#ÐMûð	Nús   ‚
 �	*–%¥*N)Ú__name__Ú
__module__Ú__qualname__ÚFORWARDÚBACKWARD_INPUTÚBACKWARD_WEIGHTr*   r+   r,   r-   r.   r/   ÚFULL_BACKWARDr1   r2   ÚstaticmethodÚstrr;   © ó    r:   r&   r&   -   sc   „ à€GØ€NØ€OØ€GØ€GØ€FØ€FØ€FØ€FØ€MØ€KØ€KàðN˜ð NÐ!3ò Nó ñNrF   r&   zK(\d+)(F|I|B|W|UNSHARD|RESHARD|REDUCE_GRAD|SEND_F|RECV_F|SEND_B|RECV_B)(\d*)c                   óˆ   — e Zd ZU eed<   eed<   dZedz  ed<   dZed   dz  ed<   d„ Z	d„ Z
ed	efd
„«       Zedefd„«       Zy)Ú_ActionÚstage_indexÚcomputation_typeNÚmicrobatch_index)rH   .Úsub_actionsc                 ó"   — | j                  «       S ©N)Ú__repr__©Úselfs    r:   Ú__str__z_Action.__str__c   s   € Ø�}‰}‹ÐrF   c                 óf  — | j                   �M| j                   D �cg c]  }t        |«      ‘Œ }}ddj                  |«      › d| j                  j                  › �S t        | j                  «      }|| j                  j                  z  }| j                  �|t        | j                  «      z  }|S c c}w )Nú(ú;ú))rL   ÚreprÚjoinrJ   ÚvaluerD   rI   rK   )rQ   Ú
sub_actionÚsub_action_reprsÚrepr_strs       r:   rO   z_Action.__repr__f   sª   € Ø×ÑÐ'àCG×CSÑCSÖT°Z¤ ZÕ 0ÐTÐÐTØ�s—x‘xÐ 0Ó1Ð2°!°D×4IÑ4I×4OÑ4OÐ3PÐQÐQä˜4×+Ñ+Ó,ˆHà˜×-Ñ-×3Ñ3Ñ3ˆHØ×$Ñ$Ð0ØœC × 5Ñ 5Ó6Ñ6�ØˆOùò  Us   ›B.r4   c                 óP   — | j                   t        t        t        t        t
        fv S rN   )rJ   r?   rB   r@   rA   r1   rP   s    r:   Úis_compute_opz_Action.is_compute_ops   s)   € à×$Ñ$ÜÜÜÜÜð)
ð 
ð 	
rF   Úaction_stringc                 óÀ  — | j                  «       } | dk(  ry| j                  d«      rµd| v r±| j                  d«      }| d| }| |dz   d }g }|j                  «       rM|j                  d«      D ]9  }t        j                  |j                  «       «      }|€Œ)|j                  |«       Œ; t	        dt        j                  |«      d|rt        |«      ¬«      S d¬«      S t        j                  | «      x}rW|j                  «       \  }}	}
t	        t        |«      t        j                  |	«      t        |
«      rt        |
«      «      S d«      S | dk(  ryt        d	| › d
�«      ‚)zž
        Reverse of __repr__

        String should be formatted as [stage][action type][(microbatch)]
            e.g. `2F0`, `1UNSHARD`, `3SEND_F1`
        Ú NrT   rV   r   rU   éÿÿÿÿ)rI   rJ   rK   rL   zInvalid action string: zD, should be formatted as [stage][action type][(microbatch)] e.g. 2F0)ÚstripÚ
startswithÚfindÚsplitrH   r;   Úappendr&   ÚtupleÚ_action_regexÚmatchÚgroupsÚintÚlenr8   )r_   Úbracket_endÚsub_partÚcomputation_type_partrL   Úsub_strrZ   rj   rI   rJ   rK   s              r:   r;   z_Action.from_str}   s•  € ð &×+Ñ+Ó-ˆØ˜BÒØð ×#Ñ# CÔ(¨S°MÑ-Aà'×,Ñ,¨SÓ1ˆKØ$Ø�+ðˆHð %2Ø˜a‘Ð!ð%Ð!ð
 ˆKØ�~‰~ÔØ'Ÿ~™~¨cÓ2ò 7�GÜ!(×!1Ñ!1°'·-±-³/Ó!B�JØ!Ñ-Ø#×*Ñ*¨:Õ6ð7ô ØÜ!1×!:Ñ!:Ð;PÓ!QØ!%Ù2=œE +Ó.ô	ð ð DHô	ð ô "×'Ñ'¨Ó6Ð6ˆ5Ð6Ø>C¿l¹l»nÑ;ˆKÐ)Ð+;ÜÜ�KÓ Ü ×)Ñ)Ð*:Ó;Ü),Ð-=Ô)>”Ð$Ó%óð ð EIóð ð
 ˜bÒ ØÜØ% m _Ð4xÐyó
ð 	
rF   )r<   r=   r>   rl   Ú__annotations__r&   rK   rL   rh   rR   rO   ÚpropertyÚboolr^   rC   rD   r;   rE   rF   r:   rH   rH   ]   sq   … ØÓØ&Ó&Ø#'Ð�c˜D‘jÓ'Ø04€K��~Ñ&¨Ñ-Ó4òòð ð
˜tò 
ó ð
ð ð3
 ò 3
ó ñ3
rF   rH   r3   r4   c                 ó   — dt        | «      › �S )NzPP:©rD   )r3   s    r:   Ú_get_profiler_function_namerw   ´   s   € à”�V“�ÐÐrF   Úpipeline_orderÚerror_step_numberc                 ó^  ‡— t        j                  | «      } | D ]/  }t        t        | |   «      «      D ]  }| |   |   �Œd| |   |<   Œ Œ1 t	        d„ | j                  «       D «       «      }t        |«      D �cg c]4  }dt        |«      j                  t        t        |dz
  «      «      «      z   ‘Œ6 }}t        | «      D �cg c]  }| j                  |dg|z  «      ‘Œ }}t        t        j                  |ddiŽ«      }t        | «      }	t        |	«      D �cg c]  }dt        |«      z   ‘Œ }
}t        |g|¢­Ž D �cg c]  }t	        d„ |D «       «      ‘Œ c}Šdt        |d	   «      d
z   z  dj                  ˆfd„t        |
«      D «       «      z   }t        ||«      D ��cg c]T  \  }}|› d�dj                  ˆfd„t        |«      D «       «      z   |�!t!        |j#                  «       d   «      |k(  rdndz   ‘ŒV }}}|dz   dj                  |«      z   dz   }|S c c}w c c}w c c}w c c}w c c}}w )zü
    Formats the pipeline order in a timestep (row) x rank (column) grid of actions
    and returns the formatted string.

    If `error_step_number` is passed in, an additional label will be added to signify which step
    that it is erroring on.
    ra   c              3   ó2   K  — | ]  }t        |«      –— Œ y ­wrN   )rm   )Ú.0Úactionss     r:   ú	<genexpr>z)_format_pipeline_order.<locals>.<genexpr>Ð   s   è ø€ ÒH W”C˜—LÑHùs   ‚zStep r   Ú	fillvalueúRank c              3   óL   K  — | ]  }|�t        t        |«      «      nd–— Œ y ­w)Nr   )rm   rD   )r|   Úitems     r:   r~   z)_format_pipeline_order.<locals>.<genexpr>à   s"   è ø€ ÒF¸$˜dÐ.ŒC”�D“	ŒN°AÓ5ÑFùs   ‚"$ú r   é   c              3   ó:   •K  — | ]  \  }}|d ‰|   › d�›–— Œ y­w©ú<ra   NrE   )r|   ÚiÚlabelÚmax_lengthss      €r:   r~   z)_format_pipeline_order.<locals>.<genexpr>ä   s0   øè ø€ ò <Ù)1¨¨Eˆ5��;˜q‘>Ð"  Ð"Ð
#Ó$ñ<ùs   ƒz: c              3   óL   •K  — | ]  \  }}t        |«      d ‰|   › d�›–— Œ y­wr†   rv   )r|   rˆ   r‚   rŠ   s      €r:   r~   z)_format_pipeline_order.<locals>.<genexpr>ê   s/   øè ø€ ÒR¹¸¸4”c˜$“i  +¨a¡.Ð!1°Ð1Ð2Ó3ÑRùs   ƒ!$z <-- ERROR HEREú
)ÚcopyÚdeepcopyÚrangerm   ÚmaxÚvaluesrD   ÚzfillÚsortedÚgetÚlistÚ	itertoolsÚzip_longestÚziprX   Ú	enumeraterl   rf   )rx   ry   Úrankrˆ   Ú	num_stepsÚstep_labelsÚkeyÚrank_actionsÚtransposed_actionsÚ	num_ranksÚrank_labelsÚcolÚ
header_rowr‰   ÚrowÚformatted_rowsÚformatted_tablerŠ   s                    @r:   Ú_format_pipeline_orderr§   ¹   sc  ø€ ô —]‘] >Ó2€Nð ò -ˆÜ”s˜>¨$Ñ/Ó0Ó1ò 	-ˆAØ˜dÑ# AÑ&Ñ.à*,�˜tÑ$ QÒ'ñ	-ð-ô ÑH°×0EÑ0EÓ0GÔHÓH€IäAFÀyÓAQöØ<=ˆ”#�a“&—,‘,œs¤3 y°1¡}Ó#5Ó6Ó7Ó7ð€Kð ô
 >DÀNÓ=SöØ69ˆ×Ñ˜3   yÑ 0Õ1ð€Lð ô
 œi×3Ñ3°\ÐPÈRÑPÓQÐä�NÓ#€IÜ-2°9Ó-=Ö>¨�7œS ›VÓ#Ð>€KÐ>ô �{Ð8Ð%7Ò8öàô 	ÑFÀ#ÔFÕFò€Kð
 œ˜K¨™NÓ+¨aÑ/Ñ0°3·8±8ó <Ü5>¸{Ó5Kô<ó 4ñ €Jô ˜kÐ+=Ó>÷
ñ ˆE�3ð ˆ'�ˆØ
�(‰(ÓRÄ9ÈSÃ>ÔRÓ
Rñ	Sð !Ð,Ü�E—K‘K“M !Ñ$Ó%Ð):Ò:ñ ð ó	
ð
€Nñ 
ð ! 4Ñ'¨$¯)©)°NÓ*CÑCÀdÑJ€OØÐùòIùòùò ?ùòùó
s   Á89HÃ HÄHÄ:H$ÆAH)c                   óÆ  — e Zd Z	 	 	 	 	 d dededej                  f   dz  deedf   dz  de	e
ef   dz  de	e
ef   ee   z  dz  d	efd
„Zd„ Zd„ Zd„ Ze	 	 	 	 	 d dedz  dedz  dedz  dedz  def
d„«       Zeddddœdedz  fd„«       Zdddœdedz  fd„Z	 	 	 	 d!dedz  dedz  dedz  dedz  deeef   f
d„Zd„ Z	 d"deedf   de	e
ef   dz  fd„Zdee   defd„Zy)#Ú_PipelineScheduleNTÚn_microbatchesÚloss_fn.Úargs_chunk_specÚkwargs_chunk_specÚoutput_merge_specÚscale_gradsc                 óâ   — || _         || _        || _        || _        || _        || _        	 | j                  d u| _        g | _        t        j                  d| j                  j                  «       y )NzUsing %s)Ú_n_microbatchesÚ_loss_fnr¯   Ú_args_chunk_specÚ_kwargs_chunk_specÚ_output_merge_specÚ_has_backwardÚ_internal_lossesÚloggerÚinfoÚ	__class__r<   )rQ   rª   r«   r¬   r­   r®   r¯   s          r:   Ú__init__z_PipelineSchedule.__init__ù   sq   € ð  .ˆÔØˆŒð 'ˆÔð !0ˆÔà"3ˆÔØ"3ˆÔð	ð "Ÿ]™]°$Ð6ˆÔð 57ˆÔÜ�‰�J §¡× 7Ñ 7Õ8rF   c                 ó˜   — |j                   r>| j                  �1| j                  |||   «      }| j                  j	                  |«       y y y rN   )Úis_lastr²   Ú_compute_lossr·   rg   )rQ   ÚstageÚoutputÚ
target_mbsÚmb_indexÚlosss         r:   Ú_maybe_compute_lossz%_PipelineSchedule._maybe_compute_loss  sD   € Ø�=Š=˜TŸ]™]Ð6Ø×%Ñ% f¨j¸Ñ.BÓCˆDØ×!Ñ!×(Ñ(¨Õ.ð 7ˆ=rF   c                 ó  — d|cxk  xr t        | j                  «      k  nc }|j                  r| j                  �|r| j                  |   S t        | j                  «      dk7  r|st	        d|› d| j                  › �«      ‚y )Nr   zLoss for microbatch z6 is not available. Available losses for microbatches: )rm   r·   r½   r²   r8   )rQ   r¿   rÂ   Úvalid_indexs       r:   Ú_maybe_get_lossz!_PipelineSchedule._maybe_get_loss   s„   € Ø˜8Ö@¤c¨$×*?Ñ*?Ó&@Ô@ˆØ�=Š=˜TŸ]™]Ð6¹;Ø×(Ñ(¨Ñ2Ð2Ü�×&Ñ&Ó'¨1Ò,±[ÜØ& x jð 16Ø6:×6KÑ6KÐ5LðNóð ð
 rF   c                 ó€  — t        |t        «      s|g}t        d„ |D «       «      }|r}|�{t        | j                  «      | j
                  k7  r.t        d| j
                  › dt        | j                  «      › �«      ‚|j                  «        |j                  | j                  «       | j                  j                  «        y)zB
        Update the losses to those in the internal state
        c              3   ó4   K  — | ]  }|j                   –— Œ y ­wrN   ©r½   )r|   r¿   s     r:   r~   z3_PipelineSchedule._update_losses.<locals>.<genexpr>3  s   è ø€ Ò!D°E %§-¥-Ñ!Dùs   ‚Nú
Expecting z losses but got )	Ú
isinstancer•   Úanyrm   r·   r±   r8   ÚclearÚextend)rQ   ÚstagesÚlossesÚcontains_last_stages       r:   Ú_update_lossesz _PipelineSchedule._update_losses,  s©   € ô
 ˜&¤$Ô'Ø�XˆFÜ!Ñ!D¸VÔ!DÓDÐñ  6Ð#5Ü�4×(Ñ(Ó)¨T×-AÑ-AÒAÜ"Ø  ×!5Ñ!5Ð 6Ð6FÄsÈ4×K`ÑK`ÓGaÐFbÐcóð ð
 �L‰LŒNà�M‰M˜$×/Ñ/Ô0à×Ñ×#Ñ#Õ%rF   Úarg_mbsÚ	kwarg_mbsrÁ   rÑ   Úreturn_outputsc                 ó   — t         ‚)aG  
        Run one iteration of the pipeline schedule with list of microbatches.
        Will go through all the microbatches according to the schedule
        implementation.

        Args:
            microbatches: list of microbatch args.
            return_outputs: whether to return the outputs from the last stage.
        ©ÚNotImplementedError)rQ   rÔ   rÕ   rÁ   rÑ   rÖ   s         r:   Ú_step_microbatchesz$_PipelineSchedule._step_microbatchesC  s
   € ô$ "Ð!rF   ©ÚtargetrÑ   rÖ   c                ó   — t         ‚)á5  
        Run one iteration of the pipeline schedule with *whole-batch* input.
        Will chunk the input into microbatches automatically, and go through the
        microbatches according to the schedule implementation.

        args: positional arguments to the model (as in non-pipeline case).
        kwargs: keyword arguments to the model (as in non-pipeline case).
        target: target for the loss function.
        losses: a list to store the losses for each microbatch.
        return_outputs: whether to return the outputs from the last stage.
        rØ   )rQ   rÜ   rÑ   rÖ   ÚargsÚkwargss         r:   Ústepz_PipelineSchedule.stepW  s
   € ô( "Ð!rF   ©rÜ   rÑ   c                óz   — | j                   }	 d| _          | j                  |||dœ|¤Ž|| _         S # || _         w xY w)aÞ  
        Run one iteration of the pipeline schedule with *whole-batch* input.
        Will chunk the input into microbatches automatically, and go through the
        microbatches, calling forward only.

        args: positional arguments to the model (as in non-pipeline case).
        kwargs: keyword arguments to the model (as in non-pipeline case).
        target: target values for the loss function.
        losses: a list to store the losses for each microbatch.
        Frâ   )r¶   rá   )rQ   rÜ   rÑ   rß   rà   Úoriginal_has_backwards         r:   Úevalz_PipelineSchedule.evalm  sH   € ð !%× 2Ñ 2Ðð	7Ø!&ˆDÔØ�4—9‘9˜d¨6¸&ÑKÀFÑKð "7ˆDÕøÐ!6ˆDÕús   Ž1 ±	:r4   c                 óü   ‡ — dt         fˆ fd„}|�
 ||d«       ndg‰ j                  z  }|�
 ||d«       ni g‰ j                  z  }|�	 ||d«       |�'t        |t        «      st	        dt        |«      › �«      ‚||fS )z*
        Pre-process/check inputs
        Únamec           
      óÖ   •— t        | t        «      st        |› dt        | «      › �«      ‚t	        | «      ‰j
                  k7  r't        d‰j
                  › d|› dt	        | «      › �«      ‚y )Nz must be a list but got a rË   rƒ   z	 but got )rÌ   r•   Ú	TypeErrorÚtyperm   r±   r7   )Úmbsrç   rQ   s     €r:   Úcheck_type_and_lenz;_PipelineSchedule._check_inputs.<locals>.check_type_and_lenŒ  sm   ø€ Ü˜c¤4Ô(Ü 4 &Ð(BÄ4ÈÃ9À+Ð NÓOÐOÜ�3‹x˜4×/Ñ/Ò/Ü Ø  ×!5Ñ!5Ð 6°a¸°v¸YÄsÈ3ÃxÀjÐQóð ð 0rF   rÔ   rE   rÕ   rÁ   z losses must be a list but got a )rD   r±   rÌ   r•   ré   rê   )rQ   rÔ   rÕ   rÁ   rÑ   rì   s   `     r:   Ú_check_inputsz_PipelineSchedule._check_inputs�  s—   ø€ ð	¬#õ 	ð ÐÙ˜w¨	Õ2à�d˜T×1Ñ1Ñ1ˆGàÐ Ù˜y¨+Õ6à˜˜t×3Ñ3Ñ3ˆIàÐ!Ù˜z¨<Ô8àÐÜ˜f¤dÔ+ÜÐ"BÄ4ÈÃ<À.Ð QÓRÐRà˜	Ð!Ð!rF   c                 ó&   — | j                  ||«      S rN   )r²   )rQ   rÀ   rÜ   s      r:   r¾   z_PipelineSchedule._compute_loss§  s   € Ø�}‰}˜V VÓ,Ð,rF   rß   rà   c                 ó²   — |s|r4t        ||| j                  | j                  | j                  «      \  }}||fS dg| j                  z  i g| j                  z  fS )zj
        Splits a full-batch input into chunks (i.e. microbatches) and returns
        the chunks
        rE   )r   r±   r³   r´   )rQ   rß   rà   Ú
args_splitÚkwargs_splits        r:   Ú_split_inputsz_PipelineSchedule._split_inputsª  sm   € ñ ‘6Ü'DØØØ×$Ñ$Ø×%Ñ%Ø×'Ñ'ó(Ñ$ˆJ˜ð ˜|Ð+Ð+ð �4˜$×.Ñ.Ñ.°°°t×7KÑ7KÑ0KÐKÐKrF   Úoutput_chunksc                 ó.   — t        || j                  «      S )z£
        Merge output chunks back to a batch state.
        If output_merge_spec is None, the utility will merge output chunks by dimension 0 (batch dim).
        )r   rµ   )rQ   ró   s     r:   Ú_merge_outputsz _PipelineSchedule._merge_outputsÁ  s   € ô
 ØØ×#Ñ#ó
ð 	
rF   ©NNNNT)NNNNrN   )r<   r=   r>   rl   r   ÚtorchÚTensorrh   r   ÚdictrD   r   rt   r»   rÄ   rÇ   rÓ   r   r•   rÚ   rá   rå   rí   r¾   rò   rõ   rE   rF   r:   r©   r©   ø   sÿ  „ ð 7;Ø>BØ?CØ@DØ ñ 9àð 9ð ˜#˜uŸ|™|Ð+Ñ,¨tÑ3ð 9ð ˜°Ð3Ñ4°tÑ;ð	 9ð
    _Ð 4Ñ5¸Ñ<ð 9ð    S ™>¨E°#©JÑ6¸Ñ=ð 9ð ó 9òD/ò

ò&ð. ð  $Ø!%Ø"&Ø"Ø#ñ"à˜‘ð"ð ˜$‘;ð"ð ˜4‘Kð	"ð
 �t‘ð"ð ò"ó ð"ð& ð Ø"Øò"ð �t‘ò	"ó ð"ð* "&¸Tò 7¨t°d©{ó 7ð,  $Ø!%Ø"&Ø"ñ$"à˜‘ð$"ð ˜$‘;ð$"ð ˜4‘Kð	$"ð
 �t‘ð$"ð 
ˆt�TˆzÑ	ó$"òL-ð )-ñLà�C˜�H‰oðLð �S˜#�X‘ Ñ%óLð.
¨D°©Ið 
¸#ô 
rF   r©   Úp2p_opsÚdescc                 óŒ   — t        | «      dk(  rg S |r|› d�nd}t        j                  d|| «       t        j                  | «      S )zt
    Simple wrapper over batch_isend_irecv from torch.distributed, which just adds a descriptive logger on top.
    r   z, ra   zbatch_p2p %s%s)rm   r¸   ÚdebugÚdistÚbatch_isend_irecv)rú   rû   Údesc_strs      r:   Ú
_batch_p2pr  Ì  sF   € ô ˆ7ƒ|�qÒØˆ	Ù"�$��r‰{¨€HÜ
‡L�LÐ! 8¨WÔ5Ü×!Ñ! 'Ó*Ð*rF   c                 óô   — t        t        «      }i }t        | «      dk(  r|S | D ]   }||j                     j	                  |«       Œ" t        |j                  «       «      D ]  \  }}t        ||¬«      ||<   Œ |S )zÊ
    Sorts the list of P2P ops by the peer rank, and then calls
    batch_isend_irecv. Return a dictionary of works by peer rank. This function
    helps us avoid hangs in case of skip connections.
    r   ©rû   )r   r•   rm   Úpeerrg   r“   Úitemsr  )rú   rû   Úops_by_peerÚwork_by_peerÚopr  Úopss          r:   Ú_sorted_batch_p2pr
  ×  s‰   € ô 0;¼4Ó/@€KØ/1€LÜ
ˆ7ƒ|�qÒØÐð ò (ˆØ�B—G‘GÑ×#Ñ# BÕ'ð(ô ˜K×-Ñ-Ó/Ó0ò 8‰	ˆˆcÜ'¨°$Ô7ˆ�TÒð8ð ÐrF   Úworkc                 ó2   — | D ]  }|j                  «        Œ y)zX
    Waits for a list of dist.Work (typically from _batch_p2p / _sorted_batch_p2p).
    N)Úwait)r  Úws     r:   Ú_wait_batch_p2pr  ò  s   € ð ò ˆØ	�‰�ñrF   c                   óÜ   ‡ — e Zd ZdZ	 	 	 	 	 ddedededz  deedf   dz  d	e	e
ef   dz  d
e	e
ef   ee   z  dz  defˆ fd„Zd„ Zddddœdedz  defd„Zde	eeedz     f   dz  fd„Zˆ xZS )r   a”  
    Base class for single-stage schedules.
    Implements the `step` method.
    Derived classes should implement `_step_microbatches`.

    Gradients are scaled by num_microbatches depending on the `scale_grads` argument, defaulting to True.  This setting
    should match the configuration of your loss_fn, which may either average losses (scale_grads=True)
    or sum losses (scale_grads=False).
    NTr¿   rª   r«   r¬   .r­   r®   r¯   c                 ó¦   •— t         ‰| �  ||||||¬«       || _        |j                  | _        d| _        d| _        | j                  «       | _        y )N©rª   r«   r¬   r­   r®   r¯   F)	Úsuperr»   Ú_stageÚ
num_stagesÚ_num_stagesÚ_stage_forward_initializedÚ_stage_backward_initializedÚ_get_pipeline_orderrx   ©	rQ   r¿   rª   r«   r¬   r­   r®   r¯   rº   s	           €r:   r»   zPipelineScheduleSingle.__init__  sf   ø€ ô 	‰ÑØ)ØØ+Ø/Ø/Ø#ð 	ô 	
ð ˆŒØ ×+Ñ+ˆÔØ*/ˆÔ'Ø+0ˆÔ(ð ×$Ñ$Ó&ð 	ÕrF   c                 ó‚  — | j                   smg }|j                  | j                  j                  «       «       t	        t        |«      «       | j                  j                  | j                  ||«       d| _         | j                  r:| j                  s-| j                  j                  | j                  «       d| _	        y y y ©NT)r  rÏ   r  Ú_get_init_p2p_neighbors_opsr  r  Ú_prepare_forward_infrar±   r¶   r  Ú_prepare_backward_infra)rQ   rß   rà   Úall_opss       r:   Ú_initialize_stagez(PipelineScheduleSingle._initialize_stage"  s–   € Ø×.Ò.ð )+ˆGØ�N‰N˜4Ÿ;™;×BÑBÓDÔEÜœJ wÓ/Ô0à�K‰K×.Ñ.¨t×/CÑ/CÀTÈ6ÔRØ.2ˆDÔ+à×Ò d×&FÒ&FØ�K‰K×/Ñ/°×0DÑ0DÔEØ/3ˆDÕ,ð 'GÐrF   rÛ   rÑ   rÖ   c                óî  — | j                   rt        j                  «       st        d«      ‚| j                   | j                  _        | j                  j                  «        | j                  ||«      \  }}|�*t        t        j                  || j                  «      «      }nd}| j                  |||||«       | j                  j                  r'|r%| j                  | j                  j                  «      S y©rÞ   z’step() requires gradients to be enabled for backward computation; it should not be used under torch.no_grad() context. Please call eval() instead.N)r¶   r÷   Úis_grad_enabledr8   r  Úhas_backwardÚclear_runtime_statesrò   r•   Útensor_splitr±   rÚ   r½   rõ   ró   )	rQ   rÜ   rÑ   rÖ   rß   rà   rð   rñ   Útargets_splits	            r:   rá   zPipelineScheduleSingle.step2  sÝ   € ð& ×Ò¤e×&;Ñ&;Ô&=Üð.óð ð $(×#5Ñ#5ˆ�‰Ô ð 	�‰×(Ñ(Ô*ð $(×#5Ñ#5°d¸FÓ#CÑ ˆ
�Lð ÐÜ ¤×!3Ñ!3°F¸D×<PÑ<PÓ!QÓR‰Mà ˆMð 	×ÑØ˜ m°V¸^ô	
ð
 �;‰;×Ò¡>Ø×&Ñ& t§{¡{×'@Ñ'@ÓAÐAàrF   r4   c                  ó   — y)a´  
        Returns the pipeline execution order as a schedule IR.

        The returned IR is a dictionary mapping rank IDs to lists of actions.
        Each action is either an _Action object representing computation to perform,
        or None representing a deliberate idle step.

        The None values are used to represent pipeline bubbles where a rank
        must wait for dependencies from other ranks before proceeding. However
        during execution, with  the _PipelineScheduleRuntime, these Nones are
        skipped since the relevant communication (send/recv) will be scheduled and waited on.

        Returns:
            A dictionary mapping rank -> list of actions
        NrE   rP   s    r:   r  z*PipelineScheduleSingle._get_pipeline_orderf  s   € ð  rF   rö   )r<   r=   r>   Ú__doc__r   rl   r   rh   r   rù   rD   r   rt   r»   r!  r•   rá   rH   r  Ú__classcell__©rº   s   @r:   r   r   ú  sì   ø„ ñð $(Ø>BØ?CØ@DØ ñ
à!ð
ð ð
ð ˜D‘ð	
ð
 ˜°Ð3Ñ4°tÑ;ð
ð    _Ð 4Ñ5¸Ñ<ð
ð    S ™>¨E°#©JÑ6¸Ñ=ð
ð õ
ò:4ð& Ø"Ø#ò2ð �t‘ð	2ð
 ó2ðh T¨#¨t°G¸d±NÑ/CÐ*CÑ%DÀtÑ%K÷ rF   r   c                   óP   — e Zd ZdZ	 	 	 	 	 d	dedz  dedz  dedz  dedz  def
d„Zy)
Ú_ScheduleForwardOnlyzo
    The forward-only schedule.
    Will go through all the microbatches and perform only the forward pass
    NrÔ   rÕ   rÁ   rÑ   rÖ   c                 óÜ  — |€|�t        d«      ‚| j                  ||||«      \  }}| j                  |d   |d   «       g }t        | j                  «      D ]ö  }t        d|› �«      5  | j                  j                  |«      }t        |d¬«      }	|	j                  «       D ]  }
t        |
«       Œ | j                  j                  |||   ||   «       | j                  j                  |«      }t        |d¬«      }	|j                  |	j                  «       «       ddd«       t        j                  d| j                  j                   |«       Œø |D ]  }
t        |
«       Œ y# 1 sw Y   ŒIxY w)	z<
        Run one iteration of the pipeline schedule
        Nz7Forward-only schedule does not support loss computationr   úForward Úfwd_recvr  Úfwd_sendú[%s] Forwarded microbatch %s)r8   rí   r!  r�   r±   r   r  Úget_fwd_recv_opsr
  r‘   r  Úforward_one_chunkÚget_fwd_send_opsrÏ   r¸   rý   rI   )rQ   rÔ   rÕ   rÁ   rÑ   rÖ   Úfwd_sends_to_waitrˆ   r	  Úworksr  s              r:   rÚ   z'_ScheduleForwardOnly._step_microbatches  sf  € ð Ð! VÐ%7ÜØIóð ð "×/Ñ/°¸ÀJÐPVÓWÑˆ�Ø×Ñ˜w q™z¨9°Q©<Ô8ð 46Ðô �t×+Ñ+Ó,ò 	UˆAÜ  8¨A¨3 Ó0ñ 
9Ø—k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!ŸL™L›Nò *�DÜ# DÕ)ð*ð —‘×-Ñ-¨a°¸±¸YÀq¹\ÔJà—k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!×(Ñ(¨¯©«Ô8÷
9ô �L‰LÐ7¸¿¹×9PÑ9PÐRSÕTð	Uð$ &ò 	"ˆDÜ˜DÕ!ñ	"÷#
9ð 
9ús   Á'B3E"Å"E+	rö   )r<   r=   r>   r*  r•   rt   rÚ   rE   rF   r:   r.  r.  y  s]   „ ñð  $Ø!%Ø"&Ø"Ø#ñ*"à˜‘ð*"ð ˜$‘;ð*"ð ˜4‘Kð	*"ð
 �t‘ð*"ð ô*"rF   r.  c                   óx   — e Zd ZdZ	 	 	 	 	 ddedz  dedz  dedz  dedz  def
d„Zd	eeee	dz     f   dz  fd
„Z
y)r   z^
    The GPipe schedule.
    Will go through all the microbatches in a fill-drain manner.
    NrÔ   rÕ   rÁ   rÑ   rÖ   c           	      óZ  — | j                  ||||«      \  }}| j                  |d   |d   «       g }t        | j                  «      D �]  }t	        d|› �«      5  | j
                  j                  |«      }t        |d¬«      }	|	j                  «       D ]  }
t        |
«       Œ | j
                  j                  |||   ||   |¬«      }| j
                  j                  |«      }t        |d¬«      }	|j                  |	j                  «       «       ddd«       t        j                  d| j
                  j                  |«       | j!                  | j
                  ||«       �Œ |D ]  }
t        |
«       Œ g }t        | j                  «      D �]  }t	        d	|› �«      5  | j
                  j#                  |«      }t        |d
¬«      }	|	j                  «       D ]  }
t        |
«       Œ | j%                  | j
                  |«      }| j
                  j'                  |||| j                  dz
  k(  ¬«       | j
                  j)                  |«      }t        |d¬«      }	|j                  |	j                  «       «       ddd«       t        j                  d| j
                  j                  |«       �Œ! |D ]  }
t        |
«       Œ | j+                  | j
                  |«       | j
                  j-                  | j.                  r| j                  «       yd«       y# 1 sw Y   �ŒxY w# 1 sw Y   Œ«xY w)a6  
        Run one iteration of the pipeline schedule with list of microbatches.
        Will go through all the microbatches according to the GPipe schedule.

        Args:
            microbatches: list of microbatch args.
            return_outputs: whether to return the outputs from the last stage.
        r   r0  r1  r  ©Úsave_forward_outputr2  Nr3  z	Backward Úbwd_recvr   ©rÃ   Úlast_backwardÚbwd_sendz[%s] Backwarded microbatch %s)rí   r!  r�   r±   r   r  r4  r
  r‘   r  r5  r6  rÏ   r¸   rý   rI   rÄ   Úget_bwd_recv_opsrÇ   Úbackward_one_chunkÚget_bwd_send_opsrÓ   Úperform_reduce_gradr¯   )rQ   rÔ   rÕ   rÁ   rÑ   rÖ   r7  rˆ   r	  r8  r  rÀ   Úbwd_sends_to_waitrÃ   s                 r:   rÚ   z ScheduleGPipe._step_microbatches²  så  € ð  "×/Ñ/°¸ÀJÐPVÓWÑˆ�Ø×Ñ˜w q™z¨9°Q©<Ô8ð 46Ðô �t×+Ñ+Ó,ó 	IˆAÜ  8¨A¨3 Ó0ñ 9Ø—k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!ŸL™L›Nò *�DÜ# DÕ)ð*ð Ÿ™×6Ñ6Ø�w˜q‘z 9¨Q¡<À^ð 7ó �ð —k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!×(Ñ(¨¯©«Ô8÷9ô �L‰LÐ7¸¿¹×9PÑ9PÐRSÔTà×$Ñ$ T§[¡[°&¸*ÀaÖHð#	Ið, &ò 	"ˆDÜ˜DÕ!ð	"ð
 46ÐÜ�t×+Ñ+Ó,ó 	VˆAÜ  9¨Q¨C Ó1ñ 9Ø—k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!ŸL™L›Nò *�DÜ# DÕ)ð*ð ×+Ñ+¨D¯K©K¸Ó;�Ø—‘×.Ñ.ØØØ"# t×';Ñ';¸aÑ'?Ñ"?ð /ô ð —k‘k×2Ñ2°1Ó5�Ü)¨#°JÔ?�Ø!×(Ñ(¨¯©«Ô8÷9ô" �L‰LÐ8¸$¿+¹+×:QÑ:QÐSTÖUð%	Vð* &ò 	"ˆDÜ˜DÕ!ð	"ð 	×Ñ˜DŸK™K¨Ô0à�‰×'Ñ'À×@PÒ@P¨×(<Ñ(<ÕXÐVWÕX÷m9ñ 9ú÷89ð 9ús   ÁB5LÆCL!ÌL	Ì!L*	r4   c           	      óö  — i }| j                   }t        |«      D ]Ü  }g }|}|j                  dg|z  «       t        | j                  «      D ],  }|j	                  t        |t        j                  |«      «       Œ. d|dz
  |z
  z  }|j                  dg|z  «       t        | j                  «      D ],  }|j	                  t        |t        j                  |«      «       Œ. t        || j                  «      ||<   ŒÞ |S )zš
        Returns the pipeline order for GPipe schedule.

        See base method in PipelineScheduleSingle for details on the schedule IR format.
        Né   r   )
r  r�   rÏ   r±   rg   rH   r&   r?   rB   Ú_add_reduce_grad)rQ   rx   Úpp_group_sizerš   r}   Úwarmup_delayÚmb_idxÚbackward_delays           r:   r  z!ScheduleGPipe._get_pipeline_order  s  € ð ˆØ×(Ñ(ˆä˜-Ó(ò 	SˆDØ,.ˆGð  ˆLØ�N‰N˜D˜6 LÑ0Ô1ô   × 4Ñ 4Ó5ò P�Ø—‘œw tÔ-=×-EÑ-EÀvÓNÕOðPð  -°!Ñ"3°dÑ":Ñ;ˆNØ�N‰N˜D˜6 NÑ2Ô3ô   × 4Ñ 4Ó5ò V�Ø—‘œw tÔ-=×-KÑ-KÈVÓTÕUðVô $4°G¸T×=QÑ=QÓ#RˆN˜4Ò ð'	Sð* ÐrF   rö   )r<   r=   r>   r*  r•   rt   rÚ   rù   rl   rH   r  rE   rF   r:   r   r   ¬  sŽ   „ ñð  $Ø!%Ø"&Ø"Ø#ñNYà˜‘ðNYð ˜$‘;ðNYð ˜4‘Kð	NYð
 �t‘ðNYð óNYð` T¨#¨t°G¸d±NÑ/CÐ*CÑ%DÀtÑ%Kô rF   r   c                   óö   ‡ — e Zd ZdZ	 	 	 	 	 ddedededz  deedf   dz  de	e
ef   dz  d	e	e
ef   ee   z  dz  d
efˆ fd„Z	 	 	 	 	 ddedz  dedz  dedz  dedz  def
d„Zde	eeedz     f   dz  fd„Zˆ xZS )r   zo
    The 1F1B schedule.
    Will perform one forward and one backward on the microbatches in steady state.
    Nr¿   rª   r«   r¬   .r­   r®   r¯   c           	      óˆ   •— t         ‰| �  |||||||¬«       || j                  k  rt        d|› d| j                  › d�«      ‚y )N)r¿   rª   r«   r¬   r­   r®   r¯   zNumber of microbatches (z9) must be greater than or equal to the number of stages (z).)r  r»   r  r7   r  s	           €r:   r»   zSchedule1F1B.__init__)  sn   ø€ ô 	‰ÑØØ)ØØ+Ø/Ø/Ø#ð 	ô 	
ð ˜D×,Ñ,Ò,ÜØ*¨>Ð*:ð ;#Ø#'×#3Ñ#3Ð"4°Bð8óð ð -rF   rÔ   rÕ   rÁ   rÑ   rÖ   c                 ót  — | j                  ||||«      \  }}| j                  |d   |d   «       t        | j                  | j                  | j
                  j                  z
  «      }d}d}g }	g }
t        |«      D ]¶  }| j
                  j                  |«      }t        t        |d¬«      «       | j
                  j                  |||   ||   |¬«      }t        |	«       | j
                  j                  |«      }
||dz
  k7  rt        |
d¬«      }	| j                  | j
                  |||«       |dz  }Œ¸ 	 | j
                  j                  |«      }t        t        |
|z   d¬«      «       | j                  | j
                  |«      }| j
                  j!                  |||| j                  dz
  k(  ¬«       | j
                  j#                  |«      }|dz  }|| j                  k(  rn™| j
                  j                  |«      }t        t        ||z   d	¬«      «       | j
                  j                  |||   ||   |¬«      }| j                  | j
                  |||«       | j
                  j                  |«      }
|dz  }�ŒGt        |d
¬«      }	|| j                  k  rÃ| j
                  j                  |«      }t        t        |d¬«      «       | j                  | j
                  |«      }| j
                  j!                  |||| j                  dz
  k(  ¬«       t        |	«       | j
                  j#                  |«      }t        |d
¬«      }	|dz  }|| j                  k  rŒÃt        |	«       | j%                  | j
                  |«       | j
                  j'                  | j(                  r| j                  «       yd«       y)a5  
        Run one iteration of the pipeline schedule with list of microbatches.
        Will go through all the microbatches according to the 1F1B schedule.

        Args:
            microbatches: list of microbatch args.
            return_outputs: whether to return the outputs from the last stage.
        r   r1  r  r;  r   r2  Úfwd_send_bwd_recvr>  Úbwd_send_fwd_recvr@  r=  N)rí   r!  Úminr±   r  r  rI   r�   r4  r  r  r5  r6  rÄ   rA  rÇ   rB  rC  rÓ   rD  r¯   )rQ   rÔ   rÕ   rÁ   rÑ   rÖ   Úwarmup_chunksÚfwd_mb_indexÚbwd_mb_indexÚ	send_workÚ	fwd_sendsÚ_Ú	fwd_recvsrÀ   Ú	bwd_recvsrÃ   Ú	bwd_sendss                    r:   rÚ   zSchedule1F1B._step_microbatchesB  s   € ð  "×/Ñ/°¸ÀJÐPVÓWÑˆ�Ø×Ñ˜w q™z¨9°Q©<Ô8ô Ø× Ñ Ø×Ñ˜tŸ{™{×6Ñ6Ñ6ó
ˆð ˆØˆð &(ˆ	Øˆ	Ü�}Ó%ò 	ˆAàŸ™×4Ñ4°\ÓBˆIÜœJ y°zÔBÔCð —[‘[×2Ñ2ØØ˜Ñ%Ø˜,Ñ'Ø$2ð	 3ó ˆFô ˜IÔ&ð Ÿ™×4Ñ4°\ÓBˆIØ˜}¨qÑ0Ò0ä& y°zÔB�	ð
 ×$Ñ$ T§[¡[°&¸*ÀlÔSØ˜AÑ‰Lð;	ðD àŸ™×4Ñ4°\ÓBˆIô œJ y°9Ñ'<ÐCVÔWÔXð ×'Ñ'¨¯©°\ÓBˆDØ�K‰K×*Ñ*ØØØ*¨d×.BÑ.BÀQÑ.FÑFð +ô ð Ÿ™×4Ñ4°\ÓBˆIØ˜AÑˆLà˜t×3Ñ3Ò3àð Ÿ™×4Ñ4°\ÓBˆIô œJ y°9Ñ'<ÐCVÔWÔXð —[‘[×2Ñ2ØØ˜Ñ%Ø˜,Ñ'Ø$2ð	 3ó ˆFð ×$Ñ$ T§[¡[°&¸*ÀlÔSð Ÿ™×4Ñ4°\ÓBˆIØ˜AÑˆLñU ôZ ˜y¨zÔ:ˆ	ð ˜T×1Ñ1Ò1àŸ™×4Ñ4°\ÓBˆIÜœJ y°zÔBÔCð ×'Ñ'¨¯©°\ÓBˆDØ�K‰K×*Ñ*ØØØ*¨d×.BÑ.BÀQÑ.FÑFð +ô ô ˜IÔ&ð Ÿ™×4Ñ4°\ÓBˆIÜ" 9°:Ô>ˆIØ˜AÑˆLð' ˜T×1Ñ1Ó1ô, 	˜	Ô"ð 	×Ñ˜DŸK™K¨Ô0à�‰×'Ñ'À×@PÒ@P¨×(<Ñ(<ÕXÐVWÕXrF   r4   c           	      óÆ  — i }| j                   }t        |«      D �]Ã  }g }|j                  dg|z  «       |dz
  |z
  }d}t        |«      D ].  }|j                  t	        |t
        j                  |«      «       |}Œ0 t        dd|dz
  |z
  z  «      }|j                  dg|z  «       d}	| j                  |z
  }
|
dkD  ri|dz  }|j                  t	        |t
        j                  |«      «       |
dz  }
|j                  t	        |t
        j                  |	«      «       |	dz  }	|
dkD  rŒi| j                  |	z
  }|dkD  r�||z
  dkD  rK|j                  d«       |dkD  ri|j                  t	        |t
        j                  |	«      «       |	dz  }	|dz  }n4|j                  t	        |t
        j                  |	«      «       |	dz  }	|dz  }|dkD  rŒ�t        || j                  «      ||<   �ŒÆ |S )z™
        Returns the pipeline order for 1F1B schedule.

        See base method in PipelineScheduleSingle for details on the schedule IR format.
        Nr   r   r„   )r  r�   rÏ   rg   rH   r&   r?   r�   r±   rB   rH  )rQ   rx   rI  rš   r}   Únum_forwardÚ
forward_mbrˆ   Úwait_for_1f1bÚbackward_mbÚremaining_forwardÚremaining_backwards               r:   r  z Schedule1F1B._get_pipeline_orderÒ  s  € ð ˆØ×(Ñ(ˆä˜-Ó(ó 8	SˆDØ,.ˆGð �N‰N˜D˜6 D™=Ô)ð )¨1Ñ,°Ñ4ˆKØˆJÜ˜;Ó'ò �Ø—‘œw tÔ-=×-EÑ-EÀqÓIÔJØ‘
ðô
    1¨¸Ñ(9¸DÑ(@Ñ#AÓBˆMØ�N‰N˜D˜6 MÑ1Ô2ð ˆKØ $× 4Ñ 4°{Ñ BÐà# aÒ'à˜a‘�
Ø—‘œw tÔ-=×-EÑ-EÀzÓRÔSØ! QÑ&Ð!ð —‘Ü˜DÔ"2×"@Ñ"@À+ÓNôð ˜qÑ �ð $ aÓ'ð "&×!5Ñ!5¸Ñ!CÐà$ qÒ(ð " DÑ(¨AÒ-Ø—N‘N 4Ô(à)¨AÒ-ØŸ™Ü# DÔ*:×*HÑ*HÈ+ÓVôð $ qÑ(˜Ø*¨aÑ/Ñ*ð —N‘NÜ Ô&6×&DÑ&DÀkÓRôð   1Ñ$�KØ&¨!Ñ+Ð&ð% % qÓ(ô( $4°G¸T×=QÑ=QÓ#RˆN˜4Ó ðq8	Sðr ÐrF   rö   )r<   r=   r>   r*  r   rl   r   rh   r   rù   rD   r   rt   r»   r•   rÚ   rH   r  r+  r,  s   @r:   r   r   #  s#  ø„ ñð $(Ø>BØ?CØ@DØ ñà!ðð ðð ˜D‘ð	ð
 ˜°Ð3Ñ4°tÑ;ðð    _Ð 4Ñ5¸Ñ<ðð    S ™>¨E°#©JÑ6¸Ñ=ðð õð6  $Ø!%Ø"&Ø"Ø#ñNYà˜‘ðNYð ˜$‘;ðNYð ˜4‘Kð	NYð
 �t‘ðNYð óNYð`B T¨#¨t°G¸d±NÑ/CÐ*CÑ%DÀtÑ%K÷ BrF   r   Úaction_typec                 ó   — | t         t        fv S rN   )r)   r0   )rc  s    r:   Ú_requires_reduce_gradre    s   € Øœ1œa˜&Ð Ð rF   r}   rª   c           	      óF  ‡‡— g }t        t        «      Šˆˆfd„}| D ]‚  }|€Œ|j                  |«       g }|j                  t        k(  r'|j
                  �|j
                  D ]  } |||«       Œ n	 |||«       |D ]"  }|j                  t        |t        d«      «       Œ$ Œ„ |S )z«
    REDUCE_GRAD refers to joint across minibatches grad reduction.
    reduce_grad frees memory and we want to schedule it just after the last "backward"-like stage.
    c                 ó˜   •— t        | j                  «      r4| j                  }‰|xx   dz  cc<   ‰|   ‰k(  r|j                  |«       y y y ©Nr   )re  rJ   rI   rg   )ÚaÚto_schedulerI   Úcntrª   s      €€r:   Ú_leaf_actionz&_add_reduce_grad.<locals>._leaf_action%  sO   ø€ Ü  ×!3Ñ!3Ô4ØŸ-™-ˆKØ�Ó Ñ!ÓØ�;Ñ >Ò1Ø×"Ñ" ;Õ/ð 2ð 5rF   N)r   rl   rg   rJ   r1   rL   rH   r2   )	r}   rª   Úactions_with_reduce_gradrl  ri  Úschedule_reduce_grad_stage_idxsrZ   Ú	stage_idxrk  s	    `      @r:   rH  rH    s½   ù€ ð 68ÐÜ%¤cÓ*€Cõ0ð ò SˆØˆ9ØØ ×'Ñ'¨Ô*Ø57Ð'Ø×Ñ¤Ò,°·±Ð1JØŸm™mò J�
Ù˜ZÐ)HÕIñJñ ˜Ð;Ô<à8ò 	SˆIØ$×+Ñ+¬G°I¼{ÈDÓ,QÕRñ	SðSð $Ð#rF   Úcompute_actionsÚmax_active_stagesc                 óØ  ‡
‡‡— dt         dt        t        dz     dt        t            fd„}t        «       Š
g Šdt         fˆ
ˆfd„}dt         fˆ
ˆfd„}t	        | «      D ]s  \  }}|€Œ	 ||| |d «      Št        t        ˆ
fd	„‰«      «      }t        t        ˆfd
„‰
«      «      }|D ]
  }	 ||	«       Œ |D ]
  }	 ||	«       Œ ‰j                  |«       Œu t        ‰
«      D ]
  }	 ||	«       Œ ‰S )a]  Given a basic schedule involving only compute actions (F,B,W,OVERLAP_F_B), add UNSHARD/RESHARD actions for FSDP.

    UNSHARD refers to fetching the full contents of an FSDP-sharded layer, requiring an all-gather operation.
    RESHARD does the opposite, releasing memory (but doing no communication)

    We abandon the "timestep lock"  during lowering

    max_active_stages controls how many prefetches we allow. It should be measured in mb and tuneable but in practice
    3 stages is probably the thing we want?
    (to account for having one f and one b active, and something else prefetching?)
    ÚcountÚnext_actionsNr4   c                 óì  — t        «       }g }|D ]â  }|€Œ|j                  t        k(  rt|j                  �h|j                  D ]G  }|j                  |vsŒ|j                  |j                  «       |j                  |j                  «       ŒI t        |«      | k\  sŒŠ |S |j                  |vsŒœ|j                  |j                  «       |j                  |j                  «       t        |«      | k(  sŒá |S  |S )zdRemove duplicates (same stage, different microbatch), find next 'count' stages that will do compute.)ÚsetrJ   r1   rL   rI   Úaddrg   rm   )rs  rt  ÚseenÚretri  rZ   s         r:   Únext_stage_indicesz0_add_unshard_reshard.<locals>.next_stage_indicesL  sß   € ä›ˆØˆàò 	"ˆAØ‰}à×%Ñ%¬Ò4¸¿¹Ð9RØ&'§m¡mò ?˜
Ø%×1Ñ1¸Ò=Ø ŸH™H Z×%;Ñ%;Ô<ØŸJ™J z×'=Ñ'=Õ>ð?ô ˜3“x 5Ó(Øð ˆ
ð —}‘}¨DÒ0ØŸ™ §¡Ô/ØŸ
™
 1§=¡=Ô1Ü˜s›8 uÓ,Ø!Øˆ
ð#	"ð" ˆ
rF   rI   c                 óh   •— ‰j                  | «       ‰j                  t        | t        d «      «       y rN   )rw  rg   rH   r*   ©rI   Úactive_stagesÚfsdp_aware_actionss    €€r:   Ú_unshardz&_add_unshard_reshard.<locals>._unshardg  s(   ø€ Ø×Ñ˜+Ô&Ø×!Ñ!¤'¨+´wÀÓ"EÕFrF   c                 óh   •— ‰j                  | «       ‰j                  t        | t        d «      «       y rN   )Úremoverg   rH   r+   r|  s    €€r:   Ú_reshardz&_add_unshard_reshard.<locals>._reshardk  s(   ø€ Ø×Ñ˜[Ô)Ø×!Ñ!¤'¨+´wÀÓ"EÕFrF   c                 ó   •— | ‰vS rN   rE   )Úsr}  s    €r:   ú<lambda>z&_add_unshard_reshard.<locals>.<lambda>v  s   ø€  a¨}Ð&<€ rF   c                 ó   •— | ‰vS rN   rE   )r„  Únext_ns    €r:   r…  z&_add_unshard_reshard.<locals>.<lambda>x  s   ø€  a¨v o€ rF   )rl   r•   rH   rv  r™   Úfilterrg   )rp  rq  rz  r  r‚  rˆ   r3   ÚfetchÚevictr¿   r}  r~  r‡  s             @@@r:   Ú_add_unshard_reshardr‹  <  s  ú€ ð ¤#ð ´T¼'ÀD¹.Ñ5Ið ÌdÔSVÉió ô0 "›e€MØ(*ÐðGœcö GðGœcö Gô ˜Ó/ò *‰	ˆˆ6Øˆ>Øñ $Ð$5°ÀqÀrÐ7JÓKˆä”VÓ<¸fÓEÓFˆä”VÓ5°}ÓEÓFˆð ò 	ˆEÙ�U�Oð	àò 	ˆEÙ�U�Oð	à×!Ñ! &Õ)ð/*ô4 �mÓ$ò ˆÙ��ðð ÐrF   c                 ó0  — g }| �r| j                  d«      }|€Œt        | «      r'| d   €"| j                  d«       t        | «      r| d   €Œ"t        | «      dkD  r| d   nd}|j                  t        k(  r�|�‹|j                  t        k(  rx|j
                  |j
                  k(  r_|j                  |j                  k(  rF|j                  t        |j
                  t        |j                  «      «       | j                  d«       n|j                  |«       | r�Œ|S )a9  Given a basic schedule involving only compute actions (F,I,W), merge adjacent I and W ops into B ops.
    (note: I = BACKWARD_INPUT, W = BACKWARD_WEIGHT, B = FULL_BACKWARD)

    B refers to running the whole backward (not separating grad_input and grad_weight), which can be more efficient
    in some cases.
    r   N)
Úpoprm   rJ   r@   rA   rI   rK   rg   rH   rB   )rp  Úmerged_actionsr3   Únext_actions       r:   Ú	_merge_bwr�  �  s  € ð €NÚ
Ø ×$Ñ$ QÓ'ˆØˆ>Øô �/Ô" °qÑ'9Ð'AØ×Ñ Ô"ô �/Ô" °qÑ'9Ñ'Aô -0°Ó,@À1Ò,D�o aÒ(È$ˆð ×#Ñ#¤~Ò5ØÐ'Ø×,Ñ,´Ò?Ø×"Ñ" k×&=Ñ&=Ò=Ø×'Ñ'¨;×+GÑ+GÒGà×!Ñ!Ü˜×*Ñ*¬M¸6×;RÑ;RÓSôð ×Ñ Õ"à×!Ñ! &Ô)ó1 ð2 ÐrF   Ústage_to_rankr  c           
      ó  ‡‡‡‡‡‡— | D �ci c]  }|g “Œ }}| D �ci c]  }|t        «       “Œ c}Šdt        dt        fˆˆfd„Šdt        dt        t        t        f   fˆfd„}dt        dz  dt         t           dt        fˆfd„Š| �rod}t	        | «      D �]M  Št        | ‰   «      d	kD  st        d
‰›dt        | ‰   «      ›�«      ‚| ‰   d	   }|�|j                  �|j                  }n|f}t        ˆˆˆfd„|D «       «      sŒo|�³|‰   j                  |«       |D ]š  }	‰‰   j                  |	«        ‰|	«      sŒ  ||	«      \  }
}|‰   j                  |
«       ‰‰   j                  |
«       | ‰|j                  «         j                  |«       ‰ ‰|j                  «         j                  |«       Œœ | ‰   j                  d	«       t        | ‰   «      d	k(  r| ‰= d}�ŒP |st        d«      ‚| r�Œo|S c c}w c c}w )zé
    Transforms a compute-only schedule into a complete schedule with communication actions.

    For actions with sub-actions (OVERLAP_F_B) we ensure that all the subactions have been
    computed and the communication is ready
    r3   r4   c                 óF  •— | j                   t        k(  r<| j                  ‰dz
  k7  xr(  ‰| j                  dz   «       ‰| j                  «      k7  S | j                   t        t        fv r9| j                  dk7  xr(  ‰| j                  dz
  «       ‰| j                  «      k7  S y)Nr   r   F)rJ   r'   rI   r@   rB   )r3   r  r‘  s    €€r:   Ú
_has_commsz"_add_send_recv.<locals>._has_commsÃ  s¯   ø€ Ø×"Ñ"¤aÒ'Ø×%Ñ%¨°a©Ñ7ò 3¹MØ×"Ñ" QÑ&ó=á˜v×1Ñ1Ó2ñ=3ð 3ð ×$Ñ$¬¼Ð(GÑGØ×%Ñ%¨Ñ*ò 3©}Ø×"Ñ" QÑ&ó0á˜v×1Ñ1Ó2ñ03ð 3ð rF   c                 ó&  •—  ‰| «      st        | › d�«      ‚| j                  }| j                  }| j                  }t	        ||t
        k(  rt        nt        |«      }|t
        k(  r|dz   n|dz
  }t	        ||t
        k(  rt        nt        |«      }||fS )Nz is not a valid comm actionr   )
ÚAssertionErrorrI   rJ   rK   rH   r'   r,   r.   r-   r/   )r3   ro  ÚctyperK  ÚsendÚrecv_stage_idxÚrecvr”  s          €r:   Ú
_get_commsz"_add_send_recv.<locals>._get_commsÎ  sŠ   ø€ Ù˜&Ô!Ü  F 8Ð+FÐ!GÓHÐHØ×&Ñ&ˆ	Ø×'Ñ'ˆØ×(Ñ(ˆÜ�y¨E´QªJ¥&¼FÀFÓKˆØ*/´1ª*˜ Qš¸)Àa¹-ˆÜ�~°¼!²¥vÄÈÓPˆØ�TˆzÐrF   NÚprev_actionsc                 óJ  •— | €y| j                   t        k(  rc| j                  dk7  rTt        | j                  t        | j
                  «      |v ryt        | j                  dz
  t        | j
                  «      |v ryy| j                   t        t        fv r‘| j                  ‰dz
  k7  rt        | j                  t        | j
                  «      |v ryt        | j                  dz   t        | j
                  «      |v ryt        | j                  dz   t        | j
                  «      |v ryyy)a  We don't put our own recv ops in the schedule, we let a sender on another rank put our recv ops in place.
        This helps ensure a sane (non-hanging) ordering of sends and recvs.
        But it also means we might not be able to schedule our next compute action yet.
        Tr   r   F)	rJ   r'   rI   rH   r-   rK   r@   rB   r/   )r3   rœ  r  s     €r:   Ú_ready_to_schedulez*_add_send_recv.<locals>._ready_to_scheduleÙ  s   ø€ ð
 ˆ>ØØ×$Ñ$¬Ò)¨f×.@Ñ.@ÀAÒ.Eä˜×*Ñ*¬F°F×4KÑ4KÓLØñ ð ä˜×*Ñ*¨QÑ.´°6×3JÑ3JÓKØñ ð Øà×#Ñ#¬¼Ð'FÑFØ×"Ñ" j°1¡nÒ4ô ˜×*Ñ*¬F°F×4KÑ4KÓLØñ ð ä˜×*Ñ*¨QÑ.´À×@WÑ@WÓXØñ ð ä˜×*Ñ*¨QÑ.´¸v×?VÑ?VÓWØñ ð ØàrF   Fr   úrank=z, len(compute_actions[rank])=c              3   ó6   •K  — | ]  } ‰|‰‰   «      –— Œ y ­wrN   rE   )r|   ri  rž  rœ  rš   s     €€€r:   r~   z!_add_send_recv.<locals>.<genexpr>  s   øè ø€ ÒVÀQÑ)¨!¨\¸$Ñ-?×@ÑVùs   ƒTz6Malformed compute schedule, can't schedule sends/recvs)rv  rH   rt   rh   r“   rm   r–  rL   Úallrg   rw  rI   r�  )rp  r‘  r  rš   Úcomm_actionsr›  Úprogressr3   Úall_actionsri  r˜  rš  r”  rž  rœ  s    ```        @@@r:   Ú_add_send_recvr¥  µ  s8  ý€ ð DSÖ-S¸4¨d°B©hÐ-S€LÐ-SØETÖ,U¸T¨T´3³5©[Ò,U€Lð	œ7ð 	¤tö 	ð	œ7ð 	¤u¬W´gÐ-=Ñ'>õ 	ð(¤7¨T¡>ð (ÄÄWÁð (ÔRVõ (òT Øˆä˜?Ó+ó 	ˆDÜ˜¨Ñ-Ó.°Ò2Ü$¨¨ wÐ.L´°_ÀTÑ5JÓ1KÐ0MÐ%NÓOÐOØ$ TÑ*¨1Ñ-ˆFàÐ! f×&8Ñ&8Ð&DØ$×0Ñ0‘à%˜i�äÕVÈ+ÔVÔVØð Ð!Ø˜TÑ"×)Ñ)¨&Ô1Ø$ò 	P�AØ  Ñ&×*Ñ*¨1Ô-Ù! !•}Ù%/°£]™
˜˜dð % TÑ*×1Ñ1°$Ô7Ø$ TÑ*×.Ñ.¨tÔ4Ø$¡]°4×3CÑ3CÓ%DÑE×LÑLÈTÔRØ$¡]°4×3CÑ3CÓ%DÑE×IÑIÈ$ÕOð	Pð ˜DÑ!×%Ñ% aÔ(Ü�? 4Ñ(Ó)¨QÒ.Ø# DÐ)ØŠHð=	ñ> Ü ØHóð óG ðL ÐùòS .TùÚ,Us
   ‹
G8›G=rI  Únum_microbatchesc                 óì  ‡ ‡‡— t        ‰ «      |k(  st        d|› dt        ‰ «      › �«      ‚t        |«      D ]  }|‰ vsŒt        d|› �«      ‚ t        |«      D �ci c]=  }|t        t	        «       t
        t	        «       t        t	        «       t        t	        «       i“Œ? c}Ši Šdt        dt        dt        fˆ ˆˆfd„}‰ D ]r  }t        ‰ |   «      D ]_  \  }}|€Œ	t        |t        «      st        d|› d	|› d
|› d�«      ‚|j                  �|j                  D ]  }	 ||	||«       Œ ŒV ||||«       Œa Œt ‰D ]·  }
t        ‰|
   t           «      }t        ‰|
   t
           «      }t        ‰|
   t           «      }t        ‰|
   t           «      }||k(  st        d|› dt        › d|
› d|› �«      ‚||k(  st        d|
› d|› d|› �«      ‚|||z   dz  z   |k(  rŒŸt        d|
› d|› d|› d|› d|› �
«      ‚ ‰S c c}w )Nz2Schedule has incorrect number of ranks - expected z	, actual z%Schedule is missing actions for rank r3   rš   rá   c                 ó:  •— | j                   }| j                  }| j                  }|t        k(  r‰|   t           j	                  |«       �n"|t
        k(  rX|‰|   t           vr.d|› d|› d|› d|› d�	}t        ‰
|¬«      }|› d|› �}t        |«      ‚‰|   t
           j	                  |«       nÁ|t        k(  rX|‰|   t           vr.d|› d|› d|› d|› d�	}t        ‰
|¬«      }|› d|› �}t        |«      ‚‰|   t           j	                  |«       n`|t        k(  rW|‰|   t           vr.d|› d|› d	|› d|› d
�	}t        ‰
|¬«      }|› d|› �}t        |«      ‚‰|   t           j	                  |«       |‰vr|‰|<   y‰|   }	||	k(  st        d|› d|› d|› d|› d|	› �
«      ‚y)zPProcess a single action and update stage_actions and stage_index_to_rank_mappingr€   ú, step z": Running Full Backward for stage z, microbatch z without first running Forward©ry   z

Full pipeline schedule:
z#: Running Backward Input for stage z$: Running Backward Weight for stage z% without first running Backward Inputz: Stage z is assigned to both rank z
 and rank N)
rI   rJ   rK   r'   rw  r0   r§   r–  r(   r)   )r3   rš   rá   Ús_idr—  Úmb_idÚ	error_msgÚformatted_scheduleÚfull_error_msgÚexisting_rankr}   Ústage_actionsÚstage_index_to_rank_mappings             €€€r:   Ú_process_actionz+_validate_schedule.<locals>._process_actionG  sh  ø€ à×!Ñ!ˆØ×'Ñ'ˆØ×'Ñ'ˆà”AŠ:Ø˜$Ñ¤Ñ"×&Ñ& uÖ-Ø”aŠZØ˜M¨$Ñ/´Ñ2Ñ2à˜D˜6 ¨¨Ð.PÐQUÐPVð W"Ø"' Ð(FðHð ô &<Ø¨tô&Ð"ð !�kÐ!>Ð?QÐ>RÐSð ô % ^Ó4Ð4Ø˜$Ñ¤Ñ"×&Ñ& uÕ-Ø”aŠZØ˜M¨$Ñ/´Ñ2Ñ2à˜D˜6 ¨¨Ð.QÐRVÐQWð X"Ø"' Ð(FðHð ô &<Ø¨tô&Ð"ð !�kÐ!>Ð?QÐ>RÐSð ô % ^Ó4Ð4Ø˜$Ñ¤Ñ"×&Ñ& uÕ-Ø”aŠZØ˜M¨$Ñ/´Ñ2Ñ2à˜D˜6 ¨¨Ð.RÐSWÐRXð Y"Ø"' Ð(MðOð ô &<Ø¨tô&Ð"ð !�kÐ!>Ð?QÐ>RÐSð ô % ^Ó4Ð4Ø˜$Ñ¤Ñ"×&Ñ& uÔ-àÐ2Ñ2Ø04Ð'¨Ò-à7¸Ñ=ˆMØ˜MÒ)Ü$Ø˜D˜6 ¨¨¨h°t°fÐ<VÐW[ÐV\Ð\fÐgtÐfuÐvóð ð *rF   r€   r©  z: Got an invalid action: z, expected instance of _ActionzGot rƒ   z microbatches for stage z, expected z(Invalid backward microbatches for stage z8: I and W must have equal counts,             but got I=z, W=r„   z: expected z( total backwards,             but got B=z, I=)rm   r–  r�   r'   rv  r0   r(   r)   rH   rl   r™   rÌ   rL   )r}   rI  r  r¦  rš   Ústage_idr³  rá   r3   rZ   r«  Úf_mbÚb_mbÚi_mbÚw_mbr±  r²  s   `              @@r:   Ú_validate_scheduler¹  ,  sž  ú€ ô �‹L˜MÒ)ÜØ@ÀÀÈyÔY\Ð]dÓYeÐXfÐgó
ð 	
ô �mÓ$ò QˆØ�wÒÜ Ð#HÈÈÐ!OÓPÐPðQô ˜jÓ)ö=ð ð 	ÜŒs‹uÜŒs‹uÜŒs‹uÜŒs‹uð	
ñ 	
ò=€Mð #%Ðð:¤ð :¬sð :¼#÷ :ðx ò 4ˆÜ% g¨d¡mÓ4ò 	4‰LˆD�&Øˆ~ØÜ˜f¤gÔ.Ü$Ø˜D˜6 ¨¨Ð.GÈÀxÐOmÐnóð ð
 ×!Ñ!Ð-à"(×"4Ñ"4ò <�JÙ# J°°dÕ;ñ<ñ   ¨¨dÕ3ñ	4ð4ð$ ò ˆÜ�= Ñ&¤qÑ)Ó*ˆÜ�= Ñ&¤qÑ)Ó*ˆÜ�= Ñ&¤qÑ)Ó*ˆÜ�= Ñ&¤qÑ)Ó*ˆàÐ(Ò(Ü Ø�t�f˜Aœa˜SÐ 8¸¸¸kÐJZÐI[Ð\óð ð ˜’Ü Ø:¸4¸&ð AØ�f˜D  ð(óð ð
 ˜˜t™¨Ñ)Ñ)Ð-=Ó=Ü Ø:¸4¸&ÀÐL\ÐK]ð ^Ø�f˜D   d¨4¨&ð2óð ð%ð, 'Ð&ùò_=s   ÁAG1c                   óP  ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddee   dededz  dee	df   dz  d	e
ee	f   dz  d
e
eef   ee   z  dz  dedz  dedefˆ fd„Zdeedf   fd„Zde
eeedz     f   ddfd„Zd„ Zdd„Zddddœdedz  defd„Z	 	 	 	 	 d dedz  dedz  dedz  dedz  def
d„Zˆ xZS )!r   aX  
    Base class for multi-stage schedules.
    Implements the `step` method.

    Gradients are scaled by num_microbatches depending on the `scale_grads` argument, defaulting to True.  This setting
    should match the configuration of your loss_fn, which may either average losses (scale_grads=True)
    or sum losses (scale_grads=False).
    NTrÐ   rª   r«   r¬   .r­   r®   Úuse_full_backwardr¯   Úbackward_requires_autogradc
                 óÞ  •‡— t         ‰| �  ||||||¬«       || _        |d   j                  | _        |d   j
                  | _        |d   j                  | _        t        | j                  | j                  «      | _
        | j                  D ]  }
| j                  |
_
        Œ d| _        d| _        | j                  d uŠˆfd„| _        i | _        |	| _        |�t"        j%                  d«       y y )Nr  r   Fc                 ó$   •— | j                   xr ‰S rN   rÊ   )r¿   Úhas_losss    €r:   r…  z0PipelineScheduleMulti.__init__.<locals>.<lambda>Þ  s   ø€ °%·-±-Ò2LÀH€ rF   zDeprecation warning: 'use_full_backward' is no longer supported. Simply stop passing it, and everything should still work fine.)r  r»   Ú_stagesr  r  Ú
group_sizerI  Ú
group_rankrš   r   Ústage_index_to_group_rankÚ_stages_forward_initializedÚ_stages_backward_initializedr²   Ú_should_compute_lossrx   Ú_backward_requires_autogradr¸   Úwarning)rQ   rÐ   rª   r«   r¬   r­   r®   r»  r¯   r¼  r¿   r¿  rº   s              @€r:   r»   zPipelineScheduleMulti.__init__¸  s  ù€ ô 	‰ÑØ)ØØ+Ø/Ø/Ø#ð 	ô 	
ð ˆŒØ! !™9×/Ñ/ˆÔØ# A™Y×1Ñ1ˆÔØ˜1‘I×(Ñ(ˆŒ	ä)GØ×Ñ × 0Ñ 0ó*
ˆÔ&ð —\‘\ò 	MˆEØ.2×.LÑ.LˆEÕ+ð	Mð ,1ˆÔ(Ø,1ˆÔ)ð Ÿ™¨dÐ2ˆÛ$LˆÔ!ð @BˆÔð
 ,FˆÔ(àÐ(Ü�N‰NðQõð )rF   rß   c                 ó  — | j                   s¯g }| j                  D ]!  }|j                  |j                  «       «       Œ# t	        t        |«      «       t        «       }| j                  D ]I  }|j                  r|j                  | j                  ||«      }Œ-|j                  | j                  ||«      }ŒK d| _         | j                  rA| j                  s4| j                  D ]  }|j                  | j                  «       Œ d| _        y y y r  )rÄ  rÀ  rÏ   r  r  r  rh   Úis_firstr  r±   r¶   rÅ  r  )rQ   rß   rà   r   r¿   Únext_stage_argss         r:   Ú_initialize_stagesz(PipelineScheduleMulti._initialize_stagesî  sý   € Ø×/Ò/ð )+ˆGØŸ™ò D�Ø—‘˜u×@Ñ@ÓBÕCðDäœJ wÓ/Ô0ô 05«wˆOØŸ™ò �Ø—>’>Ø&+×&BÑ&BØ×,Ñ,¨d°Fó'‘Oð ',×&BÑ&BØ×,Ñ,¨o¸vó'‘Oðð 04ˆDÔ,à×Ò d×&GÒ&GØŸ™ò D�Ø×-Ñ-¨d×.BÑ.BÕCðDà04ˆDÕ-ð 'HÐrF   r}   r4   c                 óª   — t        || j                  | j                  | j                  «      | _        | j
                  D ]  }| j                  |_        Œ y)z]
        Allocates the stage index to rank mapping which is needed for communication
        N)r¹  rI  r  r±   rÃ  rÀ  )rQ   r}   r¿   s      r:   Ú_validate_and_set_stage_mappingz5PipelineScheduleMulti._validate_and_set_stage_mapping  sT   € ô *<ØØ×ÑØ×ÑØ× Ñ ó	*
ˆÔ&ð —\‘\ò 	MˆEØ.2×.LÑ.LˆEÕ+ñ	MrF   c                 óÔ   — t        |dd¬«      5 }t        j                  |«      }| j                  D ]   }|j	                  | j                  |   «       Œ" 	 ddd«       y# 1 sw Y   yxY w)úQDump a CSV representation of the schedule into a file with the provided filename.r  ra   ©ÚnewlineN)ÚopenÚcsvÚwriterrx   Úwriterow)rQ   ÚfilenameÚcsvfilerÕ  rš   s        r:   Ú	_dump_csvzPipelineScheduleMulti._dump_csv  s_   € ä�(˜C¨Ô,ð 	;°Ü—Z‘Z Ó(ˆFØ×+Ñ+ò ;�Ø—‘ × 3Ñ 3°DÑ 9Õ:ñ;÷	;÷ 	;ñ 	;ús   �AAÁA'c                 ó`  — |dk7  rt        d|› �«      ‚t        |d¬«      5 }t        j                  |«      }t	        |«      D ]5  \  }}|D �cg c]  }t
        j                  |«      ‘Œ c}| j                  |<   Œ7 	 ddd«       | j                  | j                  «       yc c}w # 1 sw Y   Œ*xY w)z÷Load a CSV representation of the schedule from a file with the provided filename.
        This API will most likely get renamed/refactored so is marked as internal for now.

        format must be "compute_only" for PipelineScheduleMulti.
        Úcompute_onlyz#format must be "compute_only", got ra   rÑ  N)	r–  rÓ  rÔ  Úreaderr™   rH   r;   rx   rÎ  )rQ   r×  ÚformatrØ  rÜ  rš   r¤   r„  s           r:   Ú	_load_csvzPipelineScheduleMulti._load_csv!  s®   € ð �^Ò#Ü Ð#FÀvÀhÐ!OÓPÐPÜ�( BÔ'ð 	O¨7Ü—Z‘Z Ó(ˆFÜ& vÓ.ò O‘	��cØJMÖ,NÀQ¬W×-=Ñ-=¸aÕ-@Ò,N�×#Ñ# DÒ)ñO÷	Oð 	×,Ñ,¨T×-@Ñ-@ÕAùò	 -O÷	Oð 	Oús   ¡+B$ÁBÁ(B$ÂB$Â$B-rÛ   rÑ   rÖ   c                ó"  — | j                   r+| j                  rt        j                  «       st	        d«      ‚| j
                  D ]  }| j                   |_        Œ | j
                  D ]  }|j                  «        Œ | j                  ||«      \  }}|�*t        t        j                  || j                  «      «      }	nd}	| j                  |||	||«       | j
                  D ].  }|j                  sŒ|sŒ| j                  |j                  «      c S  yr#  )r¶   rÇ  r÷   r$  r8   rÀ  r%  r&  rò   r•   r'  r±   rÚ   r½   rõ   ró   )
rQ   rÜ   rÑ   rÖ   rß   rà   r¿   rð   rñ   r(  s
             r:   rá   zPipelineScheduleMulti.step2  s  € ð( ×ÒØ×0Ò0Ü×)Ñ)Ô+äð.óð ð —\‘\ò 	4ˆEØ!%×!3Ñ!3ˆEÕð	4ð —\‘\ò 	)ˆEØ×&Ñ&Õ(ð	)ð $(×#5Ñ#5°d¸FÓ#CÑ ˆ
�Lð ÐÜ ¤×!3Ñ!3°F¸D×<PÑ<PÓ!QÓR‰Mà ˆMð 	×ÑØ˜ m°V¸^ô	
ð
 —\‘\ò 	@ˆEØ�}‹}¢Ø×*Ñ*¨5×+>Ñ+>Ó?Ò?ð	@ð rF   rÔ   rÕ   rÁ   c           
      óÈ
  — | j                  ||||«      \  }}| j                  |d   |d   «       | j                  D �ci c]  }|j                  |“Œ }}t	        «       }t	        «       }	|D ]\  }
|
dkD  r!|j                  | j                  |
dz
     «       |
| j                  dz
  k  sŒ<|	j                  | j                  |
dz      «       Œ^ t        «       }t        | j                  | j                     «      D �]Ê  \  }}	 g }|��,|j                  }|j                  }|j                  }
|€t        d«      ‚|t        j                   k(  rV||
   }|j#                  |||   ||   |¬«      }| j%                  ||||«       |j'                  |j)                  |«      «       �n’|t        j*                  k(  r™||
   }| j-                  ||«      }||
xx   dz  cc<   ||
   | j.                  k(  }| j0                  r| j.                  nd}|j3                  ||d|¬«       |r|j1                  |«       |j'                  |j5                  |«      «       næ|t        j6                  k(  rM||
   }| j-                  ||«      }|j3                  ||dd¬«       |j'                  |j5                  |«      «       n†|t        j8                  k(  re||
   }||
xx   dz  cc<   ||
   | j.                  k(  }| j0                  r| j.                  nd}|j;                  ||¬	«       |r |j1                  |«       nt=        d
|› �«      ‚|D ]¿  }| j                  |   }d}|t?        |«      k  r||   }|€Œ*|j                  }|j                  }|j                  }
|€t        d«      ‚|t        j                   k(  r1|
dz   |v sŒv||
dz      }|j'                  |jA                  |«      «       ŒŸ|t*        t6        t8        fv rŒ³t=        d
|› �«      ‚ |	D ]µ  }| j                  |   }d}|t?        |«      k  r||   }|€Œ*|j                  }|j                  }|j                  }
|€t        d«      ‚|t         t8        fv rŒj|t6        t*        fv r1|
dz
  |v sŒ€||
dz
     }|j'                  |jC                  |«      «       Œ©t=        d
|› �«      ‚ tE        tG        |«      «       �ŒÍ | jW                  | j                  |«       yc c}w # tH        $ rs}tJ        jM                  d| j                  | jN                  jP                  tS        |«      ||«       tJ        jM                  dtU        | j                  |¬«      «       |‚d}~ww xY w)úì
        Operate on the microbatches for looped schedules (multiple stages on each rank).

        TODO: Does not use sorted_batch_isend_irecv(). As a result, this schedule does
        not support models with skip connections.
        r   r   NzCAll currently supported action types require valid microbatch_indexr;  T©rÃ   Úfull_backwardr?  F©r?  zUnknown computation type zi[Rank %s] pipeline schedule %s caught the following exception '%s' at time_step %s when running action %sz%srª  ),rí   rÌ  rÀ  rI   rv  rw  rÃ  r  r   r™   rx   rš   rJ   rK   r–  r&   r?   r5  rÄ   rÏ   r6  rB   rÇ   r±   r¯   rB  rC  r@   rA   Úbackward_weight_one_chunkr7   rm   r4  rA  r  r  Ú	Exceptionr¸   Úerrorrº   r<   rD   r§   rÓ   )rQ   rÔ   rÕ   rÁ   rÑ   rÖ   r¿   Ústage_index_to_stageÚall_prev_ranksÚall_next_ranksrI   Úbackward_counterÚ	time_stepr3   r	  rJ   rÂ   rÀ   rÃ   r?  Úgrad_scale_factorÚ	prev_rankÚprev_rank_opsÚprev_rank_actionÚ	next_rankÚnext_rank_opsÚnext_rank_actionÚes                               r:   rÚ   z(PipelineScheduleMulti._step_microbatchesm  s  € ð "×/Ñ/°¸ÀJÐPVÓWÑˆ�à×Ñ ¨¡
¨I°a©LÔ9ð
 37·,±,ö?
Ø).ˆE×Ñ˜uÑ$ð?
Ðð ?
ô $'£5ˆÜ#&£5ˆØ/ò 	TˆKà˜QŠØ×"Ñ" 4×#AÑ#AÀ+ÐPQÁ/Ñ#RÔSØ˜T×-Ñ-°Ñ1Ó1Ø×"Ñ" 4×#AÑ#AÀ+ÐPQÁ/Ñ#RÕSð	Tô *1«ÐÜ!*¨4×+>Ñ+>¸t¿y¹yÑ+IÓ!Jó Z	ÑˆI�vðYØ(*�ØÑ%Ø'-×'>Ñ'>Ð$Ø%×6Ñ6�HØ"(×"4Ñ"4�KØÐ'Ü,Øaóð ð (Ô+;×+CÑ+CÒCà 4°[Ñ A˜Ø!&×!8Ñ!8Ø$Ø# HÑ-Ø% hÑ/Ø0>ð	 "9ó "˜ð ×0Ñ0°¸À
ÈHÔUØŸ
™
 5×#9Ñ#9¸(Ó#CÖDØ)Ô-=×-KÑ-KÒKà 4°[Ñ A˜Ø#×3Ñ3°E¸8ÓD˜Ø(¨Ó5¸Ñ:Ó5à,¨[Ñ9¸T×=QÑ=QÑQð &ð 59×4DÒ4D˜D×0Ò0È!ð *ð ×0Ñ0Ø$Ø!%Ø*.Ø*7ð	 1ô ñ )Ø!×-Ñ-Ð.?Ô@àŸ
™
 5×#9Ñ#9¸(Ó#CÕDØ)Ô-=×-LÑ-LÒLà 4°[Ñ A˜Ø#×3Ñ3°E¸8ÓD˜Ø×0Ñ0Ø$Ø!%Ø*/Ø*/ð	 1ô ð Ÿ
™
 5×#9Ñ#9¸(Ó#CÕDØ)Ô-=×-MÑ-MÒMà 4°[Ñ A˜Ø(¨Ó5¸Ñ:Ó5à,¨[Ñ9¸T×=QÑ=QÑQð &ð 59×4DÒ4D˜D×0Ò0È!ð *ð ×7Ñ7Ø$Ø*7ð 8ô ñ )Ø!×-Ñ-Ð.?Õ@ä(Ð+DÐEUÐDVÐ)WÓXÐXð "0ò �IØ$(×$7Ñ$7¸	Ñ$B�MØ'+Ð$Ø ¤3 }Ó#5Ò5Ø+8¸Ñ+CÐ(Ø'Ñ3Ø+;×+LÑ+LÐ(Ø#3×#DÑ#D˜Ø&6×&BÑ&B˜Ø#Ð+Ü"0Ø eó#ð ð ,Ô/?×/GÑ/GÒGà*¨Q™Ð2FÒFð )=¸[È1¹_Ñ(M Ø #§
¡
¨5×+AÑ+AÀ(Ó+KÕ LØ-Ü)Ü*Ü+ð2ñ ð !ä",Ø";Ð<LÐ;MÐ Nó#ð ð;ð@ "0ò �IØ$(×$7Ñ$7¸	Ñ$B�MØ'+Ð$Ø ¤3 }Ó#5Ò5Ø+8¸Ñ+CÐ(Ø'Ñ3Ø+;×+LÑ+LÐ(Ø#3×#DÑ#D˜Ø&6×&BÑ&B˜Ø (Ð 4Ü"0Ø eó#ð ð ,´¼Ð/IÑIà Ø-´.Ä-Ð1PÑPà*¨Q™Ð2FÒFð )=¸[È1¹_Ñ(M Ø #§
¡
¨5×+AÑ+AÀ(Ó+KÕ Lä",Ø";Ð<LÐ;MÐ Nó#ð ð3ô<  ¤
¨3£Ö0ðSZ	ðx 	×Ñ˜DŸL™L¨&Õ1ùòY?
øôt ò Ü—‘ð(à—I‘IØ—N‘N×+Ñ+Ü˜“FØØôô —‘ØÜ*Ø×+Ñ+¸yôôð �ûð!ús;   ¾S Ã=IS%ÍAS%Î&A7S%ÐAS%Ñ4AS%Ó%	U!Ó.A.UÕU!)NNNNNTT©rÛ  rö   )r<   r=   r>   r*  r•   r   rl   r   rh   r   rù   rD   r   rt   r»   rÌ  rH   rÎ  rÙ  rÞ  rá   rÚ   r+  r,  s   @r:   r   r   ®  sŽ  ø„ ñð $(Ø>BØ?CØ@DØ)-Ø Ø+/ñ4àÐ'Ñ(ð4ð ð4ð ˜D‘ð	4ð
 ˜°Ð3Ñ4°tÑ;ð4ð    _Ð 4Ñ5¸Ñ<ð4ð    S ™>¨E°#©JÑ6¸Ñ=ð4ð   $™;ð4ð ð4ð %)õ4ðl5 u¨S°#¨X¡ó 5ð:MØ˜C  g°¡nÑ!5Ð5Ñ6ðMà	óMò;óBð( Ø"Ø#ò9ð �t‘ð	9ð
 ó9ðz  $Ø!%Ø"&Ø"Ø#ñ@2à˜‘ð@2ð ˜$‘;ð@2ð ˜4‘Kð	@2ð
 �t‘ð@2ð ÷@2rF   r   c                   óx   — e Zd ZU dZeed<   dZee   dz  ed<   dZ	ee
   dz  ed<   dZedz  ed<   dZedz  ed<   y)Ú_PipelineContextz=Context passed to custom functions during pipeline execution.Úschedule_refNrÔ   rÕ   rÁ   rÑ   )r<   r=   r>   r*  r©   rr   rÔ   r•   rh   rÕ   rù   rÁ   rÑ   rE   rF   r:   r÷  r÷  0  sM   … áGà#Ó#Ø"&€GˆT�%‰[˜4ÑÓ&Ø#'€Iˆt�D‰z˜DÑ Ó'Ø"€J��t‘Ó"Ø€FˆD�4‰KÔrF   r÷  c                   ó    — e Zd Zdededdfd„Zy)Ú_CustomFunctionProtocolr3   Úctxr4   Nc                  ó   — y rN   rE   )rQ   r3   rû  s      r:   Ú__call__z _CustomFunctionProtocol.__call__<  s   � rF   )r<   r=   r>   rH   r÷  rý  rE   rF   r:   rú  rú  ;  s   „ ØK˜wÐKÐ-=ÐKÀ$ÔKrF   rú  c                   óÞ   ‡ — e Zd ZdZˆ fd„Zdededdfd„Z	 ddee	e
edz     f   d	efˆ fd
„Zdded	efˆ fd„Zdded	efd„Zd„ Zdefd„Z	 	 	 	 	 dde
dz  de
dz  de
dz  de
dz  def
d„Zˆ xZS )Ú_PipelineScheduleRuntimea%  
    Provides a simple runtime that requires a 'schedule IR' including specified communication operations.

    Can be instantiated directly by creating _PipelineScheduleRuntime and calling load_csv, or can be
    subclassed and the subclass can be responsible for creating a schedule IR.
    c                 ó²   •— t        ‰| �  |i |¤Ž i | _        t        «       | _        i | _        i | _        t        t        «      | _	        t        «       | _        y rN   )r  r»   Ú_comp_type_to_function_mapr   rë  Úbwd_recv_opsÚfwd_recv_opsr   r•   Úunshard_opsrv  Úunsharded_stages)rQ   rß   rà   rº   s      €r:   r»   z!_PipelineScheduleRuntime.__init__G  sU   ø€ Ü‰Ñ˜$Ð) &Ò)àLNˆÔ'ä.5«iˆÔð EGˆÔØDFˆÔô <GÄtÓ;LˆÔÜ #£ˆÕrF   rJ   Úcustom_functionr4   Nc           	      óà   — |t         t        t        t        t        t
        t        t        fvrt        d|› d�«      ‚|| j                  v rt        j                  d|«       || j                  |<   y)a~  
        Register a custom function to be executed for a specific computation type.

        Args:
            computation_type: The computation type for which to register the custom function
            custom_function: The function to execute when this computation type is encountered.
                Must have signature: (action: _Action, ctx: _PipelineContext) -> None
        r6   zŒ. Only FORWARD, FULL_BACKWARD,                 BACKWARD_INPUT, BACKWARD_WEIGHT, OVERLAP_F_B, UNSHARD, RESHARD and REDUCE_GRAD are supported.zTComputation type %s is already registered. Overwriting the existing custom function.N)r?   rB   r@   rA   r1   r*   r+   r2   r7   r  r¸   rÈ  )rQ   rJ   r  s      r:   Úregister_custom_functionz1_PipelineScheduleRuntime.register_custom_functionV  s…   € ð ÜÜÜÜÜÜÜÜð	$
ñ 	
ô Ø+Ð,<Ð+=ð >nð oóð ð ˜t×>Ñ>Ñ>Ü�N‰Nð<à ôð =Lˆ×'Ñ'Ð(8Ò9rF   r}   rÝ  c                 ó¬  •‡ — t         ‰‰ �  |«       i ‰ _        |dk(  rX|D ]R  }g ‰ j                  |<   ||   D ]9  }|€t        dt	        |«      › �«      ‚‰ j                  |   j                  |«       Œ; ŒT y|dk(  rÌ|j                  «       D ]<  \  }}t        |«      D ])  \  }}|€Œ	|j                  rŒt        d|› d|› d|› d�«      ‚ Œ> |D ]M  }t        ||   «      ‰ j                  |<   t        ‰ j                  |   ‰ j                  «      ‰ j                  |<   ŒO t        ‰ j                  ˆ fd	„‰ j                  ¬
«      ‰ _        yt        d|›d�«      ‚)zé
        Given an in-memory representation for a simple compute-only schedule, lower it to a complex schedule including
        communication actions.  Stores the schedule in self, and must be called before running step_mo()
        Úcompute_commsNz$Expected action to be not None, got rÛ  z?Expected compute-only schedule but found communication action 'z
' at rank z, position ze. Communication actions (e.g. SEND_F, RECV_F, etc.) should not be present when format='compute_only'.c                 ó"   •— ‰j                   |    S rN   ©rÃ  ©r„  rQ   s    €r:   r…  zG_PipelineScheduleRuntime._prepare_schedule_with_comms.<locals>.<lambda>­  s   ø€ ¨×(FÑ(FÀqÑ(I€ rF   )r‘  r  úformat=ú is not implemented)r  rÎ  Úpipeline_order_with_commsr–  rê   rg   r  r™   r^   r7   r‹  rH  r±   r¥  r  rÙ   )rQ   r}   rÝ  rš   r3   Úaction_listrˆ   rº   s   `      €r:   Ú_prepare_schedule_with_commsz5_PipelineScheduleRuntime._prepare_schedule_with_comms}  s¥  ù€ ô 	‰Ñ/°Ô8àCEˆÔ&Ø�_Ò$Øò H�Ø79�×.Ñ.¨tÑ4Ø% d™mò H�FØ�~Ü,ØBÄ4ÈÃ<À.ÐQóð ð ×2Ñ2°4Ñ8×?Ñ?ÀÕGñHñHð �~Ò%à%,§]¡]£_ò 	Ñ!��kÜ!*¨;Ó!7ò ‘I�A�vØÑ)Ø%×3Ó3Ü",ð#$Ø$* 8¨:°d°V¸;ÀqÀcð JTð!Uó#ð ñð	ð  ò �Ü7KØ˜D‘Mó8�×.Ñ.¨tÑ4ô 8HØ×2Ñ2°4Ñ8Ø×(Ñ(ó8�×.Ñ.¨tÒ4ð	ô .<Ø×.Ñ.ÛIØ×+Ñ+ô.ˆDÕ*ô &¨¨¨	Ð1DÐ&EÓFÐFrF   r×  c                 ó¢  •— |dk(  r+t         ‰	| �  |«       | j                  | j                  «       y|dk(  rzi }t	        |d¬«      5 }t        j                  |«      }t        |«      D ]+  \  }}|D �cg c]  }t        j                  |«      ‘Œ c}||<   Œ- | j                  ||¬«       ddd«       yt        d|›d�«      ‚c c}w # 1 sw Y   yxY w)	a	  Loads a csv in simple format and then lowers it to include communication actions

        format must be either "compute_only" or "compute_comms".  If compute_only, the lowering passes
        will automatically be run to generate a compute_comms schedule.
        rÛ  r
  ra   rÑ  )rÝ  Nr  r  )r  rÞ  r  rx   rÓ  rÔ  rÜ  r™   rH   r;   rÙ   )
rQ   r×  rÝ  r}   rØ  rÜ  rš   r¤   r„  rº   s
            €r:   rÞ  z"_PipelineScheduleRuntime._load_csv³  sÝ   ø€ ð �^Ò#ä‰GÑ˜hÔ'à×-Ñ-¨d×.AÑ.AÕBØ�Ò&ØˆGÜ�h¨Ô+ð J¨wÜŸ™ GÓ,�Ü!*¨6Ó!2ò G‘I�D˜#ØBEÖ$F¸Q¤W×%5Ñ%5°aÕ%8Ò$F�G˜D’MðGà×1Ñ1°'À&Ð1ÔI÷	Jð Jô &¨¨¨	Ð1DÐ&EÓFÐFùò %G÷Jð Jús   Á+CÁ1C ÂCÃ CÃCc                 ó  — |dk(  rt| j                   €t        d«      ‚t        |dd¬«      5 }t        j                  |«      }| j                   D ]   }|j                  | j                   |   «       Œ" 	 ddd«       y|dk(  rt| j                  €t        d«      ‚t        |dd¬«      5 }t        j                  |«      }| j                  D ]   }|j                  | j                  |   «       Œ" 	 ddd«       yy# 1 sw Y   yxY w# 1 sw Y   yxY w)	rÐ  rÛ  Nz'Compute only schedule must be availabler  ra   rÑ  r
  z6Must initialize compute_comms schedule before dump_csv)rx   r–  rÓ  rÔ  rÕ  rÖ  r  )rQ   r×  rÝ  rØ  rÕ  rš   s         r:   rÙ  z"_PipelineScheduleRuntime._dump_csvÈ  s  € à�^Ò#Ø×"Ñ"Ð*Ü$Ð%NÓOÐOÜ�h ¨RÔ0ð ?°GÜŸ™ GÓ,�Ø ×/Ñ/ò ?�DØ—O‘O D×$7Ñ$7¸Ñ$=Õ>ñ?÷?ð ?ð �Ò&Ø×-Ñ-Ð5Ü$ØLóð ô �h ¨RÔ0ð J°GÜŸ™ GÓ,�Ø ×:Ñ:ò J�DØ—O‘O D×$BÑ$BÀ4Ñ$HÕIñJ÷Jð Jð '÷	?ð ?ú÷Jð Jús   «AC4Â$AD Ã4C=Ä D	c                 óL   ‡ — t        ‰ j                  ˆ fd„‰ j                  «      S )Nc                 ó"   •— ‰j                   |    S rN   r  r  s    €r:   r…  z4_PipelineScheduleRuntime._simulate.<locals>.<lambda>Þ  s   ø€ �d×4Ñ4°QÑ7€ rF   )Ú_simulate_comms_computer  r  rP   s   `r:   Ú	_simulatez"_PipelineScheduleRuntime._simulateÛ  s%   ø€ Ü&Ø×*Ñ*Û7Ø×Ñó
ð 	
rF   r¿   c                 óB  — t        |j                  t        «      }|rƒ|j                  }|| j                  v rL| j                  |   D ]  }|j                  «        Œ | j                  |= | j                  j                  |«       || j                  vrt        d|›�«      ‚yy)zQIf an unshard is active for `stage_idx`, wait() it and mark `stage_idx` unshared.z*Attempted to compute on sharded stage_idx=N)	rÌ   Úsubmodr   rI   r  r  r  rw  r–  )rQ   r¿   Ústage_uses_fsdpro  r  s        r:   Ú_assert_unshardedz*_PipelineScheduleRuntime._assert_unshardedâ  sž   € ä$ U§\¡\´:Ó>ˆÙØ×)Ñ)ˆIØ˜D×,Ñ,Ñ,Ø×*Ñ*¨9Ñ5ò �BØ—G‘G•Iðà×$Ñ$ YÐ/Ø×%Ñ%×)Ñ)¨)Ô4Ø × 5Ñ 5Ñ5Ü$Ð'RÈ	À|Ð%TÓUÐUð 6ð rF   rÔ   rÕ   rÁ   rÑ   rÖ   c           	      ó²  ‡ ‡‡‡‡‡‡— ‰ j                  ‰‰‰|«      \  ŠŠ‰ j                  ‰d   ‰d   «       ‰ j                  D �ci c]  }|j                  |“Œ c}Š‰ j                  €t        d«      ‚g Šdt        ddfˆˆˆˆ ˆˆˆfd„}‰ j                  j                  «        t        ‰ j                  ‰ j                     «      D ]Î  \  }}	t        j                  d||	«       	 t        t        |	«      «      5  |	j                  ‰ j                   v r0t#        ‰ ‰‰‰|«      }
 ‰ j                   |	j                     |	|
«       nL|	j                  t$        k(  r1|	j&                  €t        d«      ‚|	j&                  D ]
  } ||«       Œ n ||	«       ddd«       ŒÐ ‰rt/        ‰j1                  «       «       ‰rŒt3        ‰ j4                  «      dk7  rt        d«      ‚‰ j7                  ‰ j                  |«       yc c}w # 1 sw Y   ŒnxY w# t(        $ rH}t        j+                  d	||	«       t        j+                  t-        ‰ j                  |¬
«      «       |‚d}~ww xY w)rá  r   NzLMust call _prepare_schedule_with_comms() before calling _step_microbatches()r3   r4   c                 óÜ  •— | j                   }| j                  �| j                  nd}|dk\  s"|t        t        t        fv st        d| ›d�«      ‚| j                  }‰|   }t        |j                  t        «      }|dz   ‰v }|dz
  ‰v }|t        k(  r*‰j                  t        |j                  |«      «      «       y |t        k(  r*‰j                  t        |j                  |«      «      «       y |t         k(  rL||f‰j"                  v rt        d|›d|›d�«      ‚t        |j%                  |«      «      ‰j"                  ||f<   y |t&        k(  rL||f‰j(                  v rt        d|›d|›d	�«      ‚t        |j+                  |«      «      ‰j(                  ||f<   y |t        k(  r�|rš|‰j,                  vr|‰j.                  vst        d
|›d�«      ‚|j                  j1                  «       D ]Q  }t        |t        «      sŒt3        t4        |j7                  d¬«      «      }	‰j.                  |   j                  |	«       ŒS y y |t        k(  r™|r–|‰j,                  vrt        d|›d�«      ‚|‰j.                  v rt        d|›d�«      ‚|j                  j1                  «       D ]#  }t        |t        «      sŒ|j9                  «        Œ% ‰j,                  j;                  |«       y y |t<        k(  r¯‰j?                  |«       |j@                  sG|sE||f‰j"                  vrt        d| ›d�«      ‚tC        ‰j"                  jE                  ||f«      «       |jG                  |‰|   ‰|   ‰¬«      }
‰jI                  ||
‰|«       |r‰|dz      jK                  |
|«       y y |tL        k(  ré‰j?                  |«       |jN                  sG|sE||f‰j(                  vrt        d| ›d�«      ‚tC        ‰j(                  jE                  ||f«      «       ‰jQ                  ||«      }‰jR                  |xx   dz  cc<   ‰jR                  |   ‰jT                  k(  }|jW                  ||d|¬«       |r(‰|dz
     jY                  |j[                  |«      |«       y y |t\        k(  r¶‰j?                  |«       |jN                  sG|sE||f‰j(                  vrt        d| ›d�«      ‚tC        ‰j(                  jE                  ||f«      «       ‰jQ                  ||«      }|jW                  ||dd¬«       |r(‰|dz
     jY                  |j[                  |«      |«       y y |t^        k(  rX‰j?                  |«       ‰jR                  |xx   dz  cc<   ‰jR                  |   ‰jT                  k(  }|ja                  ||¬«       y |t        k(  r,‰jb                  r‰jT                  nd}|je                  |«       y tg        d| ›d�«      ‚)Nrb   r   zaction=z missing mb_indexr   zRecv twice for stage_idx=z
 mb_index=z without executing forwardz without executing backwardzUnsharding the same stage_idx=z twiceT)Úasync_opzResharding stage_idx=z without unshardingz before finishing unshardzComputing action=z before receiving inputr;  z Attempted to run compute action=râ  Frä  z is unknown or unsupported)4rJ   rK   r*   r+   r2   r–  rI   rÌ   r  r   r,   rg   r  r6  r.   rC  r-   r  r4  r/   r  rA  r  r  Úmodulesr   r   ÚunshardÚreshardr�  r?   r  rÊ  r  r�  r5  rÄ   Úset_local_fwd_inputrB   r½   rÇ   rë  r±   rB  Úset_local_bwd_inputÚget_local_bwd_outputr@   rA   rå  r¯   rD  r7   )r3   Ú	comp_typerÂ   ro  r¿   r  Úis_next_stage_on_this_rankÚis_prev_stage_on_this_rankÚ	submoduleÚhandlerÀ   rÃ   r?  rí  rÔ   rÕ   rÖ   rQ   Úsend_opsrè  rÁ   s                 €€€€€€€r:   Ú_perform_actionzD_PipelineScheduleRuntime._step_microbatches.<locals>._perform_action  sd  ø€ Ø×/Ñ/ˆIà+1×+BÑ+BÐ+N�×'Ò'ÐTVð ð ˜A’ØäÜÜðñô %¨¨ yÐ0AÐ%BÓCÐCØ×*Ñ*ˆIØ(¨Ñ3ˆEÜ(¨¯©´zÓBˆOà)2°Q©Ð:NÐ)NÐ&Ø)2°Q©Ð:NÐ)NÐ&ð œFÒ"Ø—‘¤
¨5×+AÑ+AÀ(Ó+KÓ LÕMØœfÒ$Ø—‘¤
¨5×+AÑ+AÀ(Ó+KÓ LÕMØœfÒ$Ø˜xÐ(¨D×,=Ñ,=Ñ=Ü(Ø4¨)¨°[°x°kÐA[Ð\óð ô <FØ×*Ñ*¨8Ó4ó<�×!Ñ! 9¨hÐ"7Ò8ð œfÒ$Ø˜xÐ(¨D×,=Ñ,=Ñ=Ü(Ø4¨)¨°[°x°kÐA\Ð]óð ô <FØ×*Ñ*¨8Ó4ó<�×!Ñ! 9¨hÐ"7Ò8ð œgÒ%Ù"à!¨×)>Ñ)>Ñ>Ø%¨T×-=Ñ-=Ñ=ä,Ð/NÀIÀ<ÈvÐ-VÓWÐWØ%*§\¡\×%9Ñ%9Ó%;ò C˜	Ü)¨)´ZÔ@Ø$Ü!%¤m°Y×5FÑ5FÐPTÐ5FÓ5UÓ!V˜Ø×(Ñ(¨Ñ3×:Ñ:¸6ÕBñ	Cð #ð œgÒ%Ù"Ø ¨×(=Ñ(=Ñ=Ü,Ø4¨)¨Ð5HÐIóð ð ! D×$4Ñ$4Ñ4Ü,Ø4¨)¨Ð5NÐOóð ð &+§\¡\×%9Ñ%9Ó%;ò ,˜	Ü)¨)´ZÔ@Ø$Ø!×)Ñ)Õ+ð,ð ×)Ñ)×0Ñ0°Õ;ð #ð œgÒ%Ø×&Ñ& uÔ-ð Ÿšá6à! 8Ð,°D×4EÑ4EÑEÜ,Ø0¨¨	Ð1HÐIóð ô $ D×$5Ñ$5×$9Ñ$9¸9ÀhÐ:OÓ$PÔQà×0Ñ0ØØ˜HÑ%Ø˜hÑ'Ø(6ð	 1ó �ð ×(Ñ(¨°¸
ÀHÔMñ .Ø(¨°Q©Ñ7×KÑKØ õð .ð
 œmÒ+Ø×&Ñ& uÔ-ð Ÿšá6à! 8Ð,°D×4EÑ4EÑEÜ,Ø?¸°yÐ@WÐXóð ô $ D×$5Ñ$5×$9Ñ$9¸9ÀhÐ:OÓ$PÔQØ×+Ñ+¨E°8Ó<�Ø×%Ñ% iÓ0°AÑ5Ó0Ø $× 5Ñ 5°iÑ @ÀD×DXÑDXÑ X�Ø×(Ñ(ØØØ"&Ø"/ð	 )ô ñ .Ø(¨°Q©Ñ7×KÑKØ×2Ñ2°8Ó<¸hõð .ð œnÒ,Ø×&Ñ& uÔ-à—}’}Ñ-GØ! 8Ð,°D×4EÑ4EÑEÜ,Ø?¸°yÐ@WÐXóð ô $ D×$5Ñ$5×$9Ñ$9¸9ÀhÐ:OÓ$PÔQØ×+Ñ+¨E°8Ó<�Ø×(Ñ(ØØØ"'Ø"'ð	 )ô ñ .Ø(¨°Q©Ñ7×KÑKØ×2Ñ2°8Ó<¸hõð .ð œoÒ-Ø×&Ñ& uÔ-Ø×%Ñ% iÓ0°AÑ5Ó0Ø $× 5Ñ 5°iÑ @ÀD×DXÑDXÑ X�Ø×/Ñ/ØØ"/ð 0õ ð œkÒ)Ø<@×<LÒ<L D×$8Ò$8ÐRSÐ!Ø×)Ñ)Ð*;Õ<ä  G F 9Ð,FÐ!GÓHÐHrF   z8_PipelineScheduleRuntime running time_step %d, action %szsub_actions must be setz\_PipelineScheduleRuntime caught exception at step %s when running action %s.  Full Schedule:rª  zUnused unshard operations)rí   rÌ  rÀ  rI   r  r–  rH   rë  rÎ   r™   rš   r¸   rý   r   rw   rJ   r  r÷  r1   rL   ræ  rç  r§   r  r�  rm   r  rÓ   )rQ   rÔ   rÕ   rÁ   rÑ   rÖ   r¿   r,  rì  r3   rû  Úsub_arô  r+  rè  s   ```` `       @@r:   rÚ   z+_PipelineScheduleRuntime._step_microbatchesï  se  þ€ ð "×/Ñ/°¸ÀJÐPVÓWÑˆ�Ø×Ñ ¨¡
¨I°a©LÔ9ð
 37·,±,ö?
Ø).ˆE×Ñ˜uÑ$ò?
Ðð ×)Ñ)Ð1Ü Ø^óð ð
 +-ˆðe	I¤Gð e	I°÷ e	Ió e	IðP 	×Ñ×#Ñ#Ô%Ü!*¨4×+IÑ+IÈ$Ï)É)Ñ+TÓ!Uò &	ÑˆI�vÜ�L‰LØJØØôð
 Ü$Ô%@ÀÓ%HÓIñ 0Ø×.Ñ.°$×2QÑ2QÑQÜ.Ø Ø#Ø%Ø&Ø"ó˜ð Q˜×7Ñ7¸×8OÑ8OÑPØ" Cõð  ×0Ñ0´KÒ?Ø!×-Ñ-Ð5Ü"0Ð1JÓ"KÐKØ%+×%7Ñ%7ò 3˜EÙ+¨EÕ2ñ3ñ (¨Ô/÷%0øð&	ñR Ü˜HŸL™L›NÔ+ò ô ˆt×ÑÓ  AÒ%Ü Ð!<Ó=Ð=ð 	×Ñ˜DŸL™L¨&Õ1ùòK?
÷x0ð 0ûô& ò Ü—‘ØrØØôô
 —‘Ü*Ø×6Ñ6Ø*3ôôð �ûðús8   ÁG4Ã#HÃ7BG9ÆHÇ9H	Ç>HÈ	IÈAIÉIrõ  )r
  rö   )r<   r=   r>   r*  r»   r&   rú  r  rù   rl   r•   rH   rD   r  rÞ  rÙ  r  r   r  rt   rÚ   r+  r,  s   @r:   rÿ  rÿ  ?  s  ø„ ñô&ð%Là*ð%Lð 1ð%Lð 
ó	%LðT %ñ4Gà�c˜4 ¨$¡Ñ/Ð/Ñ0ð4Gð õ4GñlG #ð G¨sõ Gñ*J #ð J¨só Jò&
ðVÐ'9ó Vð  $Ø!%Ø"&Ø"Ø#ñx2à˜‘ðx2ð ˜$‘;ðx2ð ˜4‘Kð	x2ð
 �t‘ðx2ð ÷x2rF   rÿ  c                   óz   ‡ — e Zd ZdZ	 	 	 	 ddee   dedeez  dz  de	e
ef   ee   z  dz  dedefˆ fd	„Zd
„ Zˆ xZS )r!   ai  
    Breadth-First Pipeline Parallelism.
    See https://arxiv.org/abs/2211.05953 for details.
    Similar to Interleaved 1F1B, Looped BFS supports multiple stages per rank.
    What is different is that when microbatches are ready for multiple local
    stages, Loops BFS will prioritizes the earlier stage, running all available
    microbatches at once.
    NrÐ   rª   r«   r®   r¯   r¼  c                 óè   •— t         ‰	| �  ||||||¬«       i | _        t        | j                  «      D ]"  }| j                  |«      }|| j                  |<   Œ$ | j                  | j                  «       y )N)rÐ   rª   r«   r®   r¯   r¼  )r  r»   rx   r�   rI  Ú!_calculate_single_rank_operationsr  )
rQ   rÐ   rª   r«   r®   r¯   r¼  rš   Úrank_opsrº   s
            €r:   r»   zScheduleLoopedBFS.__init__ô  s…   ø€ ô 	‰ÑØØ)ØØ/Ø#Ø'Að 	ô 	
ð @BˆÔä˜$×,Ñ,Ó-ò 	1ˆDØ×=Ñ=¸dÓCˆHØ(0ˆD×Ñ Ò%ð	1ð
 	×)Ñ)¨$×*=Ñ*=Õ>rF   c           	      óü  ‡— t        | j                  «      }t        || j                  |z  | j                  «      }t        |«      D �cg c]  }d ‘Œ }}|D ]/  Š|j	                  ˆfd„t        | j
                  «      D «       «       Œ1 d| j                  dz
  |z
  z  }|j	                  d g|z  «       t        |«      D ]8  Š|j	                  ˆfd„t        t        | j
                  «      «      D «       «       Œ: |S c c}w )Nc              3   óT   •K  — | ]  }t        ‰t        j                  |«      –— Œ! y ­wrN   )rH   r&   r?   ©r|   rÂ   rI   s     €r:   r~   zFScheduleLoopedBFS._calculate_single_rank_operations.<locals>.<genexpr>	  s)   øè ø€ ò àô ˜Ô%5×%=Ñ%=¸x×Hñùó   ƒ%(r„   r   c              3   óT   •K  — | ]  }t        ‰t        j                  |«      –— Œ! y ­wrN   )rH   r&   rB   r4  s     €r:   r~   zFScheduleLoopedBFS._calculate_single_rank_operations.<locals>.<genexpr>(	  s)   øè ø€ ò àô ˜Ô%5×%CÑ%CÀX×Nñùr5  )rm   rÀ  r�   rI  rÏ   r±   Úreversed)rQ   rš   Ún_local_stagesÚstage_indicesrX  r1  Úpost_warmup_opsrI   s          @r:   r0  z3ScheduleLoopedBFS._calculate_single_rank_operations	  sø   ø€ Ü˜TŸ\™\Ó*ˆÜØ�$×$Ñ$ ~Ñ5°t×7IÑ7Ió
ˆô 9>¸d»Ö)D°1ª$Ð)DˆÐ)Dà(ò 	ˆKØ�O‰Oó ä % d×&:Ñ&:Ó ;ôõ ð	ð ˜t×1Ñ1°AÑ5¸Ñ<Ñ=ˆØ�‰˜˜ Ñ0Ô1ä# MÓ2ò 	ˆKØ�O‰Oó ä (¬¨t×/CÑ/CÓ)DÓ Eôõ ð	ð
 ˆùò% *Es   Á	C9)NNTT)r<   r=   r>   r*  r•   r   rl   r   r   rù   rD   r   rh   rt   r»   r0  r+  r,  s   @r:   r!   r!   ê  s€   ø„ ñð ,0Ø@DØ Ø+/ñ?àÐ'Ñ(ð?ð ð?ð ˜EÑ! DÑ(ð	?ð
    S ™>¨E°#©JÑ6¸Ñ=ð?ð ð?ð %)õ?ö<rF   r!   c
                 óî  — t        t        «      }
t        t        «      }t        t        «      }t        |«      D �cg c]  }d ‘Œ }}| |z  d|dz
  |z
  z  z   ||z   z
  }|	r||z
  dz
  }||z   |z   }g }d}|	rt        nt        }t        |«      D �]ø  }||k  r^ ||«      }|
|   x}dz   |
|<   |j                  t        |t        j                  |«      «       ||dz
  k(  sŒQ|j                  d g|z  «       Œg||cxk  r||z   k  rÚn n× ||«      }|
|   x}dz   |
|<   |j                  t        |t        j                  |«      «        ||«      }||   x}dz   ||<   |j                  t        |||«      «       |j                  |«       |	sŒü||z
  |k\  s�Œ |||   «      }||   x}dz   ||<   |j                  t        |t        j                  |«      «       |dz  }�ŒO|	s|j                  d «        ||«      }||   x}dz   ||<   |j                  t        |||«      «       |j                  |«       |	s�Œ¨||z
  |k\  s�Œ² |||   «      }||   x}dz   ||<   |j                  t        |t        j                  |«      «       |dz  }�Œû |	rf|t        |«      k  rX |||   «      }||   x}dz   ||<   |j                  t        |t        j                  |«      «       |dz  }|	r|t        |«      k  rŒX|S c c}w )Nr„   r   r   )r   rl   r�   r@   rB   rg   rH   r&   r?   rÏ   rA   rm   )r8  rI  Ú
warmup_opsÚfwd_bwd_opsÚcooldown_opsrš   Úforward_stage_indexÚbackward_stage_indexÚnum_1f1b_microbatchesÚenable_zero_bubbleÚfwd_stage_mb_indexÚbwd_stage_mb_indexÚweight_stage_mb_indexrX  r1  r:  Ú	total_opsÚbackward_op_idsÚweight_op_countÚFULL_BACKWARD_OR_BACKWARD_INPUTr  Úfwd_stage_indexrÂ   rT  Úbwd_stage_indexrU  Úweight_stage_indexÚweight_mb_indexs                               r:   Ú_get_1f1b_rank_opsrN  /	  sº  € ô *5´SÓ)9ÐÜ)4´SÓ)9ÐÜ,7¼Ó,<Ðô 5:¸$³KÖ%@¨q¢dÐ%@€HÐ%@ð 	˜Ñ&¨¨m¸aÑ.?À$Ñ.FÑ)GÑGØ	�dÑ	ñ€Oñ Ø'¨$Ñ.°Ñ2ˆà˜[Ñ(¨<Ñ7€Ià€OØ€Oñ -�´-ð $ô �IÓó M%ˆà�
Š?Ù1°"Ó5ˆOð /¨Ñ?Ð?�Øñ3Ð˜Ñ/ð �O‰OÜ˜Ô)9×)AÑ)AÀ8ÓLôð �Z !‘^Ó#à—‘  ¨Ñ 8Õ9à˜2Ô8 
¨[Ñ 8Õ8Ù1°"Ó5ˆOà 2°?Ñ CÐC�Øñ3Ð˜Ñ/ð �O‰OÜ˜Ô)9×)AÑ)AÀ<ÓPôñ 3°2Ó6ˆOà 2°?Ñ CÐC�Øñ3Ð˜Ñ/ð �O‰OÜ˜Ð)HÈ,ÓWôð ×"Ñ" 2Ô&â! b¨:¡oÐ9NÔ&NÙ%9Ø# OÑ4ó&Ð"ð (=Ð=OÑ'PÐP�OØñ=Ð%Ð&8Ñ9ð —‘ÜØ*Ü(×8Ñ8Ø'óôð   1Ñ$’ñ
 &Ø—‘ Ô%á2°2Ó6ˆOà 2°?Ñ CÐC�Øñ3Ð˜Ñ/ð �O‰OÜ˜Ð)HÈ,ÓWôð ×"Ñ" 2Ô&ã! b¨:¡oÐ9NÔ&NÙ%9Ø# OÑ4ó&Ð"ð (=Ð=OÑ'PÐP�OØñ=Ð%Ð&8Ñ9ð —‘ÜØ*Ü(×8Ñ8Ø'óôð   1Ñ$’ð[M%ñ^  ´3°Ó3GÒ!GÙ1°/À/Ñ2RÓSÐà4Ð5GÑHÐHˆOØñ5ÐÐ0Ñ1ð 	�‰ÜØ"Ô$4×$DÑ$DÀoóô	
ð
 	˜1Ñˆñ  ´3°Ó3GÓ!Gð €Oùòg &As   »	K2rš   r8  Úmicrobatches_per_roundÚmultiply_factorc                 óL   — |dz
  |z  }|||dz
  | z
  z  z   }t        |||z  «      S )zN
    Calculate the number of warmup operations for interleaved schedules.
    r   )rR  )rš   r8  rO  rI  rª   rP  Úwarmups_ops_last_stager<  s           r:   Ú_get_warmup_opsrS  ¸	  sA   € ð -¨qÑ0Ð4JÑJÐà'¨/¸mÈaÑ>OÐSWÑ=WÑ*XÑX€Jäˆz˜>¨NÑ:Ó;Ð;rF   c                   ó²   ‡ — e Zd ZdZ	 	 	 	 	 	 ddee   dededz  dee	df   dz  de
ee	f   dz  d	e
eef   ee   z  dz  d
edefˆ fd„Zdeedz     fd„Zˆ xZS )r    a…  
    The Interleaved 1F1B schedule.
    See https://arxiv.org/pdf/2104.04473 for details.
    Will perform one forward and one backward on the microbatches in steady
    state and supports multiple stages per rank. When microbatches are ready for
    multiple local stages, Interleaved 1F1B prioritizes the earlier microbatch
    (also called "depth first").

    This schedule is mostly similar to the original paper.
    It differs by being relaxing the requirement of num_microbatch % pp_size == 0.
    Using the flex_pp schedule, we will have num_rounds = max(1, n_microbatches // pp_group_size) and
    it works as long as n_microbatches % num_rounds is 0. As a few examples, support

    1. pp_group_size = 4, n_microbatches = 10. We will have num_rounds = 2 and n_microbatches % 2 is 0.
    2. pp_group_size = 4, n_microbatches = 3. We will have num_rounds = 1 and n_microbatches % 1 is 0.
    NrÐ   rª   r«   r¬   .r­   r®   r¯   r¼  c	           
      ó  •— |d   j                   | _        t        ‰| �  ||||||||¬«       t	        |«      | _        |d   j                  | _        t        d|| j                  z  «      | _	        || j                  z  | _
        || j                  z  dk7  rt        d| j                  › d|› d�«      ‚i | _        t        | j                  «      D ]"  }	| j                  |	«      }
|
| j                  |	<   Œ$ | j                  | j                  «       y )Nr   ©rÐ   rª   r«   r¬   r­   r®   r¯   r¼  r   z_Interleaved 1F1B requires the number of microbatches to be a multiple of the number of rounds (ú), but got ú.)rÁ  rI  r  r»   rm   r8  rÂ  rš   r�   Únumber_of_roundsrO  r7   rx   r�   r0  r  ©rQ   rÐ   rª   r«   r¬   r­   r®   r¯   r¼  rš   r1  rº   s              €r:   r»   z ScheduleInterleaved1F1B.__init__Ý	  s-  ø€ ð $ A™Y×1Ñ1ˆÔÜ‰ÑØØ)ØØ+Ø/Ø/Ø#Ø'Að 	ô 		
ô " &›kˆÔØ˜1‘I×(Ñ(ˆŒ	Ü # A ~¸×9KÑ9KÑ'KÓ LˆÔØ&4¸×8MÑ8MÑ&MˆÔ#Ø˜D×1Ñ1Ñ1°QÒ6Üð5Ø59×5JÑ5JÐ4Kð LØ)Ð*¨!ð-óð ð @BˆÔÜ˜$×,Ñ,Ó-ò 	1ˆDØ×=Ñ=¸dÓCˆHØ(0ˆD×Ñ Ò%ð	1ð
 	×)Ñ)¨$×*=Ñ*=Õ>rF   r4   c           
      ój  ‡ ‡‡— t        ‰‰ j                  ‰ j                  ‰ j                  ‰ j                  d¬«      Š‰ j                  ‰ j                  z  }|‰z
  }||z
  }‰|z   |z   }t
        j                  d‰‰|||«       ˆˆ fd„}ˆˆ ˆfd„}t        ‰ j                  ‰ j                  ‰||‰||«      S )Nr„   ©rP  ú=rank %s, warmup_ops %s, 1f1b %s, cooldown_ops %s total_ops %sc                 ó`   •— | ‰j                   z  ‰j                  z  }|‰j                  z  ‰z   S rN   ©rO  r8  rI  ©rá   Úlocal_indexrš   rQ   s     €€r:   r?  zVScheduleInterleaved1F1B._calculate_single_rank_operations.<locals>.forward_stage_index#
  ó4   ø€ à 4×#>Ñ#>Ñ>À$×BUÑBUÑUˆKØ $×"4Ñ"4Ñ4¸Ñ<Ð<rF   c                 ó†   •— ‰j                   dz
  | ‰z
  ‰j                  z  ‰j                   z  z
  }|‰j                  z  ‰z   S rh  ©r8  rO  rI  ©rá   ra  rš   rQ   r<  s     €€€r:   r@  zWScheduleInterleaved1F1B._calculate_single_rank_operations.<locals>.backward_stage_index(
  óW   ø€ à×#Ñ#Øñà˜:Ñ%¨$×*EÑ*EÑEØ×%Ñ%ñ&ñ&ð ð   $×"4Ñ"4Ñ4¸Ñ<Ð<rF   ©rS  r8  rO  rI  r±   r¸   rý   rN  )	rQ   rš   Úmicrobatch_opsr=  r>  rF  r?  r@  r<  s	   ``      @r:   r0  z9ScheduleInterleaved1F1B._calculate_single_rank_operations
  sÍ   ú€ Ü$ØØ×ÑØ×'Ñ'Ø×ÑØ× Ñ Øô
ˆ
ð ×,Ñ,¨t×/CÑ/CÑCˆà$ zÑ1ˆà%¨Ñ3ˆà Ñ,¨|Ñ;ˆ	ä�‰ØKØØØØØô	
õ	=ö
	=ô "Ø×ÑØ×ÑØØØØØØ ó	
ð 		
rF   ©NNNNTT©r<   r=   r>   r*  r•   r   rl   r   rh   r   rù   rD   r   rt   r»   rH   r0  r+  r,  s   @r:   r    r    Ë	  sÃ   ø„ ñð* $(Ø>BØ?CØ@DØ Ø+/ñ)?àÐ'Ñ(ð)?ð ð)?ð ˜D‘ð	)?ð
 ˜°Ð3Ñ4°tÑ;ð)?ð    _Ð 4Ñ5¸Ñ<ð)?ð    S ™>¨E°#©JÑ6¸Ñ=ð)?ð ð)?ð %)õ)?ðV2
¸¸gÈ¹nÑ9M÷ 2
rF   r    c                   ó¸   ‡ — e Zd ZdZ	 	 	 	 	 	 ddee   dededz  dee	df   dz  de
ee	f   dz  d	e
eef   ee   z  dz  d
edefˆ fd„Zdeedz     fd„Zd„ Zˆ xZS )r"   aw  
    The Interleaved Zero Bubble schedule.
    See https://arxiv.org/pdf/2401.10241 for details.
    Will perform one forward and one backward on inputs for the microbatches in steady
    state and supports multiple stages per rank. Uses the backward for weights to fill in
    the pipeline bubble.

    In particular this is implementing the ZB1P schedule in the paper.
    NrÐ   rª   r«   r¬   .r­   r®   r¯   r¼  c	           
      ó¶  •— t        || j                  j                  «       |d   j                  | _        t
        ‰| �  ||||||||¬«       t        |«      | _        |d   j                  | _
        t        d|| j                  z  «      | _        || j                  z  | _        || j                  z  dk7  rt        d| j                  › d|› d�«      ‚i | _        t!        | j                  «      D ]"  }	| j#                  |	«      }
|
| j                  |	<   Œ$ | j%                  | j                  | j                  z  «      | _        | j'                  | j                  «       y )Nr   rV  r   zZZero bubble requires the number of microbatches to be a multiple of the number of rounds (rW  rX  )Ú"_check_torch_compile_compatibilityrº   r<   rÁ  rI  r  r»   rm   r8  rÂ  rš   r�   rY  rO  r7   rx   r�   r0  Ú_add_bubbles_to_actionsr  rZ  s              €r:   r»   z&ScheduleInterleavedZeroBubble.__init__H
  sj  ø€ ô 	+¨6°4·>±>×3JÑ3JÔKØ# A™Y×1Ñ1ˆÔÜ‰ÑØØ)ØØ+Ø/Ø/Ø#Ø'Að 	ô 		
ô " &›kˆÔØ˜1‘I×(Ñ(ˆŒ	Ü # A ~¸×9KÑ9KÑ'KÓ LˆÔØ&4¸×8MÑ8MÑ&MˆÔ#Ø˜D×1Ñ1Ñ1°QÒ6Üð5Ø59×5JÑ5JÐ4Kð LØ)Ð*¨!ð-óð ð @BˆÔÜ˜$×,Ñ,Ó-ò 	1ˆDØ×=Ñ=¸dÓCˆHØ(0ˆD×Ñ Ò%ð	1ð #×:Ñ:Ø×Ñ $×"4Ñ"4Ñ4ó
ˆÔð
 	×)Ñ)¨$×*=Ñ*=Õ>rF   r4   c                 ót  ‡ ‡‡	— t        ‰‰ j                  ‰ j                  ‰ j                  ‰ j                  d¬«      Š	‰ j                  ‰ j                  z  }|‰	z
  }||z
  }‰	|z   |z   }t
        j                  d‰‰	|||«       ˆˆ fd„}ˆˆ ˆ	fd„}‰}t        ‰ j                  ‰ j                  ‰	||‰|||d¬«
      S )Nr   r\  r]  c                 ó`   •— | ‰j                   z  ‰j                  z  }|‰j                  z  ‰z   S rN   r_  r`  s     €€r:   r?  z\ScheduleInterleavedZeroBubble._calculate_single_rank_operations.<locals>.forward_stage_index˜
  rb  rF   c                 ó†   •— ‰j                   dz
  | ‰z
  ‰j                  z  ‰j                   z  z
  }|‰j                  z  ‰z   S rh  rd  re  s     €€€r:   r@  z]ScheduleInterleavedZeroBubble._calculate_single_rank_operations.<locals>.backward_stage_index�
  rf  rF   T)rB  rg  )
rQ   rš   rh  r=  r>  rF  r?  r@  rA  r<  s
   ``       @r:   r0  z?ScheduleInterleavedZeroBubble._calculate_single_rank_operations|
  sÙ   ú€ Ü$ØØ×ÑØ×'Ñ'Ø×ÑØ× Ñ Øô
ˆ
ð ×,Ñ,¨t×/CÑ/CÑCˆà$ zÑ1ˆà%¨Ñ3ˆà Ñ,¨|Ñ;ˆ	ä�‰ØKØØØØØô	
õ	=ö
	=ð !%Ðä!Ø×ÑØ×ÑØØØØØØ Ø!Ø#ô
ð 	
rF   c                 óø  — | j                   }d„ }t        «       }i }i }i }d}t        | j                  «      D ]  }	g ||	<   d||	<   d||	<   Œ 	 d}
t        «       }t        | j                  «      D ]Ø  }	||	   }|t	        ||	   «      k\  rŒd}
||	   |   �”||	   |   }|€t        dt        |«      › �«      ‚|\  }}}} ||||||«      s>||	   j                  ||	   |   «       |�|j                  |||f«       ||	xx   dz  cc<   Œ–||	   j                  d «       ||	xx   dz  cc<   Œ¸||	xx   dz  cc<   ||	   j                  d «       ŒÚ |j                  |«       |
rn�Œ|dkD  rt        j                  d||«       |S )Nc                 óÀ   — |t         j                  k(  r| dk7  r| dz
  ||f|vryy|t         j                  k(  r'| |dz
  k(  r| t         j                  |f|vS | dz   ||f|vS y)Nr   r   TF)r&   r?   rB   )r¿   r  Ú
microbatchÚnum_stages_globalÚseen_opss        r:   Úneed_bubblezJScheduleInterleavedZeroBubble._add_bubbles_to_actions.<locals>.need_bubble¸
  s„   € ØÔ%×-Ñ-Ò-Ø˜A’: 5¨1¡9¨b°*Ð"=ÀXÑ"MØð
 ð	 Ô'×5Ñ5Ò5ØÐ-°Ñ1Ò1Ø!Ô#3×#;Ñ#;¸ZÐHÐPXÐXÐXØ ™	 2 zÐ2¸(ÐBÐBØrF   r   TFz)Expected temp_action to be not None, got r   z?Non zero bubbles added: total_bubbles_added=%s bubbles_added=%s)rx   rv  r�   rI  rm   r–  rê   rg   rw  Úupdater¸   rÈ  )rQ   ru  r}   rw  rv  ÚresultÚnext_pointerÚbubbles_addedÚtotal_bubbles_addedrš   Úshould_stopÚtemp_seen_opsÚ	timestampÚtemp_actionrI   r  rt  rX  s                     r:   rn  z5ScheduleInterleavedZeroBubble._add_bubbles_to_actionsµ
  sç  € Ø×%Ñ%ˆò	ô <?»5ˆØ24ˆØ')ˆØ(*ˆØÐä˜$×,Ñ,Ó-ò 	$ˆDØˆF�4‰LØ!"ˆL˜ÑØ"#ˆM˜$Òð	$ð
 ØˆKäDGÃEˆMä˜d×0Ñ0Ó1ò .�Ø(¨Ñ.�	Ø¤ G¨D¡MÓ 2Ò2Øà#�à˜4‘= Ñ+Ð7Ø")¨$¡-°	Ñ":�KØ"Ð*Ü,ØGÌÈ[ÓHYÐGZÐ[óð ð 6AÑ2�K  Z°Ù&Ø# R¨Ð5FÈôð ˜t™×+Ñ+¨G°D©M¸)Ñ,DÔEØ%Ð1Ø)×-Ñ-¨{¸BÀ
Ð.KÔLØ$ TÓ*¨aÑ/Ô*à˜t™×+Ñ+¨DÔ1Ø% dÓ+¨qÑ0Ô+à  Ó&¨!Ñ+Ó&Ø˜4‘L×'Ñ'¨Õ-ð5.ð8 �O‰O˜MÔ*ÙØñG ðJ  Ò"Ü�N‰NØQØ#Øôð
 ˆrF   ri  )r<   r=   r>   r*  r•   r   rl   r   rh   r   rù   rD   r   rt   r»   rH   r0  rn  r+  r,  s   @r:   r"   r"   =
  sÊ   ø„ ñð $(Ø>BØ?CØ@DØ Ø+/ñ2?àÐ'Ñ(ð2?ð ð2?ð ˜D‘ð	2?ð
 ˜°Ð3Ñ4°tÑ;ð2?ð    _Ð 4Ñ5¸Ñ<ð2?ð    S ™>¨E°#©JÑ6¸Ñ=ð2?ð ð2?ð %)õ2?ðh7
¸¸gÈ¹nÑ9Mó 7
örCrF   r"   c                   ó²   ‡ — e Zd ZdZ	 	 	 	 	 	 ddee   dededz  dee	df   dz  de
ee	f   dz  d	e
eef   ee   z  dz  d
edefˆ fd„Zdeedz     fd„Zˆ xZS )r#   a£  
    The Zero Bubble schedule (ZBV variant).
    See https://arxiv.org/pdf/2401.10241 Section 6 for details.

    This schedules requires exactly two stages per rank.

    This schedule will perform one forward and one backward on inputs for the microbatches in steady
    state and supports multiple stages per rank. Uses backward with respect to weights to fill in
    the pipeline bubble.

    This ZB-V schedule would have the "zero bubble" property only if time forward == time backward input == time backward weights.
    In practice, this is not likely true for real models so alternatively
    a greedy scheduler could be implemented for unequal/unbalanced time.
    NrÐ   rª   r«   r¬   .r­   r®   r¯   r¼  c	           
      ó¦  •— t        || j                  j                  «       |d   j                  | _        t
        ‰| �  ||||||||¬«       t        | j                  | j                  d¬«      | _	        | j                  D ]  }	| j                  |	_	        Œ t        |«      | _        | j                  dk7  rt        d| j                  › d�«      ‚|d   j                  | _        |d   j                   | _        i | _        t%        | j                  «      D ]"  }
| j'                  |
«      }|| j"                  |
<   Œ$ | j)                  | j"                  «       y )Nr   rV  Úv©Ústyler„   ú0ZBV requires exactly 2 stages per rank, but got rX  ©rm  rº   r<   rÁ  rI  r  r»   r   r  rÃ  rÀ  rm   r8  r7   rÂ  rš   r  rx   r�   r0  r  ©rQ   rÐ   rª   r«   r¬   r­   r®   r¯   r¼  r¿   rš   r1  rº   s               €r:   r»   zScheduleZBVZeroBubble.__init__  s\  ø€ ô 	+¨6°4·>±>×3JÑ3JÔKØ# A™Y×1Ñ1ˆÔÜ‰ÑØØ)ØØ+Ø/Ø/Ø#Ø'Að 	ô 		
ô *HØ×Ñ × 0Ñ 0¸ô*
ˆÔ&ð —\‘\ò 	MˆEØ.2×.LÑ.LˆEÕ+ð	Mô " &›kˆÔØ×Ñ !Ò#ÜØBØ×&Ñ&Ð' qð*óð ð
 ˜1‘I×(Ñ(ˆŒ	Ø  ™)×.Ñ.ˆŒð
 @BˆÔÜ˜$×,Ñ,Ó-ò 	1ˆDØ×=Ñ=¸dÓCˆHØ(0ˆD×Ñ Ò%ð	1ð
 	×)Ñ)¨$×*=Ñ*=Õ>rF   r4   c                 óf  — t        d| j                  z  dz
  | j                  «      }t        |«      D �cg c]  }d ‘Œ }}d\  }}}}d| j                  |z
  z  dz
  }	|}
| j                  dz
  |z
  }t        |	«      D ](  }|j                  t        |
t        |¬«      «       |dz  }Œ* |}t        |«      D ]N  }|j                  t        |t        |¬«      «       |dz  }|j                  t        |
t        |¬«      «       |dz  }ŒP | j                  |z
  }t        |«      D ]o  }|j                  t        |t        |¬«      «       |dz  }|j                  t        |t        |¬«      «       |j                  t        |t        |¬«      «       |dz  }Œq ||k  s||k  rë||k  r&|j                  t        |
t        |¬«      «       |dz  }|j                  t        |
t        |¬«      «       |j                  t        |
t        |¬«      «       |dz  }|j                  t        |t        |¬«      «       |dz  }|j                  t        |t        |¬«      «       |j                  t        |t        |¬«      «       |dz  }||k  rŒå||k  rŒë||}}|}t        |«      D ]N  }|j                  t        |
t        |¬«      «       |dz  }|j                  t        |t        |¬«      «       |dz  }ŒP | j                  |z
  }t        |«      D ]N  }|j                  t        |
t        |¬«      «       |dz  }|j                  t        |
t        |¬«      «       |dz  }ŒP ||k  r,|j                  t        |t        |¬«      «       |dz  }||k  rŒ,||k  r,|j                  t        |
t        |¬«      «       |dz  }||k  rŒ,||k(  r||k(  st        d|› d|› d|› �«      ‚||k(  r||k(  st        d|› d	|› d
|› �«      ‚|D �cg c]-  }|�'|j                  �|j                  | j                  k  r|nd ‘Œ/ }}|S c c}w c c}w )Nr„   r   )r   r   r   r   )rJ   rK   z0Expected w0_cnt == b0_cnt == f0_cnt, got w0_cnt=z	, b0_cnt=z	, f0_cnt=z0Expected w1_cnt == b1_cnt == f1_cnt, got w1_cnt=z	, b1_cnt=z	, f1_cnt=)r�   rI  r±   r�   r  rg   rH   r'   r(   r)   r–  rK   )rQ   rš   Ún_microrX  r1  Úf0_cntÚf1_cntÚb0_cntÚb1_cntÚ	warmup_n1Ústage_id_chunk0Ústage_id_chunk1Ú	warmup_n2Ú	warmup_n3Úw0_cntÚw1_cntÚcooldown_n1Úcooldown_n2r3   s                      r:   r0  z7ScheduleZBVZeroBubble._calculate_single_rank_operations>  s¡  € ô �a˜$×,Ñ,Ñ,¨qÑ0°$×2FÑ2FÓGˆÜ8=¸d»Ö)D°1ª$Ð)DˆÐ)Dð *4Ñ&ˆ�˜ à˜×+Ñ+¨dÑ2Ñ3°aÑ7ˆ	ØˆØŸ/™/¨AÑ-°Ñ4ˆä�yÓ!ò 	ˆAØ�O‰OÜ˜¼!ÈfÔUôð �a‰K‰Fð		ð
 ˆ	Ü�yÓ!ò 	ˆAØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFØ�O‰OÜ˜¼!ÈfÔUôð �a‰K‰Fð	ð ×&Ñ&¨Ñ-ˆ	Ü�yÓ!ò 	ˆAØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFØ�O‰OÜ˜¼!ÈfÔUôð �O‰OÜ˜¼!ÈfÔUôð �a‰K‰Fð	ð �vŠo ¨'Ò!1Ø˜ÒØ—‘ÜØ'¼!Èfôôð
 ˜!‘�Ø�O‰OÜ˜¼!ÈfÔUôð �O‰OÜ˜¼!ÈfÔUôð �a‰KˆFà�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFØ�O‰OÜ˜¼!ÈfÔUôð �O‰OÜ˜¼!ÈfÔUôð �a‰KˆFð5 �v‹o ¨'Ó!1ð8   �ˆØˆÜ�{Ó#ò 	ˆAØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFØ�O‰OÜ˜¼!ÈfÔUôð �a‰K‰Fð	ð ×(Ñ(¨4Ñ/ˆÜ�{Ó#ò 	ˆAØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFØ�O‰OÜ˜¼!ÈfÔUôð �a‰K‰Fð	ð �vŠoØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFð	 �v‹oð
 �vŠoØ�O‰OÜ˜¼!ÈfÔUôð �a‰KˆFð	 �v‹oð ˜&Ò  V¨vÒ%5Ü ØBÀ6À(È)ÐTZÐS[Ð[dÐekÐdlÐmóð ð ˜&Ò  V¨vÒ%5Ü ØBÀ6À(È)ÐTZÐS[Ð[dÐekÐdlÐmóð ð #ö	
ð ð Ð%Ø×+Ñ+Ð7Ø×+Ñ+¨d×.BÑ.BÒBñ ð ñ	ð	
ˆð 	
ð ˆùòm *EùòX	
s   ´	P)Ï32P.ri  rj  r,  s   @r:   r#   r#   û
  sÅ   ø„ ñð& $(Ø>BØ?CØ@DØ Ø+/ñ1?àÐ'Ñ(ð1?ð ð1?ð ˜D‘ð	1?ð
 ˜°Ð3Ñ4°tÑ;ð1?ð    _Ð 4Ñ5¸Ñ<ð1?ð    S ™>¨E°#©JÑ6¸Ñ=ð1?ð ð1?ð %)õ1?ðfz¸¸gÈ¹nÑ9M÷ zrF   r#   c                   ó²   ‡ — e Zd ZdZ	 	 	 	 	 	 ddee   dededz  dee	df   dz  de
ee	f   dz  d	e
eef   ee   z  dz  d
edefˆ fd„Zdeedz     fd„Zˆ xZS )r$   zò
    The DualPipeV schedule. A more efficient schedule variant based on the
    DualPipe schedule introduced by DeepSeek in https://arxiv.org/pdf/2412.19437

    Based on the open sourced code from https://github.com/deepseek-ai/DualPipe
    NrÐ   rª   r«   r¬   .r­   r®   r¯   r¼  c	           
      óü  •— t        || j                  j                  «       |d   j                  | _        t
        ‰| �  ||||||||¬«       t        | j                  | j                  d¬«      | _	        | j                  D ]  }	| j                  |	_	        Œ t        |«      | _        | j                  dk7  rt        d| j                  › d�«      ‚|| j                  k  rt        d|› d	| j                  › d
�«      ‚|d   j                  | _        |d   j                   | _        i | _        t%        | j                  «      D ]"  }
| j'                  |
«      }|| j"                  |
<   Œ$ | j)                  | j"                  «       y )Nr   rV  rƒ  r„  r„   r†  rX  zDDualPipeV requires at least as many microbatches as stages, but got z microbatches and z stages.r‡  rˆ  s               €r:   r»   zScheduleDualPipeV.__init__Ã  s˜  ø€ ô 	+¨6°4·>±>×3JÑ3JÔKØ# A™Y×1Ñ1ˆÔÜ‰ÑØØ)ØØ+Ø/Ø/Ø#Ø'Að 	ô 		
ô *HØ×Ñ × 0Ñ 0¸ô*
ˆÔ&ð —\‘\ò 	MˆEØ.2×.LÑ.LˆEÕ+ð	Mô " &›kˆÔØ×Ñ !Ò#ÜØBØ×&Ñ&Ð' qð*óð ð ˜D×,Ñ,Ò,ÜØVØ!Ð"Ð"4°T×5EÑ5EÐ4FÀhðPóð ð
 ˜1‘I×(Ñ(ˆŒ	Ø  ™)×.Ñ.ˆŒð
 @BˆÔÜ˜$×,Ñ,Ó-ò 	1ˆDØ×=Ñ=¸dÓCˆHØ(0ˆD×Ñ Ò%ð	1ð
 	×)Ñ)¨$×*=Ñ*=Õ>rF   r4   c                 óÜ  ‡‡‡— g }i Šg Š| j                   }| j                  }t        ||dz  d¬«      }||   \  }}dt        fˆfd„Šdt        dt        dt        fˆˆfd	„}dt        dt        d
t
        fˆˆˆfd„}	dt        fˆˆfd„}
||z
  dz
  dz  }t        |«      D ]  } |	||t        «       Œ |dz   }t        |«      D ]  } |	||t        «        |	||t        «       Œ  ||z
  dz
  }t        |«      D ]&  } |	||t        «        |
|«        |	||t        «       Œ( ||dz  z
  |z   dz   }t        |«      D ]B  }|dk(  r%||dz
  k(  r |	||t        «        |	||t        «       n ||||¬«        ||||¬«       ŒD ||z
  dz
  }t        |«      D ]  } |	||t        «        ||||¬«       Œ |dz   }d}t        |«      D ]V  }||dz  k(  r
|dz  dk(  rd}|rt        nt        } |	|||«       ||dz  k(  r
|dz  dk(  rd}|rt        nt        } |	|||«       ŒX ||z
  dz
  }t        |«      D ]"  } |
|«       |rt        nt        } |	|||«       Œ$ |dz   }t        |«      D ]
  } |
|«       Œ |S )Nr„   rƒ  r„  rI   c                 ó†   •— | t         f}| t        f}‰j                  |d«      dz   ‰|<   ‰j                  |d«      dz   ‰|<   y)zbHelper method to increment BACKWARD_INPUT and BACKWARD_WEIGHT counters when FULL_BACKWARD is used.r   r   N)r@   rA   r”   )rI   Ú	input_keyÚ
weight_keyÚcounterss      €r:   Úincrement_backward_countszVScheduleDualPipeV._calculate_single_rank_operations.<locals>.increment_backward_counts
  sK   ø€ à$¤nÐ5ˆIØ%¤Ð7ˆJØ"*§,¡,¨y¸!Ó"<¸qÑ"@ˆH�YÑØ#+§<¡<°
¸AÓ#>ÀÑ#BˆH�ZÒ rF   r}   Úforward_stageÚbackward_stagec                 ó  •— |t         f}|t        f}‰j                  |d«      }‰j                  |d«      }t        |t         |«      t        |t        |«      f}| j                  t        dt        d|«      «       |dz   ‰|<    ‰	|«       y)zYHelper method to add an overlapped forward+backward action which tracks microbatch index.r   rb   Nr   )r?   r@   r”   rH   rB   rg   r1   )
r}   r   r¡  Úforward_keyÚbackward_keyr^  r`  rL   rž  rŸ  s
           €€r:   Úadd_overlap_f_bzLScheduleDualPipeV._calculate_single_rank_operations.<locals>.add_overlap_f_b  s‹   ø€ ð )¬'Ð2ˆKØ*¬NÐ;ˆLà!Ÿ™ k°1Ó5ˆJØ"Ÿ,™, |°QÓ7ˆKô ˜¤w°
Ó;Ü˜¬°{ÓCðˆKð �N‰Nœ7 2¤{°D¸+ÓFÔGð %/°¡NˆH�[Ñ!Ù% nÕ5rF   rJ   c                 óø   •— |t         k7  r||fn|t        f}‰j                  |d«      }| j                  t	        |||«      «       |t         k(  r	 ‰|«       y |t        k(  r‰j                  ||f«       |dz   ‰|<   y )Nr   r   )rB   r@   r”   rg   rH   )r}   rI   rJ   r�   rÂ   rž  rŸ  Úweight_queues        €€€r:   Ú
add_actionzGScheduleDualPipeV._calculate_single_rank_operations.<locals>.add_action(  s‰   ø€ ð $¤}Ò4ð Ð.Ñ/à!¤>Ð2ð ð
  —|‘| C¨Ó+ˆHØ�N‰Nœ7 ;Ð0@À(ÓKÔLð  ¤=Ò0Ù)¨+Õ6ð $¤~Ò5à ×'Ñ'¨°hÐ(?Ô@Ø (¨1¡�˜’rF   c                 ó´   •— ‰sy‰j                  d«      \  }}| j                  t        |t        |«      «       |t        f}‰j	                  |d«      dz   ‰|<   y)z4Helper method to add a weight action from the queue.Nr   r   )r�  rg   rH   rA   r”   )r}   Úactual_stage_indexrM  r�  rž  r§  s       €€r:   Úadd_weight_action_if_pendingzYScheduleDualPipeV._calculate_single_rank_operations.<locals>.add_weight_action_if_pending@  sb   ø€ áØà2>×2BÑ2BÀ1Ó2EÑ/Ð Ø�N‰NÜØ&Ü#Ø#óôð -¬oÐ>ˆJØ#+§<¡<°
¸AÓ#>ÀÑ#BˆH�ZÒ rF   r   r   )r   r¡  FT)
rI  r±   r   rl   r•   r&   r�   r?   r@   rB   )rQ   rš   r}   r    Ú
num_chunksÚrank_to_stagesÚstage0_indexÚstage1_indexr¥  r¨  r«  Ústep_1rX  Ústep_2Ústep_3Ústep_4rˆ   Ústep_5Ústep_6Ú	enable_zbr&  Ústep_7Ústep_8rž  rŸ  r§  s                          @@@r:   r0  z3ScheduleDualPipeV._calculate_single_rank_operationsû  s  ú€ Ø(*ˆð ð 	ð ˆà×&Ñ&ˆ	Ø×)Ñ)ˆ
ä7Ø�y 1‘}¨Cô
ˆð &4°DÑ%9Ñ"ˆ�lð	C´3õ 	Cð	6Üð	6äð	6ô  ö	6ð.	-Üð	-äð	-ô /÷	-ð0	C´$ö 	Cð$ ˜dÑ" QÑ&¨!Ñ+ˆÜ�v“ò 	7ˆAÙ�w ¬gÕ6ð	7ð ˜‘ˆÜ�v“ò 	7ˆAÙ�w ¬gÔ6Ù�w ¬gÕ6ð	7ð
 ˜TÑ! AÑ%ˆÜ�v“ò 	7ˆAÙ�w ¬nÔ=Ù(¨Ô1Ù�w ¬gÕ6ð	7ð ˜i¨!™mÑ+¨dÑ2°QÑ6ˆÜ�v“ò 	ˆAØ�AŠv˜$ )¨a¡-Ò/á˜7 L´'Ô:Ù˜7 L´-Õ@áØØ".Ø#/õñ
 ØØ*Ø+öð	ð$ ˜TÑ! AÑ%ˆÜ�v“ò 	ˆAÙ�w ¬mÔ<ÙØØ*Ø+öð	ð ˜‘ˆØˆ	Ü�v“ò 	9ˆAØ�F˜a‘KÒ D¨1¡H°¢MØ �	Ù*3�¼ˆIÙ�w ¨iÔ8Ø�F˜a‘KÒ D¨1¡H°¢MØ �	Ù*3�¼ˆIÙ�w ¨iÕ8ð	9ð ˜TÑ! AÑ%ˆÜ�v“ò 	9ˆAÙ(¨Ô1Ù*3�¼ˆIÙ�w ¨iÕ8ð	9ð ˜‘ˆÜ�v“ò 	2ˆAÙ(¨Õ1ð	2ð ˆrF   ri  rj  r,  s   @r:   r$   r$   »  sÅ   ø„ ñð $(Ø>BØ?CØ@DØ Ø+/ñ6?àÐ'Ñ(ð6?ð ð6?ð ˜D‘ð	6?ð
 ˜°Ð3Ñ4°tÑ;ð6?ð    _Ð 4Ñ5¸Ñ<ð6?ð    S ™>¨E°#©JÑ6¸Ñ=ð6?ð ð6?ð %)õ6?ðp^¸¸gÈ¹nÑ9M÷ ^rF   r$   Úschedule_namec           
      ó0  — t         t        t        t        t        t
        t        t        t        dœ	}|D �ci c]  }|j                  «       |“Œ }}| j                  «       }||vr(t        d| › dt        |j                  «       «      › �«      ‚|||      S c c}w )z–
    Maps a schedule name (case insensitive) to its corresponding class object.

    Args:
        schedule_name (str): The name of the schedule.
    )	Ú1F1BÚInterleaved1F1BÚGPipeÚ	LoopedBFSÚInterleavedZeroBubbler   r   ÚZBVZeroBubbleÚ	DualPipeVzUnknown schedule name 'z'. The valid options are )r   r    r   r!   r"   r   r   r#   r$   Úlowerr7   r•   Úkeys)r¹  Úschedule_mapÚkÚlowercase_keysÚlowercase_schedule_names        r:   r   r   œ  s¢   € ô Ü2ÜÜ&Ü!>Ü"8Ü!6Ü.Ü&ñ
€Lð -9Ö9 q�a—g‘g“i ‘lÐ9€NÐ9Ø+×1Ñ1Ó3ÐØ nÑ4ÜØ% m _Ð4MÌdÐS_×SdÑSdÓSfÓNgÐMhÐió
ð 	
ð ˜Ð'>Ñ?Ñ@Ð@ùò :s   µBc           	      ó  ‡‡‡
‡— t        | «      D ��ci c]  }|| |   D �cg c]  }|€Œ|‘Œ	 c}“Œ } }}t        | «      D �ci c]  }|g “Œ c}Š‰D �ci c]  }|t        «       “Œ c}Š
dt        dt        dz  fˆ
ˆfd„}dt        dz  dt        fˆ
ˆˆfd„}| �r_d}t        | «      D ]O  }t        | |   «      dk(  rŒ| |   d   } ||«      r"|�	 |||«       | |   j                  d«       d	}ŒG ||d«       ŒQ t        | d	¬
«      D ]  }	t        | |	   «      dk(  sŒ| |	= Œ t        | «      D ]`  }t        | |   «      dk(  rŒ‰|   d   �Œ| |   d   } ||«      sŒ/|�|‰|   d<   ‰
|   j                  |«       | |   j                  d«       Œb t        | d	¬
«      D ]  }	t        | |	   «      dk(  sŒ| |	= Œ |s>t        dt        ‰«      «       | D ]  }t        d|›d| |   d   › �«       Œ t        d«      ‚| r�Œ_‰S c c}w c c}}w c c}w c c}w )aÂ  This function dry-run simulates the actions in the schedule from the perspective of all ranks, and flags
    any deadlocks caused by missing or misordered communications.  It also simulates any bubbles in time where a rank
    can not execute any action due to waiting for unmet dependencies.  The total number of simulator steps can be used
    as a metric for unit tests involving IR optimization passes as reordering and merging of IR can reduce the number
    of simulated steps.

    The simulation is not high-fidelity and does not model overlapping of compute and communication, or cuda streams.
    Future work may be to enhance this and model the compute time, comms overlap, and even memory.
    Nrš   r3   c                 ó\   •— ‰|    j                  |«       |�‰|    j                  |«       y y rN   )rg   rw  )rš   r3   Ú_prev_ops_rankÚ	_schedules     €€r:   Úadd_to_schedulez0_simulate_comms_compute.<locals>.add_to_scheduleÍ  s3   ø€ Ø�$‰×Ñ˜vÔ&ØÐØ˜4Ñ ×$Ñ$ VÕ,ð rF   r4   c                 ó  •— | €y| j                   }‰ ‰
|«         }| j                  t        k(  rd| j                   dk(  ryt        | j                   t        | j
                  «      |v ryt        | j                   dz
  t        | j
                  «      |v ryy| j                  t        t        fv r’| j                   ‰	dz
  k(  ryt        | j                   t        | j
                  «      |v ryt        | j                   dz   t        | j
                  «      |v ryt        | j                   dz   t        | j
                  «      |v ryy| j                  t        k(  ry| j                  t        k(  r)t        | j                   t        | j
                  «      }||v S | j                  t        k(  r-|dz
  }t        |t        | j
                  «      }|‰ ‰
|«         v S | j                  t        k(  rTt        | j                   t        | j
                  «      }t        | j                   t        | j
                  «      }||v xs ||v S | j                  t        k(  r-|dz   }t        |t        | j
                  «      }|‰ ‰
|«         v S t        d| › �«      ‚)NTr   r   FzUnsupported action type )rI   rJ   r'   rH   r-   rK   r@   rB   r/   rA   r,   r.   r7   )r3   ro  Úprev_opsÚ
expected_fÚpeer_stage_idxÚexpected_sendÚ
expected_bÚexpected_bwrÊ  r  r‘  s           €€€r:   rž  z3_simulate_comms_compute.<locals>._ready_to_scheduleÒ  sd  ø€ Øˆ>Øà×&Ñ&ˆ	Ø!¡-°	Ó":Ñ;ˆØ×"Ñ"¤aÒ'Ø×!Ñ! QÒ&Øä˜×*Ñ*¬F°F×4KÑ4KÓLÐPXÑXàä˜×*Ñ*¨QÑ.´°6×3JÑ3JÓKÈxÑWàØØ×$Ñ$¬¼Ð(GÑGØ×!Ñ! Z°!¡^Ò3ØÜ�v×)Ñ)¬6°6×3JÑ3JÓKÈxÑWØä˜×*Ñ*¨QÑ.´À×@WÑ@WÓXØñð ä˜×*Ñ*¨QÑ.´¸v×?VÑ?VÓWØñð ØØ×$Ñ$¬Ò7ØØ×$Ñ$¬Ò.Ü  ×!3Ñ!3´Q¸×8OÑ8OÓPˆJØ Ð)Ð)Ø×$Ñ$¬Ò.Ø&¨™]ˆNÜ# N´F¸F×<SÑ<SÓTˆMØ  N±=ÀÓ3PÑ$QÐQÐQØ×$Ñ$¬Ò.Ü Ø×"Ñ"¤N°F×4KÑ4KóˆJô "Ø×"Ñ"¤M°6×3JÑ3JóˆKð  Ð)ÒD¨[¸HÐ-DÐDØ×$Ñ$¬Ò.Ø&¨™]ˆNÜ# N´F¸F×<SÑ<SÓTˆMØ  N±=ÀÓ3PÑ$QÐQÐQäÐ7¸°xÐ@ÓAÐArF   Fr   T)Úreverserb   zWIP comms schedule:
rŸ  z next action= zSchedule is not progressing)r“   rv  rl   rH   rt   rm   r�  rw  Úprintr§   r7   )rx   r‘  r  rš   ri  rÌ  rž  r£  r3   rˆ   rÊ  rË  s    ``       @@r:   r  r  ·  s“  û€ ô ˜>Ó*÷àð 	˜.¨Ñ.Ö@�Q°!±-ŠqÒ@Ñ@ð€Nñ ô
 $ NÓ3ö2Øˆˆb‰ò2€Ið HQÖ.Q¸t¨t´S³U©{Ò.Q€Nð-œcð -¬7°T©>ö -ð
8B¤7¨T¡>ð 8B´d÷ 8Bòt ØˆÜ˜>Ó*ò 	,ˆDÜ�> $Ñ'Ó(¨AÒ-Øà# DÑ)¨!Ñ,ˆFÙ! &Ô)ØÐ%Ù# D¨&Ô1Ø˜tÑ$×(Ñ(¨Ô+Ø‘á  dÕ+ð	,ô ˜°Ô5ò 	&ˆAÜ�> !Ñ$Ó%¨Ó*Ø" 1Ñ%ð	&ô ˜>Ó*ò 	,ˆDÜ�> $Ñ'Ó(¨AÒ-Øà˜‰˜rÑ"Ð.Øà# DÑ)¨!Ñ,ˆFÙ! &Õ)ØÐ%Ø*0�I˜d‘O BÑ'Ø" 4Ñ(×,Ñ,¨VÔ4Ø˜tÑ$×(Ñ(¨Õ+ð	,ô ˜°Ô5ò 	&ˆAÜ�> !Ñ$Ó%¨Ó*Ø" 1Ñ%ð	&ñ ÜÐ)Ô+AÀ)Ó+LÔMØ&ò I�Ü˜˜˜˜~¨n¸TÑ.BÀ1Ñ.EÐ-FÐGÕHðIäÐ:Ó;Ð;óW ðZ Ðùòk Aùóùò2ùò /Rs&   “G= G8¨G8¬G=Á
HÁHÇ8G=c                 óN  — g }t        | «      D ]\  }t        | |   «      D ]I  \  }}|€Œ	|j                  t        |«      |j                  t
        t        t        fv rdndd|||ddœ«       ŒK Œ^ ddl}t        |d«      5 }|j                  d	|i|«       ddd«       y# 1 sw Y   yxY w)
aï  
    This function dumps a schedule IR into a chrometrace format so it can be visualized.

    It is currently very basic and only serves as a graphical alternative to dumping the schedule IR as text.

    As future work we may extend this to include more accurate heuristics for durations, or let users input durations,
    add 'flow events' to let the UI show the connection between sends and recvs, and model cuda streams for comm/compute
    as separate streams on the chrometrace view.
    NÚcomputationÚcommunicationÚXr   )rç   ÚcatÚphÚpidÚtidÚtsÚdurr   r  ÚtraceEvents)r“   r™   rg   rD   rJ   r'   r0   r)   ÚjsonrÓ  Údump)Úscheduler×  Úeventsrš   Útimestepr3   rá  Úfs           r:   Ú_dump_chrometracerç  <  sÀ   € ð €FÜ�xÓ ò ˆÜ )¨(°4©.Ó 9ò 	ÑˆH�fØˆ~ØØ�M‰Mä ›Kð "×2Ñ2´q¼!¼Q°iÑ?ñ &à,àØØØ"Øñõñ	ðó& ä	ˆh˜Ó	ð . Ø�	‰	�= &Ð)¨1Ô-÷.÷ .ñ .ús   Á=BÂB$rÐ   c           
      ó  — | D ]„  }t        |j                  t        j                  j                  «      sŒ2|j                  j                  «       D ]6  }t        |t        «      sŒt        d|› dt        |«      j                  › �«      ‚ Œ† y)zþ
    Check if the schedule is compatible with torch.compile.

    Args:
        stages: List of pipeline stages to check
        schedule_name: Name of the schedule for error message

    Raises:
        RuntimeError: If any stage uses torch.compile
    zThe ze schedule is not supported with stage modules that have used torch.compile. Found OptimizedModule in N)
rÌ   r  r÷   ÚnnÚModuler   r   r8   rê   r<   )rÐ   r¹  r¿   Úmodules       r:   rm  rm  `  s�   € ð ò 
ˆÜ˜%Ÿ,™,¬¯©¯©Ô8Øà—l‘l×*Ñ*Ó,ò 	ˆFÜ˜&¤/Õ2Ü"Ø˜=˜/ð *0ä04°V³×0EÑ0EÐ/FðHóð ñ	ñ	
rF   rN   )rG  )r   F)r„   )lr�   rÔ  r–   ÚloggingÚreÚabcr   r   Úcollectionsr   r   Úcollections.abcr   Údataclassesr   Úenumr	   Ú	functoolsr
   Útypingr   r   r   r   r÷   Útorch.distributedÚdistributedrþ   Útorch._dynamor   Útorch.distributed.fsdpr   r   Útorch.nn.modules.lossr   Útorch.profilerr   Ú_utilsr   r   rt  r   r   r   r¿   r   Ú__all__Ú	getLoggerr<   r¸   rD   r&   r?   r@   rA   r*   r+   r,   r-   r.   r/   rB   r1   r2   r'   r(   r)   r0   Úcompileri   rH   rw   rù   rl   r•   r§   r©   ÚP2POpÚWorkr  r
  r  r   r.  r   r   rt   re  rH  r‹  r�  r¥  r¹  r   r÷  rú  rÿ  r!   rN  rS  r    r"   r#   r$   r   r  rç  rm  rE   rF   r:   ú<module>r     só  ðó Û 
Û Û Û 	ß #ß ,Ý $Ý !Ý Ý ß 2Ó 2ã Ý  Ý )ß <Ý 'Ý *ç Rß TÑ TÝ %ò€ð 
ˆ×	Ñ	˜8Ó	$€ôN�s˜Dô Nð. ×
"Ñ
"€Ø!×0Ñ0€Ø"×2Ñ2€Ø
×
"Ñ
"€Ø
×
"Ñ
"€Ø	×	 Ñ	 €Ø	×	 Ñ	 €Ø	×	 Ñ	 €Ø	×	 Ñ	 €Ø ×.Ñ.€Ø×*Ñ*€Ø×*Ñ*€ð €Ø€Ø€Ø€ð �—
‘
ØRó€ô
T
ˆjô T
ðn ð¨ð °Cò ó ðð %)ñ<Ø˜˜d 7¨T¡>Ñ2Ð2Ñ3ð<à˜T‘zð<ð 	ó<ô~Q
˜ô Q
ñh+˜˜TŸZ™ZÑ(ð +°°d±
ð +ÀdÈ4Ï9É9Áoó +ð 37ñØ�$—*‘*ÑðØ%(¨4¡Zðà	ˆ#ˆt�D—I‘I‰Ð
Ñóð6˜$˜tŸy™y™/ó ô|Ð.ô |ô~0"Ð1ô 0"ôftÐ*ô tônqÐ)ô qðh!Ð'7ð !¸Dó !ð$Ø�'˜D‘.Ñ!ð$Ø36ð$à	ˆ'�D‰.Ñó$ðF ñPØ˜' D™.Ñ)ðPàðPð 
ˆ'�]óPðf#Ø˜' D™.Ñ)ð#à	ˆ'�]ó#ðLtØ˜#˜t G™}Ð,Ñ-ðtà˜S˜E 3˜JÑ'ðtð ðtð 
ˆ#ˆt�G‰}Ð
Ñó	tðn'Ø�#�t˜G d™NÑ+Ð+Ñ,ð'àð'ð ð'ð ð	'ð
 
ˆ#ˆsˆ(�^ó'ôD2Ð-ô 2ðD ÷ð ó ðôL˜hô Lôh2Ð4ô h2ôVBÐ0ô Bð\ ØóFð^ ñ<Ø
ð<àð<ð  ð<ð ð	<ð
 ð<ð ð<ð 	ó<ô&o
Ð6ô o
ôd{Ð$<ô {ô|}Ð4ô }ô@^Ð0ô ^ðBA có Að6BØ#+¨S¨E°3¨JÑ#7ðBØEHóBòJ!.ðHØÐ#Ñ$ðØ58ôrF   