)i!>uUddlZddlZddlZddlZddlZddlmZmZddlm Z ddl m Z ddlm Z m Z ddlmZmZddlmZmZmZmZddlZddlZddlmZddlmZddlmcmZ ddlm!Z!m"Z"m#Z#dd l$m%Z%dd l&m'Z'dd lm(Z(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.m/Z/m0Z0m1Z1dd l2m3Z4ddl5m6Z6ejnjpZ8gZ9e:e;ee#d2e#d?e;fd@Z\e%e?jeFd(e#dAe#fdBZ]e%e?je1eFd2e#de#fdCZ^e%e?je1d eFd(e#d2e#de#fdDZ_e%e?jd2e#dEe#de#fdFZ`e%e?jd(e#d2e#dEe#debe#e#ffdGZae%e?je1eFd(e#d2e#dHe#dIeOdJeOdKeBde#de:eJdeJdeJdseJde#f dZe%e?j jeF dd(e#d2e#deeOde#fdZe%e?j"e?j"jj%e8j&e?j"jj%e8j(dAe#dkeOdeeBfdZe%e?j*e1dddAe#dkeOdeeBfdZe%e?j,e1de#deJdeBfdZe%e?j.e1dde#deJdeBfdZe%e?j0e1 ddEe#de#deJdeBdeBde#f dZe%e?j2e1d(e#de#deJdeJdeBf dZde:eJfdZde:e#deJdeJde:e#fdZde:e#fdZde:e#deJfdZde:e#deJdeJfdZe%e?j>je?j>jg dde:e#deJdeJdee#de#f dZe%e?j@je?j@jg dd2e#de:eJdeJdee:e#dee:e#f dZe%e?jBjFddAe#deJdeJdebe#dffdZe%e?jDj ddAe#de:eJdeJdebe#dffdZe%e?jFjFdd2e#deJdeJdebe#dffdZe?jHjJj%e8j& dd2e#de#deJdebe#dffdZe%e?jNe1deFdd2e#de#de#d%eJd)eJf dZe%e?jPe1eF dd2e#de#de#d%eJd)eJdeBf dZe%e?jRe1deFdd2e#de#de#d%eJd)eJf dZe%e?jTjeFd(e#dAe#de#de#dee#deJdeJdeJdeJde:eBdebee#ee#ee#ffdƄZe%e?jTjd(e#dAe#de#de#dee#deJdeJdeJdeJde:eBdejFdejFdejFdebee#ee#ee#ffdȄZdee#dee#fdɄZe%e?jZjde#dAe#de:eJde#de#dEee#dee#de:eBdebee#ee#ee#ffd̈́Ze%e?jZjde#dAe#de:eJde#de#dEee#dee#de:eBdejFdejFdejFdebee#ee#ee#ffd΄Ze%e?j^jdAe#de:eJdEee#deeOdebe#e#ff dτZe%e?j`jde#dAe#de:eJde#dEee#de:eBdebee#ee#ffdЄZdAe#dEee#dee#dee#dee#dKeBdeOdeOdeBdebe#e#e#ee#ee#ffdՄZe%e?jde1ddd׫dAe#dEee#dee#dee#dee#dKeBdeOdeOdebe#e#e#ffd؄Ze?jdjj%e8j(e?jdjj%e8j&dAe#dEee#dee#dee#dee#dKeBdeOdeOdebe#e#e#ffdلZe?jhjj%e8j&dde:e#fdڄZe%e?jljdAe#dEee#dee#de#de#deOdeOdebe#e#e#ffdۄZe%e?jnjdAe#dEee#dee#de#de#dKeBdeOdeOdebe#e#e#ffd܄Ze%e?jnjpdAe#dEee#dee#dKeBdeOdeOdebe#e#e#ffd݄Ze%e?jtjdAe#dEee#dee#de#de#dKeBdeOdeOdebe#e#e#e#e#ffdބZdAe#dEee#dee#de#de#deOdKeBde#fd߄Ze%e?jxjdAe#dEee#dee#de#de#deOdeOdebe#e#e#e#ffdZe%e?jzjdAe#dEee#dee#de#de#deOdeOdebe#e#e#e#e#e#ffdZe%e?j|jdAe#dEee#dee#de#de#deOdeOdebe#e#e#e#ffdZe%e?j~e1ddeFddZe%e?je1ddddddddee#e)fdTeejdeejdeBdeBdeejf dZe%e?je?je?jge1dZe?jjj%e8j(e%e?je1dddddAe#dEe#dee#dee#dee#dKeBdeOdeOfdZdZe%e?jjde#dAe#dEee#dee#dee#dee#dee#deBdeOde:eBde#debe#ee#ee#ffdZe%e?jjde#dAe#dEee#dee#dee#dee#dee#deBdeOde:eBdebe#ee#ee#ffdZe%e?jjde#dAe#dEee#dee#dee#dee#dee#deBdeOde:eBdejFdejFdejFdebe#ee#ee#ffdZe%e?je1dddǫdAe#d(e#dEe#dee#dee#dee#dee#deOfdZe%e?je1dddǫdAe#d(e#dEe#dee#dee#dee#dee#deOde#fdZe%e?je1eFdAe#debeJeJffdZd2e+de+de:eJdeJfdZe%e?je1d2e+de+de:eJfdZe%e?je1dAe+de+de:eJde:eJde:eJf dZe%e?jdudde+deJdze+de+d)e)f dZe%e?je1dudde+deJdze+de+d)e)f dZdudde+deJdze+de+deBd)e)f dZe%e?jje?jjj%e8j&ddZe%e?jde+deJdze+de+fdZe%e?je1de+deJdze+de+fdZde+deJdze+de+deBf dZe%e?je1ddMeFd2e#debe#e#ffdZe%e?je1 dde#deeBeJeOfdeeBeJeOfd eejfd Ze%e?jdd Zސd Zߐd Ze%e?jje%e?jje%e?jje?jjj%e8j&e?jjj%e8j(e?jjj%e8j&e?jjj%e8j(e?jjj%e8j&e?jjj%e8j(dAe#dee:eJdee:eOde#fdZe%e?jje%e?jje%e?jje?jjj%e8j&e?jjj%e8j(e?jjj%e8j&e?jjj%e8j(e?jjj%e8j&e?jjj%e8j(dAe#dee:eJdee:eOde#fdZddZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOde#fdZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOde#fdZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOdeeOde#f dZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOdeeOde#f dZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOdeeOdeeOde#f dZe%e?jje?jjge?jjj%e8j&e?jjj%e8j(e1dd ddAe#de:eJdeeOdeeOdeeOde#f dZeF ddAe#de:eJde:eeOdeBde#f dZdZd Zd!Zd"Z dd#Zd$Zd%Zdd&Zdd'Zd(Ze%e?jje?jjj%e8j&e?jjj%e8j(d)Ze%e?jje?jjj%e8j&e?jjj%e8j(d*Ze%e?jje?jjj%e8j&e?jjj%e8j(d+Ze%e?jje?jjj%e8j&e?jjj%e8j(d,Zd-Zdd.Zdd/Zd0Ze%e?jje?jjj%e8j&e?jjj%e8j(d1Ze%e?jje?jjj%e8j&e?jjj%e8j(d2Zd3Zd4Ze%e?jje?jjj%e8j&e?jjj%e8j(d5Z e%e?jje?jjj%e8j&e?jjj%e8j(d6Z e%e?jje?jjj%e8j&e?jjj%e8j(d7Z e%e?jje?jjj%e8j&e?jjj%e8j(d8Ze%e?jje%e?j je?j"jj%e8j&e?j"jj%e8j(e?jjj%e8j&e?jjj%e8j(e?j jj%e8j&e?j jj%e8j(d9Ze%e?j"je?j"jge1 ddAe#de:eJd:eBdeeOde#f d;Ze%e?jje?jjge?jjj%e8j(e1 ddAe#de:eJd:eBdeeOdeeOde#f d<Ze%e?j je?j jge1 ddAe#de:eJd:eBdeeOdeeOdeeOde#fd=Zdd>Zd?Zdxee#d@ee#dAe#de#fdBZd@e,de#fdCZeFdAe#de:eJd:eBde:eeOde#f dDZe%e?j0jdEe#dFe#deBfdGZe%e?j2e?j4ge1dHZe%e?j6gdIZe%e?j8gddJZe%e?j:gdKZe%e?j<gdLZd2e#dVe#dEee#dReJdaeJdebe#e#ff dMZe%e?j@e1ddbd2e#dVe#dEee#dReJdaeJdebe#e#ff dNZ e%e?jBe1ddbd2e#dVe#dEee#dReJdaeJdebe#e#ff dOZ!de#dPeOde#fdQZ"de#dPeOde#fdRZ#dSe#de,fdTZ$dUe,dVe#de#fdWZ%dVee#de#fdXZ&dYeJd:eBdTejdejfdZZ'd[e#d\eJd]eJd:eBfd^Z(d[e#d_eJd\eJd]eJd:eBf d`Z)d[e#de:eJd:eBfdaZ*d[e#de:eJd:eBfdbZ+e%e?jXe1eFd[e#de:eJd:eBfdcZ, ddEe#dde#deeJdfeJd:eBdgeBde#fdhZ-e%e?j\e1eF ddEe#dde#deeJdfeJd:eBde#f diZ.e%e?j^e1deFdjZ/e%e?j`e1dde@jjfdkZ0dlejFdmejFdneBdeBfdoZ1e?jdjj%e8j&e?jdjj%e8j&e1dpddqdrZ2e%e?jfje?jfjge?jfjj%e8j(e1eF ddAe#debeJeJfd:eBdseeOdteeOde#f duZ4e%e?jfje?jfjj%e8j&e?jfjj%e8j(e1eF ddEe#deebeJeJfd:eBdeebeOeOfde#f dvZ5e%e?jle%e?jne%e?jpeFe1dEe#debeJdfde#fdwZ9e%e?jte%e?jve%e?jxeFe1dEe#debeJdfde#fdxZ=dEe#debeJdfdyeeJeJeJge#fde#fdzZ>e%e?j~e%e?je%e?je1d d{ZBe%e?je1d|d}ddd~dZCe%e?je1ddddZDe%e?jje?jjge1dejddddre)dTeejdejdeejdeBf dZHe%e?jjgdejddddqe)dre)dTeejdejdeejdeBf dZJe%e'dZKe%e?je?jjj%e8j(e1dudude@jjfdAe#dVe#dke)de)dEee#dReJde#fdZLe%e?je?jjj%e8j(e1dddAe#dVe#dReJdebe#e#ffdZMe%e?jj ddddde#de#d0e#deOdeBdee#d*eeOdebe#e#ffdZOdZPe%e?jge1deFddZQe%e?je1dZRe%e?jdZSe%e?jje?jjgdddd2e#dTeejdee#de#fdZUe%e?jje?jjgdd2e#deeJfdZXe%ej|j~jddZYe%e?je1ddddZZe%e?jjddd2ejFd eejdejFfdZ[dddZ\ddddZ]e%e?je1dZ^e%e?jddZ_ePe?je?jePe?je?jNePe?je?jRePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?je?jePe?j e?j ePe?je?jePe?je?jePe?je?jy(N)CallableIterable) nullcontext)Enum)partialreduce)chainproduct)AnycastOptionalUnion) sym_floatsym_intTensorregister_decomposition) out_dtype)IntLike NumberTypesuggest_memory_format TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_out out_wrapper)_pytree)tree_map__all__ceZdZdZdZdZy) ReductionrN)__name__ __module__ __qualname__NONEMEANSUM^/var/www/pod-logistic/pod-ai/venv/lib/python3.12/site-packages/torch/_decomp/decompositions.pyr"r"1s D D Cr,r"Fftype_promotioncompute_dtype_onlyinclude_non_tensor_argscNtjfd}|S)Nc\ r ttjjfntf}t j |i|Dcgc]}t ||r|}}tj|d i\ fd} fd} t||it||} r|St||Scc}w)Ntype_promotion_kindcJt|tr|jS|SN isinstancerto)xcomputation_dtypes r- increase_precz0type_casts..inner..increase_precOs"!V$tt-..r,cJt|tr|jS|Sr6r7)r: result_dtypes r- decrease_precz0type_casts..inner..decrease_precUs!!V$ttL))r,) rtorchtypes_Numberpytreearg_tree_leavesr8utilselementwise_dtypesr)argskwargs allowed_typesr: flat_argsr<r?rr;r>r0r.r1r/s @@r-innerztype_casts..inner@s.EVU[[(( )6)  ++TJ;V1W W ' !  r,c.tj||Sr6)r@ full_likeselfvalues r- fill_scalarr~s ??4 ''r,r}ctjjdk(fdtj |S)Nrc,djdS)Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrP)r}sr-zfill_tensor..sRSXS\S\S^R__jkr,)r@_checkrPatencopyr{s `r- fill_tensorrs3 LL qk 99T5 !!r,r|cftjtj|dzdddz SNrminmaxr@clampr|s r- hardsigmoidrs) ;;u{{4!83 ;a ??r,cHtj|dkD|dkz|dzdS)Ngg@gUUUUUU?r@rjrmr|s r-hardsigmoid_backwardrs0 ;; $y!  r,min_valmax_valcBtj||k||k\zd|S)Nrr)rmr|rrs r-hardtanh_backwardrs$ ;;DGO L r,c6tj||kd|Srtr)rmr|res r-threshold_backwardrs ;;ty(![ 99r,negative_slopeself_is_resultc<tj|dkD|||zSrtr)rmr|rrs r-leaky_relu_backwardrs ;;taxkN.J KKr,grad approximatecd}d}d}|dk(ri||zdz}d}||z}||z} |||| zzz} tj| } d|z} d| z} d| z}d| | zz }|dd|z|zzz}| |z|z}|||zzS|}||zdz}ddtj||zzz}|tj||zd zz}||||zzzS) Ng;f?g;f?gmBP ?tanhrgHm?r#rg)r@rerfri)rr|rM_SQRT2 M_SQRT1_2 M_2_SQRTPIkBetakKappax_sqx_cuberL tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs r- gelu_backwardrs'%G&I'Jf*$s*d{/0ZZ& TzJ+j:55 AF T(9$9:/14DD)99::Y&,Q4&=112eiit d 233sTCZ'((r,inputctjtj|}tj|}||zd||zz z}|||zzSr])r@rFsoftplussigmoid)rmrinput_tanh_softplus input_sigmoidouts r- mish_backwardrsV **QZZ%67MM%(M - 1':=P'P#P QC -3 44r,c2|tj|zSr6)r@rrs r-silur"s %--% %%r,c\ddtj| zz }||zd|d|z zzzSr])r@ri)rmr|rs r- silu_backwardr)s<1uyy$''(G  AG (<$< ==r,weightc<tj|dkD|||zSrtr)r|rs r- _prelu_kernelr1s ;;taxv} 55r,c~tj|dkD|||z}tj|dkDd||z}||fS)Nrrr)rmr|r input_grad weight_grads r-_prelu_kernel_backwardr6sE TAX{F[4HIJ++dQhTK-?@K  $$r,noiseloweruppertrainingcx|r||z dkDr|j|S||zdz }tj||||S)Ngư>r$)mulrr)rmr|rrrrrrs r-rrelu_with_noise_backwardrAsKEEMD(u%%%-1,'' ~~  r,bufferc|dk}tj|dd}tj|dd}tjtj| }||||d|zz zz zS)Nrr#rT)r@rjriabs)rmr|r in_negative max_derivsignrks r-log_sigmoid_backwardrVsg(K KA.I ;;{Ar *D 599T?"#A )da1q5k&:: ;;r,otherctj|jstj|jrtj n |j}|j dd|}|t j||zS)Nr+@dtype)rEis_integer_dtyperis_boolean_dtyper@float32new_fullpow)r|r two_dtype two_tensors r-ldexprcsh  ! !$** -1G1G 1S  ZZ r3i8J %))J. ..r,loss reductionc|tjjk(rtj|S|tj jk(rtj |S|Sr6)r"r)r}r@meanr*sum)rrs r-apply_loss_reductionrosHINN(((zz$ imm)) )yy r,rc|tjk(rtjS|tjk(rtjS|tj k(rtj Syr6)r@ complex32float16 complex64r complex128float64rs r- to_real_dtyperxsK }} %// !}} %"" "}} #r,targetc*||z dz}t||S)Nr$)r)r|rrrs r-mse_lossrs 6Ma D i 00r,c||tjjk(rd|jz nd}|||z z|zS)Nr)r"r)r}numel)rmrrrnorms r-mse_loss_backwardrs; #,y~~/C/C"C3 D 56> "[ 00r,ctj|||}|jtd}tj||d}tj |}tj |||S)N)rPrz-infTrPkeepdim)r@softmaxeqfloatall zeros_likerj)r|rPrrmasked masked_rowszeross r- safe_softmaxr s[ --#U 3C WWU6] #F))FT:K   S !E ;;{E3 //r,rhc||z j}tj||kd|dzz|z |d|zz }t||S)Nrr$)rr@rjr)r|rrrdrs r-smooth_l1_lossrsO 6M   D ;;td{C$'MD$8$t:K LD i 00r,c|tjjk(rd|jz nd}||z }t j |}||z}t j ||k||z|z |t j|zSrg)r"r)r}rr@rrjr) rmr|rrrdrr:abs_x norm_grads r-smooth_l1_loss_backwardrs{ "+inn.B.B!B3 D v A IIaLE{"I ;;  A EJJqM! r,cht|||||}t||jt||dSNT copy_fromcopy_to exact_dtype)rrshaper)rmr|rrrdrYresults r-smooth_l1_loss_backward_outrs3%[$ 4 PFj&,,/ FJD QQr,deltac |tjjk(rd|jz nd}||z }t j || k| |z|zt j ||kD||z|z||z|zSrg)r"r)r}rr@rj)rmr|rrrrr:s r-huber_loss_backwardrs "+inn.B.B!B3 D v A ;; UF  e# AItk1E94!8k;QR r,cht|||||}t||jt||dSr)rrrr)rmr|rrrrYrs r-huber_loss_backward_outr!s3!dFIu MFj&,,/ FJD QQr, ignore_index total_weightc|jdkrdnd}|tjjk(r||z }|j |}t j ||k7|d}t j|} t j| ||d} | j|jcxkDrdkDrnn|j |}|Nt|jD cgc]} d} } |jd| |<|j| }||z}t j ||k7|d}| |zScc} w)Nr$rr#g) rPr"r)r}rVr@rjr scatterrUrreshape) rmr|rrrr"r# channel_dim safe_targetrYrW new_shapes r-_nll_loss_backwardr*sxxzA~!1KINN(((!L0   k *F++f 4fa@K!!$'Jz; TJJ~~+//+/a/!++K8   %dhhj 121Q2 2!'a + *!F* ++f 4k1EK  ##3s* D<c|jdkr tdtj|j|}|j |}|dzdk7rtd|d||dz}|j |d|}|j |||}t j|}d|z |z|z|z} ||z}t j|| g|S)Nrz*glu does not support 0-dimensional tensorsr$z.Halving dimension must be even, but dimension z is size rhr) rPAssertionErrorrEcanonicalize_dimsizenarrowr@rcat) rmr|rPwrap_dimnIn inputSize firstHalf secondHalfgradInputFirstHalfgradInputSecondHalfs r- glu_backwardr8s xxzQIJJ%%dhhj#6H ))H C Qw!|H MMr,c ^d|jcxkrdksntd|jd|jdkDrtd|jd|jdk(xr|jdk(}|sE|jd|jdk(s&td|jd|jd |jdk7r*td |jd |jd |+|j|jd k7r td|tj j k(r|jdk(rn|jdk(r|jd|jdk(sztd|jdd|jd|jd|jdkr|jdk(std|jt|||||||S)Nrr$%input tensor should be 1D or 2D, got Dr#A0D or 1D target tensor expected, multi-target not supported, got size mismatch (got input: , target: ):expected total_weight to be a single element tensor, got: z (z elements)rTz J  INN(((TXXZ1_!Q&;+<+!+C I+J[J[I\]  T669lL r,c ^|jdk7rtd|j|jdk7rtd|j|jd|jdk(r>|jd|jdk(r|jd|jdk(s%td|jd |j|jdk7r*td |jd |jd t |||||||S) NzSonly batches of spatial inputs supported (4D tensors), but got input of dimension: rzUonly batches of spatial targets supported (3D tensors) but got targets of dimension: rr$r#r=r>r@z ( z , elements))rPr,rrr*)rmr|rrrr"r#s r-nll_loss2d_backwardrEZs; xxzQabfbjbjblam n  zz|qcdjdndndpcq r  1 a( JJqMV\\!_ , JJqMV\\!_ ,( Jv||n M  q  &&'s<+=+=+?*@  M  T669lL r,c "|dz tjtj| |jddz|tjtj||jddzz }|||z}t ||S)Nr#r+i)r@maximumlog1prlogr)r|rrrrs r-binary_cross_entropyrJs QJ%-- TEDMM"d3 uyy b$0GHH IDf} i 00r,cd}|||z ztj|d|z z|z }|||z}|tjjk(r||j z }|S)Ng-q=r#r)r@rr"r)r}r)rmr|rrrEPSILONrs r-binary_cross_entropy_backwardrMsgG D6M *U[[T9JPW-X XF &INN((($**,& Mr,crtjtj| |z}t||Sr6)r@rHrir)rrrrs r-soft_margin_lossrOs. ;;uyy%&1 2D i 00r,c||ztj||zdz z}|tjjk(r||j z }|Sr])r@rr"r)r}r)rmr|rrrYs r-soft_margin_loss_backwardrQsM+%v})E)IJJINN((($**,. r,pc6tj||z |S)N)rR)rr)rrrRs r-distrTs 99UU]a9 ((r,x1x2c|jdjdd}tj|tj}|jdjdd}tj|tj}tj |j d||gd}tj |||gd}|j|j}|jdjS)Nr$rTT memory_formatr) rrr@ ones_likecontiguous_formatr0rmatmulmT clamp_minsqrt) rUrVx1_normx1_padx2_normx2_padx1_x2_rs r-_euclidean_distrgsffQimmB%G __WE4K4K LFffQimmB%G __WE4K4K LF ))RVVBZ&12 6C ))R)2 .C ZZ F   A  # # %%r, input_sizesstartendstepcX|j|}tj||||||Sr6) new_zerosr@ slice_scatter)rmrhrPrirjrkrYs r-slice_backwardros/&&{3J   z;UC NNr,r#cddlm}|j}|dk(r tdt j |j|}t |j}t |j}|dkr td||nd} ||ntj} | dkr| ||z } | dkr| ||z } | dkrd} n | ||kDr||} || tjk(r||} n| | kr| } n | ||kDr||} |j| ||zz} | | z } | |zdz |z||<||xx|zcc<|jr td|j||| S)Nrstatically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver#z>r,cn|j|dtffd }||dd}|||}||fS)zn Normalize start and end such that both are in the range [0, x.get_size()[dim]] and start <= end. rQcL||S|dkr|z}tt|||Srtrr)valrrdefaultdim_sizes r- clamp_wrapz(_normalize_start_end..clamp_wrap(s0 ;N 7.C3sE?E**r,r)rint)r:rPrirjrrs @r-_normalize_start_endrsJwws|H+#+ ua1 -E S%8 4C #:r,srcc |tj|j|}|j|}t ||||\}}t |j}||z |dz z|z||<|j |}|dk(r||k(r|dk(r|jSdg|jz}tj||j} | |z |z||<tj||jtj} |dk7rtj| | |k\} ||k7rtj| | |k} |dk7rtj| | |z |zdk(} dg|jz} d| |<| j| } t j#| t j%|| |d|S)Nr#rdevicerrrT)rEr-r}rrruexpandclonerPr@arangeronesbool logical_andviewrrj_unsafe_masked_index) rrrPrirjrkrsrc_sizeindicesidxmask mask_shapes r-rnrn6s  S 1C{{3H%eS%=JE3EKK H5[D1H-$6HSM **X C zcXo$!)yy{'+fuyy{&:G ,,x 5C%KD(GCL ::hu||5:: FD z  se|4 h  sSy1 qy  e t';q'@Auyy{"JJsO 99Z D ::dD55c4!Le TTr,indexcT|j|}tj||||Sr6)rmr@select_scatter)rmrhrPrrYs r-select_backwardr_s+&&{3J    Ke DDr,offsetdim1dim2cV|j|}tj|||||Sr6)rmr@diagonal_scatter)rmrhrrrrYs r-diagonal_backwardrfs- &&{3J  ! !*k64 NNr, input_dtypecF|j|k7r|j|}|Sr6)rr9)rmrYrs r-_cast_grad_to_input_dtyperos&K']];/ r,outputc~||z}||tj||dzz }t|||jSNTr)r@rr contiguous)rmrrPrnew_grad_outputrYs r-_softmax_backward_datarwsK "F*O 6EIIS$-$J %[*k J U U WWr,c~|tj|tj||dzz }t|||Sr)r@rirr)rmrrPrrYs r-_log_softmax_backward_datarsA uyy0599d4 J %[*k JJr,c||dzz||dz zz }ttjtj|}|d||j d}|d||z|j d} || zS)z/Utility function to implement im2col and col2imr$r#rrrrT)rr@rint64rV) input_dkernel_d dilation_d padding_dstride_drblocks_d arange_kwblocks_d_indices kernel_grids r- _im2col_col2im_indices_along_dimrsQ&x!|)DDH EKKGI!Hh7AA!DAx*4jAKKBOK k ))r, kernel_sizedilationpaddingrvc tjtdk(dtjtdk(dtjtdk(dtjtdk(ddd}|d|d|d d |d |jt}tj|d vxrt dddDfdt dt ddDtjt dDfd|dk(}|s|jd}|j\}} } } \} } \}}\}}\}}t| |||| |j}t| |||| |j}tj|||||f}|jdjd}|dddd||f}|jdddddd}|jd}|jd}|j|| |z|z||z}|s|jd}|S)Nr$cy)Nz"im2col(): only 2D kernel supportedr+r+r,r-rzim2col..r,cy)Nz$im2col(): only 2D dilation supportedr+r+r,r-rzim2col..rr,cy)Nz#im2col(): only 2D padding supportedr+r+r,r-rzim2col..rr,cy)Nz"im2col(): only 2D stride supportedr+r+r,r-rzim2col..rr,c|rtdDntdD}tj|fdy)Nc3&K|] }|dkD ywrNr+.0rRs r- z1im2col..check_positive..(Q1q5(c3&K|] }|dk\ ywrr+rs r-rz1im2col..check_positive..;RqAF;RrcdSNz& should be greater than zero, but got r+param param_namesr-rz0im2col..check_positive..ZL(NugVr,rr@rrrstrictconds`` r-check_positivezim2col..check_positive4,2s(%((;RE;R8R V r,rrrFrrvrrDc3&K|] }|dk7 ywrr+rds r-rzim2col..:!qAv:rrc dtS)NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tuplersr-rzim2col..--25\N<r,c3\K|]$\}}}}}d|d|zz||dz zz dz |zz&yw)r#r$Nr+)rrpaddilkersts r-rzim2col..sF "Cc3 S1s7]SC!G_ ,q 0R 77s*,rZc3&K|] }|dkD ywrr+)rcs r-rzim2col..s 'aAE 'rc Fdtdddddddd S) Nz!Given an input with spatial size rZ, kernel_size= , dilation= , padding= , stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r)rr output_sizerrrvsr-rzim2col..sL3E%*4E3FG"m;xj9)9VH-]D Fr,rDrrTr#rT)r@rrrrrziprVrrrrpermuter.r&squeeze)rrrrrvrr} batched_input batch_dimr'input_hinput_wstride_hstride_w padding_h padding_w dilation_h dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indices padded_inputrnum_blocks_rownum_blocks_colrrs ```` @@r-im2colrs~ LL[!Q&(TU LLX!#%ST LLW"$QR LLV!#OP ; .8Z(8Yu568$ KKE u:D LL :3:uRSz:: < &) "#J;' K  LL '; '' F FAIM "/4{{,I{GWHh"Iy%J $Hh9:y(ELL::y(ELL 55Iy) LML+55b9CCBG !Q 24FF GF ^^Aq!Q1 -F',,Q/N',,Q/N ^^;)H4n~6UF " Mr,rc !"tjtdk(dtjtdk(dtjtdk(dtjtdk(dtjtdk(ddd}|d |d |d d |d|d|j"t"}tj|dvxrt d"ddD"fdddz}tj"d|zdk("fdt D cgc]"\} } } } } d| d| zz| | dz zz dz | zz$}} } } } } |d|dz!tj"d!k(!"fdtj!dkD!"fd|dk(}|s|j d}|j"\}}\}}\}}\}}\}}|j"d"d|zgz|z}|jdddddd}t||||||j}t|d}t||||||j}t Dcgc] \}}|d|zz}}}|j"d"dtzg|z}dd||f} tj|| |d}t!j"|| | | | f}|s|j%d}|Scc} } } } } wcc}}w)Nr$cy)Nzonly 2D output_size supportedr+r+r,r-rzcol2im..rr,cy)Nzonly 2D kernel supportedr+r+r,r-rzcol2im..rr,cy)Nzonly 2D dilation supportedr+r+r,r-rzcol2im..rr,cy)Nzonly 2D padding supportedr+r+r,r-rzcol2im..rr,cy)Nzonly 2D stride supportedr+r+r,r-rzcol2im.. rr,Tc|rtdDntdD}tj|fdy)Nc3&K|] }|dkD ywrr+rs r-rz1col2im..check_positive.. rrc3&K|] }|dk\ ywrr+rs r-rz1col2im..check_positive.. rrcdSrr+rsr-rz0col2im..check_positive..rr,rrs`` r-rzcol2im..check_positive rr,rrrFrrvr)r$rc3&K|] }|dk7 ywrr+rs r-rzcol2im..rrrZc dtS)NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: rrsr-rzcol2im..rr,rr#cdddS)Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rZz and kernel_size=r+)rrsr-rzcol2im..!s#==B2YKH"m%r,rTc:ddddddddd S NzGiven output_size=rrrrz , expected input.size(-1) to be but got rT.r+Lrrrrrrvsr-rzcol2im...F$[M }M:Zy &B))*9U2YKqBr,c:ddddddddd Srr+r sr-rzcol2im..4r"r,rrDr accumulater)r@rrrrrrVr&rrrrXrmprodr_unsafe_index_putrrr)#rrrrrrvrr}prod_kernel_sizerrrrrcolrout_hout_wrrrrrrrr indices_row indices_colorRoutput_padded_sizerrr!rs# ````` @@r-col2imr0s LL[!Q&(OP LL[!Q&(JK LLX!#%IJ LLW"$GH LLV!#EF ; .8Z(7Ie468$; . KKE u:D LL :3:uRSz:: < #1~ A6 LL b $$) %'* (K'    "Cc3 S1s7]SC!G_ ,q 0R 77 C  AQA LL b Q B B  LL A B B AIM " KKELE5Hh"Iy%J $Hh MM58U1X1A%AB[PSVV WE MM!Q1a +E2 xY%,,K$K3K2 xY%,,K14K0IJ1!a!e)JJ __ q58tK0014FFF {K 0C  # #FC4 # HF UU6YJ YJ K LF " Mk VKs 6'M 9Mrcz||j||zzjtj|}|SNrX)type_asrrEr)rmrrorKs r-native_dropout_backwardr4]sB  [1E9 :AA11+> B A Hr, input_size dimensionr.ct|dk(rtj|dStjt||}tj |||j tj}|jd||j}|jd|dzj||dz}|j|}d|z|fz}tj|||djS)NrrrTr#r6Tr$)rr@ squeeze_copyrEr-rrint32unfoldflattenmovedimrmrr'r) rr5r6r.rkrPrrYrs r-unfold_backwardr=ls  :!!!$**  Z) lohis r-logit_backwardrCs    2X{{   dbj$"* 5 43:. /   {{   dck43; 7 43:. / MM"eEl +  r,traincd|r|dk7rtj|||dS|jSrt)rnative_dropoutr)rrRrDs r-dropoutrGs3 a""5!U3A66{{}r,out0out1c|r|dk7r|dk(r:tj|tj|tjfS|jjs t dtj ||kD}||ztdd|z z z}||fS|tj|tjfS)Nrr#rz?result type Float can't be cast to the desired output type Longrh) r@r rris_floating_pointrt rand_likerr[)rrRrD bool_maskress r-rFrFs a 6$$U+U-=-=e5::-VW W{{,,Q OOE*Q. %%sQw"88YuuEJJ?@@r, half_to_floatcBddlm}|j}|r6|jtj k7rt d|jdtj|tjj\}}|j|}||jdk(rt j|}n0t j||d}t j||z }|t j||dz }|s|j|}|SNrguard_or_falsez%half_to_float is True but x.dtype is z, expected torch.halfr4T)r)rsrSrrr@halfr,rErFELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr9rriamaxr) r:rPrOrSr;r> unnormalizedx_maxrs r-_softmaxr[sE A 77ejj  7y@UV ',&>&> uDDLL'#|  Aaggi1n%yy|  1c40yyU+ EIIlCF FF <( Mr,)rcFddlm}|j}|r6|jtj k7rt d|jdtj|tjj\}}|j|}||jdk(r|}nt j||d}||z }t jt jt j ||d}||z } |s| j|} | SrQ)rsrSrrr@rUr,rErFrVrWr9rrXrIrri) r:rPrOrSr;r>shiftedrZshifted_logsumexprs r- _log_softmaxr_sE A 77ejj  7y@UV ',&>&> uDDLL'#|  Aaggi1n% 1c40e) %))EIIg,>T"RS ( (F <( Mr,r padding_idxscale_grad_by_freqsparsec|jdk7rtd|jd|jdkr4|jd|}|jdk(r|j d}|S||S)Nr$z'weight' must be 2-D, got z-Dr#r)rPr,r} index_selectr)rrr`rarbrs r- embeddingresszz|q9&**,rJKK||q!!!W- <<1 ++a.C gr, num_weightscttj|tjj\}}|j |}t |t j}|rZ|j|f}t j|}tj||g|d}||} || jdz }t||k(|j} |j| d} |j|f|j |jdz} tj| |g| dj |S)NrTTr$rTr)rErFrVrWr9rr@longrmr[rr'rVrXr} masked_fillr) rmrrfr`rar;r>countsrgrad_weights_scalerr grad_weights r-embedding_dense_backwardrms,',&>&>)N)N)V)V'#|..!23K%guzz:G""K>2w''' 4D'Q#G_!$6$@$@$DD W 3[5E5E FD  " "4 +D'' **7<<>::K  ! !+y$4 ! P S S r,c"d}|D]}||z} |Sr]r+)r:rKis r-r&r&s$ A  Q Hr,tensors num_chunkscZg}|D]}|j}|||zdz |z|z}|||k7r;dgdz|j|z dz zd|||z gz}tj||d}|d|t j |dgz}|j |j||S)Nr#rr$rT)r.r}rconstant_pad_ndr@Sizeappendr&) rprPrqpadded_tensorstensor tensor_size pad_along_dimr view_sizes r- _pad_chunkr{$s N 9kkm $S)J6:zIJV K, ,#'V[[3.23 C 007C))&#q9F% J3C(DD fnnY78 9 r,cR|dj}|D]}|j|k7syy)NrFTr})rpr}rws r-have_same_ndimsr~9s2 1:??D ;;$  r,c|djd|}|D]-}tj|jd||k(d/y)Nrcy)NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr+r+r,r-rz+leading_dimension_matches..Frr,)r.r@r)rprPleading_dim_sizesrws r-leading_dimension_matchesrAsN )$3/  KKM$3 #4 4 ]  r,ctj|dk\dtjt|dkDd|dj}|dj}|D]r}tj|j dkDdtj|j|k(dtj|j|k(dtt |r(tj|dj|}nEtj|dk\d|D]&}tj||jkd (t|||S) Nr#cy)Nz&_chunk_cat expects positive num_chunksr+r+r,r-rz._preprocess_chunk_cat_inputs..Orr,rcy)Nz0_chunk_cat expects a non-empty input tensor listr+r+r,r-rz._preprocess_chunk_cat_inputs..Qrr,cy)Nz#_chunk_cat expects non-empty tensorr+r+r,r-rz._preprocess_chunk_cat_inputs..Vrr,cy)Nz8_chunk_cat expects all input tensors with the same dtyper+r+r,r-rz._preprocess_chunk_cat_inputs..Yrr,cy)Nz8_chunk_cat expects all inputs tensors on the same devicer+r+r,r-rz._preprocess_chunk_cat_inputs..]rr,cy)NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr+r+r,r-rz._preprocess_chunk_cat_inputs..drr,cy)Nz3_chunk_cat expects dim < ndim for all input tensorsr+r+r,r-rz._preprocess_chunk_cat_inputs..irr,) r@rrrrrr~rEr-rPr}r)rprPrqexpected_dtypeexpected_devicerws r-_preprocess_chunk_cat_inputsrJs-  LLq"RS LL G qTQZ%%Naj''O   V\\^a')VW LLN * N   MM_ , N  w$$WQZ^^%5s; 1H a  F LLfkk!M   gs+ Jr,rct|||}t|||}|tj||dzStj||dz||S)Nr#)r)rr{r@r0)rprPrqrrvs r- _chunk_catrosS 'wZ @Cj9N {yyq11 .#'s3 r, split_sizesctj|||}|.|Dcgc]"}|jtj$c}St ||D])\}}t ||jt||d+ycc}w)NrrXTr) rsplit_with_sizesrr@r\rrrr)r|rrPrsplitssrsplits r-split_with_sizes_copyrs " "4# " >F {HNO1e&=&=>OO f- NMFE fekk 2 UF M N Ps'B split_size.cDtjj|||Sr6)rrr)rrrPs r- unsafe_splitrs ::  UJ 44r,cDtjj|||Sr6)rrr)rrrPs r-unsafe_split_with_sizesrs  ( ( S AAr,c.|j}||}|dk(r%|dk7rtd|d|jfS||zdz |z}ddlm}||}t |Dcgc]}|}}|||z|z z |d<t j|||Scc}w)Nrz split_size is 0 but dim_size is z , expected 0r#) guard_intrT)rr,detachrsrrUr@r) r|rrPrhrchunksrrors r-rrs**K3HQ q= 28*LI  #a'J 6F@ v F',V}5!:5K5 J$7($BCKO ;;t[# ..6s" Btensor_indices_or_sectionsc |jjdk7rtd|j|jtj k7rtd|j|j t j dk(xs dk( fd dk(rS|j}t|ts!tdt|j|j||St}tjjx}r|j x}r |j"}|5|Dcgc]}|j}}ddd|j|Scc}w#1swY xYw)Ncpuz/tensor_indices_or_sections must be on CPU, got z.tensor_indices_or_sections must be int64, got r#rcddS)Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr+) split_dimsr-rzAtensor_split_tensor_indices_or_sections_py_impl..s<X>_>_=` a  "''5;;6<=W=]=]<^ _  +..0I LLQ()q. M A~-224(G, 7X8O8O7PQ   3//779 9I 9",, ,I ,99C U E)CDAqvvxDGD E  #..E E Es:E7?E2E72E77Fmat1mat2c|js&|jst|}t|}|tj||z}|dk(r|S|||zzSrt)rK is_complexrr@mm)r|rrrdrnrs r-addmmrs]  ! ! #DOO,=4yE  %((4& &C qy   r,use_geluct|||||}|r8|jrtj|dStj|Stj |S)Nr)r)ris_cudargelurelu)r|rrrdrnrrs r-_addmm_activationrsO dD$ .C <<99Sf95 599S> ! 99S>r,vecc|js&|jst|}t|}|tj||z}|dk(r|S|j dk(r||zS|||zzSrt)rKrrr@mvr)r|rrrdrnrs r-addmvrss  ! ! #DOO,=4yE  %((4% %C qy  yy{ad{  r,rrstdgammaNCHxWgroup output_maskc Dtj|||dtj||dtj|dtj|j zzk(fdtjj fk(fdtjduxsj k(fdz} tj| zk(fdtj||jjdg} |jjdg} d} d}d}| d r*d | zz }tj| jd j| jd}tj| jd j| jd}tj|jd jd | }n| j| jd}| j| jd}tj|jd tjd | f|j }|z|z |z|z|z|z}| z||z|zz }|jd }t|d}t|d}tj|j| |tj|j| |z|z} | j|j j|j } | d rk| j| | j| jd zz |jd zjd gj}| dr| jd g}| ||fS)NF)allow_cpu_scalar_tensorscdzzdS)NzExpect input to have z elementsr+)rrrsr-rz,native_group_norm_backward..6s'A }I>r,c.dddjS)NzExpect mean to have shape (, z , but got r)rrrsr-rz,native_group_norm_backward..:s-aS5'DJJ<Pr,c<ddjSdS)NzExpect gamma to have z elements but got rT)r)rrsr-rz,native_group_norm_backward..>s-'s*.Ds,QC/[\a[bcr,r$rrrhrTr#rrD)rEcheck_same_devicecheck_same_shaper@rrrrrrrVr&rrrXr9r)rmrrrrrrrrrcpgdsdbd_inputd_gammad_biasrds_valdb_valc1c2c3s ` ````` r-native_group_norm_backwardr!s UD$ 5+N 4F LL Q$> LL q%j P LL  +!+i u*C LL S5[c ; & + +Aq# 6 : :s : CB   !Q $ ( (aS ( 1B $G $G#F1~ 39   YYr5??1#56>>q%MQQRSTFYYr5??1#56>>q%MQQRSTFr" a,B ZZ5#.2215FZZ5#.2215Fr" Auc?4;;?Btmf$ ,t 3d :Q >S4Z&4-!+ + \\"  r1 % r1 % IIk))!UC=r Bii aS92> ?   //%++.11%++>1~E3'"''!UC*@4>>RTCU*UU..$%SaSS\ WQZ 1~QC Wf %%r,out2c  t|||||||||| } | | | f}t| D]2\}}| t|||jt |||d4|Sr)r enumeraterrr)rmrrrrrrrrrrHrIrrrYrorKs r-native_group_norm_backward_outrzsz"(UD$q!S%Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r,c,||j|S|Sr6r9)r:rs r- _maybe_castrs}ttE{ Hr,grad_outnormalized_shapebiasc~"|j}|j} tj|j""fd||||fD\} } } } | t d| t |z }||d}|d|}g}g}t| D]*}||k\r|j||j|,t|}t|}ddl m }||dk(s ||dk(rN|dr|j|nd|dr|j||dnd|dr|j||dfSdfSt|| j}t|| j}| t d| |z |z}| | | z}n| }||z}tj||d}tj ||}tj||d}tj ||}||z |z }d}d} d}!|dr||z |z}|dr0| .t |dkDrtj| |z|d } n| |z} |dr8| 6t |dkDrtj| |d }!n| j#}!t%||jt%| | |jndt%|!||jfSdfS) Nc3hK|])}|!|jtjn|+ywr2)r9r@r\rr:r;s r-rz-native_layer_norm_backward..s>9  =  e.E.EF  9s/2 grad_out_cast should not be Nonerrqr#r$zinput_cast should not be NoneTF)rrPrEget_computation_dtyperr,rrUrur&rsrrrmrXr@rrrr)#rrrrrrrr input_shape input_ndim grad_out_cast input_cast weight_cast bias_castaxis inner_dims outer_dimsinner_dim_indicesouter_dim_indicesrorMrrx_hat grad_x_hatabrrrrLrd_weightrr;s# @r-native_layer_norm_backwardrs++KJ33EKK@9E640 95M:{I ?@@ ,- -DTU#JUd#J#%#% : ( 9  $ $Q '  $ $Q ' ( ZA ZAKQ!V$(=a1f(E,7NEOOK (3>q>EOOK. /t3>q>EOOK. /  HL  T:>>#3 4D T:>>#3 4D<== $ $ &E"[0 " QA */6A :u %B 2($ /B 5" B EBJE $G!%H#F1~!8u$1~+1  !A %yy!68I5QH$u,H1~)/  !A %YY}.?GF"((*F GU[[)Hf.@flldKF$*:DJJE AEE r,c t||||||||} || | f} t| D]2\} }| t| | |jt || | d4| Sr)rrrrr)rrrrrrrrrHrIrrrYrorKs r-native_layer_norm_backward_outrsw(%)4vt[Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r,c&g}tt|D]'}|j|j|z dz )t j |j }|j|}|~|tjtjfvr.tjtjj}n0tjtjj}n|}tjtjj j"j%tj&|dj)|d|} |j+| } || j+|} |j,xs|duxr |j,} t j.|} | tj0tj2fv} | s"| s | j5} | j5} | j7|}|| fS)Nr#r$Tr)rUrrurPrErrr9r@rrfinfor>rrsqrtopsraddScalarrrr is_nestedr channels_lastchannels_last_3drr3)rrrr>dims_to_reduceror;upcasted_inputeps_val rqrst_inputupcasted_resultrrYis_channels_lastrs r-_fused_rms_normrs!#N 3'( )3eiikAo12333EKK@XX/0N {  @ @kk%--044Gkk%--044G++  !! IIna ( - -.$ - OQX K%((5O )--f5LF$$6$K6;K;KI//6M$  ) -)446!,,.  $ $U +F ; r,cL|j}|j}tj|j}|j |t j} |j |t j} |!|j |t jnd} | td|t|z } || d} |d| }g}g}t|D]*}|| k\r|j||j|,t| }t|}ddl m}||dk(s ||dk(r4|dr|j|nd|dr|j|| dfSdfSt!|| j}| | | z}n| }d}d}| |z}|dr)t j"||z|d}|||z |zz |z}|dr0| .| |z}t|dkDrt j"||d}n|}t%||jt%||jfS) NrXrrrRr#TrF)rrPrErrr9r@r\r,rrUrur&rsrSrmrXrr)rrrrrrrrr;rrrrrrrrrorrrSrrrrsum_vald_weight_full_shapes r-_fused_rms_norm_backwardrFsa++KJ33EKK@KK)@)@ M+5;R;RSJ    #53J3J K  ?@@ ,- -DTU#JUd#J#%#% : ( 9  $ $Q '  $ $Q ' ( ZA ZADa1fQ!7,7NEOOK (3>q>EOOK. /  GK  T:>>#3 4D"[0 " $G!%H  E1~))EJ.4EtTg 55=1~+1+e3  !A %yy#):EH+H GU[[)Hekk* r, running_mean running_varmomentum functionalc Ddgttd|jz} tj|j } |} |} |r"tj|j } |j | } tj| | dd\}}tj||z}||z |z}tj|| }tj|| }|!||zd|z |zz} |s|j| |;|j|jdz }tj|| }|||dz z z}||zd|z |zz} |s|j| n|| td|j | d}|} |j | d}|} |}dtj||zz }|j j"d k7r|}|}n"|j%d }|j%d }t'||jdz }t'||jdz }||z |z}|2|j)}t'||jdz }||z}|2|j)}t'||jdz }||z}|j j"d k(r8|j |j }|j |j }|j |j ||| | fS) Nrr$rT)rP correctionrr#z:running_mean and running_var must not be None in eval mode)rrrr)rurUrPrErrr9r@var_meanrrcopy_rrr,r`rrrmrXr;)rrrrrrrr>rreduction_dimsr;new_running_meannew_running_var input_acc biased_varrrr save_mean save_rstdn squeezed_var unbiased_varinvstds r-native_batch_norm_helperr'sS4a 566N33EKK@#!O!77 DHH#4H5  >> >a  D{{:+,$,$&MM$7 MM$7  #')3q8||6SS ""#34  "  A.A!==^DL'1A;7L&5X8TTO!!/2  ;#6 L $->TJ '!nn+<4nH %ejjs!234 <<   %$II-I-I uyy{Q7"6599;?;$,&( !"6599;?;& ||~ uyy{Q7$ ||E!LLu{{L3 LLu{{L3   $  r,r! save_invstdc >t||||||||d \}} } } } || | fSNFr' rrrrrrrr>rr!r"rWs r-native_batch_normr-s=*B vt\;(CQV*&FIy!Q 9i ''r,c ||tj||||||S| td| td|rtj||||||||Stj|||||||S)Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r_native_batch_norm_legitrt$_native_batch_norm_legit_no_training)rrrrrrrr>s r-native_batch_norm_decompositionr1s 3,, 648S   <   <  ,, 64{HhPS  88 64{Hc  r,c~|j|}||zdz |z}|dk(rc|dk(r^t|Dcgc]}|}}|||z|z z ||dz <tjjj j |||Stjjjj|||Scc}wNr#r) r.rUr@rrrrrr)rwrrPrrrWrs r-unsafe_chunk_py_implr4(s{{3HV#a'F2JQ8q=+0=9az9 9", V0Ch0N"O FQJyy~~55==fkSVWW 99>> & & - -fj# FF:s B:c Ntjj|||||d||Sr*)rr/r)rrrrrrr>s r-r0r04s5  ( ( 0 0      r,c >t||||||||d \}} } } } || | fSr*r+r,s r-r/r/Js=*B vt\;(CQV*&FIy!Q 9i ''r,c >t|||dd|||d \}}}} } |||fSr*r+) rrrrrr>rr!r"rWs r-!_native_batch_norm_legit_no_statsr8[s<*B vtT48S%*&FIy!Q 9i ''r,c vt||||||||d \}} } } } | td| td|| | | | fS)NT#new_running_mean should not be None"new_running_var should not be None)r'r,) rrrrrrrr>rr!r"rrs r-#_native_batch_norm_legit_functionalr<jsl" ! vt\;(CQU  BCCABB 9i)9? JJr,c Ttjj|||||d|}d}|tjjjk(r tjj ||}tj |tj|j|jS)a Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`, which support a variety of backends including cudnn. We create this tensor here to get the correct shape in the traced graph if we detect that will call the cudnn kernel, and rely on DCE to avoid materializing this tensor. Tr)rlayoutr) r@_C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8r>r) rrrrrr>rbackend reserve_sizes r-_get_batch_norm_reserve_tensorrHs hh11 vt\;cGL%((,,222xxHH 8  ;;EKK U\\ r,c dt|||||d||d \}}} } } t||||||d} ||| | fS)NTFrr'rH rrrrrrr>rr!r"rWreserves r-_batch_norm_with_updaterNsa*B      *&FIy!Q- vt\;dG 9i 00r,c t|||||d||d \}}} } } t||||||d} | td| td||| | | | fS)NTrJr:r;)r'rHr,) rrrrrrr>rr!r"new_rmnew_rvrMs r-"_batch_norm_with_update_functionalrRs ! vt\;hT  - vt\;dG~BCC ~ABB Iy'66 BBr,c dt|||||d||d \}}} } } t||||||d} ||| | fS)NFrJrKrLs r-_batch_norm_no_updaterTsa*B      *&FIy!Q- vt\;eG 9i 00r,c|td|tj||kjtj}|j ||zd|z z}||fS)Nz=generator must be None for _fused_dropout decomposition, got rrh)r,r@rLr9rEr3)rrR generatorrrNs r-_fused_dropout_decompositionrWsnKI; W   OOE "Q & * * * =D ,,u  %q 1C ;r,)rr>r pin_memory non_blockingrYrrXrYrYc|r!|tjk7rtd||r tdt|tjt t ttfs!tdt|j|0|.|,t|tjr|jS|Sd}t|tjr|}ntj|}|c||jk7rT|1|jdk(r"tjj||}d}tjj!|||}|$|s"tjj||}d}|tj||S|S)Nz*layout must be None or torch.strided, got z:pin_memory=True is not supported in _to_copy decompositionz x must be Tensor or scalar, got FrTrX)r@stridedr,r8rrrrcomplexrr%r scalar_tensorr_primsconvert_element_type device_put) r:rr>rrXrYrYdtype_convertedx_tensors r-_to_copyrc sZ&EMM)I&RSS H   a%,,UD'B C?Q@P@P?QRSS ~%-M,A a &779 HO!U\\"&&q) f7  !5||885IH"O<<**8V\J <<44XuE {{8=AA Or,c,tj|Sr6)ralias)r:s r-nop_decompositionrf5 s ::a=r,out3exponential_average_factorepsilonc tj||||||||\}} } |r%|| | |jdtjfS||jd|jd|jdtjfS)Nrr)rr-rmr@rE) rrrrrrrhrirrrs r-cudnn_batch_normrk= s$$  " GAq!1aU[[ABB  EKK0  r,ct|D]>\}}|dk(s ||jkr|j||k(r.|j|}@|Sr])rr}rrV)r:broadcast_maskrrs r-_broadcast_batch_norm_backwardrn_ sO/" d 19dQVVm 0E D!A" Hr,rMc *t|||||||||| Sr6)native_batch_norm_backward) rrrrrr!r(rDr>rrMs r-batch_norm_backwardrqf s/ &     r,c &|j} | |j} n| } tj|j&&fd|||||||fD\} } }}}}}|j}|j }|dkrt d|d}t t|||z }|}|}|r||4t d|| t d|}tj||z}dg|z}||||<g}t|D]}||k7s |j|t||}d|z }tj| |}tj| | |z z|}t||z|}ttj||z||z|} |t||dz}!nt||z|}!|r| |z | z}"| |"z |z |!z}#n| |!z}#| dr||z}$nd}$| dr|}%nd}%|#j| t!|$| t!|%| fS)Nc3HK|]}||jn|ywr6rrs r-rz-native_batch_norm_backward.. s,  $%= a7 s"r$z*rank of the input must be at least 2, got r#z1mean and invstd must not be None in training modezDrunning_mean_cast and running_var_cast must not be None in eval moderh)rrErrrPr,r&rur@rrUrurnrrr9r)'rrrrrr!r(rDr>rr weight_dtyperrrrunning_mean_castrunning_var_castsave_mean_castsave_invstd_castr input_rankr num_featuresrr&rmreduction_axesrorgrad_output_sumdot_p grad_mean proj_scale grad_scaleprojrYrl grad_biasr;s' @r-rprp s++K || " 33EKK@          ++KJA~I*VWW D[)*[->>L D F <6> !TU U  $(8(@ V !-34!"j 0N&t,N4 "N : % 9  ! !! $% *$ ?D  Dii ~>O IImzD'89> JE./E~VI/ %$,0J 3FNKcQ 3 [ .  T!Z/$t+y8JF "Z/ 1~fn  1~#    k"K.I|, r,c  t|||||||||| } | | | f}t| D]2\}}| t|||jt |||d4|Sr)rprrrr)rrrrrr!r(rDr>rrHrIrrrYrorKs r-native_batch_norm_backward_outr s"(    Fd#J&!Q1 = jmQWW 5 Q 1 4 PQ r,save_varc Btj|||||||d|gd SNT)TTTrrp)rrmrrrr!rris r-miopen_batch_norm_backwardr s5  * *    r, reserveSpacec Btj|||||||d|gd Srr) rrmrrrr!rrirs r-cudnn_batch_norm_backwardr0 s5  * *    r,c|j|jttjdvfd|jddD]}tj|dk7fd d|dzdk(rxd|dzdk(rjt dt dd|D}t dt dd||D}tjjj|||Sd d fd }|d|d\}}}} |d|d\} } } } |d t|d | f}| s| stj|dSd}||||| d\}}||| | | d\}} d}tt|jdt|jdD]!\}}| |d |dd|f}||d |dd|fz}#||| zz S)NrcdS)Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r+r}sr-rz%adaptive_avg_pool2d..U sKD6Rr,rZrc"dtdS)Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape rrrsr-rz%adaptive_avg_pool2d..Z s99>uaIr,rTc3,K|] \}}||zywr6r+)rror.s r-rz&adaptive_avg_pool2d..` sG$!QqAvGc3:K|]\}}}||dz |zz ywr#Nr+)rror.rs r-rz&adaptive_avg_pool2d..a s' '1aAQ! O sc8tj||z|dS)Ntrunc rounding_moder@divrrrs r- start_indexz(adaptive_avg_pool2d..start_indexf syyQ99r,cJtj|dz|z|zdz |dS)Nr#rrrrs r- end_indexz&adaptive_avg_pool2d..end_indexi s&yy!a%1q1,awGGr,ctj| tj}|||}||zdz}||z}|dk(xs||zdk( }|r|dz }n |dk(r|dz}tj| tj}|jd|z}|rUtj|dz |j |j } tj|| } |||} | |z } n|} || ||fS)Nrr#rrTr)r@rrrVr]rrminimum)in_sizeout_sizeorangei0 maxlength in_size_modadaptive range_maxrmaxvali1lengthrrrs r- compute_idxz(adaptive_avg_pool2d..compute_idxl s hvU[[I 7 3x'!+ ( #q(GH{,Ba,GH  NI A  NILL6M ll2* ((! 399SZZF--V,C68W5B"WFFFIx//r,.rD)rrTrct|tr||fS|dk\rtd|||jdk\}|dk(r t |d}t j ||d}t || }||fS)Nrz)dim should be negative when masking, got rTrZrDr)r8rr,rVrXr@ri)valsrrrrPrs r- maybe_maskz'adaptive_avg_pool2d..maybe_mask s fg &< ax$'PQTPU%VWW 0 0 44Dby(q1$$T45D&vt4F< r,)rrPr)rrrr@rrrnnr avg_pool2drXrr rU)rrrrvkernelridxhlength_h range_max_h adaptive_hidxwlength_w range_max_w adaptive_wrrretrojrrr}rrs @@@@@r-adaptive_avg_pool2drK s. \\F KKE u:D LL R[[   F I   Ry;r?"a'E"I B,G1,LG#eBCj+*FGG +.uRSz;+O  xx""--eVVDD:H0@/:%)[QS_.U+D(K.9%)[QS_.U+D(K 'a0$6 7D jzz$H--   h jbND(  h jbND( Cdjjn-uTZZ^/DE+1 ;sAq!|$CS!Q\**C + (X% &&r,ctjd|dt|jd| t|z}t d|Dr|j |St tj|jd| }t tj|}dg|jz}|jd| |d| |tj||jj||zzjd}|j |} tj| jd|g|jddj| jS) N max_unpooling d_forward_outc3&K|] }|dk( ywrr+)rrs r-rz _max_unpoolnd.. s (a16 (rr#rrTFr$)rEalert_not_deterministicruranyrmroperatorrr}rrrrr&r') r|rrrP output_shapenchwindices_nc_shape indices_flatrs r- _max_unpoolndr s= !!M#m"DE 5SD)*T+->>L (< ((~~l++  djj3$/ 0B  k *BsTYY"jj3$/Usd$++b+5::;KLrQQ gbk^^L )F  ! !r\NDLL,< "  d6<<r,ctjjtjk(fdtjt dk(fdtjj dvfdtjj j k(fdtdj D].tjjdkDfd 0tdS) Nc"djS)Nz2elements in indices should be type int64 but got: r)rsr-rzmax_unpool2d.. sDW]]OTr,r$c"dtdS)NzMThere should be exactly two elements (height, width) in output_size, but got elements.rrsr-rzmax_unpool2d.. ;'(  4r,rc$djdS)NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with dimensions.r}rsr-rzmax_unpool2d.. s%%)YYK| =r,c<djdjSNzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r)rr|sr-rzmax_unpool2d.. s,PQUQ[Q[P\]229-- Br,r#rc*djddS)NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got  with dimension being empty.r)ror|sr-rzmax_unpool2d.. s%::,&6qcHr,) r@rrrrr}rrUr.r)r|rrros```@r- max_unpool2dr s LL $T LL KA  LL V  LL gmm# 1dii   IIaL1     w Q 77r,c tjjtjk(dtjjdvfdtjt dk(fdtjt dk(fdtjt dk(fdtjj j k(fdtd jD].tjjd kDfd 0tjd d kDxrd d kDxrd d kDfd tdS)Ncy)Nz(elements in indices should be type int64r+r+r,r-rzmax_unpool3d.. rr,rDrc$djdS)NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with rr}rsr-rzmax_unpool3d.. s^_d_i_i^jjvwr,rc"dtdS)NzVThere should be exactly three elements (depth, height, width) in output_size, but got rrrsr-rzmax_unpool3d.. rr,c"dtdS)NzRThere should be exactly three elements (depth, height, width) in stride, but got: rrrvsr-rzmax_unpool3d.. sdehioepdqq{|r,c"dtdS)NzSThere should be exactly three elements (depth, height, width) in padding, but got: rr)rsr-rzmax_unpool3d.. sefijqfress}~r,c<djdjSrr)rrsr-rzmax_unpool3d.. s,PQVQ\Q\P]^229-- Br,r#rc*djddS)NzZmax_unpooling3d(): Expected input to have non-zero size for non-batch dimensions, but got rrr)rorsr-rzmax_unpool3d..) s% ;;-'7s-Ir,r$cdS)Nz5strides should be greater than zero, but got stride: r+rsr-rzmax_unpool3d..2 sGxPr,) r@rrrr}rrrUr.r)rrrrvrros`````@r- max_unpool3dr sD LL $&X LL fw LL KA  LL F q| LL G ~ LL w}}$ 1ejj !  JJqMA     LLq A 9&)a-9F1IMP a 88r,)rnrwc$t||||d|S)NTinplacern _index_addr:rPrrwrns r- index_add_r8 s aeVT GGr,c$t||||d|S)NFrrrs r- index_addrD s aeVU% HHr,rc6 tj|jtjjdkfdjdk(rj dnd |jdkDr|j nd tj k( fddk7retj |j tj tuxstjt  fd|z}|jdk(}|r|jdn|}dzfz}|rtjntj} | |||d} |r|S|r| jdS| j!S) Nr#c$djdSNz(Index should have dimension 1 or 0 (got r?r}rsr-rz_index_add..] :5::,aHr,rcdddS)NzNumber of indices (z') should be equal to tensor.size(dim) (z ), for dim=r+)rP index_sizerxsr-rz_index_add..c s!%j\1XYdXeeqmplrsr,c(dtddS)Nzalpha argument of type z cannot be safely cast to type !)r)rn python_typesr-rz_index_add..j s-d5k]:YZeYffghr,r6Tr$)rEcanonicalize_dimsr}r@rr. dtype_to_typerris_weakly_lesser_typerrVr index_put_ index_putrr)r:rPrrwrrnzero_dimrUrrrrrrxs `` ` @@@r-rrQ sM  ! !!&&# .C LL aH#(**/AqJ&,kkAo&++c"1K LLz!s z))!''2  4  E**4; D h %vv{H#QB C-5( "C#*I BV 5C!)s{{1~?s~~/??r,c  tjt|dkDdt|}|dj}|dd}t d|D}|r||f}n||f}||z}|dj ||}dt|z} t |D]j} || } tj| | d|| jdz fz|} |rtj|| d| }Rtj|| d| }l|S)Nrcy)Nz#received an empty list of sequencesr+r+r,r-rzpad_sequence..| rr,r#c3>K|]}|jdywrr.)rr:s r-rzpad_sequence.. s/!&&)/s)rrrPr) r@rrr.rrrUrrsr) sequences batch_first padding_valuesequences_sizemax_size trailing_dimsmax_lenout_dimsr dim_paddingsrocurrseqrows r- pad_sequencery s# LLY!#%RS^N|  "HQRLM/Y//G"G,^,-'H A,  - 8CC ..L > "@A,"" \Q',,q/(A$BBM  %%c3AQ%?C%%c3AQ%?C@ Jr,c"t||||dS)NTr _index_copyr:rPrrws r- index_copy_r s q#ufd ;;r,c"t||||dS)NFrrrs r- index_copyr s q#ufe <. rr,rr6) rErr}r@rrVrrrrr) r:rPrrwrrrUrrrs ` r-rr s  ! !!&&# .C LL aH vv{H#QB"'**/EOOA uE C-5( "C#*I BV $C!)s{{1~?s~~/??r,c*tj|jd|}tjtj| }|j s |j r|jd}n|}|tj|z |fS)Nr+r)r@rrmrirris_xpurH)r|rrkrs r-log_sigmoid_forwardr sn --r*D 1C 599T?"#A ||t{{% Q  ''r,lowhighrVc tj|j|jt |t ||j |j |S)N)rvrrrrrV)prims_uniform_helperrrvrrr)r:rrrVs r-uniformr" sF   xxz cN t_ggxx r,c<|jt||||Sr6)rr")r|rrrVs r-uniform_r$ s ::gdCy9 ::r,c"t|dz }|>tj|dudtjt||k(d|S|tj|dudtjt||k(dg}t|D]Z\}}t ||k(r$|j ||dzt |z8|j t ||dz|z\|Stjddy)Nr$cyNz9Must specify exactly one of output_size and scale_factorsr+r+r,r-rz.upsample_compute_output_size.. rr,cyNr+r+r,r-rz.upsample_compute_output_size.. rr,cyr'r+r+r,r-rz.upsample_compute_output_size.. rr,cyr)r+r+r,r-rz.upsample_compute_output_size.. rr,Fcyr'r+r+r,r-rz.upsample_compute_output_size.. rr,)rr@rrrrur)r5r scale_factorsspatial_dimensionsrors r-upsample_compute_output_sizer0 s Z1, T ! O   S%);;ZH  4  O   S'+==zJ m, CDAq1v{"":a!e#4s1v#=>""7:a!e+ sB ){M RE& TFSZ,?  UE6 >>r,cg}t|}|rdnd}t|D]}||}|j| |z} ||||dkDr | | ||zz n| |z } tj|tj |j } | |z| zjtj} t|dz |z D]} | jd} |j| |S)Nrrrrr#rT) rrUrr@rrrr9rrVru)rrr2r=rnum_spatial_dimsrrr8isizerooutput_indices input_indicesrWs r-!_compute_upsample_nearest_indicesrD' sG;'SsF # $&A --12 ay$Q UVAY& '  e5==V(61U:>>u{{K '!+a/0 8A)33B7M 8}%7&8 Nr,)preserve_memory_formatrr2ct|||gSr6r6rrr2s r-upsample_nearest1drIM s UK& ::r,c"t|||gdSr;rGrHs r-upsample_nearest_exact1drKY s UK& FFr,scales_hscales_wc t||||gSr6rGrrrLrMs r-upsample_nearest2drPg s UK(H1E FFr,c$t||||gdSr;rGrOs r-_upsample_nearest_exact2drRt s UK(H1ET RRr,scales_dc"t|||||gSr6rGrrrSrLrMs r-upsample_nearest3drV s UK(Hh1O PPr,c&t|||||gdSr;rGrUs r-_upsample_nearest_exact3drX s!  {Xx:$ r,r=cDt||||}ddg|z}tj||}|jdk(rdt j |}|j d}|jjdk(r|dkrtj}|j|}|S)Nr<rDr#cudarX) rDr _unsafe_indexr}rErrrrr@r\r) rrr2r=spatial_indicesrrrY n_channelss r-r6r6 s8 {F%OTl_,G   w /F {{a33E: [[^ <<   &:>!33M"""? Mr,c |r|rd}n |rd}n|rd}nd}t||zdk7rtdt|d|tdt||Dcgc]}t||||zc}Scc}w)NrrDrr$rz len(params)=z is not divisible by group_size=)rr,rUr)params has_biaseshas_projections group_sizeros r- gather_paramsrc so     6{Z1$3v;-'G | T  49CK3T ./fQZ()  sA2c~|r'|d|z|d|z}}|d|zdz|d|zdz}}n||||}}d\}}||||fS)Nr$r#NNr+)r_hiddensro bidirectional cur_params cur_hidden bidir_params bidir_hiddens r-params_hiddensrl sk!'AAJ %+AEAI%6A 8Jl !'GAJJ %/" l z< ==r,c||krtd|d|d|j|jd|||z |jdd|S)Nlast_batch_size (z) must be > batch_size (r?r)r,rur/)rilast_batch_size batch_sizerfs r-update_hidden_for_packedrq sc*$00H TU V   NN:$$Q Oj4PQR   Q: ..r,c ||k(r|S||k\rtd|d|dtj||jd|||z fS)Nrnz) must be < batch_size (r?r)r,r@concatr/)rirorp inp_hiddens r- update_hidden_for_packed_reverseru sn*$*$00H TU V   <<    a*2N O  r,c X|d}|d}|r|dnd} |r|dnd} g} g} |r|dn|d} |jdd| }tj|t|}|r|ddd}|D]V}|jd}| |k(rn|rt || ||}nt || || }||||| || }|} | j|X|r| jn!| j|| jtj| d}|stj| dn|}||fS)Nrr#r$rrT) r/r@rrurrurqrureverser0)inphiddenr_r` hidden_fn batch_sizesrw ih_weight hh_weightih_biashh_bias step_outputrfrori split_inpror hidden_outs r-one_layer_rnn_datar sRq Iq I%fQi4G%fQi4GK"$G)0k"ok!nOq!_5J Ck!23IddO ' IIaL a   9OQJ2OQJsJ 7IwW :&#'&z" ))K #C.57A&:J  ?r,cfd}|S)NcDtj||||zSr6rlinearrorir|r~r}r nonlinearitys r-rLzrnn_cell..inner% s AHHZGDqHIIr,r+rrLs` r-rnn_cellr$ sJ Lr,cfd}|S)Ncrtj|||}tj||||zSr6rrs r-rLzrnn_cell_data..inner, s2 HHQ 7 +AHHZGDqHIIr,r+rs` r- rnn_cell_datar+ sJ Lr,c x|d}|d}|r|dnd}|r|dnd} tj|||} |r| jdn| } |jd} g} | D] } || | |||| } | j | "|r| j t j| d}|| jdfS)Nrr#r$r) rrfliprVrurwr@r0r)rxryr_r`rzrwr|r}r~rprecomputed_inputrirrors r- one_layer_rnnr3 sq Iq I%fQi4G%fQi4Gi95<)..q1BS!!!$JK 'q*i)WU :&' ))K #C  ""1% %%r,c|d}|d}|r |d}|d}nFtj|j}tj|j}|djd} |djd} g} d} | jd} d}d}d}d}|j }| j } | j } tj j jj|||||| | || | | |||||}|d|d|d}}}||jd|jdffS)Nrr#r$rF) r@r r.rVrrrmkldnn_rnn_layerrr)rxryr_r`rww0w1w2w3hxcxr{mode hidden_size num_layersrgrrDoutputsr[hycys r-mkldnn_one_layer_lstmrI sS B B AY AY [[ # [[ #   Q B   Q BK D''!*KJMK E .. C B Biinn--55         !G$ GAJ 2rA rzz!}bjjm, ,,r,c |r|jddn|}g} t|D]} t||| |\} } }}|r || dz kr|nd}| || | |\}}| j||r!| ||||d\}}| j||r*t j |g|j dz }n|}|dk7s|s| |dz kst j||d}|r|jddn|}|| fS)Nrr#rT)rw)rD) transposerUrlrur@r0rPrG)rryr_r`rrGrDrgrlayer_fn final_hiddensrorhrirjrkfwd_inp fwd_hiddenbwd_inp bwd_hiddens r- _rnn_helperr{ s"&1EOOAq !eEM : >=K FA}> : J l$ QU(:'&uj*jQZ( "*|\:t# GZ   , IIw0'++-!2CDEE a,&1EOOAq !eE - r,c |jd} t||d}t|| |||||||ttt t j \} } | t j| dfSNrFrz) unbindrcrrrrr@rstack rrr_r`rrGrDrgrryrrs r-rnn_tanh_inputr tYYq\F 6:u 5F$   %**)=> C  M1- --r,c |jd} t||d}t|| |||||||ttt t j \} } | t j| dfSr) rrcrrrrr@rrrs r-rnn_relu_inputr rr,c |jd} t||d}t|| ||||||dtt|t t j \} } | t j| dfSNrFr{rz) rrcrrrrr@rr datar{rr_r`rrGrDrgryrrs r- rnn_relu_datar {YYq\F 6:u 5F$    ##EJJ/ C  M1- --r,c |jd} t||d}t|| ||||||dtt|t t j \} } | t j| dfSr) rrcrrrrr@rrrs r- rnn_tanh_datarrr,cltj||||z}|jd|}|dj} |dj} |dj } |dj} | |z| | zz} | | j z}||ntj||d}|| fS)NrDrr#r$rrrchunkrr)rxrrr}r hr_weight chunk_dimgates chunked_gatesin_gate forget_gate cell_gateout_gaterrs r- lstm_cellr(s HHRG ,s 2EKK9-MA&&(G"**,Ka %%'IQ'')H r Wy0 1B BGGI B ahhr9d&CB r6Mr,c (|d}|d}|r|dnd}|r|dnd}t|dk(r|dnt|dk(r|dnd} |djd} |djd} tj|||} |r| j dn| } g} | D](}t || | ||| d\} } | j | *|r| jtj| d}|| jd| jdffS)Nrr#r$rrrDr) rrVrrrrrurwr@r0r)rxryr_r`rwr|r}r~rrrrrrrs r-one_layer_lstmr6s*q Iq I%fQi4G%fQi4G[A%q F q8H6!9d   Q B   Q Bi95<)..q1BSK 3B 7IQRSB2 ))K #C A 1 . ..r,c |d}|d}|r|dnd}|r|dnd} t|dk(r|dnt|dk(r|dnd} g} g} |r|dn|d} tj|t|}|r|ddd}|d}|d}|j dd| |j dd| }}|D]}|j d}t j|||}|| kra| j|j d|| |z |j d|| |z f|j dd||j dd|}}|| kDrXtj||j d| || z fd}tj||j d| || z fd}t||||| | d\}}|} | j||r| j||f}nZ| j||f| jt| \}}tj|dtj|df}tj| d}||fS) Nrr#r$rrrDrTr)rr@rrur/rrrrursrrwrr0)rxryr_r`r{rwr|r}r~rrrrfrororig_hxorig_cxrrrorhidden0hidden1rs r-one_layer_lstm_datarQsyq Iq I%fQi4G%fQi4G[A%q F q8H6!9dKG)0k"ok!nO Ck!23IddO QiGQiGq!_-q!_- B  IIaLhhsIw/   NNIIaOa$78IIaOa$78  YYq!Q'1a);B  W^^AO8KLMqBW^^AO8KLMqB3B 7IQRSB236"X Bx =YYw*EIIgq,AA ))K #C  ?r,c4d}||||rtStS)a*Check whether we could use decompose lstm with mkldnn_rnn_layer. All the below conditions need to be met: * ``torch._C._get_mkldnn_enabled()`` returns ``True``. * All the input args are on CPU. * The dtypes of args are either torch.float or torch.bfloat16. * Inference. * ``has_projections`` returns ``False``. Args: * input: the input sequence to LSTM * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM * params: the weight and bias tensors of LSTM cNtjjsy|gt|ztt j |z}|Dchc]}|j }}t|dk7ry|j}|tj dk7ry|Dchc]}|j}}|D]&}|tjtjfvs&y|jry|djd|djdk7} | ryycc}wcc}w)NFr#rrr$T)r@r?_get_mkldnn_enabledrur from_iterablerrpoprrbfloat16 requires_gradr.) rrr_rptdevicesrdtypesrras r- use_mkldnnz2select_one_layer_lstm_function..use_mkldnnsxx++-'DH$tE,?,?,G'HH%,-188-- w<1  U\\%( (#*+a!''++ EU[[%..99    Q%**Q-2a5::a=8 ).,s D#D")rr)rrr_rs r-select_one_layer_lstm_functionrs!:%V$$$r,c t|dk7rtdt|t|||djd|djdk7}t t |d|d} t |||} t|| |||||||| \} } t t | } | tj| ddtj| ddfS)Nr$$lstm expects two hidden states, got rr#) rr,rcr.rurrrr@r) rrr_r`rrGrDrgrryrrrs r- lstm_implrs 2w!|CCG9MNN 6:r!uzz!}1 1 /M NF #beRU# $F-eR@H$   Cm,-M  M!,a0%++mA>NPQ2R RRr,c t|dk7rtdt|t|||djd|djdk7}t t |d|d} t || ||||||dtt| \} } t t | } | tj| ddtj| ddfS)Nr$rrr#F)r{) rr,rcr.rurrrrr@rrs r-lstm_data_implrs 2w!|CCG9MNN 6:r!uzz!}1 1 /M NF #beRU# $F$   #= Cm,-M  M!,a0%++mA>NPQ2R RRr,c&|jdd}tj|||jdd}|d|dzj}|d|dzj} |d|d|zzj } || z | z| zS)Nrr#r$r)rrrrr rxrir|r~r}rchunked_igateschunked_hgates reset_gate input_gatenew_gates r-gru_cellrsYYq!_NXXj)W=CCAqIN #nQ&77@@BJ #nQ&77@@BJq!^A%6%CDJJLH  !Z /( ::r,cPtj|||jdd}tj|||jdd}|d|dzj}|d|dzj} |d|d|zzj } || z | z| zS)Nrr#rr$rrs r- gru_cell_datarsXXc9g6<> 1 1 umWg r,ct|j||}t|d}t|d}tjj j |||||Sr)r0r.r3r@rr_upsample_bicubic2d_aars r-upsample_bicubic2d_aa_vecrbsV ){M REmQ/GmQ/G 99>> 0 0 umWg r,czt|j||}|r|ndgt|z}t||||Sr6)r0r.r_upsample_linear)rrrr.r8r2s r-_upsample_linear_vecrns= ){M RE+]$#e*1DF E5- @@r,rc t||||gSr6r)rrrrMs r-upsample_linear1dr|s E; z JJr,c"t|||||gSr6r)rrrrLrMs r-upsample_bilinear2drs E; (?S TTr,c$t||||||gSr6r)rrrrSrLrMs r-upsample_trilinear3drs!  {MHh+I r,cL|r|dkDr |dz |dz z SdS| |dkDrd|z S||z S)Nr#rhrr+)rrrros r-_compute_scalersB5=\# (S.1HqH#/EAIsU{U7XCUUr,c&|r||zS||dzzdz SNrr+)ro dst_indexrs r-_compute_source_indexr s$y   C(3..r,weightsweights_precisionctdt||Dd|dz zz}||z }tj|ddj tj S)Nc3K|]F\}}|jtj|jtjzHywr6)r9r@r9)rrrs r-rz%_sum_tensors_uint8..s826!QU[[ADD--sA Ar#r) _sum_tensorsrr@rr9rE)rr rrs r-_sum_tensors_uint8rsd:=c7:K  1$ %'F( (F ;;vq# & ) )%++ 66r,ctj|j}d}tj||j}d|d|dzzzz}|dk\}||j z S)Nrrr#i)r@rrrrr)r  max_weightmax_weight_precision precisionsvaluesrs r-_compute_weight_precisionrsiW%))+J2:;L;LMJ :zA~!67 7F g D $((* ,,r,c Ljd}jdd}t|}tjtjj \}fd}t t|||D cgc]\} \} } }|| | ||dz | z } } } } }tt| \} }}g}tddgg|zD]c}ddgt|Dcgc]}||dk(r||n||c}z}tj|}t|}|j|ett|D]p} | | || z j!ddj#}t|ddd|dddDcgc]!\}}|t%j&||z |z#}}}rt|dk7rt)dt||d}tj*}j,j.d k(r|d krt$j0}t3|t$j4s!t)d t/|j6|j9| }j;s|j=}|Scc}} } } wcc}wcc}}w) Nr#r$rTc|t|| |}tj| jj }t || j d}|j|jdgdg|z}|j tj}|dzj |dz }|||fS)Nrrrrrr#r) rr@rrr9r rr&rr) inp_sizerr2nsqueeze scale_factorrox_f32r:xp1rrrs r- get_valuesz$_upsample_linear..get_valuess%h-P  LL%,, 7 : : : G%lA}EKKPSKT ekk!n@sh/?@ HHU[[ !1umm1 m-a}r,rrrhz+Expected vs to have exactly 1 element, got rZz$Expected result to be a Tensor, got rX)rrrErFrV INT_TO_FLOATrrrur rUrr[rrureversedrr9r@rr,rrrr\r8rr%rrKround)rrrr2r] inp_sizesn_dimsrWr"rorrrxs_f32xsxp1svsrkrvxscalev1v2rrYrs` ` @r-rrsQJ ABI ^F'' !AANNHAu 09  ; /0  +A+(F 8XvvzA~>F CL)FB B 1vh' (TluV}U!qtqybed1g=UU   uc * #Au - !  eFm $ )be#**3477>b1gr!$Q$x0 B 27F+ +    2w!|J3r7)TUU UF//6M ||F"zB// fell +24<3H3H2I J    ]  ;F  " " $ MYV sJ J &J rrc4|j|jk(Sr6r)rrs r- is_same_sizer3s 77agg r,c.tj||Sr6)rr)r:rrGs r-_reshape_aliasr5s 99Q r,c.tj||Sr6)rr)r:rs r-r[r[!s ::a !!r,c2tj||||Sr6)rr)r:rr}r%s r-r'r'&s >>!WeZ 88r,c|D]F}|tj|jtjtjfvdHtj|jtj k(dddlm}||jdk(r.1rr,cyNz*tensors used as masks must be bool tensorsr+r+r,r-rz&_unsafe_masked_index..6rr,rrRr#r)r@rrrhrrrsrSr_meta_registrationsmeta_index_TensorrrrUrrr.rr[ri)r:rrfillrrS meta_resultros r-rr+s   LL  EII66M  LL ejj < Eaggi1n%//AA!WM zz+++T22 3w< ?   q A >GAJ?   a ) 5 5teT BBr,cL|D]F}|tj|jtjtjfvdHtj|jtj k(d|j dk(r|jStt|D]B}||}| |j|j| |j|dz ||<D|j|d}tj|||dS)Ncyr:r+r+r,r-rz5_unsafe_masked_index_put_accumulate..Mrr,cyr<r+r+r,r-rz5_unsafe_masked_index_put_accumulate..Rrr,rr#rTr$)r@rrrhrrrrrUrrr.rirr')r:rrrrro masked_values r-#_unsafe_masked_index_put_accumulaterEGs   LL  EII66M  LL ejj <  wwyA~wwy 3w< H   !&&)QGGAJH %%teQ/L  ! !!Wlt ! LLr,c|j}d}|dkrd}|6|dkDr*dg|z}|jd||<|j|}n|}||z}tj||k7|d} | j |} tj ||| j| } tj||k7| d} |tjjk(r|dkDr|jdd} | | fS|lj|j}tj ||| j|} tj||k7| d} | j} n"||k7jj|} |tjjk(r| j} | | fS|tj jk(r| j| z } | | fS)Nr#r$rr+r)rPrrr@rjrVgatherrr"r(r}rrrr9r*r))r|rrrr"r(r'rwr( safe_target_rr#wsums r-_nll_loss_forwardrKasXXZFK z   A:E"(aE+  E"AAax++f 4fa@K((5Lll4l;CCKP PF [[</ ;FINN(((VaZ}}R- |##  HHTZZ ||A{L9AA+N{{6\14;xxz ,.33588> IMM''' <  inn** * , < r,c|jdkDr|jdkstd|jd|jdkDrtd|jd|jdk(xr|jdk(}|sE|jd|jdk(s&td|jd|jd |jd }|A|jdk(r|j|k(std |d |jt |||||S) Nrr$r:r;r#r<r=r>r?rTz/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rPr,rrrK)r|rrrr"rA n_classess r-nll_loss_forwardrNs9 HHJNtxxzQDTXXZLPQRSS zz|aOPVPZPZP\~]^ _  88:?8vzz|q'8L TZZ]fll1o=( Jv||nA N   2I 6::<1#499T=i[I//5||n >  T669l KKr,c t|||||Sr6)rK)r|rrrr"s r-nll_loss2d_forwardrPs T669l KKr,Ac0|dz|z|dzz |z|zdzS)Nr$rr#r+r:rQs r-_upsample_cubic_convolution1rTs( UaK1q5 !Q & *Q ..r,c<||zd|zz |zd|zz|zd|zz S)NrrDr+rSs r-_upsample_cubic_convolution2rWs0 UQU]a !a% '1 ,q1u 44r,rcd}|jtjdk(rtj|d|z gd}tj|dzd|z gd}t||}t ||}tj |d\}}tj |d\}} ||| |fSt|dz|t ||t d|z |td|z |fS)Ngrrhrrr)rr@rrWrTr) rrQtt1tt2w03w12rrrrs r- _upsample_get_cubic_coefficientsr]s Axx5<<&&kk1cAg,A.kk1s7C!G,!4*32*32cq)Bcq)B2r2~ )S! 4 (A . (q! 4 (q! 4   r,coeffstscPt|}tdt||DS)Nc3,K|] \}}||zywr6r+rrrs r-rz+_upsample_cubic_interp1d..sEHRREr)r]rr)r^r_coeffs2s r-_upsample_cubic_interp1drds$.r2G EFG0DE EEr,c6ttj|Sr6)rr@r)r_s r-rrs %))R  r, num_stepsc|dkrtjd||S|s|dz |z nd}tj| ||||S)Nr#rr)stepsrr)r@rwlinspace)rfrrrrs r-_linspace_from_neg_onerjsIA~||AfE::-:)a-9 $A >>1"ayu MMr,thetahrHc|j}|j}t||||jd|d}t||||j|dd}t j d||}tj jj|ddd}tj jj|ddd}tj jj|d dd}||z|zS) Nr#)r#r#r#r)rr$constantrrrr}r#r#)r$r rrrjrr@rrrr) rkrlrHrrrgrid_xgrid_ygrid_ones r-_make_base_grid_4drus KKE \\F$A}eV D I I!QPQ RF #A}eV D I I!QPQ RFzz)5@HXX $ $VjPQ $ RF XX $ $VjPQ $ RFxx""&&xV*TU&VH F?X %%r,rc|j}|j}t||||jdd|d}t||||jd|dd}t||||j|ddd} t j d||} tj jj|ddd}tj jj|ddd}tj jj| d dd} tj jj| d dd} ||z| z| zS) Nr#)r#r#r#r#r)rrrnrro)r#r$)r$r#)rrrq) rkrrlrHrrrrrrsgrid_zrts r-_make_base_grid_5drxs5 KKE \\F #A}eV D I I!QPQST UF #A}eV D I I!QPQST UF #A}eV D I I!QPQST UFzz,eFCHXX $ $VjPQ $ RF XX $ $VjPQ $ RF XX $ $VjPQ $ RFxx""&&xV*TU&VH F?V #h ..r,c|\}}}}t||||}|jddd|jjdzj d}|j|||dS)NrrTrr#rZr$)rurr^rVr) rkr.rr#rWrlrH base_gridgrids r-_affine_grid_generator_4dr}sgJAq!Q"5!QmLI NN2q! $uxx'9'9!'< < A A" ED 99Q1a  r,c|\}}}}}t|||||}|jddd|jjdzj d} | j||||dS)NrzrTrDr#rZr)rxrr^rVr) rkr.rr#rWrrlrHr{r|s r-_affine_grid_generator_5dr"smMAq!Q"5!QOI NN2q! $uxx'9'9!'< < A A" ED 99Q1a ##r,ctjt|dvdt|dk(rt|||St |||S)Nrcy)NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r+r+r,r-rz'affine_grid_generator..2rr,rDrz)r@rrr}r)rkr.rs r-affine_grid_generatorr,sJ LL D VU 4yA~(MRR(MRRr,r|interpolation_mode padding_mode _expand_gridc  !"#$%&'()*+,-tjdvfdtjdvfddtdtdtffd -dtdtd tdtfd +dtdtdtf+fd dtdtdtf -fd }j\$%|j\})*}|d k7rt d|r(|j d)*|j)*d }dtdtdtf$%fd &tjjj dddtjjj ddddtdtdtdtf&)*fd dtdtdtffd "|d} |d} dk(r|| %} || $} | j| jc'('dz(}} '(dz}}| |}}|| z || z z}| |z || z z}| | z | |z z}| 'z | (z z}t"fd'(|f| ||f|||f|||ffDSdk(r<|| %} || $} | j}| j}"||dS-| %} -| $} | j'| j(| 'z ,| (z }s",jd,|jd}dtdtdtf "$%fd #dtdtf#'(,fd !t!fd t!d!D}t#||S)"N)rr#r$cdS)NzInvalid interpolation mode r+)rsr-rz"_grid_sampler_2d..Ks-.@-ABr,cdS)NzInvalid padding mode r+)rsr-rz"_grid_sampler_2d..Ns-B<.+Qr,coordsr.rQcBr|dzdz n|dz}|dzdz }||z|zSr r+)rr.rofsrs r- unnormalizez%_grid_sampler_2d..unnormalizeQs8%2tczCs Sj3|c!!r, twice_low twice_highcP||k(rtj|S|dz }||z dz }||z j}tj||}||z j j tj }tj|dzdk(||z||z|z S)Nr$rr#r)r@r rfmodfloorr9int8rj)rrr coords_min coords_spancoords2extraflipss r-reflect_coordinatesz-_grid_sampler_2d..reflect_coordinates\s  "##F+ +] !I-2 J&++- 7K0;&--/222D{{ AINEJ. j0H50P  r,cdk(r|Sdk(rtj|d|dz Sr|dd|dz z}n|dd|zdz }tj|d|dz S)Nrr#r$rTr)rr.coords_reflectedrrrs r-compute_coordinatesz-_grid_sampler_2d..compute_coordinateshsx 1 M Q ;;vq$(3 3#6vq!tax.#Q #6vr1t8a<#P ;;/D1H= =r,c(||}||Sr6r+)rr. coords_unrrs r-compute_source_indexz._grid_sampler_2d..compute_source_indexts- "9d33r,r$z4grid last dimension must be 2 (for x,y coords), got r#r*ysc tjd|ktj|ktjd|k|kSrtr@r)r*riHiWs r-in_bounds_condz(_grid_sampler_2d..in_bounds_condsF  GU&&rBw0A0A!r'2PR70ST  r,rwsc||rndt fd|jtj|jtj|fDS)Nr#c3pK|]-}tj|dj/ywr)r@rjr)rrrrroHoWs r-rz1_grid_sampler_2d..clip..s7  KKa # ( (Ar2 6 s36r)rr9r@r) r*rrrrrrrrrrs @@r-clipz_grid_sampler_2d..clipsYb"% A1 ee%++e.EKK0H"M   r,ixiyc8 |||\}}}||f|zSr6r+) rrrHidx_xidx_yw_C_idxN_idxrrs r- get_summandz%_grid_sampler_2d..get_summands0B?ubue+,r11r,).r).r#rc3:K|]\}}}|||ywr6r+)rrrrHrs r-rz#_grid_sampler_2d..s( R B " sc<|}|}||dSr]r+)rrr:r[rrrrs r-get_value_boundedz+_grid_sampler_2d..get_value_boundeds*#B+A#B+Aq!Q' 'r,rc|dz z}dz ||dz|dz|f}t|S)Nr#r$)rd)riy_ofscsrix_nwiy_nwtxs r- get_coeffz#_grid_sampler_2d..get_coeffs[cAg&F!%!)V4!%0!%!)V4!%!)V4 B ,B3 3r,c3.K|] }|ywr6r+)rrrs r-rz#_grid_sampler_2d..s:#y~:rD)r@rrrrr,rrrrrrrr&rVrrUrd).rr|rrrrrrWtwor:r[rrix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se ix_nearest iy_nearesttyr^rrrrrrrrrrrrrrrrrrrs.` ```` @@@@@@@@@@@@@@@@@@@r-_grid_sampler_2dr:s LLi'B LL !#Q"F"#"&"  F  s    PV   >F ># >& >4V4346477LAq"bZZNAr2s axB3% H   yyAr2s+221aRC 6 v & LL188 , 1 1!Q1 =E LL188 , 1 1!Q1 =E    V    4F   22F2&22 V A V AQ !!R ( !!R (xxz288: uqy%ueaiueu urz*U urz* rEz*U rEz* t$t$t$t$      q !!R ( !!R (XXZ XXZ :z155 B  B    %Z %ZaBaB (& (f ( ( (  43 46 4 4:q::'33r,c"t|||||S)N)r|rrr)r)rr|rrrs r-grid_sampler_2drs   -!#  r,c&tjjdk(xrjdk(fdtjjdjdk(fdzj dS)Nr$r#cLdjdjS)Nzmatrix @ vector expected, got rrr|rsr-rzmv..s!0 BswwykJr,rcvdjddjddjddS)Nzsize mismatch, got input (rr:r#z), vec (r?rrsr-rzmv..s<,TYYq\N!DIIaL>RURZRZ[\R]Q^^_`r,r)r@rrPr.rrs``r-rrss LL  a*CGGINJ LL !  #` 3J    ""r,c|-|dz |zdz}d|z |z|tj|zz }nd|z |ztj|z }|||z}t||Sr])r logsigmoidr)r|rr pos_weightr log_weightrs r- binary_cross_entropy_with_logitsrsv  1n.2 F d"j1<<3E&EFF d"Q\\$%77 f} i 00r,tensor1tensor2is_outc  |j|jk\r||fn||f\}}ddlm |jdk\r|jdksy|jr|sy|jdk(ry |j dk(ry|j }|j }dg}t|ddD]}|j||dzt fd t|tt||DS) NrrRrr$FTr#rTc3VK|] \}}}|dk(xs ||k("ywrr+)rrrr.rSs r-rzshould_fold...s9 D% tqy!B^DEM%BBs&)) r}rsrSrrrrvr%rurrru) rrrt1t2t1_shape t1_strideexpected_strider.rSs @r- should_foldrs$+<<7<<#?gw gwEWFBD GGqLRWW\ ||qbhhjAo&xxH IcO!"&;tob&99:; !$ tH_56"  r,) pass_is_out)rc ddlm}m}|j}|j}|dk(s|dk(rt d|d||dk(r|dk(rt j ||S|dk(r|dk(rt j||S|dk(rC|dk(r>t jt jt j|d|dS|dk(r|dk(rt j||St|||r>||kD}|r |jn|}|s|n|dk(r|jn|} |j} t| dd} t!t"j$| } | jdk(} | r| j'| jd|j)| | d}| rWt j*j,j/|j| | }|r|jj1S|St j*j,j/|j| | S|dk\rp|dk\rj|dkDr|j3dnd}|j3d}|jdd}|dkDr|j3dn|j3d}|dkDr|j3dnd}g}t5|dz D]"}|j'|j3|$|d k(r|d k(r{||d|dk7rj||ddk(r'|j6rt9|jd|S||ddk(r'|j6rt9||jdStt j:||}|||gz}t=|}|j?|j)|||}|dk(}|r7||gz}|j?|j)||jd}n)|||gz}|j?|j)|||}|} |dkDr| j'||dkDr| j'||r/|jA|jdjC| S|jA|jC| St jDd d y) Nr)rS guard_or_truez9matmul does not support 0-dimensional tensors, got dims: z and r#r$rTrZrFcy)Nz/both arguments to matmul need to be at least 1Dr+r+r,r-rzmatmul..rr,)#rsrSrrPr,r@dotrrrrVrr^rrrurrrrur&rr _unsafe_viewrr.rUrr]broadcast_shapesr&rbmmrr)rrrrSr dim_tensor1 dim_tensor2rrrsizes_1r folded_dim1 t2_is_matrix t1_foldedrr#m1 batch_tensor1m2rR batch_tensor2roexpand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded vector_rhstensor2_expand_sizetensor2_expandeds r-r]r]6sT++-K++-Ka;!+G }TYZeYf g  aK1,yy'**  kQ.xx))  kQ.}}UXXeoogq&A7KQOO  kQ.xx)) Wgv . +- $WZZ'$G+:J799;PW ((GCRL) X\\<8 vvx1}     ,JJ{GBK8 YY^^00b1A<PF-6699'') BF B99>>..y||B/?N N  kQ.!,aGLL Q \\"  cr* !,qW\\" gll26F +aGLL Q#% {Q' 2A  a 1 2 1 q mA.-2BBCmA.!349N9Ngooa0'::mA.!349N9Ngwq'9:: $  " "=- @ 3aW<#$89#>>*=>FF !R !A% "6""= 23-r21  #7"a"@ &~~.ABJJ$b!  , ?    " ?    " #''(89AA"EJJ<X X#''(89>>|L L UUVr,rrcnj\}}t|d||}t|d||}tjtjj \}}t j|djj|} t j|djj|} t|| |} t|| |} | jd} | j} | j}| |z jdd}| | z jdd}| jt j} |jt j}|dz ||dz|d zf}| dz | | dz| d zft|t|}d \}j t j"k(rt%t%|}Dcgc]@}|dzzt j&|d zzjt j(Bc}|Dcgc]@}|d|zzt j&|d zzjt j(B}}fd fd t+fd|D}j t j"k(r| t-dt/|||}nt1dt3||D}tj4}|j7|}|Scc}wcc}w)Nrr#rTrrrTrrhr$rerctj|ddz }tj|ddz }tjdd||g}|Sr)r@rrr[)rr*y_idxx_idxr.in_hin_wrs r- load_boundedz0upsample_bicubic2d_default..load_boundedsO B4!8, B4!8,   utT5%&@ Ar,ctfdD}jtjk(r t dt |St dt|DS)Nc30K|] }|ywr6r+)rx_ofsrr[s r-rzCupsample_bicubic2d_default..get_x_interp..sBl1e,Bsz4weights_precision_x must not be None for uint8 inputc3,K|] \}}||zywr6r+rbs r-rzCupsample_bicubic2d_default..get_x_interp..sJRBGJr)rrr@rEr,rrr)r[src_xrixs_ofsrweights_precision_x weights_xs` r- get_x_interpz0upsample_bicubic2d_default..get_x_interpscB'BB ;;%++ %"*$J&eY8KL LJCy4IJJJr,c3.K|] }|ywr6r+)ry_ofsr s r-rz-upsample_bicubic2d_default..s;%,u%;rz4weights_precision_y must not be None for uint8 inputc3,K|] \}}||zywr6r+rbs r-rz-upsample_bicubic2d_default..sL(2rb2gLrrX)rrrErFrVr$r@rrr9r rVrrrr]rrErrint16rr,rrrrr)rrrrrrWh_scale_factorw_scale_factorrrorx_floaty_floatr:r[yscaler/iys_ofs weights_yweights_precision_yrHsrc_yrrYr rrr rr r s` @@@@@@@r-upsample_bicubic2d_defaultrs{{Aq$$D+a.-QN#D+a.-QN'' 5#H#H#U#UHAu  [^ELL9<<5<IA [^ELL9<<5<IA#NA}EG#NA}EG#G A Ak c *Fk c *F U[[A U[[A1uaQA&G1uaQA&G08I08I/9,, {{ekk!7 B7 B !** +ejjmc.A A E Eekk R  !** +ejjmc.A A E Eekk R   KK ;7; ;E {{ekk!  & !WX X#E96IJLc%6KLL//6M   ]  ;F MI  s9AL-AL2c 6tjt|t|zdk(d|Q| tdt t t t ft dt|jdd|D}|r|nd\}}t|||||S)Nr#cy)Nz:Must specify exactly one of output_size and scale_factors.r+r+r,r-rz(upsample_bicubic2d_vec..rr,z7scale_factors must not be None when output_size is Nonec3PK|]\}}tt||z ywr6)rr)rrHros r-rz)upsample_bicubic2d_vec..&s*Au ! u,-s$&r$re) r@rrr,r rrrrr)rrrr.rrs r-upsample_bicubic2d_vecrs LL [D//14L   I  #s(O  #AGGABK ?  )6}<GW %amWg VVr,c(fd}t||S)Nctj| ||zj}|dz |dz |jz jz S)Nrr#)r@rrrrmiddlerdim_idxrs r-rz_reflection_pad..idx5sF,,ufunQXXFzVaZ'++-7<<>>>r,_reflection_or_replication_padrrrs` r-_reflection_padr(/s ? *   r,c(fd}t||S)Nctj| ||zj}tj|d|dz S)Nrrr#)r@rrrr"s r-rz_replication_pad..idxFs6,,ufunQXXF{{7Avz22r,r%r's` r-_replication_padr+@s 3 *   r,idx_fncn t|dz tj|j dz dzfv fd|j d}|j z }t Dcgc]}|d dz |z z}}t Dcgc]}|d dz |z zdz}}|}t D]E}dg|jz} |||||||| ||z<t j|| }Gtj|} |j| }|Scc}wcc}w)Nr$r#c(dddzddzdS)Nreflection_padz d requires r#zD or r$zD inputr+rsr-rz0_reflection_or_replication_pad..Ys$.[q sQwiwOr,rX) rr@rrPrrUrr[rErr) rrr, inp_shapenc_dimro padding_left padding_rightrrrYrPs @r-r&r&QsH g,! C LL C!GS1W%%OI UUWs]F8=c C1GAq1-.CLC=B3ZHWQ#'A+.23HMH F 3Z1&**,. a)A, a@PQAJ##FC01 //7M   ]  ;F MDHs 1D-D2c t|dz|j dDcgc]}|dz  }}tDcgc]}|ddz |z z}}tDcgc]}|ddz |z zdz}}g}t|jD]c}dg|jz} d| |<|j t j |j||jj| e|d | d} dtDcgc] }| |||z} }tDcgc] }||| |z } }tDcgc]}d||z||z| |z } }tDcgc]}| |d||||z||zf}}tjtjtDcgc] }||c}}tj|| zd}fd}tjtDcgc]}gd c}D]}|t!dgzk(rg}g}tD]t}||dk(r | |}||}n=||dk(r| |}| |d||f}n$||dk(r| |}| |||||z ||dz f}|j |j v||||}|Scc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}wcc}w) Nr$r#rTrcF|\}}}tj||k\||kSr6r) index_rangerolbubs r-index_range_conditionz7_reflection_pad_backward..index_range_conditions( 2r  b!r'22r,rrc .t D]*}||d||dk}t|ts%|s(|cStjt j |Dcgc] } | c}}t j ||zd}||zScc}w)Nr$r#r)rUr8rrMrrrr) rr index_rangesroupper_less_than_lowerr6rgrrPrmr9s r-r%z,_reflection_pad_backward..accumulatess A$0OA$6a9K$K !/6;P      CO PK "; / P   % %k4S# Fax QsB )rTrr#)rrrUr}rur@rrrrMrrrr itertoolsr r)rmr:rrldhwror2r3r view_shapexyzcenter left_reflect right_reflectrange_crrr%rWareaoutsr;rr6rrPr9s` @@@r-_reflection_pad_backwardrHms` g,! C''3$%. )Q1q5 )C )8=c C1GAq1-.CLC=B3ZHWQ#'A+.23HMHG 166]SS166\  1 u||AGGAJqxx@EEjQRS #A 3$%.C316c ;1c!f|A& ;F ;6;CjALOc!f,ALADI#JOqQQZ,q/1CF:OMO NSSVZHIAs1v Q/-2BBCG    eCjQ0<Q D  $ $[$F C HD !!c #C1J#CD4 5!s# #  s -AAw!|Qi%aj aB"1o"1vq,q/: aA#A&"1vs1v a0@'@#a&1*M KK     , -$l3-40 Kc *CH2<AOR&$Ds5 KK$$K)K.8K3K8K=L & Lrrrcjtj|||}tj|||}||fS)Nr)r@aminrX)r|rPrrJrXs r-aminmaxrKs2 ::dW 5D ::dW 5D :r,rctjtjtj|d||||S)Nrr)rrr@rjisnan)r|rPrrs r-nansumrNs2 88EKK D 11d;S'QV8 WWr,rr>rrXr>c Ntjjd|d||||S)Nrr#rOrr start_step)rjrr>rrXs r-arange_defaultrSs/ ;; ! ! 3vf " r,c Ntjj||d||||S)Nr#rOrQ)rirjrr>rrXs r- arange_startrUs/ ;; ! ! sAU6&Z " r,cddlm}||i|S)Nr)out_dtype_dense)!torch._higher_order_ops.out_dtyperW)rGrHrWs r-out_dtype_decomprYsA D +F ++r,marginct tjtjjd jd tj|dk(xs|dk(dtjj dk(xr dk7fdtjj dk(xrj k( fdQtjtjj dk(xrj k( fdjdtjd}||z z}|jd}|dk(r|n||z}|z}tj j }tj|k7|d}|tjjk(r|j!S|tj"jk(r |j%|jdz S|j!d S) Nrr#r$cy)Nz only p == 1 and p == 2 supportedr+r+r,r-rz#multi_margin_loss.. rr,c"djSNzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: rrsr-rz#multi_margin_loss..s_`e`k`k_lmr,c(ddjS)Nz#inconsistent target size, expected rr)nframersr-rz#multi_margin_loss..s5fXYv||nUr,c(ddjS)Nz#inconsistent weight size, expected rr)rPrsr-rz#multi_margin_loss..s9#i ~Vr,rrr)r@ atleast_2d atleast_1drrr}rrVrGr_rrrjr"r)r}rr*r) rrrRrZrrurkrrPr`s `` ` @@r-multi_margin_lossres   U #E   f %F [[^F ++a.C LLa!16#MN LL a$C1Hm LL q5V\\^v5U!!&) KK1  63!6 V   a F U0A UA AA !VQA  v  ,,s5<< 0C C6M1a(AINN(((vvx imm)) )uuw##vv!v}r, is_targetc |j |jtj|}tj|}|jd}tjt dkxr|dk7 fdtjt dkxr k( fdtj ||j }|dk(}tjtj|||dd }||k}tj||d}tj|d| } tj||d} tj|| jd k(d } d | jjd z |z} | jd} | |z } tj| d| } |tjj k(r!| j#d j%} n@|tj&j k(r| j#} n| j#d } | j)|j*j-} | | fS)Nr#r$rcdSr^r+)orig_input_shapesr-rz0multilabel_margin_loss_forward..;s_`p_qrr,cddS)Nzinconsistent target size: z for input of size: r+)riorig_target_shapesr-rz0multilabel_margin_loss_forward..?s,->,??STdSefr,rrTTrrrrh)rrT)rr@rbrrrrrJrjrGrrVTr_r"r)r}rrr*r9rr&)rrrrPris_endend_idx target_masktidx0rdtidx1rfrkrirks @@r-multilabel_margin_loss_forwardrr,s{{    U #E   f %F ++a.C LL "/saxr LL !#M(9=M(Mf ,,s6== 1C r\FjjVS#6BMG-K KK VQ /E U%0A KK VR 0E #R!88a@I accmmm##e+A AA CA Iq!$AINN((( EEgE  # # % imm)) ) EEG EEgE  U[[)112CDI i<r,) attn_maskroquerykey dropout_p is_causalrsctjtjfdtjjdk(xr(jdk(xrjdk(fdtjdk(fdtjjdjdk(xrjdjdk(dt j j||d|jdjdk7 \}}|jd d ddjtj jdd d d}||fS) Nc"djS)Nz-query must be FP32, FP64, BF16, FP16 but got r)rtsr-rz.xs? }Mr,rDcndjdjdjS)Nz,q, k, v must be a 4 dimensional tensor, got rr)rurtr}sr-rz.|s3>uyy{m2cggi[XZ[`[d[d[fZghr,rcdS)Nz&dropout probability must be zero, got r+)rvsr-rz.s$J9+"Vr,rcy)Nz&q, k, v should have the same head sizer+r+r,r-rz.rr,r#)rsrvrw dropout_maskro enable_gqar$rrX) r@rrKrPrr"_scaled_dot_product_attention_mathrr.rrr\) rtrur}rvrwrsrorattns ```` r-*scaled_dot_product_flash_attention_for_cpurksV LL &M LL q@SWWY!^@ q0@h LLSV LL A%++a.(KSYYq\U[[^-K8 ::BB   ::a=CHHQK/C LFDF q!Q" %"9"9 : Aq!  4<r,c.t|fd}|S)Nc<|i|}|dj|Srt)r)rGrHr outplace_ops r- inplace_opz$register_inplace..inplace_ops%4*6*Aw}}S!!r,r)aten_oprrs ` r-register_inplacers"G$"%" r,cB|js&|jst|}t|}tj||}t |t jr|dk7r||z}|dk(r|St |t jr|dk7r||z}||zSr3)rKrrr@rr8numbersNumber)r|batch1batch2rdrnrs r-baddbmmrs  ! ! #DOO,=4yE  YYvv &F eW^^ , % qy dGNN +tqyd{ &=r,c2tj||dS)Nrrr)r|rs r- floor_dividers 99T5 88r,c`tjtj|jdSr])rMrrrr)rs r- sym_numelrs   HLL!''1 55r,rrc|"tjj|g|Stjj|g||S)Nrr)rr dim_IntList IntList_out)r|rrs r- sum_defaultrsC {xx##D"E#::xx##D"Es#CCr,c t|tjs|S|@tjj |t t|jStjj ||gSr6) r8r@rrrdimsrurUrP)r|rPs r-squeeze_defaultrs\ dELL )  {||  tE$((*,='>??||  u--r,c2tfdtt|jD}|jt j k(rt jnd}|jd|d|}|||j|jz z|fS)Nc3.K|] }|k7s |ywr6r+)rrorPs r-rz)_weight_norm_interface..s@1qCxQ@s r$T)rr) rrUrrrr@rrrr9)r.r=rPkeep_dim norm_dtypers ` r-_weight_norm_interfacerst@c!''l 3@@H !5>> 9tJ 66!Xt:6 >D DGGAGG$$ %t ++r, assume_uniqueinvertct|tjs!tj||j}t|tjs.|rtj ||Stj ||S|jdt|jdzkrt|||St||||S)Nrg$@g(\?rr) r8r@rr]rnerrr isin_default isin_sorting)elements test_elementsrrs r-isinrs h -&&x 8L8LM mU\\ 2 88Hm4 488Hm4 4tc(..*:E&BBBHmFCC m=  r,)rVcB|?tj|jtj|j}n?tj|j|tj|j}||kj |j }|S)Nr)rVrr)r@randr.rrr9r)r|rVraw_prRs r- bernoullir sq  499;emmDKKP IIK--;;   $**%A Hr,rc|jdk(r%tj|tjS|jdk(r ||k(}|r|S|S|j d|jzz}|j |}ttd|j dz d}||k(j|}|r|S|S)Nrr)r#rTr#r) rr@ empty_likerr}rrrrUr)rrrrNexpanded_elem_shaper:rPs r-rrs~~1 ;;Q-'t&3&"..4-2D2D+DD )*A b=---126 7C  " "s " +CC4"s"r,c|j}|j}|rtj||g}tj|d\}}|dd|ddk(} tj| ddgd} |r| j } tj | } | jd|| } | d|jj|jStj|\} } tj| |} tj| | jk| d}| ||k(}|r|j n|}|j|jS)NT)stabler#rTrF) r;r@r0sortrs logical_notrrrr&r searchsortedrj)rrrr elements_flattest_elements_flat all_elementssorted_elements sorted_orderduplicate_maskrsorted_test_elementsrWrtest_idxcmps r-rr,sN$$&M&..0yy-1C!DE (- <(M%(,0DD..~1vuM +779N/q,?A()11(..AA"'**-?"@a  !5}E;;s%9%?%?%AA3J"8, =#)coos{{8>>**r,c.|jd}||SrS)r&)r|r flatteneds r-takerIs R I U r,c|tj}|tjk(r t|}tj ||j |Sr2)r@r\preserve_formatrrresizer)r|rrYs r- resize_asrPsD// ----e4 ;;tU[[ ; FFr,)FF)noner6)r$)rNNr#)rTFFrrrp)r#r#F)Fr)rrhN)rr#N)Fre)NNN)rrFT)rrFr*)rF(rMr>rrrwcollections.abcrr contextlibrenumrrrr r typingr r r rr@torch._meta_registrations torch._primsr^r torch._prims_common _prims_commonrEtorch.nn.functionalrrrrrr torch._decomprrXrrrrrrtorch._prims_common.wrappersrrrr torch.utilsrrCtorch.utils._pytreerr? DispatchKeyr rustr__annotations___opsrrr"rVrrOrWcompute_only_pw_cast_for_opmathpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_argsr$pw_cast_for_int_to_realrrXrarcrlrrxr?rr~rrrrrrrrrrrrrrrrrrrrrrr)r}r _safe_softmaxr rrrrYrrrr!r*r8rBrErJrMrOrQrTrgroslicerrrnrrrrrrrr0r4r=rCrGpy_implCompositeImplicitAutogradAutogradrFr[r_rermr&r{r~rrrrrrrrrrrrrrrrrrrrr'r-r1 unsafe_chunkr4r0r/no_statsr8r<rHrNrRrT_fused_dropoutrWrcrrYrlift lift_freshrfrkrnrqrprrr_adaptive_avg_pool2drrrrrrrrrrrrr" Generatorr$r0r3rIrrPrVr9_upsample_nearest_exact1drRrXr>rDrKr6rcrlrqrurrrrrrrnn_tanhrrrnn_relurrrrrrrrlstmrrrrgrurrrrrrrrrrrr rrrr3r5rr[r'rrErKrNrPrTrWr]rdrrjrurxr}rrrrrrrr]upsample_bicubic2drrreflection_pad1dreflection_pad2dreflection_pad3dr(replication_pad1dreplication_pad2dreplication_pad3dr+r&reflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardrHrKrNrr[r>rSrirUrYrerr+_scaled_dot_product_flash_attention_for_cpurrrrrrrrrPrrrrrrrraddbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r hardswish_ hardtanh_hardtanh hardsigmoid___iand____and__ __ilshift__ __lshift__rr index_reduce_ index_reduce__ior____or__ __irshift__ __rshift____ixor____xor__ldexp_ leaky_relu_ leaky_relulogit_logitrelu_rrenorm_renormround_r&scatter_r% scatter_add_ scatter_addscatter_reduce_scatter_reducesilu_r+r,r-rsN ."%$-- #,,07 *(hh"" c zz~~   %$) ''99''" 'T#*88@@# uDDLL&-88@@ &" "uDDQQ   c f **+ \2F2v2,2--. \4v4&4/4../ \SS6SS5S0S ))* \           + 2))*+(,())*+"V","(() @f@@*@112 \fF3../ \PP%P05P@EP0P ' GFGvG(G//0 F&V1//0 \:F:&:U:1:001 \LL%L7<LNRL2L **+ \))f)3),)<**+5v5f5,5 " &v&&&#&**+ \>v>V>>,> **+66666,6334%% % % 66> %5%667                8 $112 \>3G3G1 1 1-01 1'1../ \11 &1061CF101**+0,0++, ^^)) 1  1  1 1  1- 144<<=  % /5 BE MR > 44??@ R R  R  R R  R  RA R00889  % /5 BE NS : 00445 R R  R  R R  R  R6 R$$ $ $ V  $  $  $$ $@))* \NfNFNNN+N(../ \.. . . V  .  .  .. .0.b001 \$$ $ $ V  $  $  $$ $2$N112  $^^)) 1 1 1 V 1 1  131&::; \ $^^))       V      <  --. ^^))1 1 11 1/1667 \ ^^))         8  " ))v)%)#),,- &&F&v&.&++,  O Oc O  O  O O  O- O ))* 3? 3? 3? C= 3? # 3?  3?+3?l  ( 4B  % ,4UO   5 & %kCCDk2236eHTN4E&++, VV A& AU A8D> A- A & S'4))* FT+6' $        ((556     7: DI  &\  &\ *T&\ tF| # " &\" ""J00$//2E2EFG !  &\     &    H   '')C)C)G)GH "&   c    $v,   d6l   ))00155C5c5%PS BT52544<<=67B B $S B03B 63;B>B  ))*//C/c/% :L/+/(--55)) // // &// // 63; ////f # fF##$"../            0" #   f 6   $ 77??@S&S& S& S&  S& F  S&  S& S& S& S&dS& 8F Xf-x/? ?@S&AS&n77;;<     F       d ,, ,, ,, 8F Xf-x/? ?@=8 8F# x/? 77??@QQ Q3iQ  Q  Q V  Q 6 QdQ 8F Xf-x/? ?@QAQj77;;< 3i     V   6 d ,, ,, ,, 8F Xf-x/? ?@=4,,4453 33i3 V 3 % 3  66> 363l55==>HH H3iH  H V  H d H 8F Xf- -.H?HVR R V R 6 R6" R &! R  RR RR 6668F#3Xf5E EFRj../ UK/ (  ( V  ( 6  (6" ( &! (  ( (  ( 666 !" (00 (4'' (<(<='' (M(MN    V    6   6"  &!          666 !"  O>  F"";#H#HIG4<GJGAAIIJ  V  6        666 !"K*55==> (  ( V  ( 6  ( (  (  ( (  ( 666 !" (? ( 55>>? (  ( V  ( 6  ( (  (  ( 666 !" (@ (@@HHIK K V K 6 K K  K  KK K 66666 12KJK4  V  6       :44<<=1 1 V 1 6 1 1  1  1 1 6666 )*1>14??GGHC C V C 6 C C  C  C C 666666 9:CIC822::;1 1 V 1 6 1 1  1  1 1 6666 )*1<14++, VV- & $( %)37+ VZ  + EKK + U\\ " +  ++E//0+'+bdiiAB C &&{';';<--. VVVV,   6 6"  &!   !&-/=> 00889  V 6"  &!   &!  d 68F#Xf%5 56:677??@ii i V i6" i &! i  i&!i i idi 68F#Xf%5 56iAiZ77;;<"" " V "6" " &! "  "&!" " "d" ,," ,," ,," 68F#Xf%5 56"="J778 VVV$  6"  &!   v%90667 VVV$  6"  &!   v%82112 d'vd'E#s(Od'3d'N )8(!2( % $'$'+/   tS%  c5 !( &" &;'; 8 //334//334//334$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9:$$[%J%JK$$[%9%9: 3  3$s)$ 3DK( 3 3;L;L;L555 366::;66::;66::;##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A##++K,Q,QR##++K,@,@A ?  ?$s)$ ?DK( ? ?BSBSBS<<< ?#L0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;#; ;c; UO; ;<?PX; ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;#G GcG UOG G<FW G0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;!% $ G GcGuoGuo G  G<?PXG ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;!% $ S ScSuoSuo S  S<FW S0088$:Q:Q:U:UVW  (()N)NO  (()=)=> Dd;!% $ $ Q QcQuoQuo Q uo Q  Q<?PXQ ##++T-K-K-O-OP''// 0U0UV''// 0D0DE Dd;!% $ $   c uo uo  uo   <FW    c % !    4&>/$FK+\&,/-d& R ++,[BBC[112.3D-.8 ++,[BBC[112.3D-.8 **+KAABK001.2C,.@ **+KAABK001.2C,.@ /6=@/d (>>?--.S/@)S@ ' ==> ,,-S.?(S>;; &{<<={++,.->'.6' ==> ,,-..?(.644889!!))+*O*OP!!))+*>*>?@Q:33778  (()N)NO  (()=)=>?P90044511556##K$I$IJ##K$8$89%%k&K&KL%%k&:&:;&&{'L'LM&&{';';<A=N<M:K76A //779O9O9S9STU  !% K KcKKuo K  KVK %%t'?'?'C'CD!!))+*>*>?  !% $ U UcUUuo U uo U  U@ U &&(A(A(E(EF  !% $ $   c  uo  uo  uo    V/7 & 7$,V$47IO7 7-'9-f-I IcII % ! I  IIZ))112Fv$3,,d.?.?@A B++,-"."//019292234C5C6AABCMDM23 3 3  V 3  3  3  66> 3 l--. X~&L L L V L L  L  66> L'/L>//0 X~&L L L V L L  L  66> L'1L/F/u//5F5u55  3E (F%7FVFF !Xf%!&!NN#'N05 NEJ\\N&f&&&T&"/f////T/"!V!49!T!$V$49$T$223 SStCySS4S  f4 f4 f4f4 f4  f4  f4 f4R,,-             .     #! #==> $)..:N:N 1? 1 dt@[BBC>>? ',yW@DyWx0088$:Q:Q:U:UVW  (()=)=>  $# T TsCxTTe_ T e_ T  T?XTn//334$$[%J%JK$$[%9%9:  48 W W%S/*WWE%,/0 W  W;L5 W4--.--.--.  v c3h F /// ../../../   sCx V 000   38_ c3_f, - 8667667667 \T888Tn % UEu& $ X4X%X,,dkkoo>? $( ==%)    EKK   LL  U\\ "   @ **+, $( ==%)      EKK  LL  U\\ "  -  ",#, ../'' (<(<= #^^)) ) ) ))  ) V  )  ) )>0)X;;<$$,,44[5I5IJ X{#- - -- 66> -$K=-xHHPPQ  A#'!A A A A A  AA E?A 66>ARAH'  ( ))* 9+9'6(6))488<<89$( D  D EKK  D &  D  D: D--t||/?/?@A.&.x}.B. ==>,?, " 38 # $../,0 ,, (  \\  0 $5: #