From 1446e24d13cc28fbea22beb19f68ade24523960d Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Fri, 25 Sep 2026 03:21:34 +0800 Subject: [PATCH] [AMD] Add .co for deepseek v4 fp8 decode kernel and add group decode opt (#41120) --- docker/rocm.Dockerfile | 6 + .../mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co | Bin 0 -> 129992 bytes .../grouped_verify_streams.py | 139 ++++++++++++++++++ .../dsv4/unified_kv_kernels/runtime.py | 29 ++-- .../deepseek_v4_backend_hip_radix.py | 94 +++++++++++- 5 files changed, 251 insertions(+), 17 deletions(-) create mode 100755 python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co create mode 100644 python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/grouped_verify_streams.py diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index 3682a5264a..50cd8b0429 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -586,6 +586,12 @@ RUN git clone ${AITER_REPO} \ git revert --no-edit --no-commit e708f6c15; \ fi +# ROCm/aiter#5195's .co with a mask assembled in between the q rows sharing a +# tile, which the grouped dsv4 decode needs; bit-identical at max_seqlen_q=1. +COPY --from=local_src \ + /src/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co \ + /sgl-workspace/aiter/hsa/gfx950/mla_v4/ + # The pinned AITER revision uses std::optional in topk_per_row_kernels.cu without # including : ROCm/aiter#4702 removed the torch headers that previously # supplied it transitively. ROCm 7.0 therefore fails module_top_k_per_row, while diff --git a/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co b/python/sglang/kernels/ops/attention/dsv4/asm/gfx950/mla_v4/mla_a8w8_qh64_qseqlen1_gqaratio64_nm.co new file mode 100755 index 0000000000000000000000000000000000000000..410af4a1e3e43b20ac9282013d3581d3ea17a56e GIT binary patch literal 129992 zcmeHw4SZD9nfJ-e5W-g+D7G;q7}AKqVmgydCKHT0lS~430*w?E6%Bzz5)2SRbkU{Z zCL{=!)tK^H?fSMXwO#AlmRj5TcDqTWw)F!opxa_=yKIZCTH9LJcDrr2^ZuXnap&Hf zJCm6tD6o?shiC3{KJK~a-h0mf{?Busv$LkI){&7BbTRSgWG7jM{7oV#fA9ID(0*hR z=>jYZ|Ic94SSI2ruS#c?!?G|XvyhhlNUhGv6_)>Ioi7whGPgLUe5e9r?-$2xzD0gV zo={0cKB_DGv5fz-S+>h!MBzA)`5gF}Og@!gUEj)AJDd3RC3!ws8+H})@$;$rBq`}! z{Crel`m@NqSwgL(YWvAhXCbZtGPFFrwEk+wdcM@Ssi|dKWAQD`9h;lmw%xn$bH$Av z8@9c|mhApgapU^VTN~RN8rQFH-nQ+Lj^@UuhQIA-?QBL&Q&R_Y4cpoq*EfHDL)(^h zjcuPR-rTsU`5!;>PsLkqzIj`7C;MFSw$?kEv;MjGw#K$wn;ULv-Q4s=k+fmmtvBD? z+|ehpe73!@NoC5i$@K8L+Z%Whs-(`_+nYbzT3RyJDqf||=C(Q8s8!6d$&_>zzrb5@ z%jUM*6E6BiHmHVa7X`S~yHCBy zXayKMl+JPH-#*U#JI0Xzp-t=BZczog1#uhlzJKh5JjJE)cDl`adJ`NNLw-{eJT%?{9v*K2kB*^$gH^F6I5dX* zrY3lNyahZl-U7Zgh5`;&$C_Yx4Eaq>@YHwaGkoe6+sqO8sShKEor(No_-_+LF&=~mJK*QD+zpr8Iw&tyE&6~Xq8@4uf zG zuKlOtZMSb)x25gn(o|1hd`lC)mGqLHcd#!P|83)K%{O;!L4(nC2NmgaTXV;@)-9U{ z6eA^gBkTVZJIe{CYQ8z91cx)hm<_)n@&{!!9jTB@x+w@@p_}^jpC6ub@cAd4P*D&$ zV-yxt;xE;!W~pXCGpTd^FOeD1jNQ>FV;cMJV3a*G=x}CEjX zoQ17`bU{*@h0%o~&4LA*ws2w3MbV4IaZ!<`U35{;st<@fvmq(Z%J+-o0Z2NgbFI1Y zXWBZ~w4q&#`i9oAR@wIv$pcU)dTC%hBI03P~SC2wCe8XwWZ6KYs>4F-?Mz> z)S=}om-o4lZwlfakdq-NLr#KR?(lUv9saH!zrTpN+fUBP&C-@OO&dDAun*_`;o#Dy z6WYq^(7U!yxel6#)7YYyu>9`QNY9Jc#- zZn=Zymit1Dp613v%!U66BF#_`sl7G zr;jn`3`fuE)kOyo7g@c!g~vlOcm2uv7iVfD7r&jU-BefCa+A_8koy0O>FfJ5hx&(W z`sxp44jrhe>AL~i&o zFGR!<8HnP#DQ!_hJs$>rzgSoUF(H)u_?%#Nb64fd-wjhH=D=*WA>7HD6xsaD%!R4Ju--mnG%a#=6qKy2Rt*(5!r;{f+qDz)c zr~8p#R?reHC}^Q_=-%>jB0G4wbWD9}BuCxHY3BQg{Ni3r(|Dax?k?t^4?J>wjfF=dKlD!;Z(u3i=>I1pGeed?v*jnQzpWf)^qcVJ0T zcOX!ZcWGe3u0UXZUQK1o+?tY&yGlxmde}7xg+)=kM+|8rg@rAj!S(L)`NeZPWs+kK zl|6Hls}F5Bi++D#lPg*@e-X8N58hYQXFmPrWuRWOk%mpaBx82jf<+zY?ZdEl*U;o; zQ-=1iKJ;yDr~rK%EBNb=ry`GANS7vLR=<$b**qTab_+RM6Ede?$UNkKxe!f2eeg-N zL2hZG<3AwhK>iFe5Ax@b`H&mg3r8=Qm%)C{*s-5yd?Djz)-yD@Th=2a>mj5|3-aTv zej%r8LhtSuayF~t@i}fG^N`+D4>~{9gX%L{J;GFv9$Ak@Sq~vynvhxjLQZGvc)Z&! zI8#&StxKe2!blJmfdE zM=JHe(=^m>cwX}4^Fm`GV}JcIy1N3mkVWUqdq{D^St9PJkVVJqU@EyBlwH5pGf)aAoYv%o(EA$ zW1r4KI7Xk&CHi!d%yH$R^P>@tD`q?LN+OQAUFU7gJ9ZwPO>M=BTpC~hgnswp`i$tG z&<9_XM}}kLYsYdw5pePNRWkmXSiH*^zgos$AB)d4#;=j_H^t(!WIT;K zOnc`i4z`o2F^7&_mg7vzahCCz#^QyU8Xt?_iH*7R&Z<7)eB9$+7A&B5v|v&O+nDj! z!JOdBFQPxmxj@QXDd$L;CuP2rbEUjc$|q6AA>6NS^!uW(XUB8V*VFysV`n7R-}C#0 z{+>(p_arrT%^|P6Kc1X7OCyfo?EEj>pZ}u!bL!PObPwMC;)%(XhBl({We@sHF;)bM z@>JhUV}vij`)|@k{bG#R&Fcp|Kp4l`$0N@H^zp|2eTM9VwfcJf`B)&Ij|Fj_j|KAi zSP@=GviXnMtn213}gHlFiSj;ujs4G{b}op+>@=;|MzMu`c_0&+{va-pT4Ukr(i`+&I0WhzvTD#KXHFA zGhE|6xL-8x_TyWwm-#skm@@-ovm7_0z^1G{gGGGrZ=q*-4xZ)7yCx60@Gff`-{|`c z3(W5g1d5`8Knwd>2Afv#ex~}GIT$Z!4Bv$`7@v#saj)l0M%lm?biTYxm$ZyH&yqzg z2No^H^Ca*GrJ;Pe({4q+Vva+i#hw;wTaJkr$@lR0vG-7GAMVoHpF(&V;RwP}gqILb zAiRQb65$lWYY1;5yoK-%!XFX-(zUF6+E5AJU#*?()Y@GLSqRe++z7J~auD(m3J{7A z79cD}@F4gQ0tl5mm-S2oCZKYHI1VE$L#RWjM_7fh8et8>T7)Kq7KAp0c7#rZ?Fc&% zG=yG+JqY^{`Vsac+>h`e!U2Sb5FSQ&6yXrU;|Nb6d~4@zhOs9L^^s#viyU*>3dH!* z?h&$+FbFVAJunQ(RRs&UT<@)pod(2NXRwA(EpCH;87vNC!xpj+P8#kLisP_ z@jS%}I#2wDu{TjZ9lutvfFJ)F-~%{O#R=VNe3OM;z>#R%;*Ub&-Ja*@9a3XT0sHl_ zQ_q#UPC>rr?3cQ}u}m8|_G zFGu##3vK#nD&v>m6J-Rwhx4rCFV8Rg@P#&gcp+;&USI8CAIuW62*+39_#zyW^sukJ z*b+jyZ1Cl20MooY+2=2`>GP?)m9jjNRNhBrc_gX4wkr?kxn+5DFJ61G)b$$V>&NI` z9N$T0tdeDrq%y9SWssz6ehv4^hkLaQ_o_gaMb~b`8gJl!y@C5h)ABkt+*Oq7C4ct#EXZ5_}d34^@^1LMJ zyw}O|lBDzg3ipoEuk{|!V-=!}Dc|)nA4$r0gUm>dcU9Ry`*0+^m(MGaT4jl+aJ5IS?Kdgzhnv5my&*i&>PG5k-l|g z`DLWPWn}q&(r+4BemUv4I5KWP{}RGA-w=&x=t$Vo$aCG1$*a0oO>^pR;SMf+w+QLq za)+y+RrcYnqQ9Q&^mWa3`nxi4FXw#K*OlvB(ly5!;K%fg@$pm0Rdc@P>&pLHpo_}R zEf;0xT`I~%y44<(4Y?+W{`te#JJ{SCE`R@pH~3wX{Qjv?f6orwp3o&%RJlaIospqc zRk2I1xSmbAJ~(w!Rn=6SPmFg$E}j>v>hiGXj(MhOyG6tWc3 z2kC<>gDiujGBFl<+(P>N(EFh;hb)Kmgw}R>L+iSzT%@bVHSn>sB-GSZ8oIelLq0x+ z3R&q3wRDw*Hg+9AIzE;PxyB!A>naaz?wWVwnyJG`%g0qA!`?=pqomB|C`IrgoJ1Zz z&I;L72AvG!~@shZu|AP5kj1+W$4Q|Lc&iORO*!>@i-fkt(*ZK7(V$8rEkwjXf}?r?E#G zdjN}I+)iVUH1^;;`l7QOd%&h-sD?c~=q?m;HRiBCm)AGc4I6=F!v6_ON5%*X$?GgVGYe!Rs(jq{X@SGZ(LSz=}yI zJ@Ffqeyw*_K9z~|_fdMZ;}Vw1^HuV?CYWa`TQM%46XXQKD?je@<#^A<18@nCzF@J||ND7J%T#Izu8+&Hh@p?^&+tD~MBNrSA z@nmx(PnL&c!m0U^Cu>1Et??i}$H3uWB977%|3&FZQhMUQC_TwGr0;EfYzF7SWO~A# zl%6D|C$5XqlWa%&J&lKFaIQf zNo5c(MP-oOjxzcie>Q{jQu3U{M^So`l%Dt~N>6eJ((iA4dj{vFWP0MGC_PC^Pka=m zC#fO*_EtwW=cQzN;-e@%NlH(A6s0HGi}X8MXJvDK3h5_`Zwi9DxUhR99~2_}u919D z73up%@-^zu#A7C{?6h}t4#!v|4Y zRN3%B)E3pGxA0BW7Gaz6sV!=3%BQxdwJD$4Vkz{BZwl^mOd?LG>Q2E4g+DH2?LBw6 z_TahES_`wcegw97hPl~&(D$~MWEmjGoa;k~ z-?4QdI~k7aVWicz9;A8+o=M16iu)q2i|UNJz4j>5_HKPFI|z;{2#zWQ8G@{Wtb(kD ztcDCjh9PSpYanYOYay3HE`>aVa`tRJoUNhWoWl~bG8odjLZP0ns!%V^DR{ugk#^tK znr{aN+_)SL5OLaqsQ`y5r^t*Q~i2;V}w z{afG8uEFnGgrx|?VFkfe;W=R;$f`RXtO_#xaR;l0ti8vDYzar>2HE)v^!sn`I4^@;`;BW?yM`7|?v0IINnkOW z%Ss80!EV5c#i$IL&k72Rat9N*Lt(KXjopcC42)Qe%A$ELDhoJ3$6~g20>BQ&c`Z$x zzb7cp-z%|L6+eFh8wL;Bkk4O{oeAc*sT`bJ$6^#u^HGG=7GvIfSe!>;v2Y?R#^-sG z#A1o;D6Cja%|pe(VzovrrskR`tpSTIi-*PZxh69f`>@1fb-X_l~8L`-BJ{Sj!btk}L-AQ7x?i8_DcZyi7J4GzkJpov(J4Gzkogx<7n<5t5 zJMmb|gK<60zn!~te;kYLl~~MRN5!$&-jZyiJyk#Yj6Hh|SnTH#iwP;P*xn?t*j`|< zUr8(`WDJX;EFFvOO#+ME3oQ0?iN%EUv7Z~T*u5!Wv3r5VekHM(kY+4~y6RZ$-p8`7 z^N^#>J6h)%4^eyG3oM3ub1WugrJ8@F`No1@8L-&B$FphvQO!3R=NCB^yBAmt_2yVi z$Tik^MvleqeLLGYznIK?BFAD60E?lH9E%BQ#$pdhEcO7f7|zeJn2-XCp)=uF>;Z|z z9;iBR*dwsmQ6U8u!#pL&Vh>0x_Q1OHP6h-PdrL@x#U6-Zv1eSVVX>5XF@wDk@hHji zV#(SZ#pA{7Y>?>u$?{?g6WA&R@%*)fBrk^bH&!f0@j5StbvYI+HetNjLdlES*+W^e z*n7>3ElgxUvD^J3D*`EhF7WOy;upJOp0C!QBW9XS>g(u~E7yco{Uv6zr?c`>W)d^}#v zYEy6J#cqaew?3zHI(RYIu#JTmOJwhs3@>JDiFP3Ot-HOHDYhG+o0{g;=F}z1&+m%;l)sIj>UwWcwP*3 zaemM;U#aZCM&rfoY|OZ=(`mFFv)Z#$9hKb}t(DWnJYmdkOxkzxdQF7An8gmwI9Dm{ z#?H6QRR(l>Fx1u7UQF7FS?4r)oyTD}X0;Qu&RrVkDm(MyV6r&;*l6}*O?a=!t}I!5 zu@B+tB+uK*^Bk!nC!>Wc4HI2PBCpahWE$7k6G=*I3|PL80@&jngVH0W?budKfMpI8&ep} zY8R$>F|yI(YaOLsnbBU1$|u}*fS!HWjU{6jrnoZX6>A@bTx{gOl)c%*D3@%=Qnd?H zycy~))=mmpXSEljYeW5CJB0I)-I&o%%xD+JaTqu;sx!x&Li&vMV#=m7|V=vf;#j_tffplay#xWS|#H{vX%5Ln_{}aPxqhYX^|AQnk*t=jeRzL6X>^K+< zR&A(s!HB;~fWc~cy(9*+^S6^UAC>@vNqe!`77VsbVz3WWdzyR_ zC_6EM!RqWVSRx-lqw!)2gZc0*lYLks4EDaF;rKySeggZ*G+b!9n(CR2A^1IJ2CFZ z3;w#OE&ey!iP>T>;=wHTVkuUcz8$pQ+qdpW5PV^_gno!BVsjv&CSdG;gkY*lC z+K(m8gP~3wg9$ly9t?Hi7)(er1~c+tI1k5QLdNC6;@OE=c`&P;n8ILx_lFn;`#ac& zjW!>dG8gt9`4h9V39I)y&L_Ki_&L-4R!pQ#7=ta>2N)aSPmFB5pjSRDeDQFY;=-t$ z9QYEWe2{muaoU722S0zmCgcF|Nk-pfHa21Tln3XI`4Uq;!VETHtG%<%r!tZ5KIJ>i zZa(tjcgZH~x3l8lurwED#$jojurwE@eA=H&Tv&>}#B6a`+9r(KU2z-+J1EGJT-Z5a z6Q;oy42C9wejzKBPcc3pX|M@9fHvUk283Lre2Pu9P1ptR#BkVX7%XKj>|L-4v-2aS ze1aKm!s@+;XInAYMB0P}-2#JoFz2JMmn1)8La#7bI35O5To{!@7>x26FxZ%F!h*8| z1{)SKAqJCNSRUmuU@%*ou+`q~92f0j$!K6)CKIJiBFx$0}#Dh_p1`Jkd$Ai6FHer`!$H8D}9?Xov(mYt22Qy%> zv`v`U?{PobOQvnYvaz?Zz+gg-{I1e@^o3Ijr!Gfiu zV=%>o4I_^MgT>{+s7wO}v*p3wEt{|-)8b&TG!JIRU}+vK&4U>**g3<4rES6{h6jWF z)46UFX5CNRG#6>)!O#{Qg9#az2OFqYx+l0OYdeLp9we+}!LDIFX4O&S*Y?;@Kq=yiW$!=ikB|42IK`w~e%5(Q|R@z#Oz zliiypO8MSRUR_F+3j<-i}#4c|4x z(9_;!Pf>n@FJ`{Z*{~PcOOzizWw^bj!FP=Dog4mbG^*aYT8#~5png5Bo*|c(Idmc3 zyLs$=$Jm7#tf#7~XmW4l?)vT{THVl*U6H!TZpZGryCJnmPvq`MRaFb^t2J-#46Pme z=|w9mr~aqwYoSXnyZrr^Tv6qk$0i+{H;u7*pI~mimUxy#jpu4el;V5$}6r}!^@e6^Uayj;-c%#!1We~)=XW7 zeR6wPWl?l;WPo9De5A4x*Oq%?E(%cZpzJ4duf?;GMemx}`*zN?xW}}=uG|ARPws)6 z&!5BKS6B?wd(I?{K$}Jmzq3B>8U+cN@u<9q!E}f8ubrll%{d zyMyGb4tFQX*B$QLNdDU4-cIs&4)+}-fA4VbAo<@8cNfY3ak#0j>rgjs8ICiYIXxsN zIdghRPI2b!CON~IvxnqNXU<(D&v)kRBYA-{=Wdeu&YXUd^PD*YBrkI2>?gU zOPo3Plk_@sK1H(3ne!mY3TMuzNnYm6p*9Ml9tTjbypYp%57=P1d4bmkl<`8j9KOC-PG%=rPyFFJEhko<}>=SL*J=FEA8f4j;w-%%95W8;aI946rQ5u4-62aNXL-KqM1Aeo+ z2Jd0M`Mo|@Q7`iqM18*Ll;2V2YH>A01{}3<0~l2WltYtlcqS_ z?0=5sP6@eG{lg6V>CP)?p*m~0mPkQCOHV-ozrJ4I?CF%Bu9K%be|8I9KhDpu-Hm$Y zLZ|1QJ-el6_Uxhzr>kr2R$te;tpW5kgD<4?| z9Wg!i6TioExwhemR^R=+w!9zs4EkVjz3NL&&IYyUw6>hmIusjc>vdT zklI4Fv%ZG9N2(^W5%SUniC29-VmWOcs80D93`8bZhPWfe9K7r$JQhwQ% z-@@@A-{{+QvE!Y-(%?K`;ngd^d9KDeZKh*q3_rGc**f*oBY<*h`%3YTa|G&#y$EKQdl(iTD=8Z?uW;m-*Wezu6|m{e6R+;UKE&T`6R++;KjH^$;?+IakNEp+;?+I4AMu~EiC6dFLBxOBCjN}L z2N-Jx30I?z2W{%8?!iNd|GZ7Sx(5#<{t=sabq^jz{FiLv)jc?b_{VJG)jfC|@n5%z zSNGrv#DCK!UfqLlA%4&%{!F+B!>HR~o4TpCe+u#6wTV}4|1{#CwTV}4e+2O_*u<;0 zKZ^L{Hu0+MUqbv3Y~oehpFsSNY~oehzk>Lm+Qgp;ZNCzH(|UYMx$qrN9EAE-uYFxTZ^MA4Yfz;c0{;2uBfK zLO6l&3c^W*QwXmiyovA@!aE3mMEDB=+o`p?5V8=aBe)S}Bjh0DArv4KAuK>xjNn1= zAp{UA5rPO|gk=bI2=xf75LP3sL0F5>gwTS}hR}}CiLf1E2ZDysi?9b_A3{IEeuVoG z9z-~R@DReo2#+EhLUPT!&-jN7@DEo2{nVL8w*WF`4D#_{SyLavGY?6Q_&5f?lv zWcZ|rTli#CE6dJ_!L zJa9qtGQN;8n%`K$l+*CNNaavNUBTroc&_n21iJ2FmGf((k=;?Cv50o0yRLHg$6Vmy zHfH>FP_?mYL)AvA4JiIX+yjay`6Tj0`YmNti2J0jN!3}^sQ}oY@_MACGA8e0Lw^Eh zp?-q;3*wgFOTJw6SC77U>PaW+DfdUgnrPY&n2tR&59iT(r(b+W9FskWas5-!dpM75 zGH9UD=K8q)Q-w1jJJXbg}BL2!Yx_`=F-q}1aZ(ZkWH%DRJ+rj8JB^iIg#=RL*^#g46 z_Z!c@uhEZhJiook`sby&+cJEIECc3xKd_jZb0!Qt8xMinZ%W~_5p%Mm%;~-f4r3|J zEn-e)CI<4P%N@sD@0W4TMH6w(cVP}VjJl@hJk9%MJoiV~8rawwOw`y<*kQc$)HL>{ z$9@fS?HGfbM~=u^}lKL$dWKlC~a2($=F$+Ikd8TaO}X>ro_aJ&L5QN0GGkD3Z1wMbg%z zNZNW7Nn4L1Y3orWZ9R<w#}TJ;>IjNZPs-Nn4j9Y3ouXZC#3_txJ)#bt#gzE=AJT zrAXSk6iHi`B5CVVByC-aq^(Plv~?+xwk}1mbs?Ld!VF>aQ#eW3{1i?RHa~?kgw0Rk zOkwj=c)qatDZD_~{1oO3o1emY!se&&B4P7WxKP;q6kZ~1ehR(9=BKbs*!&b$2%De6 z%bYoDas7qT_Nh?XJ{3yar$TA_R48qq3Z?B+p|pJ}l(tWW()OuP+CCLZ+owWl`&1}x zp9-bzQ=zndDunIR#(ZsX^`6Y3Zmp)T7PbsEkdwQo4t)UQ9UZ$N!K;F+YC>p1XhUd6 z=tS6#umeFu=tbCrun(ahVL!tC2oH8m2B-X1R^QO%p3I>+Gx~;>&W!mywBt?{-lO&2 z^%;&?y6vgtp|tnNzoDJqLaOU8zvpCB*Pz#tZ?zd!_PFF{`r^~D=dB&LOm$NG-L6La zkxeM)LFpquoBYyetRa zJL*9;xQhFw^ojWKPNb)OHxx(w$sY-B|JN37f3-X(Njm3s@|+}xKX{XLG1)A{{DZ4tEL9;q$P z*1bCDnc4c0eM6eTHzDaN*PqNx0 zfgt6f8rPsY~nC_jRUa1G5_+2We_$Y-FPFF=KBsID}B76;c<(RjV!H0&dr z2-omAxg>GTf(h75)`)A=eBfxfhSII|VjfYh`Qhy(am|98GNPDrMc>eok+t!)W4WIQ zxCqy%IYsKDjC+qMT%+b1jkxBwe~E)@;;eb*xCZ;Z(zshM$6b;%?p`IwU6LHvV4v5N za1Hi(O$pavpVyRd4c0A>8`of;*OYJ#_IW*9aE*uXLgPr>5t~ak^PVw2+01*!_~bON zN^Ma}_{e9No5kEcam+p&+(aC68R-Lt^npMNam;?wFEQvbcTXI1IrPc!sx;?5QZ`$+$HRNjPeI;f}77oCaIk$a}nxe{_ZsHf3S-Gp&Es2k^WgdEA~;JlpE5po2l zgLTE6({T$qg3}qfM`{fF4E#7X>=T#Md5`?x+4;3o*e4lICz(A_)p*UZ&w`QIN7-5% z(=CQSykxLXDjd(4eBz0@MBPV=5&MjX<4NS}E)mD0+DKs^JFce_wtBhn??FM?h;o~T zG}mK~ebQVH>j(Eh>j~0akHkc14{D1N8!qOIdzwALW z2Sk0jasFouZ1`+3L7ES-#{}?Sm(u(X{M?=H`5$^8$7`=ZWms(z)Vpr9D^NO-zVpOy z82hzRH~MeE4S)DVtdCGwbcCHA=CkAU3qd{~D*J^6N%RYLbJTJAg;D3EdB3nIPQO6= z*2VGbNbjU^4_|%_3Feo_(l0C+WsW%6exddEar%Yyw}O4Ya0Y%W(7QO^`9A)hTjuoC zJ8qnpQ#z4;?EOOche`Acm=B86FL>ykS}6O4g-P@ac5^&&`h^l+wzO4Qcufq8xFi-) z-{8D|u$jv-#vA8wlEDv&=4ZyzPmDIlV(lkX-4(xJ#Su5Y6Q{pO;|Tly;!MU76U3ME zzQDqXtA4`BYb%{dKlZ+(><3Bo9pF>r^c@^iERuc4q9pnbJB~C?-@!3OB2HAjU%c$UWzGKCI#_2oKSi`>WIFqr)1o2J0FR*Y*s-H0OLP{sn zkG=1h|8KFrgWffNVc*b;@YfXg+XQQf;`AdNgIt^fXQB8AYu{nVWe^5Qv<`^(A7kJw zc)1DI@L2l^>$eQWbIw9?6cnEcJNX^a9h%$47%zX6bvZh=u-LC!u}0O~ar%omSYtF> zRkcr?kV*Puz63ZUx3YfO8o+C-pAc9oI(3{(YvbF z=c)HL70!b4FdI8%(ifa|-a_;XA9yi|eqjPRi+Fg!j?19;H<2Bx@w>v#{?r(sh_m45 zNnk@d5%^&=&O+5S+5W=wR-FDKjU&wa{b4=u8IL2zz*+FV!^X~!_Z_F5w-9|t*6}3z zjtS%};$aFqE`#3bcZ0Lw=aGHKqU&S4YNq5>C!+5djk8F#@0j=7IDJPNYnc0vvjJ<2 zfwSO!hmD;K?>kOAZz1}Q|2`V)JLviSDC{iG1Zsqi zI00wDuxB00>DTwLv&_FzJj&az_Ov^loW=RSiPK-Cv4*+7I2*7=l71n0AMqY@7H@q& z)<>M}oJG)W=oiS&LLA>kdg>SGnD(xuej(Ai{opLoFANJgVC5_*p85rfhrG|OUzjM) z;=EtS=@-&C!rU*M4LBmnb=ACIc#k=YH(p4hUpU)1ivrO!0LeWpK4{|x8pP+c^Cn!E%KQYmqh4alg{Y4r_nEQ*f0Y{vroW+lyPonQQ z%Q%Z+k4@i!wPr!D=sPH$`VKl)eMdsOiW1Rx1WWDvj$y=8-$C*5`i_a_EZ%t|PT!Hn z8s@&^Y`_|4DQ9uyNUZOmak_o3;4DV?kcH3m^RXs-!hFcWckH4meM18?#^^g%>_h%R z;vleAV|&_%Y|PG`$^Abj$`?r@PDkx~Mr)C2za{b?Qa^K6{;;?&`WhOl)B70n8`tHI z$0tdm^)`h|eY4;8jm@v@me=F-$#L+UvQ-&v?poQXBs+JlY*dn+yH++T$$HE_iQnJiT=Xxy2hWYxT9|FM zf7%CqZ)-`m(a&r@bbDH>vUR^C7Cv!5;`g<#%ucjE_kN`5Z(WydUEigz(M6f$pX5QL z+uz!eooH_j$!Q87C27C2j_NX>BaA@pa<=)MZGAR|#m3}!){2SJ{%7^Ie-aag6JerM ze3T^Ov#pqj%B4Lh<6xp%BPJS;uaZO@xWYu_hlKXIBn~}UpR|j983z-^`A+4SDD9J0 z-%s~?>6e!z`Q^Pq`sF3bF;NOWX*nj^n<6IKJMowZt_jk~9FwqE~wN^}&_FWs-ckM*;Mv3O~tQhIt z;*Ao`|0#?V<``+@ene^>Q`s|9I_thf|9&bCR!Z|n_G>3k#vD zWPIGF#3fnm<8|C5ecdL_CBZi<{I#Y1-GW0JHaZc2QUWo?j-o1~xIr1>WJb>+B8NT1OsO4?Vh&yms|N$;Vr+&$lp;U*flZu>yr z&=290b;Mc?D?U2Q{m&@wM}yrC{Kmw5=SsVs(bjg2nA@_f?V|Yc_@hx6LtzN({FT)w z&Bsp0!4q+Kn9=Na_?#B@k51Nprv(^CYfRDiL}QA+C$Nw7cqPb$d@O)HhSgV%vcXW_#a3GkvL6t( z72Jo_o9b9e=~HS)TNgGFeM&NO;>q?YE&m>; zPl@x5l2o6P_Gj~AKfYyB>QgXZo>HHJ`EpaAg86V0_DpepQ{hXipBvt%U=CAZM(bRr z>K`V~Pp{}xZg?_3ji?>eO;Szy=~&Zv+7Tl71Z4X4oWoMr3J z2(w^aiMijg;d~Qen?yYBMD#n!aI&fPJ0JQx{rV7A{ToA4V=r=|~nB_GB{ZPK9L?$WzmqCQGy9b|eN7tUnERTu4da~U9L?mxSpRaaa5V9J z^(ca$1oDWh^Ur`vZ=s(niIM{)X;G=4GnDQ6pgIm6cY-9BmWW+WJIhxb#Gia>+&Ua75>37l?$K3CnZ5Zbq;%L79%~-!f<7x)% zXo#bk+%sipQbwO{PxH2;s!MBMi_nD7g3yN0j?js)9bpH8hR}5`w<@O zBEE<)2kn=ju!jZ=^~n>}MH|_(N8ilIVDn(RH!tH0_@xaIPm*X)N=h@9y&HHvEOsya zTnYAJD8zc;daMKfV3v?YIKB$U7vY#BFOvXSDA;?@-LfF5M@b`F%nveSms?;=S|MXC{?C!1{bq-gKW3d#BZ2$Y*1pF!jr0 z^bLr45%LN63-Bv|1D~bHkaQ}E2j#-o! zW#~4!)-^!!)=nG!%8c-}hwJziu7mVzy;$ok_lbNv9P^u});BvaCK=c2eD~kte1F8@ zJ(TV2X!dZ*W;R(H+cvaiFKlKxrpaZD$|o!{2jf3U^0h)(CLhNn+mU_`Y-Z0TA9*@% zk=IXng|43@T|eO!x_*+BR?QKoedDEl<6Yg{tkIh0L~EPrnNdN zBiANBB7M3DDSW!X`6Jy23L4flyBTZg7qX3Ied@&Qs#yF1tZhbo`ynAaMLuD_@#U|_ zxPdYGxr;Nt(;8;a%)@zc#`mCce5bU5c_OXd_)hUb@8P^S<9n5dkMAPhIKGp z$&Byu{LNX%_jqf3jpO?Wzj1PWC;eJ4{Jj~*cOU9VKHx~wI$Z1c{=$oK#`ieW>6=Hcku4?t2BFWT za^N?O%JGrDb!7Qvq))HGc=+iv+bPl+4hacQybq$_-uPJ-(Aa_Y1p6j=v$&?-Kev8m|KZK3)UjAG@$m z=<`VrpAR&SSCjrup*N22Vbbp%Sw8vWO8X{*|Jc*)o6K%}=VP(c*EQGa=X0lXzUu4B zbuQ_e;|%a)idWy~Jnbmo<&-8_9~YCZ4^Ev_RW)^9sEVH}$eueEoTl}LAVZK2&USSgx70^2x*EeEME3bj-&X!#AS~*J$K`YmTPJ z5M@(9yfe*J6YolrcvqUMCf=3gc6?{|pMhhDdUi=B;`S)s%HOHH?BGQCN1*peydM`E z-j5OX_le()V3O}f)#-dUy1pOh-AMB%_V31N^kv4M-3r*xq&PpKo^O(gzKePJmTzB` zcjDIh$KH*#Ux|$!DZU$L#JKT|7vj7d>2DAFcjF8nH`KeKp81K9VjTF~q1dy2&|n`l zR-B~f=Be+;TzpSZ;GEvmITrl+`P82G(dNk2Tzax|OC&_n%RlMW6wmq%Uy6+btG_gUK54Y)odGwIe+o^=yXe^LkAlbdYYr|4$nON=_tP=t zePm;3<&(bhT>5!Gm!9`C2`7Exi?R3PbmJuDXJOdQpZ&Y6eif|#66l!TjbR~e-;Fn( zjq`4#agxrl@^6SUXuaRrf|HIsI_7sHb(~Zn#tr2cK*KSO8_FMmb)7#Q)4Rbw7=PUO zV7zxD4xg0NH$>XcGoP<;8+cP%Ge`3qBx!zw*38lT2FX5 zao%;sm5uh!D~>E#PEB!S$#QCnBTJT3Q+(KXIJMI~=cs(OCu^J9i+f7G+!e2uUVoBa zf0E4l6Yi(GwJ=-x`tQecw5PQyTYX`XCmAXzrzWPk5tHAl}PO`l`^`w)Hwr8Fd z8=Q&jQzSMBC&C8fU8f?kL9G!RB)eWkVFMrD>qKi-GQJlF8^rlW=GY*;W<|G)pf&S^ z?@1EAC;u6Q?@4lek3AQX{T@*Mh*QD_*pp$Ru|az6irkkCYi+14JhWECU)dI9QV-ix zTF2wHSzAKucuH*6me4w$Qqubj<)D08$K$gp|8%Zh;n-lW#0GmyvN<-`Ta~T2cIzAz z#|C?ozy{dE!@75ejt#IjM#lzwJF-W`2I(~|=fs*8$HS>%gAr?5(!9Zo^)to#71$T( zw#M%UZ}6w5;$VX`Z}6_$oTYh#^tzY1UpW2iUgGfv3LB)t6R5F2Sw4Vx>-2h<w?eQ*e_VdPO|aG7;*>M9blcx4zgF6#aQa|QOJ*S_6*B)G4fm3M|KS;SNFH1 z#;9SDULU*Y*tkwceXCk+8?OI$YU9sne6`}WljW-wznv^!t@!O^`D(>)kB6`BB;M{U z@)I48k5v7Nm6ueUqv}Jf+@qT7nz;2e!Zv8t=VI^H2-}C`-m7ujipuvY9Ou2N9qW75 zdpOQ}6^HXl>b*+akDgZhQN;}@u7}SH8TRAk@0HEBy!v*xe$(^!>OI(Vv+JR8zgK7M zc=e?xlNyi4F2(rs_-AA9$yYx6?qNFy zY-X}i9cAw{vQb6)xHhZtu-(sx)9(ts|B2Q=r?H*%5intErNz7TlY_B$>)pb3R{K_k z?W}gL@|&r4G<(;0@70WW?^PPxov}8r=L)u)JpT78b!=B)wvCnVmCtG;E8i>sXtuKP z-mCJV^muiyk5^WX^K@am%l&9{9k)npn-jjDNvP{~Dz~qfY(40zY;-r?Xxs zV_8y}DP_TI@nf1ih8;NnnEiaiAjKGe2l^M>Y*hW5tJmd%Zunj5xlY3OWe>}=?0-?Xh^edFea zZMU|!Z|UeniuJcNZfM=S0rAZZP0j0W-O$jw`Q|MRH*e`^Xl!e1*s`smy<^MGt!>TQ zKD4c4{fFAxZrgMTG#i`Ock+XR@^t}^r_o#C^ZQFmD%J&jR+?y;SUUdsE?(b6;i()1_{Q$>!iNOqKD&Q{AIGBMbB?C^lL17kisH8 z3>b`mef%Jkb^483nuw&oO!kmP{|{21B`%O3(SM(p`W%b?D^l;dlK;^E`_w>$SjzdS)Ni-k-`A!7#J`x!c}MEkTFMbGVkQg!!kiw?vGAA4_Eehd zdA`&i5)H0rDvMIq$Vn|J2py^y{R) zzQL^DB=xVrOP%p=yVSRL7`2S;lKOZ4&aA&%>bHvrNsoL$>i=@LuHru)lKS5J&H67( zebzy<{+m+2@1M>3??}D-Yi9jXsee#BV0z?_r9NoU|3d03E&AU{{qU=L5&Xwrq<-VuYYV8oBVO!kDOoCQ)J7&Mn(D)n_kX8q+--)_-gDfRDI^sA)4 zZP;8+gVYaO^c$uArNid*w@Upfi(U+KnXK*^bNY|V^vf*z`=q|;s5$*VN`26xe^lzj zub9(+L+bbZ$gDps^|LMd=cV3l(Vvid&yUUJ{9NjLE$M$F^+8L0{#&L0nYkPn#&o*x zZj1grsh@7q=SuzJpPS3MNa{69dXLmsTKbJkrTz&^|5hvY4_WkANqw8>0QHlvk@|u; z{6*098>Idv(b4Jp4yj-Kx~}3sJ|^|9x6S%JQs4jIX8k9m{t1iz(^CJ+pUmmMAocH9 z^p8uuJ8Di3Cw}v~q|AGhfLP3liL&FN1{{aY6OucUr@mO1?&q~87aX1#+0 zv`prjZr0C``aHK;pCk2a-e=Z-Q0jMB^u+AjDNeO{+-X7^#fA>M4nmyf2IDY7@+l{9+rA{ zkvaWWrQT!F4@&(j7X33)&n_~T^ODrRGvBQLcd4&iVAj7O^=%gY@1;Ixp*cO9M3<7u zDlPhHQtz?6kMEcIyf2!|DU|xQOU(L9q`uc{)(52iDW6$iE%m1?`V~^&UuI5!ozy>W z(Kku`!{z4mTcn;&^NesXws6tiN08vu-i#KPB}GEc(w&{c9HeV^Tl8&0Nl))X&~*);}xt+7`3^ z2U35)qW_uHuWdJ{|Bcl5TJ(RC`h6Yd^qG^z^WgfSSwBna-|95$=SqF}cC&tw)IYkz ztoKX(5sSV`>i2!joPLGWAF}ALm-;7mn$zDb^+nxgeTUSCE&490clDUl-zD`6Ec*MT z-qUMNe^BaAo;2&fB=vvUW7dCD>K_&#O8O)EU8&!5pE><;sc-qTS^rb1fAk@<{tc;r z`k&1DKS=$lN6dN$CzLYTOP2mWTk79BWKN$e_0u0W>*q`TUo8D$iPYza4>$dKFPD1X zznaVWu+$ekY1UsO^_9<<_3Na5^)a)4v(%rkJkNJXy+?ci>h;+z^?QD5E@wdMAO8=t z{?k%F{Z+I65vgx_&8(j@1*dZ0Cj8l~|2opoa^j_&uUCct_$Bl+F^&(vA<_p`bn)gb zoz1Mc>GsXrZr{X;H*CJOxTSGh3zPqozO#cBcQ$YDn%f!zHMKtZ(^KlHR@!h#{q$7S=syExdfGr( z*H-v-ZOE@{L*9TYH&hx+Q07tEYF+E~_*Gi3r$YbftyP72OZ1v~YrU#4Z%Iw;cOaw+ zMhcY=7pH!fSCpxr-ddk3qN27+ovtRVUr}keTqP*0idC&vzxt9A{S+19DwVc0tfwuh z(5qAGEzuJMLh5WjkAC&Oa_pfcZicV6L{+lPucs}mDN)({Wg%6VKU|`+mHR4Gw(?L+ zTT!jjmRD4(v=z1biOWO!iOWN=3#!pCsJyCFFSn{n|5+VYg_T$94^MfxTvf6>T&pLj z(VIHpDN_jovHKnH>xBjU)#?<1uzvA)&eT;@l*g{Z@6oko73ypiWdT(}g2|pX!eRXuhD!8u zLlxyJZ7B8{hGNfemB*(NRK=RFsziUws>*8ARaE)CdfIaR9#@t7RFhPpA>?!BE%od7 zr7Eo7mujzGpK70eoz*`57FEX@xVlWgjOy|V)jZV!y|bvU(BIkWifUEM>I(fWtFF?^ z_4>;7%2(Iu`NAdos}nBMI~mkhoi6MzRhJa@>u+Z`;8Ujy$KJzm>>UWlx~XtTe|o~z zdb!>*PpMu+jebLGO2SH8r#A+E>(3rKaQzDKJ63N`Aok4pO9HAzyq;?PJ@a}!VRgw~%<`zm6~ALW zwzo|0pS<2O{iX1F%k{?adINgLh2MrUEA+?H>&3gS-T`myO+_)W>-E;?Po1}b)S|EB(#$`a-dCd^LKXh-Y2D?bNh-`gW`EzoV=RFiUs;#-z1sldcPWu=Xl$YjyRW>yrtmHD8mS_Mv7l)^iIZ}TfB#jE-&c_HG7@>KC^zFNtfWqu{h=$WyL z5kWfsSxbo&j>du${wF4e= 0, pi + swa_pages, -1), mask=m) + + # Then the swa from this group's first draft to its last. + slot = tl.load(slot_ptr + g) + pos = tl.load(pos_ptr + g) + n = tl.load(indptr_ptr + g + 1) - base - n_tail + i = tl.arange(0, WIN_B) + m = i < n + abs_pos = pos - n + 1 + i + val = slot * ring_stride + abs_pos % ring_stride + tl.store(out_ptr + base + n_tail + i, val, mask=m) + + +def _group_index(R, num_draft, block_q, device): + """ + Per-group gather index and group size, built on the device. + + With R=3, num_draft=7, block_q=4 the six groups come out as + + last_token = [3, 6, 10, 13, 17, 20] + group_size = [4, 3, 4, 3, 4, 3] + """ + n_full = num_draft // block_q + remainder = num_draft % block_q + n_groups = n_full + (1 if remainder else 0) + ends = ( + torch.arange(1, n_groups + 1, dtype=torch.int64, device=device) * block_q + ).clamp(max=num_draft) + base = torch.arange(R, dtype=torch.int64, device=device) * num_draft + last_token = (base[:, None] + (ends - 1)).reshape(-1) + group_size = torch.diff(ends, prepend=ends.new_zeros(1)) + return last_token, group_size.to(torch.int32).repeat(R) + + +def build_grouped_verify_streams( + *, + state_slot: torch.Tensor, # [N] int, per query token + positions: torch.Tensor, # [N] int, per query token + hca_len: torch.Tensor, # [N] int, compressed entries per token + hca_page_indices: torch.Tensor, # [N, Wc] int32 + win: int, + ring_stride: int, + swa_pages: int, + num_draft: int, + block_q: int = 4, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Returns (indices, kv_indptr, qo_indptr) for the grouped call. + """ + assert block_q in _ALLOWED_BLOCK_Q, ( + f"block_q={block_q} is not one of {_ALLOWED_BLOCK_Q}; aiter's v4 " + "dispatcher would not route it to the tile this path needs" + ) + dev = state_slot.device + N = state_slot.shape[0] + assert N % num_draft == 0, f"{N} tokens is not a whole number of drafts" + R = N // num_draft + last_token, group_size = _group_index(R, num_draft, block_q, dev) + G = last_token.numel() + + slot_g = state_slot.to(torch.int32)[last_token].contiguous() + pos_g = positions.to(torch.int32)[last_token].contiguous() + tail_g = hca_len.to(torch.int32)[last_token].contiguous() + page_g = hca_page_indices[last_token].contiguous() + + lens = tail_g + torch.minimum(pos_g + 1, win + group_size - 1) + kv_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev) + kv_indptr[1:] = lens.cumsum(0).to(torch.int32) + + Wc = page_g.shape[1] + # Sized to the worst case rather than to kv_indptr[-1], which would need a + # device read and so could not be captured in a graph. The streams are + # packed by kv_indptr, so the slack past the last one is never touched. + indices = torch.empty( + (G * (Wc + win + block_q - 1),), dtype=torch.int32, device=dev + ) + _grouped_stream_kernel[(G,)]( + indices, + kv_indptr, + slot_g, + pos_g, + tail_g, + page_g, + ring_stride, + swa_pages, + TAIL_W=max(Wc, 1), + TAIL_B=min(triton.next_power_of_2(max(Wc, 1)), 1024), + WIN_B=triton.next_power_of_2(win + block_q - 1), + num_warps=4, + ) + + qo_indptr = torch.zeros((G + 1,), dtype=torch.int32, device=dev) + qo_indptr[1:] = group_size.cumsum(0).to(torch.int32) + return indices, kv_indptr, qo_indptr diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py index 426bb8bb6b..5b60c02202 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py @@ -293,6 +293,7 @@ def decode_fp8_2buff( attn_sink: torch.Tensor, # [H] fp32 v_head_dim: int, qo_indptr: Optional[torch.Tensor] = None, + max_seqlen_q: int = 1, num_kv_splits: Optional[int] = None, compress_ratio: Optional[int] = None, ) -> torch.Tensor: @@ -310,6 +311,9 @@ def decode_fp8_2buff( ``compress_ratio`` names the stream (0 SWA, 4 CSA, 128 HCA) so the split count past 40 tokens can fit the stream; None keeps the fixed 4. + + ``max_seqlen_q`` above 1 puts several q rows in one tile, which the v4 + dispatcher serves for gqa=16 at 1, 2 and 4 off the same 64-row tile. """ from aiter.mla import mla_decode_fwd_v4_nm @@ -339,24 +343,21 @@ def decode_fp8_2buff( if qo_indptr is None: qo_indptr = decode_qo_indptr(T, q.device) - # num_seqs comes from qo_indptr.numel()-1 and the kernel writes - # num_seqs * max_seqlen_q rows into `out`, so both have to be sized off q's - # own T. A qo_indptr built from a padded token count writes past `out`. - assert qo_indptr.shape[0] >= T + 1, ( - f"qo_indptr holds {qo_indptr.shape[0]} entries, kernel reads {T + 1}" + n_seq = qo_indptr.shape[0] - 1 + assert kv_indptr.shape[0] >= n_seq + 1, ( + f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {n_seq + 1}" ) - assert kv_indptr.shape[0] >= T + 1, ( - f"kv_indptr holds {kv_indptr.shape[0]} entries, kernel reads {T + 1}" - ) - qo_indptr = qo_indptr[: T + 1] rows = unified_kv.shape[0] - out = q_rope.new_empty((T, H, v_head_dim)) + out = q_rope.new_empty((n_seq * max_seqlen_q, H, v_head_dim)) # Left None, the wrapper's occupancy heuristic picks it, folds the cross-split # merge back into `out`, and leaves the final bf16 there whether or not it # split. Pinning it to 1 costs 6.9x at bs=1 kv=2048. - if num_kv_splits is None and T > _DECODE_SPLIT_TAIL_MIN_TOKENS: - num_kv_splits = _decode_fp8_tail_splits(T, compress_ratio) + # Sequences, not tokens: the wave-fitting rule counts the workgroups the + # kernel launches, which is n_seq x splits. They are the same number on the + # per-token path and differ once q rows share a tile. + if num_kv_splits is None and n_seq > _DECODE_SPLIT_TAIL_MIN_TOKENS: + num_kv_splits = _decode_fp8_tail_splits(n_seq, compress_ratio) mla_decode_fwd_v4_nm( q, q_rope, @@ -366,7 +367,7 @@ def decode_fp8_2buff( qo_indptr, kv_indptr, kv_indices, - 1, # max_seqlen_q; qo_indptr is per-token so every sequence is one token + max_seqlen_q, sink=attn_sink, num_kv_splits=num_kv_splits, ) @@ -374,7 +375,7 @@ def decode_fp8_2buff( # ReqToTokenPool reserves, so the builders can't emit a zero-length one, and # the compare + masked_fill_ was costing a launch per layer for it. One would # come back NaN now (all-sink denominator); the guard UT pins that. - return out + return out[:T] @triton.jit diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index 06ef59b6bd..b9b05beea3 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -3,6 +3,7 @@ from __future__ import annotations import enum import functools import logging +import os from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -104,6 +105,14 @@ class UnifiedKvMetadata: csa_indices: Optional[torch.Tensor] = None csa_indptr: Optional[torch.Tensor] = None + # Grouped target-verify streams for the asm decode: one per (request, + # draft group), laid out [compressed tail][that group's window slice] so + # the kernel's own causal bound and band mask land on each draft's real + # cutoff. + gasm_indices: Optional[torch.Tensor] = None + gasm_kv_indptr: Optional[torch.Tensor] = None + gasm_qo_indptr: Optional[torch.Tensor] = None + # prefill/extend per-token mapping pf_state_slot: Optional[torch.Tensor] = None pf_chunk_start: Optional[torch.Tensor] = None @@ -131,6 +140,9 @@ class UnifiedKvMetadata: "swa_indptr", "hca_indices", "hca_indptr", + "gasm_indices", + "gasm_kv_indptr", + "gasm_qo_indptr", "csa_indices", "csa_indptr", "pf_state_slot", @@ -160,6 +172,9 @@ class UnifiedKvMetadata: "swa_indptr", "hca_indices", "hca_indptr", + "gasm_indices", + "gasm_kv_indptr", + "gasm_qo_indptr", "csa_indices", "csa_indptr", "pf_state_slot", @@ -511,6 +526,24 @@ class DSV4RawDecodeMetadata: self.out_cache_loc.copy_(other.out_cache_loc) +_GROUPED_ASM_BLOCK_Q = 4 + +# aiter serves gqa*msq off one 64-row q tile and only whitelists msq above 1 +# at gqa=16 -- the TP-only shape. DP attention gives a rank all 128 heads, so +# the tile is full of heads with no room to group, and none is needed either: +# one kv read already feeds 128 heads rather than 16. An unsupported pair does +# not downgrade, it fails aiter's kernel lookup outright. +_GROUPED_ASM_GQA = 16 +_GROUPED_ASM_MIN_REQS = 16 + + +def _grouped_asm_enabled() -> bool: + """ + Grouped target-verify decode through the asm kernel, off by default. + """ + return os.environ.get("SGLANG_DSV4_GROUPED_ASM", "0") == "1" + + class _GraphBucket(enum.Enum): DECODE_OR_IDLE = "decode_or_idle" TARGET_VERIFY = "target_verify" @@ -714,7 +747,9 @@ class DeepseekV4HipRadixBackend( # per-token (num_draft*bs -> bs) req-slot map produced by the prefill # expansion above. self._attach_unified_kv_decode_streams( - core_attn_metadata, req_pool_indices_repeated + core_attn_metadata, + req_pool_indices_repeated, + num_draft=self.target_verify_num_draft_tokens, ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) @@ -923,7 +958,9 @@ class DeepseekV4HipRadixBackend( num_draft_tokens * bs, ) self._attach_unified_kv_decode_streams( - core_attn_metadata, req_pool_indices_repeated + core_attn_metadata, + req_pool_indices_repeated, + num_draft=self.target_verify_num_draft_tokens, ) indexer_metadata = ( self.init_forward_metadata_indexer(core_attn_metadata) @@ -1469,7 +1506,10 @@ class DeepseekV4HipRadixBackend( self.forward_metadata = current_raw def _attach_unified_kv_decode_streams( - self, core: DSV4AttnMetadata, state_slot: torch.Tensor + self, + core: DSV4AttnMetadata, + state_slot: torch.Tensor, + num_draft: int = 0, ) -> None: # state_slot maps each query token to its request slot; # target-verify repeats request slots for the draft tokens. @@ -1521,6 +1561,35 @@ class DeepseekV4HipRadixBackend( ring_stride=pool.unified_swa_ring_size, swa_pages=pool.unified_swa_pages, ) + core.unified.gasm_indices = None + core.unified.gasm_kv_indptr = None + core.unified.gasm_qo_indptr = None + if ( + _grouped_asm_enabled() + and num_draft > 1 + and N % num_draft == 0 + and N // num_draft >= _GROUPED_ASM_MIN_REQS + ): + from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import ( + grouped_verify_streams, + ) + + ( + core.unified.gasm_indices, + core.unified.gasm_kv_indptr, + core.unified.gasm_qo_indptr, + ) = grouped_verify_streams.build_grouped_verify_streams( + state_slot=state_slot, + positions=core.positions_casual, + hca_len=hca_len, + hca_page_indices=hca_page_indices, + win=pool.unified_swa_window, + ring_stride=pool.unified_swa_ring_size, + swa_pages=pool.unified_swa_pages, + num_draft=num_draft, + block_q=_GROUPED_ASM_BLOCK_Q, + ) + # SWA ring write target, same value for every layer this forward. req_slot = state_slot.to(torch.int64) core.unified.swa_loc = ( @@ -1691,6 +1760,25 @@ class DeepseekV4HipRadixBackend( "the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is " f"at {self.softmax_scale}" ) + gasm = ( + unified_metadata.gasm_indices + if compress_ratio == 128 and q.shape[1] == _GROUPED_ASM_GQA + else None + ) + if gasm is not None: + return runtime.decode_fp8_2buff( + q=q, + q_rope=q_rope, + unified_kv=unified, + unified_kv_rope=pool.get_unified_kv_rope(layer_id), + kv_indices=gasm, + kv_indptr=unified_metadata.gasm_kv_indptr, + qo_indptr=unified_metadata.gasm_qo_indptr, + attn_sink=attn_sink, + v_head_dim=layer.v_head_dim, + max_seqlen_q=_GROUPED_ASM_BLOCK_Q, + compress_ratio=compress_ratio, + ) return runtime.decode_fp8_2buff( q=q, q_rope=q_rope,