From b760f7fb1949604af2137c8284f1f121d2f0f8ff Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:54:55 +0800 Subject: [PATCH] docs(cookbook): add IBM Granite 4.2 cookbook (#36286) --- docs/cards/logos/ibm.png | Bin 0 -> 13633 bytes .../autoregressive/IBM/Granite-4.2.mdx | 231 ++++++++++++++++++ docs/cookbook/autoregressive/intro.mdx | 6 + docs/docs.json | 6 + .../ibm-granite/granite-4.2-benchmarks.jsx | 110 +++++++++ .../configs/ibm-granite/granite-4.2.jsx | 192 +++++++++++++++ 6 files changed, 545 insertions(+) create mode 100644 docs/cards/logos/ibm.png create mode 100644 docs/cookbook/autoregressive/IBM/Granite-4.2.mdx create mode 100644 docs/src/snippets/configs/ibm-granite/granite-4.2-benchmarks.jsx create mode 100644 docs/src/snippets/configs/ibm-granite/granite-4.2.jsx diff --git a/docs/cards/logos/ibm.png b/docs/cards/logos/ibm.png new file mode 100644 index 0000000000000000000000000000000000000000..ab7a3945cc87ff5d48b4b39615d473b2613a214d GIT binary patch literal 13633 zcmeHtX;_l!|F12R(_}eQQ)X_>o6%Nkc97+;T%x5mS*GCz~eI2HX{y zav^ce9XBc`4HpYe}{` zS!IMo2ZUQaCj0w4fzOVAbNt}#D|V`iY5P=iuTDMP_f783V7d*wNAb5GwqGrNHtaip zZt?87Pnw7Qk2oB>vwwni$HG&qb8O9%{a|8xD%4*Y-GfypwL{T^3qCAUkyeF|2TH;lb)vy!q=LEIc>xFFDmDS79D zu)B*E*63MEV*DXN$q3y2s%M1pc;V3J;)_ef%MDv8x)xE=d;IpD{)rUv(TS8<E?kn;3SH#c=r-<{twZ}0k<@;f=swWRG4idO{qSm zU(tMFh&sSwne2!Y@&;DgthSiKKc^ndvleo6^r+3XQ&=K42<235fiXx>|0eI;V1nf* zp)N0G^P}@1Te9SeIU-~C&z7&SE5t@XzS9z9ySbd_j5n$dusmL4mHO^G8ClER2y_;D z=zM4ans7lxeJFmR#m3gMW#eYHxcUN*KQcu zBS7XW?KsoJF?Kf@dPpj;FeYcvkt=hyZvU0a0}(b*JZ|W(Kvu@i?R?4(H3{h>Y*yKM zZz{R#l=9^k=JTW@|3Q%M!4_CrEcp&~H+#KYLG4}hsBQS=onmIi+|u(Djcy`XMdqNJ zjK@*IPFt~Ma(RAL+cCBS<>9EbFjq1|>G*7lvi5Z1t)S`~BR^CoZ@MCkyiU(8(r{U{ zuI0Z!vZ^;$EO#DGx;lTl6yo)A=)=1cRQaEG5A4^GQLdQ--GNv0_7>Xs%i6YDf3TKqP5aNH&1t>6P@bUMd2t&?v1K=JM*DFB|0I{%fwHH z4Ma)_ZnHp+&5OGr@40_(iAL|66myr_5!PBoTVHnjTb4f=rSKnOwwmuodY8Sp8)BSC zEuVEBxI;}CP#>=z3LP5W0`U~tfwU4Q%6$V5QPN-g9-oxSXnaKDIEDDL&X??)%Bq1^#^rFM};(R>pp)v3DuB70XSnk+63Uli#g zaatW;iRQw_!F;B_>AS~TqosTexcmKY9-kb2qa)?J|2Shnu@@OJm%3)B$sMzQ4%{r9 zo6HpBr{EhUMrU4N@|IWMNiI1J&)v#ICORSePEI85Sn1T;o2qJ#CC|}0QR8SA`>BQ4 z&&y+2{!&z3eUHg6mS08593}}pvujq7?l1eJ#MOlk--NL$Pw&-hJ2TN!zw4Q1N6tP) zehRlgcAtc6o}KaFB&lum+nT9_jq+`C(I0hZ;T^W?zbLa`fcF1itGozmFQfB@|K|P7 zXx7KvE}P+@`hdUZ{?X9@$9_gl=hY9)UN=%SDhfBt>ebzws1moa-vgW}lJuVc;6H2( zX%mA?kqg45NfUdh)vwv37tGnOF@{b4-u9s`iWcVT_BEaBJ#q>;AkH}v4eJxLc^!|p zU^=)K4ax5$T!OOs;mz}u%1x{K)i2T1CGK3&b7uz>^R~TqPo-_O6HGWTy2P+{JMz7U z|ME_@KEpO{{PlaYJ1yo9+0NKME3K1Vxs*ACJYxU$A5sCmD4MdPcu_t36(;rQe`W*2 z0@fyML=(x2rYIYz_-jkC-V=O21e&sjY`BA3itDiz$CyvDdz{5^$*!Gm$A$eo7d4RN z5^rsdG-*nd4CQfECNusV>6}xM3R|bv^Ow2%bV@1@*8Licpy+(?FMM=t*jSvNR&qcn zNbPJ1WKoQaiqy&oFR;oTMDG$y&h@B4x|*bNYIkC}g7-0LRuzVWp@~@2a&B?H;d!vP z4);|A&Wijt1m&Ay9&lj`=AG~#g_Fk|{Hir~)~7C0#aaG!vcoB4 zSf8d{ad6Z&%(9p|P|hjZ7{D(~SK}ttzV)=|$tHK_P?Ci@Myi`M&Patt55cy`!xf<% zQujeQPA1x8p$nU`u$5C3|6V6%c-FK$RFeS;CCgP1j8xS_eAi96bFuT*Nx5%8yJ@TD{ zc%kSyu6WwfrQq?QgFrp|{5g)00#tTI*6JkqpVPfu$n*66voxi4b5XuRqwv=c!||UR ziY0{fQ|LZj-~N8ViK}}*G;8b?VrJ8q1^@6=%Qraq5Gi!I`E%Lw zJ1xujfDDVRB%?f~iDMm0*WLF65cc`op};JsmbvsJIDfsVHUqPXQ@d|^JI?nOThaO= z-)}|9cAf9!l!o3o1KW|m`|qOEe=I9pt)q~g6GL6B>fIT2yFEcKC21gFKN@Af`MdeN zMnyY0pG9m{X{~Q-+b^pN0NcACVl-=bz)c6JiaR0_Z<~f%CRe}}emq3Zr*7}uhPUgT zzX;UT)m?}=zbM4H`9RY0DPWk)Q$u*VU1}Ah<0qiJzY(Q=yPT&DZ&Re?SaL7v4?GB= zI_jNoEJfz%MHC@d8-}sRWz=Cw7%P)#A}VN&iCpxbT|`%O~P7mS)hK_OOx4^znHGK{Q2(>y}@j@IY+x&xm)Gr)7&6$FFA2lBvg}U3* z-7U@Meq1Kwtv1F1JY)wO6vl}tm--n$pxbaGDz8quy=+jbDm5Yr4qi8lI6*0;IqeS0JS>*W!CLE}+V z@^7_h+m6-!@saqP%GWj7q1`i)wS5DF4-WzZADKjBkBv^<@`ltHJ4X@Lc73J#oN9{; zgRPBQ(NSmsMs*t4H4g}&(7qq6)lI=nNU@YxOHP}m+AC%-h*flB&L(K*Qp7Sbz{w`wI$JtaLcTVF1@!ntxGhZ2Xe9 z?^s36@posSTLggJ>W7564yO-O$U)hq>YW^SEyeIg*@M4faO;04#|nwS_M6m}n_?X! zcM;%)yWZ3jRH>wB2dm%zVVe9m>En(T+%%tY*&Zs5(do@{#jQ-tH@%#izHp-gv=9w(bwThhE_FJ%0c67C1=q<2%5?V5E`^QF z)rV=~DnPHdWTdX&aLWlI%$cT3b-}B#)oIXXr;r%sIKzri&%$I&d;HkohiQgU0m(+a zIn1$Q(xP0sxVw>(@T#yuPm-OyMj~mIJXvB*q858r!G2cr;NH3MhT{Auoh2ik@SacV zQ0)qC8(rb`qg1s+*)lSd20JzOT&H?=v5uB>`A4SY&TZykbV^nLb$b&yQnRN}Etb;J z01_LA=Vuu!YFqBrC!I=T%)2&04$4ceF0B5Hv0@p)fg&o-b(DyHsrkblL#K0IKvj`5 z;YLN3uI_Fp_PC~*@(do_%t|P`R`oNM0008kNG}-@oIx1QvPHc04{h2^U$Hz_=xnZ2 z^8Rdg_4p$|K`9r1#!SUBmqpz=mHro?A;zZzY0vQN0Xz;n_fI6-hkGCs$#-rPZ3M~y zB)!TTqRm<7EfINrQwV|eV|aE2p%kF7$Sr5p`sPXkStrH>J3d&Uusii7-cP=N4|x2_ z4us#}IBFogMI@slPp?-eJlt>jEeD%<#`LvF;xa%)#Leb!^s9%9MO9iM%p8c~iv_D^ z05#e7sQY^l1+~^V2AkD#`q4-rTw@eevU--6G5s9K({2DHC0n%8sT9N{R&DwF4(ntNE!QYeM32k4zhcrdBa< z%!6e^UGUmIp@P3ax|_CeVDmky?GjK9iLOlJl172Q3KV@@{`kW;&4rUMr{heaZPi|p z;RUcn8o24Zoc$J^`RnEF+TAc6#BCGEFb>>|oKFSRxmSI{$~~Yo7{*Ll&lM5W@z$I& zbB<@q1xC$OgH3|A~+$NU8VdtUrkjl!E{K$6Wy?V`*rHLlv#}l z;&T0Zyb2~u6Ve}?3oS5)W)nhP(qN+^A$Xi`QF9ZjA82+OZ;yYX_Mk{|8+eyDYa^CA z*<9d}o>Zjhsl#iZ^m4BSXB3d3YQjO|pyQ5xt;Z0C>`l)p7@X%eW@y#l3Db2Sd7kXV<+xt(#Y55BWfTHd9 zu{Bus*y^chs28Jx&3#iA+TGV&mS~arUy(rOtBlXrZ^DJ&*1GqyBmQWfj!GfGaD|ewU+fWcwhPc&V3az5=TMHXgK%I&7n!i`SFDT`z zA8EInK#tvy`OfG1a(h8fR&!Gl6S6sN^l2$$4Z`T;Yue$gt9@i$z6=NU>K@S!XnJ({ z4Zvwf>zyu7#G4>*V{$DH*44$Iw1{P;p=g&I72yFJged9|z!OCiLXJ&e zlUk=PG#g4hGm>>HsWZ!lmX(5)rT_{uf7W7umeb2c4+n4;FnGTWgGs- zvcRtRXJt^2?xF}8zX>fmoE&Y?7ZjCD543ngC!ah2c2x1jp*?#C@DW^dQkrZAOf>GG zBDC<{UkP%X*&|Tj!GBE$#aTnZ9-{3GsHH^l+Y)`blP4gVtrKqEfLo&~SaM&HuaK9k z0lcEO6A8Pkz?>CTUIzeApzddXxG@f1h>sQ9vCz&^vIa(UlIh#BdpExFDs-#Cz@qON zn`G8&6O1g-ly3`>cCaPMeFhbbO&_I}h=1$e(15XOV3;_L-#-7xF0;1t0_0AI?=3%X z$)oW|)^2&|4Ni5fAdhXGWO#t*DD1wJS=cI1fbQxpHv2U1f4&djrwA6CT)d9^&`B?PWJHD_k@w9C?)SW0?vB_I)h0)y@ zBk&ukw@j2{@2N@;IDZOdZ3bYL-$9eC1MvKi6zX|*?{O-Fd6KHl7 zBPLO(gbB_n4V!)H=z=CRGiG@ZHF*4}niW92N_9hd*-rq6h0(Q!*s-Za-r~1E^dC&4 z6YNK$%ynpX8f8FrA6^P3V9M1^Q@AgP+bl^?@QMSkIIf+aNApBP-Pfk&n_nPuQ={w& zrF{Aw#CjV!t)m><>r00mV$xn<0LoH1=vPCYN3BoA;mb8KubYbp3Ik#6XsI-K&45Cc zesnNw*jdcFb?$AmS(O{yP?fSESZTc|NGiD8c{IUbY>tLnTZ+R6Pg<3$PasyVq30Y> z;$q{HmHzwVrD2Q*BaKVruv3giZQ>@nRucm3^jX|)>Pq;=P6+`s^x@i}Sf{fxGG8IY z8X1|Vl6Atty0q$?+|DF}L2jglW&wJ%b4&2Ms_Ut!7qtZPcgnJEq}${$VnFx5n&7{q zh?yfGC@xp=t1_sZCo7dL%X#_Rb326VXfd>08IR|!{5n7e{!z1 zmkEy^o-zhUd1oID<(1OsdJoKDkW&27n!ZN|}j*w+k z$n*MYOLd4XMX+3kJ>A*m&c1(E=d!SbE>zjZU*^<0!-!u>MYh}ny$hQ86I1;(ln@DK4!+J$c7Hi89`^!jRHQw53E2<|f)flYqADjlvP#`UTG@J) zguOTZ7Bk-oZ)A1&H{C+7@;gFDU6yA;tLD9p+kkPDMLFN}ZW4wK^j|q^R7f3kNI`{9 z3nm2(8*A;Of%lg$W@wka2)745r=Dka?79RTwW+q-44vGAQz0t2sr; zA+mOWb4g@j@M>`{6OkU%e=-V;@^DxIdg|_7WG6QjHNu5{nRn?aSs$Yl1zZH+D{eyg-f;Ir!1CYy`NQ4b611Vc zWFm_7lBRFwcHdjup`ub-I`_xSu&%(<1{oedl?ZQh6`$i;3pGpPgk-oHGPWRQ&N}Y9 zJ&Wk~qt!uKw>j)5QLTo(xOTmui>DjUzb)gdry?P+?QK+X#1YsbyMeKl5%IusF%_Ph z0&_XWc>J%g-tiCUO!}}d>YC4XnnMGBF)>D3%6{H-u0b!YvC?|GMNwBB3JasU8D&!vr~oZ1{@FVaX>`r9w0IfT1jK_~C#_6{G9hf00*<&2-p zYq2)zOd^l4Rc=FrDAtOf>Svdi(jYa&xh=KZhJfgm$+&{$#{s4#yC|gdg0pCKrC^GB zp{_aL&~2?jGotWZtpg$~CW<}jRH?z7j@x9$`T~mrSPFlS4=BF}&Qk~w#&~UG)cOvy ztDRT>Dx6<6P-|P?kgkmRtPuzJU+~7^i0meNY?~{)Aaj?~UGu}0 zBXQX>qidzQSaNA;4FkdQYTC{ZQ=q7VW*zz6mAsl~^Jo46khl&Hcz39u+19+V*yXYo z;Hd<4^z~$Ylz2439Up$CK2?k;RQDCVTVWTi2+MBMCoQK7G&zv1hS_GN`i-?}u&U(V znxq*v>(OcEaKdy;M*yo%8{Vi0Ml=U- z6Pi6wHdUdAe!Wx%39xS=CH7`&f`PhzbvP^a{@O9VH8yGIlhht;vLwJ$8BX9j5J zu;Q_vis8_iga67nDZ7bER`F#p`kFmKr}jLnK{8ADMJ_}E-G*#fDZd!F@Fw4D)y6lQ z+9_Te!Iq|bnw3Wo1YcUZv}n9cXTbbHUw{_uZjg|{@zaTZ0evoFHGm-vAa>`}f?;D` zgv79o-{^);QEeD(3ra&OVs-pn@_ye)%v)HY)6)D$l4w;N09RsI!n0?(k_0t4kz`=` zqDZ6z)QU)|HN1YO^f`^ys5f|i+3QpVI;)gFvAntgpNRYN%|tS{CD_>Juv~TFkEY-# zO^#EO)Utl2RRBD3g+PDmTF>R9?)yHS>(lAQrf z#VS?HPK^=}YB0keOni48p^?@{!t=x@WTLl?0M6;;^sw&;x>a`N(M630+dAXUb$2i6 zYHbjeiNG@oox-=mY`(^ZWI5RYO$w%tZht@odv(%m@m&F|=Lorkg1nVzL{~Q;Hi>>X z^>$h+@LLhV3dp`KQAuQ7_YQ4wKKg;t{jIGidl1|v)*>W!Py@%1TJ53K=d%rN*ww!R zyANmf4`uq8?qJ^ue?Pz&rrQ2V%i*HvPbDE}5VW7|2r;rpA+e0E?dGM9LD#W=eX-wq z2G4fp&$Tu<|KK5iabrdL@VtFXU6OPCcc~KDmK=gKIH`%^%yVl3~LOvciSFm z15l0LVsXL~v+w1-{H=a;s2ZHhHb|Y&^t9P%1DyBa(a85@rDHe>-y({76xSiS`dA5g zW-`Yl=^fjj%Leh5574<&sDy2!ulJOIdAEWs9d&#!WnZkOuX&}rl|Dp-c=3gJK zE!y~Sl~QKX)Eue$rHaJlPDCn$)Nu~sn*-Ece5J1a+{8?3KFb?iJPvqCLD2Lg;S;5qZ0)AXl*IGgr zKhAUu*7@_EvyX|t@U(d_w}(pzop@NPUEzQ=R6PhBBWO7R&3S<@C;b$9Bb2RUNKl_X zx&2bebLt!E0j!i19d|%M;ebF2&V~NcAFvl=l(P6;qv~+e&~qB|`bJ0IG8a6=#Ny;D^qfHY}J z1@WpzP}7P6*Xj#EgGiOhtUpG{Mt~dkZP6ZVEe)NRbjbz`K^Ma!rSQ7i!m}*l zPD^laWJsXp#ZdOLk_I*fyTcSz*Rq>+SQwY)5Pw(^f!k;rSXN5l$+0(^V=TTVjL$7l zCAsTkYT3gdF3mqqi+u`X@LfD3NGaC731j`SON!0|ey^;M6`?&GCwRZS5Ewzxt=ny zpgRx21Wh!zUw}>Aw}dL~{r`Wx%pw4_a?*N^dsp#Ti=p+S@=8qKOR`~jL8*36R`-K+ zwXbz4r+d=M9ff`NZ!Cng8GgXtT#cFYK>EyU9x>nXRJ(lV>Vxg-s^I$XhO>Z`M)pTb z{ydtE_l=GyH*qXWSBmVdU#GUY2KldC162LyQ-lzP&L4z3A=jP>=f{Ziv4}te#!@XU z1_*hfuf^ZKtOr~b1Blii*=8g$lD7to(V^7TP5|8f5`mr5IDnT zYjIc5GF5E&qlY|@*#+O5Tgoxpj?D|LAPApKVn?m+W32~W~ueN!$EWhI!6nxTNm?* zj#Y=)AiTxdcH{aw;+TpbAz(t{TCB$YwJ$b+{!nVz;(Hn)M>yKu^GXaLc<~*hj`mAQ zS*xYgs=rfF+_1N(?D}1YPcrhC~U4*#pz&!mWH{seqZ2tlUo%pvc@WbC{=6BCy^{csqK}-9H_Pi&_gs`X#J9j$Uxhl zzH?de;ic2Q;`fO1ED+sr*-K~kme7bfg&Y*cvY|?HRX-Ctt^qjKHI6RtXBk3W>`yuzLQjouc z!PleR*Gy+`VXJXFf8R5kdH3DUBi;8p<`}MHv31I_9ZF{}hdze%TfQxp+spxiTW5&y zcc(oY>|MQ%MkfarKeag@AzJIi7~HF5KYsZ6JKlGP+R9Kot~|%=Tt)o6m1BIvpP}E zQQp{c<+i8#kmhu)G2hJ`v3%lQhW==t>cI&%ej}i7kwT6;{YHmkvuh<^3uzpP&(+vB zaom)Jcq+TC-?}yv__4YHN8?9jGk?h$Hg!>|eYY_YqCS|Xm3-G+BHtumc_Sn##xy&{~t@V??xKZtt8w+lbB5d7!B1lMv15CKIGUr*mrIgPw$Hg7v zr;HjFQ{(7^O}P?DpmHT9xZ^eJVvr?H6Aq-^4oo zABqM+#h;$DSkZS!qlzhhKj|c*l*1b>astC_V?N?YhNj5iB!ckWsW~nTXv4s7^vlxA z8rD9(A&@l-0P(kJx${waxk4z5AcbN^u$+ra4zTUN-M6+1Pl}!3*nC}{rqbHd_xq2@ zhnRYRl`@v+(pStZAT%?{G2469E@j4$n)dZJxnQ;TOO^H%z#eW~|b_b6(!nny5ix0Jmr(J=Ud1IDdeb#0(k>@^@|^Iq6g|oGD1k1Sify)^yYq z!i8ZcMDc84W|Zd+#~01e2C5`jEwwI@QnZdJjZ z_kMhb+umDrDLZ6$snYNaHy5fo8We$+>kP(usKG5$V*ufA!=O@dMEUpocArT-@F(;>DTuVu9AtF)vkdBBf3NPkAZO zopLCNru@z^+MgAf&w#^#VDWa-Tft2?4Q*}hlAVEw$s%t?&7dqw@OtX&GYk3dJVVM< z&DFG6VNgTs^Va}UEZ=k={PLt8$>=3=ZEXs*c0U&ylEPmSiY0AF)C6~Yo!Jg6DbSv7 z^3O|OK8R<;yJh618ld?FTb@6naK-P*Woufsg*ea)J^2h36tbxS#TSoAA5GTew$X6A zD*JhVGP8S1;s`JkWgBdvwU{M%%t{D~p90-v=2x}WEakaya<}l6RRe3A=U}gm!GHx4 zwf48#Zt88gDI72Qi|UOq4T6nL15Um^U%oBUJQU7N8Lq&%SUx^%rYvV!SAPO{4e5oq z9ePT39IIE#Y0lQGqbp=z$-1*zS6yEi#{`V_Bt;|mOySO6G8z;o4Sl6pM$#iLt4Ytc z!_=BD#qI?;|8W1gq*}@yy@Ohhe_?3uH~N{MaYj>RTYm%nFX#qN%vXOpZfqAk#O*8leUIYp-540>d#CGg(k>QljahapSN$?EYZU} zjx=3d*Nt55TmaI7p=1~6bI)HLz%49ljRkD*1+LAX<1Pc-$JfE(>qMJ3FfN`^E0{+} z9VKiM$O^ZkhU-hX4~6<5|MtF6fwmwuYazS^2o5#QwWlP#@M@xwhNq2K2#8{g=YN=> z{a>%BD({y8y8yYgkv-lBn_nUX`jNw}*VKRpyQJ7=WRNp@iR4Zvg!R{&HyW(dpDUQ} zMIJG0CwyN%nYhIzIJt5GU4d}=HD03@z^iC8qUqtulM(h{TDl7q97O=|*tEf7YoQXg zr6zcSkUW4%hxz1J?X7D;(CpNsLt{Qdj@S0uRRW=qf7_JNl48=5BH^nyAK2S%BxYqWmpz%e%TC@v=FS03ZcCQ#am)K;&Y8-xTA$Fqz;8By4~b$O z6wofV`$wFJd{hnPedEY#X(*tlCJh2^?f0ngn^fOx)-EU=yucjk1atw`iBxE4AJ2LK zhlzJR0jp_+0>Mq>ypK5drsm1gBmT>w|#rpQb%~ ztH978zS$$xzAxIOWT)MT#Lih_NSc5q)qWy^=Uy*Ps|Nm30EB%n(yL6Y|JnTy-TX?? zy^mtgUm0y5CXF$fM1;r;(57}oasLX~^^|_rV$y#t-v9H(|Lnm3+YZPsN@X1zTUUO^ SJ%R7d0KV?@pPFlazy2?#VHiUI literal 0 HcmV?d00001 diff --git a/docs/cookbook/autoregressive/IBM/Granite-4.2.mdx b/docs/cookbook/autoregressive/IBM/Granite-4.2.mdx new file mode 100644 index 000000000..6c87d597d --- /dev/null +++ b/docs/cookbook/autoregressive/IBM/Granite-4.2.mdx @@ -0,0 +1,231 @@ +--- +title: Granite 4.2 +description: "Deploy Granite 4.2 3B, 8B, and 30B dense models with SGLang on NVIDIA H200 and B200, including thinking modes and structured tool calling." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install --prerelease=allow sglang +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:dev +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick a Granite 4.2 checkpoint to generate the launch command. The verified matrix covers BF16 serving on one NVIDIA H200 or B200 with tensor parallelism 1. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/ibm-granite/granite-4.2.jsx"; +import { benchmarks } from "/src/snippets/configs/ibm-granite/granite-4.2-benchmarks.jsx"; + + + + + The H200 speed results use `lmsysorg/sglang:dev` at SGLang commit `d59c1ddf7` and the B200 results at commit `d10a656ad8`; the launch recipes were verified end to end on both GPUs against the release checkpoints. Each speed point uses 80 fixed-length random requests at 8,192 input and 1,024 output tokens, 8 warmup requests, a flushed cache, greedy sampling, and ignore-EOS. + + +## Playground + +The Playground layers SGLang features on top of the verified recipe. Any override changes the badge to **Not Verified** until that exact configuration is tested end to end. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model introduction + +**Granite 4.2** is IBM's dense decoder-only language model family with 3B, 8B, and 30B checkpoints. Each checkpoint uses BF16 weights, has a configured context length of 131,072 tokens, and supports default thinking, non-thinking, low-effort thinking, and structured tool calls through its chat template. The repositories declare the Apache-2.0 license. + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantTotal paramsPosition in family
Granite 4.2 3B3BSmallest checkpoint
Granite 4.2 8B8BMid-size checkpoint
Granite 4.2 30B30BLargest checkpoint
+ +**Recommended generation:** IBM recommends `temperature=1.0` and `top_p=0.95` for general chat, reasoning, and tool calling. The release checkpoints ship these values in `generation_config.json`; send them per request when you want to be explicit. + +**Resources:** [Granite 4.2 3B](https://huggingface.co/ibm-granite/granite-4.2-3b) · [Granite 4.2 8B](https://huggingface.co/ibm-granite/granite-4.2-8b) · [Granite 4.2 30B](https://huggingface.co/ibm-granite/granite-4.2-30b). + +## 2. Configuration tips + +- **Thinking is enabled by default.** Set `chat_template_kwargs.enable_thinking` to `false` for a direct answer. Set `enable_thinking` and `low_effort` to `true` for a shorter reasoning trace. +- **Give thinking enough tokens.** At `temperature=1.0` the default thinking mode can run past 1,000 tokens on multi-step problems. Use a `max_tokens` of at least 2,048 for thinking requests so the answer is not cut off. +- **Reasoning parser.** Launch with `--reasoning-parser nemotron_3` so OpenAI-compatible responses separate the trace into `message.reasoning_content` and the answer into `message.content`. Without a parser flag the reasoning markup stays inline in `message.content`. `--reasoning-parser auto` also resolves to `nemotron_3` for these checkpoints. +- **Tool-call parser.** Launch with `--tool-call-parser qwen3_coder` so tool requests are returned through `message.tool_calls`. Without it, raw `` markup stays in `message.content`. `--tool-call-parser auto` also resolves to `qwen3_coder` for these checkpoints. +- **Single-GPU sizing.** All three BF16 checkpoints loaded and completed chat requests with `--tp 1 --mem-fraction-static 0.8` on one H200 and on one B200. Increase TP only after validating the new topology. +- **Image selection.** The validated path uses `lmsysorg/sglang:dev`. A stable image tested during validation had an incompatible dependency set before model loading, so use the recipe's image until a newer tagged release is confirmed. + +## 3. Advanced usage + +The outputs below are verbatim captures from Granite 4.2 3B on the verified server. Sampling is stochastic, so a repeated request can produce different wording. + +### 3.1 Thinking modes + +The `nemotron_3` reasoning parser keeps reasoning and final content in separate fields. Granite 4.2 accepts three chat-template modes: default thinking, non-thinking, and low-effort thinking. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +model = "ibm-granite/granite-4.2-3b" + +modes = { + "thinking": {"enable_thinking": True}, + "non-thinking": {"enable_thinking": False}, + "low-effort": {"enable_thinking": True, "low_effort": True}, +} + +for name, chat_template_kwargs in modes.items(): + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "user", "content": "What is 17 * 23? Answer briefly."} + ], + extra_body={"chat_template_kwargs": chat_template_kwargs}, + ) + message = response.choices[0].message + print(f"[{name}]") + print("Reasoning:", getattr(message, "reasoning_content", None)) + print("Answer:", message.content) +``` + + + + + +```text Output +[thinking] +Reasoning: Okay, the user asked "What is 17 * 23? Answer briefly." I need to calculate 17 multiplied by 23. + +Let me do the multiplication. 17 times 23. + +I can break it down: 17 * 20 = 340, and 17 * 3 = 51. Then add them: 340 + 51 = 391. + +Alternatively, 23 * 17: 23*10=230, 23*7=161, 230+161=391. Same result. + +So the answer is 391. + +The user wants a brief answer, so just state the number. + +Answer: +391 +[non-thinking] +Reasoning: None +Answer: 391 +[low-effort] +Reasoning: Compute 17*23 = 17*20=340, plus 17*3=51 => 391. + +Answer: +391 +``` + + + +### 3.2 Tool calling + +The `qwen3_coder` parser converts the model's tool markup into OpenAI-compatible structured calls. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "The city name"}, + }, + "required": ["city"], + }, + }, +}] + +response = client.chat.completions.create( + model="ibm-granite/granite-4.2-3b", + messages=[{"role": "user", "content": "What is the weather in Boston right now?"}], + tools=tools, + tool_choice="auto", +) + +choice = response.choices[0] +message = choice.message +print("Reasoning:", getattr(message, "reasoning_content", None)) +print("Content:", message.content) +for call in message.tool_calls or []: + print("Tool:", call.function.name) + print("Arguments:", call.function.arguments) +print("Finish reason:", choice.finish_reason) +``` + + + + + +```text Output +Reasoning: Okay, the user is asking for the weather in Boston right now. I need to use the available tool called get_weather. The tool requires the city parameter. Since the user specified Boston, I'll call get_weather with city set to Boston. + +Content: None +Tool: get_weather +Arguments: {"city": "Boston"} +Finish reason: tool_calls +``` + + diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx index dca685146..cabe49dcd 100644 --- a/docs/cookbook/autoregressive/intro.mdx +++ b/docs/cookbook/autoregressive/intro.mdx @@ -61,6 +61,12 @@ metatags: href="/cookbook/autoregressive/Google/Gemma4" img="/cards/logos/google.png" /> +