From bf66b7b6da33a6a4b17227fa4fc13afe4a907d89 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 6 Jun 2026 11:14:28 +0800 Subject: [PATCH] [diffusion] model: support Ideogram4 NVFP4 (#27379) --- docs_new/cards/logos/ideogram.png | Bin 0 -> 37910 bytes .../cookbook/diffusion/Ideogram/Ideogram4.mdx | 82 ++++ docs_new/cookbook/diffusion/intro.mdx | 6 + docs_new/docs.json | 7 + .../configs/models/encoders/ideogram.py | 13 +- python/sglang/multimodal_gen/registry.py | 5 + .../runtime/distributed/__init__.py | 15 - .../runtime/layers/quantization/__init__.py | 5 + .../layers/quantization/bitsandbytes.py | 383 ++++++++++++++++++ .../component_loaders/text_encoder_loader.py | 12 + .../component_loaders/transformer_loader.py | 17 + .../runtime/loader/fsdp_load.py | 28 +- .../runtime/loader/transformer_load_utils.py | 44 ++ .../runtime/models/dits/ideogram.py | 184 ++++++++- .../runtime/models/encoders/ideogram.py | 62 ++- .../runtime/models/encoders/t5.py | 15 +- .../runtime/pipelines/ideogram.py | 170 +++++++- .../multimodal_gen/runtime/server_args.py | 5 + .../runtime/utils/quantization_utils.py | 78 +++- .../multimodal_gen/test/server/gpu_cases.py | 9 + .../sglang/multimodal_gen/test/test_utils.py | 2 +- .../test/unit/test_ideogram4.py | 159 ++++++++ .../test/unit/test_transformer_quant.py | 98 +++++ python/sglang/utils.py | 2 + 24 files changed, 1342 insertions(+), 59 deletions(-) create mode 100644 docs_new/cards/logos/ideogram.png create mode 100644 docs_new/cookbook/diffusion/Ideogram/Ideogram4.mdx create mode 100644 python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py diff --git a/docs_new/cards/logos/ideogram.png b/docs_new/cards/logos/ideogram.png new file mode 100644 index 0000000000000000000000000000000000000000..f39ef114535752fa760c7888de4be7a54afc3e3a GIT binary patch literal 37910 zcmeFZc{r5+|2908M3!VPQuZPHPN-~UH)G#-A_gjeW_!WnadU z82eJz&~m?hzQ6l--+w&M@%;T9$M34c)GXIE*L!)r&ewUKFVT8BkEpKRyb6IpsMH^; zz#))J?;((jRAd*yGt~&kE%0#1{xMt|0tw)OKwgAHAgAD|7i$oRj|c>^VFQ84r9&W0 zo|rnAJa~cBPVe{MTHb^hiP(o0ofyLk{8hcwE-~U|ck|Oy3 z`Ei>^(r*f!7^1H7(7-Q=OGo8&Fkb`1Np1*fzQIPxv*Mr?2j=n%n<5J(zQ3VBs z;N6`$e0@l(+UzqH7M2PY=^d4Y;RCIban~YjQn3biQA5Eo$ztOb+Vy|_R5)96b>|m9 zhH$~&J_x#Rn9u*JpI28gqx;uN&%((qrT=`1|M~epBk(^X@INE)|2YC*b=C^lAdr^Y zFnB{#Da+tV0oz4Li(Gpa2}C}U1oqKPpwjKSiaHI+LmR*9dT9(Z&SfnV!CagVjj>xh zEnuUAv;?#fE<)supje}lft?TL0!A=2Gu;h)Ex)n$nHT+zk!9;KaQUF%pAC`Kt!mmK_Sd_vi;xf0B6Q%ymtgdBY_$VBHgSpNJfZ8-e=f0+ z(UA<|^Pw1fF=mLAR6oh3mdHmps!d86OAP5R?_F`ENxx*}oDVfCHHsaNq4tJ+7-s+X zJ$%(>rD1nwU0NBNPlVrJdr0ptb=T4F4DuqLs@x9pVG*XJs_yX0aY1oVw!nZ!rT@~U z7PY8jO32htD8uC+JJw(H$4JUKd()vYBt#m>ht}wX05EmeVrbh)Lg9--j@Mp2-He9w zzq&Zc2XXTzYlVE5d4q;&R7ABaKj6dCFK}FJiHu?rJD=_;ZUSitV#=O_x)YMq&EKIXQ4|>#1?h4l(QwSr{~i!|@_Fng)?M`$f`nvbQyT z-qHRE1oFuk+$v-fCx2N$J!x_&TJ7$pt84J{+;LY)E%9ufYt+=*I@*q6rlS&kzl)3W zEcRQNeejs-aY^lgf#=QE^xV~N-@<#MdwSN_*AGUL@-~UFbV?$(DIwoqoDb5(C0>kY zA~^#?;_KJ1k7{Z%H;VP3%Sah6bqU!ahN~#lewvWAY{;LswW)Wunc-RJkt||!Ls{8T z%b(wYzPm;5Udd$ZgkH_WG3=d~$ScaqN{Wa~O)lJ5P$=CbW$$hQGk6t@{nw^o!>k8< zJ~P?zR8&#dZp~CVKT}q2?AlIJfzkI?(^yCk;5s{tH~6F4tppfWm{xqWSHtx*RKWaQ z;?0pc>L1mWmXt6RNP@lsLqwa5QXLh~*iuc>5^AwwQM|?Wj|VU?tbpI8mOfVdJjk z5lT|^?tIyp%uG0aik=dm*VOhS>f3&Y#57-DUs*RJb$SxD%bifiI>cM?46IhF-pUU@ zR_aGQ$pa2m#l_e6p4f?C$3)`Xc=UzzNg-2j!0_6%2Xe9hoZu4SR-x?xH~8|(H3HazrXu|U@27C!J*$KyNIEAXrq(X&e3OFZZ)t{$s>LF;k#EVf4=Gd z{aAvEnsefmP8;yj**|If(!J&}f3|&Zw;^!n*GZ6~wY9ZJEde=gA=l`Drii^Q2bQ^? z{2SJbkgtFA+O&IR^SEfKw`a!3>m2*my4O~%gG9v@l%W~Sx!nSeRslaEwh3~qzATG8 zoEX%OuT0j>tQo>g>PUtk$D+5a+zi3o+a!W}B(h|Ig@l@jTklUHL9VN#q zE-p?%L6KpRDvE<{Fj?nNM=G(E*B0ej(ci$LJ=>V~{TnQ^@132Jt}rpmBc2%>rwwgl zMd+;s&ylY1O^vLw@tC%w3A zATI*Lz^3&aLLkVxxYc8f40j0iwFQj|DLUe6CrBBm%(9{Wp=?Ez5dGD&hIzeiU2<-; zr;>wdS!qcXVLP(S+`VRI#>_OeFM+mgwqekC`%V3U9 zBTwC=#1$2#L?qrpd$*T3!tYRleSi7<{uRFUvD}OeVv8%g# z{3y{$>uWd;zJ5*d^izHPy{tadQls+8dY?_ktU23s3)oo3^tAP#y;5=KAxE(rog|&< z15SiWpFY%(;_uli-A8x%xzxR7FwaA5Y;D;SAEPz6mB<=RwYA7jjFd$fJ1?{>T>!yB zsv(LdOm_9Nfbl#$6#kH=xTHi^{>SId^2Qo$bqAsfro@c(t#7Qu;i{{vEf)XjBWlK& zLFs9uKhMr$^51D@xOs&9J-hw<=kTN3!UCUVRx@#eolENnaG_jl3oVN#Si(Gn&-l3k>g=rwL&C@CqqJvEH4 zDk;I^Vf5)`E_Hb+XW6EuQpw3G9HR2_BLu=EVN&|beUbb3g=3T~gBvGT@1|%}RaNcp zQ{ZgjYJCK&aOJ$>V*adwRGq>?9FL|(QDuIm`&08k@P{z5l8UqWA}mr2RaF+3;Z5RF zuQZfCS(};~FEK8+Y?PCck+}!v$x(9dldNIzKEt0HiCQBxWp@6-Aw7gfouYgHFS94i`Cn3JaMQ7JNz&`S0kWKLEQrKP?#)%>?3 z&1z<%V!){?MFU&ZQzWdL{pVYsaMYtNxTa=IaS|~&L3S~m#%SKFef99;M+r9Yk+tD= zAD?Q*5^cevOUf^se~p`iSYQ>`C8;f~tt0CkXZn8PfymhW{PVT1tqS$Ns%^F!E*#fy zv#@M$ZtiYw=3+1!JjwW=CO!U49#Qhb@Gn=MPd_?GgO}=*K`pJ6x+J`4K?8QtzJH^j zpkTf4&TNytxwcRBoKvzO<1`1njVZZESMg-Iw)reLcZj0{DH?Y(XKIqOX%$1w| zqPOcSrvh_8Kl$A6l)AF&%*13LVHYiI9YH&4uzwPkKPV`mrK2r~9W8G-u!AzcGExhB zZoPW6eG+7gKs42#op!{gU5ScHTE{9?JYgnfU)ZUX1-S1b#S+A(-8?G73*)H+T}Dl2L3@-7E#E!)!i4z>B;g8x#9*dAJ;>-IK5;>8 zY|T`F-Htjt(t`?UTom;|Jo)96L9%MW$5#;%p`<7r!?AkcbeWVpJT{-1JmP434z>2B zwQlC7IxV?$pLPV_s!_@8%m`O+e%a=<#dBX@9b+HUB>ua1bApf7opbyviz;%E$?2r{ zX~IL=@Zu*2U;q5ZTj-3&pF~{NmPX98VbL04CIa+K_OTj84w*cQ7!-In_bf;3@RN^~ zmBmFx`25Psva*%lloI9kc6`vhh*tlVOD#N6U>9YvtcKIT=!30vL1;KYtS$pB|L>He%>{!RB}s2JM^p^QSdq zIs>Hj`~Le^z-khxw6Nl;ZTNAyFkBzRH;Wn*j8GbvIc>B7e})l5wLFI!-=az?w)6DV z{!s$S@8>Xj1a->wUNTFz+t{&nPnDAV#F!X2sd}^yL#*oH$?eE_NuPuVvY^IfJFmpD zZt{NK-1qhIY5dt1Ynh$>JonL?aMb?FA6jb<>4$vPCIYQC-Rqty8qvr$BF?44q&R=L zG4SYP%|{V6`&ciHBGpuPqbvbUFi-t^9|d_oyH?eFln87@D191Te1bq!nw5s5+B`bk z?J{vV7c9K!9a;+kfB3}JRqlfafP*jrdIrNhwAWzYBQlapm2UVOwatl+vV3W-KO34} zzou4}rfhHU{G%kF@dg?Lc`ZK&>2PP&ap8uNqQEFbBiA&VL2~;}N5>)bLo41VA zECxTv!Lr70md4f1?U=|5(P)s_8jxL9Ce%LrSnFYFrg4pu@-3WCJ#`MDD?)Q{(jb)9 zZd*IOC1sj=`Jv54=eU%8aV+L4nmkUd^$S(?TgDW)V(ySb)%qA*EJH&OYiM}r*T|6g zSQE8hb5w)%c6IVCv%tAANA?km-@p4{DkyzdqDe7ZlTT7RXTT|zyFvMWQyql)k zp#)`ib3x2El?H2WGBz*pYK{!d&8;DMZI!sx?ZRwmb)dFrl9zvuUe~PPa3E__FbVlS zJ7&Df-3H~wh^~AfT)GhAsH)0EOHOi0s8AU;7TVf6jhF2WpKAL?@MsqQQrE$6z84WB zpBUF+huYtdR>{{xEIoko&N@{?u}bdl+5kq>xK|XduLnKP$5>qsjZS);8k?M!{GkWm zgU8+IaR~79`#E#C6B^ojV$yF{;qB|o7V^y`kT9`c)Ds;YZCCWukW$Ty1mbq{{4Q^J z4PJiu{Uk^`K3*rvKFICY!TM->dpkE48g1v{vHRuA*B#4x!c5J?_(Y@O7jFS7>>Y=tVc%|u_*jDF0)$S+0o%j$}H=vD2-nXyg~`Z*f}~n8h-2*>F+7(nX(hG2H+o8 zlv(`lBCgDyxb^cf3VuhnHV`F1WOUGIQ z)c)gaYG`_&d#{W%m9ivd`xrTI7*vc$;E}n;jRMTRQiTCD70Nv;;bX}gP7&6U?pcLM zr%?hW8@w}p4pClK=KAE_ZD!Wj9dvPE;x_hD?h4hGR5Z12?#FQG8%iek=xFO49vvQE zqZAYn5D;{fuw2D#;%+2#72*mvHaGK!i#Ljk{p8D@${BTEXmNUZo}sRAXPu$TCI_9lkOyy)yH%-5;h<3l17QSAY4RA zreJIxSvQB^5t$Rw=z!~A{>2Hf( zlU|&W(P@yPI<2M#AKt>K;vWs+^}2_A1zLB)^)-Aftu=&iM6Ms7aVEaH^NQ6GEY%ah z47Kp6-tdvTf|tteKm6JxntQX+ zv>HVVtAvpeunPH2m))dTVg-dd?bf}E|GB$X$3{sCFh3}}9MqGzia6klKm3^OF*NPV zni*PU%^`MnBX)^=*i0O8-`~)X+!0N#MI-ZfW8O>Wlgrh(>22RM!T@u3MHee5 zc-_=6D2*a$%td>nl8!~s$4TCS>JVXGH4P}1XYoYI>NzzVXSRmVBcM1YaO(D1T` zJ7X+?SZU^{;bLfb(7pIls1U`@nCuwawOVB&Kpn+NPibVNMpnSdS#Uc9VAYW*sip;{#dq`UXa2*tG!uMkEW0*=y0O#QdayWYM1A z6P$R_!g5kx$XJ!@+XuqhfM)|Mg{&x*WzyTnTw}9i7KVlllq9Ua^*!oR$+y^UY;Ui} z^M1o*0p$CtC7nQ=nXf1FBSo65?n zv4CyBV(~?lVjEudj;1R-!i<^#S0W0R`ZGpPKp+j^Nf|USP$A1J)KGx&awx2mj6I2q^krz>sa!3AL6@s?+QR?X{xu1RE|mW+S2P@v z`9p;xL*uI5Q%lS3*V}|uCZ<9h&d{*Bo3VUyRL{B@wK|x6uO0-cV77#IL^5|YTunw= zYjnw%5BqI)>tK0otRt3b+xL;aezHqukAthX!`t$54+jUXoLBX8Wxc&Ibu(sBbDfXz zN0a83mcfS`)k>lI09hw-jZcjI46;(a|o zvx-Jdf)p2Oj*cwrpH)jpN42vTwzu}OK(k1Bd3eLGZ+PK1T%Wpb zx}Kh%0qpaECN@gRc2Hd|+bpglsVBUwsMY3s_hP%QyjPIs)7Sx5+c+A8nV?bgkTBUr z_CCuSBoD~|=b9CtPKq7l(4;y%>`J)J8y<9WQZzk{@90pm{`=q#z=glGY<@V{*#$Ki z^xu9zXHC5ZPIHsUdiQGoHnJ{z@Wq0$>3MbyINTV2I$6%0>=M;CK0NI8EqWaOa7S<6gfW>>s?W96BK=NL~2*=Wu&*{>*z&$1zh! z0ZrzPNa&x3r4S;Tri~dC-?0N_>j7W`uf$eLD3J|I80yo6L8MjI<+BQBS8dl#vQR7oGxAXn%&HXpLPU{BK88<^<`~305_5ll%97^J0QYrA5Nq53HqV;iT7@k!lT!$^A*A7T&$@M(erNo56IYou2ux)Z+1sab2S+7 zuScq<9t+{nG@;j{=8TNZ-D^!PbqXgp8$ExFjP=JM)`+D`l$N*I*lymuSzJ<7LD(j4 zulLHH)(G_R6$5X)%@+TNI^SmiCmc?aV$4{XLZ54WakS2LV* zcD81Bkg^@f#a(w)Px#f7ATNZX)7nZYEor-mle6>wiM;JacDWN*sY$d7Bl(qUtW&$l zdR6to;9oF~3<;{2Fd$7Xe;zy8XZmH83yoRyG@(~fl40jr{bl6;**GyB=!HylJz(U6 zfBxWYY`hd+ zy(YZCn1}wdC-FUgetx0(@1NOUm@<~|FUXFFymj+hx1#b=+u9De1{pgB=2T%;DwWl9 zGW$zO{<^VY1WbcWR2wDAsELKgiIz{^eZ>J-bDdskwZ7w*>5)a7Y885zoa}q3mD!=c z+f0*nfvJ_I&o|W&^Rs3gUjEv)QptSBMwfsp^KY@8?Q#=uQ4{j!enwobVsLXa6V@5gzp2E4gx!j<=_3rPlie04(ql(II(9!AN4*BKMvbxY%5(^*1<1OB{kUJW@ z2?HpgUV%DF--ctc&wqSb-WqDLyOWcZ`*&|FjUXpCGcyAM>!r{)qb^LU!x97VBD5Aa zFE4Mo^={LAu)Tw~8~8SQ2BQ+>Qn8m_Xj7gqw(>oLr}#z_cI$Mg77uOD@NhJWN$7GI zd)laNwHdUat}IPMLnuv?Tfd@Xs!a)03F)|=UQ|aot-N^l+%0Cg5Ovs||Vr7=^$ZgYM(wL&P-5&9iI1^?aZP z81$FQy1~j?=e<6%y!^I!eukt>J|JgkHMYY}p6zySy8N?0`IDzf94+z&AOb0N|*rYe^ zfQnMI)6J-4+=)RoIt!N%#lqDmeU!jev2l8Bi7zhLj4Gc)r+9rO8DcnH8*{bEmK5*+ zc>2j(C4d!TdV@2wY!!8)1yn*}4A!o~WW;AcM5}W4K^U5#PKFM!*vZVyjCmX};B;0z z@q!kFIDO;#;Ir3SJms768CGJSDjGyy(`i6C$CeIjK7QPp8aB1I-u-azMqcgdqp8w~ zn3$NEnYyMwtx8QcH#ImPB7B9bgZ6XE+D>mn#q+5z`@Y* z!BLKEAg;Ojezuv|C$D|xF$rzc)XmM!E_=9yYqccRXoyt6qC&MM$@Nbqq>cIy_(zG=8mTSa4(ijIEI^)PDIw7HrD$l#J}eK z=B3If8amhD$bU6|X!Yqro#RlB!maIXLqk|vd#j4kA5rP(e6*AEYMFKObD)2+NF&~+ ziDe?t9+{)%*%W}Cx_Ew1_XhC;PETE2gHAIFeRnsCi^_cK=X^`zE3=aE7&;~GW@%b# zsfo#*`Dy>gv(Bj7&%L*2d<-XL7W-0Kd5i&@Y9gksQ`Ca@y4xwHQCjI%O$Ks#$=sMbT&z`MAPEj!8a%7vVsN3(J1UIW zZ9OKBTAJ2|U$ zbs1m{{v@_LE!27p(dn|56c>G~Q(N0k?nY;ErS(x!Q)SDdRz_0puJ zNg^pEi|l-V`yDfd^$qmBubiEqKl%j(!^5$jm|jpTc?H%KgnSBn&l=D7OosK(iv@{+ z;fe|iA?HjU?3l!WQrfL?{8#+sT%tf6q9Av0H}r2XuKO; zm+?JVdwq-0z$T5~o{%lfbmB^m-pOIf4f%6YG+M}unM6P z;Ob)Fm8{bq)Y$^ZIH$0jOl|DVUI?WA7^n6o*8$Ol<8xCJNlmWC7fiPoVkzQOJ z*7ELw?CKI79e%;dY1K&w<7anVW|`>u*|p*WfyYAi;YZU_G&)ODPe+D#%&AHv%sguOq~}KUBi>jOJegY9Q8aZen=2#^s76`UU7$Ngz@Kq>yH%QK;XOI4nB4{PdnV1d_mT&Vas!V)LOjweE8OG(o}E z7HmYvfa=T;C1(K^-O`bXpjab5;>Um$HHJnhnlCzRXo)(iJ?<)6U!9g4#u5IcIP)>j zP|m&F$6R8X8SPQjps+D<>_aNs!UwSpvzI zk(DQyJ2ClOur|s$sI&$}RQvO0G>r?)_^?tZqBf4)@1qOy#G2{+2 zrLT{#j+T~=mQGQzFF{g|1ypnAYWq3)0HePY`oP}a-p7Xl>+RruP&K5uySd%xghJHT zV~f0o<{RUwe5?`(PE~VKH?;VKuh=2hBFi>7=zb|$`?&i+>A&%)0|vFjL@N0P8>^9~ zMxLGw^5dCs zn5B&32z?+zLng+<^)0$+U48D0kEJAKK||^6fT@GopC~#re!e2_0B`SKdntVYe5_-+ zvj*m8W&*w~(+vy^jE|2GXlsW)0MdD@NIxO=B@k{eoljJp%xc&j10w??W8->@9S1SM zp>`RqHA0Hdtc&h=KF zMT>IO#3GLAs;bhBRTabP=$@jX=AV1u{aM*rR8j4RROg4`_;~08J%_e~gM$+arhtGz zCmfEKhvzP5;^W7RB%yk)={THgO#bqIL{ZWDd{a}&WZCi2L4SWgeGGSOhn%P=_pAKL z&HYbvK*GWm^Jl!;tTWPCPB^u_0P*uXx7CP$!-#h(<$+4=Z8%s2Irl?s1Dw<*snk=c zMltC*Ihon(>)UgH>IYI2s82yCR4kvgt4_y&R19V*HccoY?sY)$e8Hmx#lz+3n8)F$ zvUeM^v%kK6wb9ibYH9g8wrue1IKDTgxA;j)hn*v)FfA45>?CQ~EPJ9MsSR}8$5=RV zde%-IAf5)V0^Zyq4?;~T9_!e3v|Vam0|=1!jCTv%M4O&Gn_^t5NF1)%Ulo?yqITaj zN*h(@yUV8`mu+QYl8(vT+1|z@i5pWtKNW4FMPaE_I>7WYh*78wK05+gRQAceTe{TIp2mj^ZOF&5lB~$Un#>__Mi=iRBZKj2xMqFZX@w=|2qSDgatgHqaF~#eu zsy;WI*pss|b8;}45eavHDEdjAlqm3bDWvpxn>eZ)j>BO^Aas_My1Td(l@|HdVbDlb zzTW)D%!i*Who4{x+c%hRDx!{p@=P)@IaEGzB!s;PWfRV;d76@_nr#sFrT*&fZ}l=si>?(I**qb%le zn)%C~GItAL^#hFGuZT^%<>{rTW8_12f&^;bY?9*rpGMk#ruzY+Q^0IlXQW#4z8+WG z4Zslh7`O&F0sc9pWpbj)$2>5rZ}DxM*uyt*HzM&g=JLTS7SE2UVmmZsgo`vP3=A~D z4>8K@*)2wx)nJ#bv{r$M05Ik(BmH4XsxW?!;?Q&ciR#~og+?lUa@oMS@MGap$uNVI zg^9d4xLqbtaz&l0r?^acFv}=BI-l|C<$*8JEo6@9iX?dN3Nj}DYTRIdf3{rk(H0(L zKT-8mOCjKtIy?z}UOu$$%^xEsrKl((^Uu(#htVB9_|D>+Voj|-8HdUsMvHdC!(OlCK zfDH#sG_310<+eNfF)B1os;2>sN*e)tA z9#~NC{3hfHUvgd?vP~7{IduzBOK_?#&dk&)1XVCQF=K^r{A8&Lh0@H}`uhD9S2WfR z=={rj{+6TUpgjAb(q>ofMwoz^o0mXc1sDMm1%*Ay!;gg0L{F%h;rcZa2UC8uM#a>( zwzeR-6d^{_X1Hg`z{D555YN9&sS_)nTwHxf{8w8$q^su> zKad64Yu>rkxiAB2KAJ0rIxSW#)k>pl$*W!D%WW!%6#KuKr8n^JAcz=)gO_Vh03q9G zr7O>7%%F-^X54pvRiPEGwB zJMg~qRoWY*yXQ}L?VdkrAC_#w zr1e>blP_YjctjISQxz6!Bm`oJysPLheT}D9!NphknzHc5F5I4txWqBT!XS*iz}(1 zol~2NDxNAnikc0HpY5<~-8__%8hf_0=H}w^TBVJ;-jbeVP?j5`=USmrcfGi@)C+Gh zyBtnuXjtg%6iFJz&w0ImZpU+quOPDwczL-Qr@#5)FJ5R~*rgzO$p%(um20M=sOa=4 zfk~IE!D{H}_0j36oN20N#|WG!ON(bI+CeO%-=Q+5ICFLx6Pw@R1Q`9-5fM`$?<|i2 zKi2hR6rv6@SX^A}30?NBR}D?tn@5GuZ2Q*sncpg%1_nFe_IecgD5>Xd2Zx8dhli`1 zzzRjXa(l&8zAf`w0>Icb=L=Qpz~}}LA8#_Vx*-tr^&Sf#BLY5ILVN%0wU=OEBMkOZ^wG;Y!h9&$#vbfFsuIP0nXWr3kUqflSB6|C<4PfqS)=+~b=RjfT0>;#-LNx9<$bfwb@F|w=S-}fd4E#CAm`O0RO*@);d z!~?~!lg{fQ&Bn%Nw$3E6&-nE8G^Qip1=)Fy1J3>3FSOrbPCoSvL`R>~Sw?CFz+zfb zTujegMQ{UCNdu^fbe@x9_nW-oG=re-<)!UV3Kb2d)gb3vrYI;VXf$MK)${!O=l8f| zb&r7=ttu#)_fLL3&t2HvKWSh~Gp{jW2HF?CasK%Enfai0c~||jp`6UjchH!~R!Qw4 zClrVT8Cp0H5ahB4-x_M~2grl;$JO)u(xrAz&S^KqNsb<{D}u=?bE!=|8qRuF*?=4b z5{^YFn1mChRuRfxlV{I{fG6S6qb{xF0(wBq^Pca=&oVtkUFgqk76YXhA|I4a&XxsH&>99}1TlCeAFa&cLbmeLU zz%1U+7SeOAMYO7dd^$~}vcPdSg&u+IQra#4rz;BcUlxE-wmWDd*g)G2|AACSB@N@R;f4$9tl#YLR$?V1^st6W(< z@o{nC7oQh8i2M7mUR6@jAWv3ts(yt?o8ywm&`C&m4d^{5JG)8~_}bn-P^>`i5D0ld zv+tl;zE=CdNxHsCRHO35%|mygzWHPTPrecag@>pg#P z6mV62{8)twG&D4zS5=Z(CF;4-0>=QFW)P3t0FLmYw|bq3Tiwe}JXzbfSt+w+mkvK+ zEG-df$(}k|gR+GOD_a7NJgc$M%j@FKcCmoHqAfcAXC@2FR?P-7L8=n-UN$cr&JXwE z%er;*CcqTl9@ZW_Swzfe){iV2z&^Q2m2Au)jbqAJ!kS}xaR?n7b=i1I|KwBpJukH!a6=Y!V?`OMLxN$wP9ZK?S)*6MHSlU$!n#;n$*cS%SbM1EIip1Wld83HoAzz&;p19n0)(3 z*Dk-jpiWOFEbU|;4)0BOy4mA^wDDYv)h|^1vplMqCM?WD8)-X#OnQU3gPhjtlo&X> z2l{bt|6BWf-w`Jf9!g3=DnM49D)e3n3V{L;h&Y)%X%3qH>nwDj!2%mcXSb%tlU4tG zsZ+@fHvW;2#8rqvjrOnZ49d@>14?L)~MhDi{GZOBfELRQP&rMJNBJsP%(c`1MxO2#UyO`e06Ri=;&niB`EH+BL6Wz8=t!d_O+k<`L>8W zek0x*KKnivTX${}iJ1EK{rlsEmIrQLiDE0BbtDhphI{GA^77bnt7co9yb|n$a--oI zvg{4Ba`C8~AtIFf+%B=<3cTr#9o(wx>$m)l8yA|6&yKHe zMN?t(-`{{|I3(-_opvz&{{8!hijtwB5;Sk?;nmRVVHdvsc#&xXfhct&IVHdzSl86d zuuyFNzzoWxCg1GdL&7RCr7r zG(~9Ml@=YI{Z8T%hw(4()6vlZpjzjB;;QSh(7$C-@%9TLK!zEYAxz)J!xIxzNn<`c zJ&u1iv=bQ_P}t6=UIzeWQBg*Awl-H%OnxRQFZyc!liGV>p?dDh+dFgh)LRRy6eRSj z8D{Aj7|hMA^z@93j7;!1^R=MR3dQBQFZiByW3RtX^$zFew%GI54LVg(GfU1svDMA% zo2kz*U(s+F4VU`IX}3jB88cH;hIqBR1u4T%xZt)GW@%#UKYsjB0sJnMIZ}u0db&*_ z@xh%hQ&VGOV_Y%cwzjsucYEr33;~6*tBZG3^TwRW4g36-CwPl=I-O5tEI5~|`Cq;8 zKuwjam#cl|{jo;k(cQb`L}ifFIXQb&;DUUSJ>=N1JJW7QHg^mx&9{F97#Kzi3IN%5 zU}#Wx2o%r#0-baxb`K5DF*HKRqpgAMzkT1{mLX#a@{pF_=VUdNN4ByEyMrwGf=YBt z`W=zPXy2?-;1Mlow4dCWGp|KZkp?G0X_Ux>JxW@kj+tc)S0hw7WaZ_9)<>``3et>r zqMam#eeM+|+U@--l|3qe6hp#<=}17v^p;hHjTs1L^=#BxT?fVPlLb7=S@S6hO7A(f zoexD;HSn==8@xfg)MY&8I=ylV#OkrRIfeW8tpymV+P_dKOZS(n>H7KS1X^1C_|FRa z*X&(;J*+trrNfsOC)!2=D#5sL&M4|zLBy0aGc)roqr2?kbcmZy6$UqN-@YHR*p?yg zOnT)?TB=xav47*dLf{cb#=o)##{*DiQ_#uWvjyNg@SdX~VW+>^^mBxw{&>p^KRY@q zZXNXJ1W0D^9<+caBCRAZZ|D1>^4ZZ&X?Zz76Bp@v_3I*%0X$mGVA-dKwGoR#oRp<}9!;uwTL}vC~k>VRY1!&`EM~ zaz}mOGT&l{fT;oWC}`{SHb(T6I-(Yg$U56(Abynp2}l4mV$shJ$8T_lr{h3iPZQG& z-+S_LWc%ndqrLTW|Bdb11;ACf$|vg4$J5wVPx)@o9D=j%5*qt<_f5EZ!1{mBXgFS1 zUtd^Q__X+`Rp9pcErQ&vP!v=wYkcD{z|F{MQR*lv2Wv@xsdh1|H9T5iH zdcICaFR$JI^>azr$;rvUlKbQAeyHFs$Fcj)rE{}#!WHi*&=2QKv}(vK8yFhMRrIMV zt1+rE0gWKlo0C^aucW4qvGsO z*6Z~U3Q&fC-iLc`tk)o(%}^p{br1jN-@~rL00vt`%5kgH zst#`+vda5;E;K2O8>WCw44>C~Nuz;0nFRxsmYT|?-n&vcS!OvzL=u&OIbks$3d{t@ z+Y5mQYwYDioa*_isdMKRCkuE?@bBQp#>R~PBp0FG=l$fOebO&LE9wWSbaGP2r~7D7 zE$GOAfLkAQQarHY=02BSJoDwtKQ4Q~5e%H+H6C-{yrY^{ZwAlb zcIx}Jbr4Qc&{9PWVA#+$-EV&5a+6(a{39@59pC@evi-wtbB*S*8ael8dXklW|5}fQmBX`V zyE&j4Cww%bV=97ziS$a;UId5Z)+T#;0S_o6OTXL#$h;wHjAMJV$(8XTpCJQx<41I# z)1_9Mr<#4Kf~5?JuLPnop69)xbuOv&K1kQoWZnB`bGsqAI)&Q6Zxv40{snYbNYyKR zJtnyO0z});AWTX}QcKu84A35Yr1l6nDMnco6f)dW;WhG+S%SlSo;4<{!u^%-iL$93`Y@_^z34mVI0WQYaUZW95g{Sy^2Nm0?mAK>19_3#2O zSe-462DvR!dr5wCeq`=_2Zo^K5J zulIKE#nU3iFN_j4*qgS`;?6hWg)Pto!7{HrC5jO2trd#NZ+F`FQ&`md1c+Kz@_SoS zhWblCJ_FC!LBEzWD#|cMPX;7q+|9POGR(XcZXn3u4m@42eXk z_5-J<)`R4WXt<76TeLtQTWyzXwdg|;?*bEkr>YFQA_t_tO`PE51y0bkArA#{8T3aJ zx2JorW8l5aOW}8m&G@-{s~f{jl~5fCG^_JxQmO0KQs0P-dT8GeFqH> zy}B^N=J{XZbjxW$IBoB{2MF_kR*$o5r|~;Z#M}`i+{xN98fj;DmlE>A=R9J4P+YsM zGd@y2G+QR0sJR=YTG!A{T09AXOfdO^Z}M+rvN-7O4H;{aD)`m);pa8TSJ4+4plk6; zgw0hj(>6T+b`W2=1aT8M-*^9St;}C5UL?&%kvp?Mq!iB!$L})iz=n)Fd3U9fFCFK? zObQ2VzkpPd$xogQ@Y0`&Bq@+XRM$^)%Q93Shz+n&1 zI&tS>i#-JFb2d7F+a+Xq<3Lv+bN+X^JT!99P!$x#Cl)%BhmRCEz(Bu98H0Q`xlYo$ z=qYERW17n6NDLuk4~Gj7&FsIwt=!GZC{Hna2+VFKCJe&P8$K0=RVq$-@tz zUoXot)$t(>8Ta|(Faqj!M?S7NA>ak%SS<+5^(SdGTs=wd-ZM5uq{HeUPykZqY$qk$ zy+kJ#CXY{6b1fVE*fSo>Z`hunv2F2tUi+hKx8%lVXTKd^Nx!pPnbFhJL%}qdeY)58 z=Z|8$2-_j^g`EpdVYV-Ux!dDUPcEB+Zv(L8Zyo~2F^pb&R3Jqo12|Dn@ZyEKjlVao zEgJ)fumpO@*B@TXYZa3VKXr^(2R*sAka$N^9D1WJb!auYXY4`9!j;><{%l+SYiClTxGlIjbA-Mg|7yJV18Ld`nnv$Vs+iP^_uu@{1T$RI4C zqw4|0abw+_ppnhPjk@P&3B52AL&MmP%7Uu>vxN|p?ejmqDSZh6-H@kkY=3aPaTJ zGRu^dVFtz8EF53o-mraFW@gsq$R`(AF69(`ur)BAl#r5&{V>Qrq;Alz%PS}0C?i=J@<^yukh)ei7lOXeMt_DJAA?^+imZBV$@bG@teAs6V`lC9T&(xUMG>Yz!#pGiYeKr~w8e*C3K#KS*tILzz?0=jc z%QS=@L z)Cds*gccBxP6&aJoQv;o=AC)Z%=!NucXxJnhw|ik?&rR*>(j2wSz0l1aXGsi(WJ1= zm2V2p7azBNJ*)NmVOMaO1u8@~u@gc^gw>0S*#K+G{al^IbN+nTs`M)L51QHv6_v)O zHTxa$_S)JSf8XWO;$q8O&otdGS%4!*!R|h`IJ-l?nI$yg?wsfNuk}|^+;_^E$R$nPe+_}E1)r)Z+4U1)@BUYtwSZs zJeHaN@$ZMs;np)CH+1^npCIQhn~GO?cIlCM0-r#iwzDnp=4`accbii)Gq!Be5xS-m zC#X7crzVvy(zUi12r%sacleNkMx-?>LxhSV1Dxe6%!ihuynnSqCW$LOoa4sQ_2n7I z)oxNh>8N>ltm#wx!{bDuwt)WYYdu`!o8r=Xlz@U@IA4GQPsz|#u#lQVxCllq-zN9b zzh@v^!WFLYvh`clgnQSo*yaD%DJy;20>fLa^0IXbEPb(t1`(~+pg*fHK6blbvgy%} z!oec29Sl?qjHI2nz;;Wg_su8kE?8$X-)0v4Fp!=;`cP(?w3nq-XcvF%>K}OBopT>O z%uje75EA4N5xhnZ@J^x3G@h;Z)E~dsN7fk#cC6ZbG^NxiAjP|STNa$)Yl6L;ZNXPPRQbt zZtAeoO_h5HWoD!o;LAdDha5m(58%^jl}kKvSepulyvG+F6zO+4+Tzf**b3`Iy^c?4WLijFOTcaWQ!>Sj6EiTmZCKQZ?1!gafL}zzDmuf4E`% z>Z^o;;t^NBr#GZ-C{186q5HeU=81{4G$#w}CVA|xSw1^x!UP4E7MJQ=4`iFBdS;xj z^+7%sJtX)hufRE9_eMr{8yhsk4k`~KPn{y+t~c-xBK^>yz63*%sXP*PlyXJA(A z?&j4suh&OL(#R-?CaG(l`G@Y}*`iku zKE%XmE-!P7l$8PdcJ5rVR_bQP=2nt!YyYW;Z-crIFx8S;C-u~TyKPvd>~;~<@r`P5 z$_|&gNtGt8LiLHs~jZwFywu5G8N6R zFgNGo_)Ejn5H!4xTxi{VZJW(Wx))QP{H%<_L(;#74yZ2!{Rg2+Xe=zvzX|-O!v8y{ zd49}qrnx#e1c4^U$IdRm)zt@{zk|aZHM`xtt^}3utG<*XpL3&qgKXXxlU-Ew8X3y7 z8K3|k2P7XWfBr%jtgENTLQl8s*8uF0HS0!%ouI6;w6uhT=gb3A45#!1l}IC*qc52Y zF$Q{O2X?HHks4nYGdBwh0WQDa+<5|mSFWV?iHeA5YHFQk+#+soZ;?nbv2Pu!n6?g2 zUIfI7gE8_Bd=gy?0}CVkBlvAapXlk1SR%1go^XV;zb{{k0EwKKU1gfFYnWYtozMS1 zM@UG>!RH7A0j9kFN^x09qJ1}mSYxF1M?GN8%3&{y+jTu$dh%fT_4J6)$cTM}goH#_ zf=}J2JJ&7D-|p|0S^44rNUc`*Gui%}^1S!KpihZJ0c90_c)~JUN{Baw+2)U&?N9jV z<16j$1uWNcGPARTX#03HF&e?i%PVwFy-j!BOeE@mIT8_<<7Yaw6*m7?fgKq9!DhbY z-7h2{NgE_c3J5L}cze4dlijy}Nt>dGwQ{;l~Qx@x+but`7@RG@zdI+X>^6&L) zjIDr_g~A=&->zFdeP+w?=J|adwED@Y%gDyvQLFtSEBtaUKrs8tv?+8WR10GaEqMZg zp%Ztb&Y;)Z{h4^=T0!RuELxcyy;*zxg)D6ZLx{KhK_6rtgb#--x1P#86 zc46)-fSpc+&M&X5fPSswo7gw**LgdmocKU8!Kle{bv#V6@8xNSp3Ovq4x&<^;k6c(U!Dnga`iq>tnLVVB?y9aI8C{1JT@ZCaw)gehq{2ny}3{82pwZjfoNH>M<_@?r5rL(WO z4IjvK|SW1Z7Z$eYct&yM7*KXUJlou5#`u*L3 zI@`?DG&Bs|jZ##Y4VgE;UAG$)jqBPMeB}In{0t0?q$T&K%j&3Q zXymGk^gpllDP$`O0p1idBO}MEk<}g7#(hh8%^}W8S_*O;(3DMi4_Z)2$X}k7)w?SdKUUq4^+epp72dyT34P-ZFR2bc{$ynF|x{wcBMMt?jNSW9XgjfyX~Xb3{0 z_XsMr=A%A!W)h5dS?{Y2_%cW_!ebBE(e$TWCuFi{ttd`F0uXv@L#R(-&s<#{(m^FP z6OqFkZ@~r)QgK|x2aH2iD7kGO_6EGJB3J&_p9M{e3(&M){HX9lGE@8x zJvg4EGI|JJD8=k2)r(e$-Nd4zB9GquG95ln6A`?M#`G{(gb(4O5<$6D>x&|5&XT z6t&qrMU7V&zx7@r20?hsuM7{WC$z}!ar5ql@w&wV`5?%=GqoMZ;G^O=zdq9e0ZmqB zW}IckUUC-UeD4cTwe#(~3YYQ(c0A2hVbqO)+%S3h zhGx>fG!wTHG@G53=1Pe|yzg|K0h8psFhNOFmSmhMv!dF#yZbHoJ@;U;FKsTTe($_S zTE^kW26c6Gz}1|_w-y{?h!3(GcZd4R2Q?;|x`_{?R>dh(|76d-AUlgS&dwcMnWufWYiYDRF14HZ3)2@e-+-47+4(f? zF3o7Wto?0+Z)tJSGGGeW{k=Ha&WE3a3Ah2!PNwo^AsjXe19=S(WEU2xG4LPOhXdz0X9z8*w!S9Cx zratbD_)03mH7PE6 z{oc^-2iyAhgZ{X(a9!8B|2SJL0jQAH9?20sEr%+pw2+NZa zwo+LD+mj-DFqJ~Cnz6Md?cbhm(NXhXo!G3Stt-3XYMT^pH^=G>NoZ#D_)&IPDuw$r zw3!&UE}v+1h#ckq{WrX&zXaJt1%yZ6$VmBd_MTL82s_=>*OnKwZrVnUyuVLMw8Zez z51zPrBz4~k_I5i|rA9idmvoWci9e?})JbM1`XL$8S9CL!VQS8IIUo@^8sCCWGdbu3KNSto{Z0ttJ ztX_dCs@0*YiQ4XehFk9MCV7}(d8<72iVc8UKKVQLU((5Oz>Wg^_CxZVW*%Pr%Q03DyOkxOdb= zwwaH9S~Hwayq$+ScQupc_G54t4`J~#UbDZ;B5X>|H2dH*Ja5K-f$9w81rZz7fk*uJf8PxbsTj4ZKInJ1tBZ>FHf1vBO^F?oH81`}D?;=U%_G3eI9~VIo4p z2rkh0@hSv-tSt|V@LhTdUHqH2>Dk%u1oc!0hXRnT33o#k*;E#a+g^6Ba~#s8(_uZ& z;=cOOzQ#0vkHIoC<4n}dpNbift5)8uZ!ct?Z!{M8+0&bZMoVdW2LXgxuF}J?KEHYY(mAP&Wp{k7ckhdq#~@rS9}jM(Om8M*l_e#mm7(u| z^Z-#0>(3jL=WUa28W^ax9Sr0~e$|2EV)(zxn?Jo_<9hqxgMWkGLG+%-*q>|= z&W&gKD(}t!-==Ya>)kmZlwZ5?R4;vYP_)rrQfj^Et1Xhf^QN! zh1+Z@z*L(1$Ww+3!j5Lv<+!vO#>2FeYf2LMQMt`ikhJIS>THU#eP+#pa`p3a-k6Tq zWEaiM$$SqzcLIMzYdoY3v_{2M*VOIpA6^j6N|w|F{3vvV%wBGeYP>_wc7OfPz>HYa zR#)-Us*rMqh;O=bARCcZR?g{hxTnLQ@(rNqfBF7+${6koj)}kBuHBZ#QxdZ|rRV17 zSNuS0m!6RUz(@T?U3;XpIm(NSrp?5Kk#M9RJ!A(>%+>Y|$b34nyXQ?F^GEN6nxw1- zfZ(ynm@>XPvDemz06}k930XAonV_ed6Zzz34_j@fH-vuNhi1<^5rk}y!P~j;YwK6F zbqLXX$T!p`;fh-e+N83!*XXtvWd+Qof9l44*mJ~ntZ z#t_bVgA#fl3uIXtu7UKG=z>M(6Kvk(u`(PpFz5R0zZxI9hRME zi_WjAT7uUVRE`wfmnxTP4l4=~i64a?qHoXOwiI~gV(#PuhRT{R0p<%u#hNc)-kWqi zNq4Rq6%-VLRst}+CMEoMc3<5JfXp$P2jXbh;F$JS4!g3!GgsA7W8#we0tH6ROAa+B zsHPTB>zEwMCZRi3w}>_OrOwrET~zWubRNrp#;j|~%|_e>ScBHvQcFr)46|X$7w8uG zPXNG-AVHu=;;z( zf9Do!<-j%Dc7TWr&?{_ZSV`k4ESfT#1MgI?eN`rl+H1a7HNL)(`Zpad)?{vSzDZGS z!b!gXH{-PWyB7LgF(sOCBC0i|_H6~YWjFLm47UiN!oRv1jo2d>0zcljPp6Yf^ZL5?AU zd|YQ;%+jCs0(<_VpkPkK4ClTda}Wcv@plBGQ8QPX!`%G*p#8>3XkcbD#UETz-Pk)S z*1WZ`Ewc{eYNH`_J{Ff27+mzjrn0)T$Pppgk}Qha^SJcR&1_HGoU7Y9|F z%REJ%0Nk~%a2~qo;GlYz^)LM0fUN--qU1+8S%^U`4qZ_NjK_+YfUgDcupjSF9UFPf z$5Hg@%g~lwvKnb?YzI7Ehp;sVMtQ13+2Af*51)9`Fb|n=py}%SjS1&1+pyUv4d}fvefoh{8#sPI%pCK z3P~oW_4_KolHFhfao#cb0Z`eIh99Q*#FyEl+t;?6EEOc8fTo5tm*ZG|&>ESp0XTA3!m zNEmaC?Nmf~E~mz0HWNQNiR)dLL_> zkPryK&C(iQX8{RrAI+ut(GMD^D|4XPV32+9z2b#}w=h%*k^9IPS(&rpT?C!CG&*qp z=Yk$Y!!G!!>9DpNYxl*#$uU3Ykb{_tfKEzO%9ss)5Dy3p!_w z+!BkNM%k@Gwi6N5LZD+qHw%l4!AWxPcW-ZyqGibtTg(Gq&q}aEx`R)yx%qYq;oMCX zxh$pN?Vo+XO|?9KelAu@QIWWCjaOQ-E>nTbpZ_;QJ}Vh}botNUMxTXg>AT{#!mJFw z%VQ=M7T$aQ>;B(mX&^pBP&acdntj$|=bZt7Xd+Dz$rJRiyOw9p$i_Z zuaD1Y2p^Bty+BWwsWsoFYN`W4trI?iP2WDx&&?e_u?rzxxtG2z5sx3at;33K;{VV5 z8cZf3E)?c8()REc^}t_il<$L2%M$HJ1QDT9809rsdmtLfvf1aUo!v|1&O)rWWpi73 zRs*1Dg9I<++Q)NrEd(!so7-lhmnxq|3cOKQSBG>TaK8F0@BV2WNlU{a0Jxo5e_1R5 zdoSSPE((tYlOaXnTQHD&GjU3p85w({hZ8&RDdpg6-Ga%t&B?OB>`2CNB z`BVI@3${F4)_c^ZMoUxEve9#Dx@{9IrCrWUEkK;Xf#wDoanK=cu#NdQ@Xt41()|aX z$J97oz)N6~ZY9_SVpl-K6F>;;@w!HfpPGWzu?O&AX~<8&^3~bJTyf1!fTkUeg0Q%{ z;65Im1VbJ(D~~?dKO4P{h&DX^aPZ|^@_y#^m>Zgj-Ia~@E;w#SG*G@!n+zF9LOh#A z#j)qsf52$98b+ERdpjC&XoNgnXQypfLB~$sr=HUd3)B)lC_AhDtkBJ0IQHg5z%m(F zqw9JXYkO)lU-xN#H^+mU$+~J+aN~RZI%E(wqu~y{s8QGOuil$kVM z+CH!H{??2<&pYp`hP@z!mB)OnRt_G$xCJqga=~+3+F=_r2iQ++j;;oVi;L^e!iu|- zQ(38bQ*p=AIy4+AoDgOWJXn>sGSHX@*}tnxW_uD8${08VYtqsdAh03%?tz6l;#=IB z`w@jZe^_8&9kLmzv@hvfwF>M?0Zgb%ew7*|fL?08s`8+6wJQ*REEXAg+=%+bI^6+i zQUZGp!YzJe92kyY!^0EnDha~T;h#53vpWMJ?_kTISVp5&;Cg?ply37?~Nuc7FUjFSb10#RjUp zcsLnvq=EVAP{j@@RE04Syyg-Y++1BvbD8c2u6V*9xHB;wKZQj!wY3-OsXhMky|OIt zu}j`~r>nmms#bniv@Wfe5{|Z}VG8FRjCd1@iyiy2(=y*1qcrcKqA@9DZs2J4Z+w=! z)c&TTrtzva>-~Du(SoEDw2bA{b?XbqtyT@5W+t|@Jm=lVB0yRm_HO2VL9Wy2{?L@b zn@VT>v-dlp#ovi!1+}A-FdJtMhZXAVYj2)t5!hl4{3{NE>waozYL;g{lbzq(^j242 zsG(DeZoAYuSj-+w&)C{8xoqk0X!rK%H*XZBc)%oqzhRup*CcaDN;K*2*8F&$RI`DFxB^%0TWXU9JtBW;GnK(s)f3g z9(JwYvlmmOf6>ag`!HlY2zUDL>44|cRuA){QTRJMdq+nH8yg!B#m=;>cB63jvA`_p z1VstmG4CtK760%)d|h#lnLs{x^?c;i^9Sqdh)iE0ZS3)HpHe9^dwYH33awb?JJ&^K zH;*3OXQ~o?r+eYGKAj=%FvujTMq+v7;Q)(}oIPuS zfB!z*LAr42zh9S#FoXqs7fvOi-!2lCjX2D5m^B1Ux*Kha^v!3|$a52y{Oj2pjfIPg z>r+k-b~iZV*CXNiJU@Ek0CwFozodF{es*`LWOaXA8sa#v=TraXV`U^CeGau^>^%r7 zfxiGZJxJdShf3Fc+?^xRnxiBvhqUKW^>&nAe+Hb@~ko( zMu=>Z4LQtEwnj$S-tzJ+(o@R0Sl23?2;0&!!nO`~dFmC2jpmQ5If4B;EqmBErql%Z zcaiUgXIc&tsli~{H?Li&SI0CV(F57+ZN{zw|c_Kd`NGy-h=7inEy?m)e_B4wq_Rd;y2{x@P!r2x9r9C3xgQ>>^fv3C(1>6#H%eO$)A1 z;d=$P`vsqSm(ICyb7#rR;YQ^l9_;AzuhKcnq^-6iwi*d`u%-?URZP77$owX<-syb4 zzCkhyh4R=7UPP`3G?6M)zP|X7^rKQ(4Y8K~xCXc!Wn2FzrQ0>$27<{p6%oPdoa*^{ zRxb4+(prU>C1L>v@#sm5td;p&O$`l5I6`8BJL=ZFZa}K*J_IrVFmOu zl(O@>K{GMcKOcsx-|iHkX1J9?e3pi#3NhCqctbT$ zpxiY0^Gyite8nVtEv=xlhe)|USy1s0dc82It1`c`a{9->%mMG!^r3ZT9WMRNU@`9; zHh^Ma#dMfQ9qPe_S9y=?a2#feEFAUm-S$|2QD=LCT$BMWEL2?@*qOB`ynl2aq(fsX z6|At;4gNV_sz3(PhqSc#Dhs#DxoZ4IJ8|sG(}n4XYdJwd5$~>bZveikB|{$R3)`*X z`s*oU8M;g~vI_oCS-IivU&R#;$nHjulpE7a@78|o(#VzW6mEIBoKPwXR@~4~4aigo zUder(p{vW3p3TpZa^b`_SraVt&yyzDqe^s1g8(xsZT>xgYDz%>645G}T0(j5*2H5{ zsjwWA23y0yW3<-zY+P1StD2W|p+HBFjd!kCM%OaFY9jQ@@DK5~$mN`CZ}NtmExIW= z?i_8w3Px{=%a@k|L^~^og65fyO9#~}(I~zjCT?*C32>G+8hwaUM z)w3^J9=R-~uw2uEIIXLygX71wuZysqg1#}0)+)j=+#=W31%pFAbj0#128H0M`>>q_ zmMlk68Y4XJ%5W)puB`#23a}aW_ivx2cniGt96kDfNKL`r^DF=o;g`I6B`pv?@HxP# zK5R~(%@iu$t+}ak^>ea+X8rOaXirE!#c;^PNG5AT`1!c~(z|;KVbC)|DbFiUeTi9U zG~2kPS%u2auPt8zW*WBjn`w~f{MyjK&1u}GW>jX%zrhBj`r}{PS2(}1gFDj~VX0MA zEaU^luuNQqk}jog!dSg*aaj_%B$Z3Qq{95NtLsl9!12)hqrB%u+kcPpj+i0J*W=r4 znKBOadP*X2-YCQiW`fTtm~wgHh(&&$ZX&PSG7fkVrHPOjY1!JdUkh ziP2Sqg2=8C1ZsMTiMMWH@^#mTuoFOQTclnw{4Dw{{s-Lo%~W6<1cg4hG^&wJ00sa6 z?Ap~1$AUslcPZ%_E+sFYh@Q^FQnCk!B%_5G>#2fHBURVTThB*aYgL%Tv1|F)AS1Vp zc)=`i+30FjQV;d^uUm{wA4@0hnd6Kb*>}|Y$!_P z9oyT?mWX}%y=`7&)Jt2D=l_|Dh)YY8maJ%t^U8bJdp4RD!aN1Q?N84pcEsg?yue)c zu%6m^+qOH_5D4BLvyz+J0@6ymOYxm{gX&IEtxvc9unY}J#_gVQ;~?}LE34hhQaYIO zkV0snZs=Ab@Xrq@7wBOOcuj6I>l<*NKfkiPaQ*@p2y#+< zY|H$tg98?!T%OGb0YK9PotP7TAM&Q_?TfJ6hOC z-%x4R@^q1%^YuiNsUslVI!K}n+xkG;#Aqued_2L;(`3T3)?<0P&E`h)mrqM2jU(Q@ zZc@QeT_vncU1?ioqPv@GAl5G1XLqaf9DBZwTA87XudlB-$L^f#)^d)(%J9jZVeL(F zh*SnQCwJg{mKBrCuy)%C3a=3rtz$M1SIZ{f*5@p2X^1(ItNDwH2FK* z`!#|3(ZjC%2WQx);&inur(2E5O*tSvK0ro!ys#6-t{0OvN-1u@K=~4hT7`O;Ed!Rt z(a_CMeZ8SJV?fT+OO_j*t8m3G#elSrdWdqJiMl&wAO~uTUQa9Y2m?88>b{+&{(QN?SK(g1qw$gq8=q`;z0^}X=iE}B# zaI)n~hA7Ar3GB+dNSY^rYf-2@=`m@YBv?R0gM;-jVy?jH?u(hy0`T z^b7DF^8U%mN_*YL26%7l7(c;sXlOW@Gp{UUad$5WTJ0B`^L@!w&!)?ir)R;Gd#>!Q zp=?&~Ikw~C2N2!?^azlXjzDSiIOCzQ(+4kY#2ssAuS$Kv_#p>rP|lQ>7nK*C=j62L zwvoh$^M*u)hk#F69HPp`+(``A;Vc5y#tQrc0PxWawNrHkL3}4IB{k|ApX{C6;|@(} z-~Z{Gx;j|ta$V-j(?>^>ZOu_@wsv-QVQt%Xz&>ED5VkZJpoqj$+UaMaqiGw7IY1za zsOx^r`!=&a>RRjdI=aB=jc(Wc1~hm9mnuFVq+His+sHvP2U(C!IB(rz-sU8y!egm4MW$2cs#O5xT66SkjobUaCs^I;nl?A z;uL@0Cft?twxX#^KzYvCLcGNXQ0CoWqdyKG^abLUEW`#YDEu0#@s21r$lP3(_v4cD z*#Y4#$kK6pO4Y`~mJLCK{+3q2p@d#O#z;WH4w|msEX5{gdOw*=P#A!+*Q(W32yjr7 zsb_C1wzp%AEQ80F*F)SYF6YA!h<*DuWNYdK^!4~bav-Rl^wF^REG;iN?)Y^ zcEOB~UA+y!F$fW9j(oidg(0BFFItjQ8sHGCtt|u5TlBq4LfQ>pz3Nx;L8{>3u_vT{&tLC6(Ufo<^M9FnE1y2$^W_X zIr^Jmq8@avi-J}fH`*y-2m#gy2#IIsCL@@nE^F!`t%Z;_Mp?U?q}3wuhckMplOD35rG~WcM@a@3jLt_6OiWN$sf)-sF2%X^(ju&J z8h>Z`f=g1r6eZDbMKYB?&L&Vge#tWMc8lHoa9306!^7e;H4*?r~{`woQxt|M# z*+B(OA8dzCbytS0^b@=536DINbRVmE6QZOHt8zTl zZN7ILm}z%RJ{&l0qr<~ge<*BVZSaPyg?aSFBzuh8R8-g0@GAHO^kp~sMa|GQX;g$o zz0XrZzSGNhx)lM7zu3#l%1)f&F(rRX8Z4NbyQbZg;{`z80zS;Qp2V7Yg&z4gF;bfc-MDBIP==M4|GwHJrA)&0AB|N;uY2RTx$)Q z4W%@1aWL{Ih48}}=RR^%N(y4H4x}E!?sbLD&DP-Mm6wxulae>Vx!bwikeA=v!&7T> za|Z}sWouFn=>HtF-nwcmT!~rOAcufhpt(60 z9vG<7;BLRkGe`SLvPH+llVRkUy21t1kyruHK%6 z{;cxmXhqo!sLvYe{ZtdGQv&S+03L%dKG366l26(G1dZ2v)@oqmgo{zRu&8K?+WmGq zP~tK~)VaF_Pk(ph2`XZ1MV}9e9WJ5u1PmA5?JX6AN7h3gl%vY`_yMc&Sts2U&bcku0zAKp5~a- z@N{N(^6TW*mcCrFzJVs|{is%*STx*MkiJ8dFc>)`uIC?!n<2?}XXke@u^Bc?m>z^{ z3f9K)o)w$BT()@S0*~1N$(B(cMm6{mf?|p#_S8=0#=Mdu^F1h~a{~qzV z=;UMN-M8hV?Z0}LGyHJOan%Pus9`+%`Zq9}s|amv+AidJukxOW_?6`R@pN~kd)iGq z6GKyNm-LiDT~on)u#q`|HAi0(f?;RY_pZrgXTIm)<~FIcQ=NO#x$L)!Bh9At006lJ z(=cVurd$bJ7_`rj@)-GS{Q|$*H)lMVty+k=`{KpAvG8WM21cKi5U;{xfes5bGjlO> zagpIlu+RVe*>imNG~hjJyCHreQ{;(@lVK{%Z~KrCwH%WnmFh8`LMY{vH|*`~1g;25 zH5l#8EF5(?;j7?BWcf2>NU$j78?A#bCFK@brskGa?VfjI@`&RP4M+4pM zl&2pHoKDX#S%v!2s;@+SfZc73-=TQ18>luDT{_};}j365YABTh;+YJIz zp(+#?%YBYlnYDKggr~n#&vz7t@QM^IzJBTDLdG~o(y?9PsJoeRoMy~)g&1&K&~UcMxLDf z8+&Qs2cg@ouvjZwG1$R%3uE!0ohi1yq1@alczE}L6wBy6oksg}iW|HuX6Xerbyf?j zNC6R%jBU&A=A5Y?M{YfURUn3u$hZJf^ZCho;t~S`Llwey&d%^Q^PJts{>oC&9kxc{ zx94}z=2N-Sxz{gm86D}3d^iVNm*03G0bjSPkgF~(hSY8(db@Bm-Fs?{YwNd2Q{hsOqe=%S z679F~T?JFl?G}Cig9v-NOJ9~|{x6hU9Jl3xxjeFb3C2IOvMN=RraTbyVD&{}lrb_M~{c?DRz9z~rYy|p)CH#3F#6ks=UL|xZm#ndaw z$h_0^I<`i>2M?+N(k*C#WUOY4Ciu{#*crMj@3cJj*7Mf0Z5?}10^>22X$#s~1_q~@ znDQ|TsHSYEbi;R6pQ|dpiZ7h1>-XJ%LBh*_rX>TV5572AMv~t3nhzglVx%o;o192T zW9}Kh@o@|E{xRvpaphzvD%;3^L9_l@&w`(b|XUfvW3l>9GZ`)SYY8JhwUL~xQQ@d zhr5&Ee00~Y#qr8-zhJjg@O+H$@DRJb7CQYcVh(PWo{Gxe&lOMMZnb>SvRy;$z!!y) zjhYmjHkFcDz9`4DFv=u!ekX4dBc0yH#Doz_ti*(ceaw-^Zitwq=s6dE_^?0OA|1!Q zzt)v3q*Zu}i$O$GFunEN9c)c=^TbhP^B`OWo<&<}eKs*7BAV`YqY{B8`YP{9;YqOX z3`074mcoc2#QaWPEeK7`x(4Nzn1pl_Z6Y3CP|Kzu$@o`sjD6c=`8iO=LR6;m-U0$a z+oOFBRf*`E@1AQORjQ1%_Vf1-U84r8#D#fQ5W?+EV5JN^k+^m(>gVH0Ho6w$hp-9A zWr?Pm=@gis(q}!Ein-F=W#J~v69=8|y-Sc5TmjeMI#G@C9LVGJR&&n|y)?$3rRDGE zCm|`7sT5jYy!idc+GqQ;*Y`pK?7n>ZvJ*=TU3Toh#aVx9ix?9ZbK*!SELZ};Oj==Q z^36G+{V=qX$uQARPDz$sa|fb950sbN+djU2wRKia{bRody}(-5``-;%t2H%e&YXE0 z%Lr2F{*;ZfB_3lVbKOhbmE%qYP9v{tq)ol_f?;g?w<%8#4{bPiy>0ajhF-6%l%J0? zBFxP#k2X0PI4vx@1pdyMf1jt+4hgvx+s}RF{q(s)mNY>PNwzXcfTY@Pf*1WaZuXXQ zbWu*}kfVCmF}uSxeGY2W6TFQ`CsP5AG0;2cJgwkYmClS6b*zslXdP4H}m& z^S%2zWA6TPip0woS}rntt9EM`V5-dfV!pzdBT9d!RowpmKi|n*>~Z(LT~m@1Rp03E z?+4+iAsjG9N75GOAF^K2R6lm)c|k#By7#sX^8nGq3MF^ z9|L4~!KRyKFMI4uptrSkZr?m9Kmj196#KnPb{q@(YU=#c;19J6*x6dpQ&FaTt|1P_ zEYrPcS=2KacmnfdOzW(~>7wjSU{>0#==Oxpdv{yOb8`h7WjLPDtyA)Xf`;YB3w@o$ zAV}@QEqHZS3aciBa^KX{!}?la^(FJKCcSu(Os#5d02=kN-8Msyr=Nb>jy#QzS!Ym$SC*q z^Z6+wb?$z}moKwZDmbrcNRA$%#sECeWARj>5g=X3b-&?+vm5SjSc#L(J$ z>P@oH<|zi4%JNhD>LBA538o-A2*_TN;VQ5{efmsnZ0yes=C`qLxx(hN{R^rQ&#gQ1ycr6Gi6$m7QNgax&KgAQ zUrd)$3+!H(;$OApK@CeJ`mS;Z)-8KwJPkc@y7h^ROIb--R%T|SZGEcVQ@?5}&8vxq z?MFtlEV629<|))nM&-u(0^*iAB;;@6JpSGOaj5mqCDjDX!r4O3+zMf#EGQcDr;>F)NHJJ%}yVc*6=eZvrVfKLRe&J&ZzfQ_I`H-KK0r;i$yT77o>#J^V zWpyDVtT1VYBSS(&9a}x(_zw4>5gZvUC#2m$R;o@OmMp=}&d<-!&UGcg)O&GOu0N~j zAfg()qircEj|4tBjf$+hTUuJ`=&b9*xQv6jVR3OwyGez~Anhm>=&H6Ax+7P%RG+QM zZGK0{d4l zVE5$j#ynu$p4Az)Zn<-tnQ<&FPaq79Um~I)k{WZpvr>RQrS;}uWPwws-Aw>(Z3Y6- z8iH@8AveRl4r0*+Au8v-Dr-cXO8D}I<+k8g__FwiA;qWJ=mwTHcN;BJaWGzu^?n7X zchj)3DaHphY?9}r+Z<4(C4BU72XXj7YQ$<7vby^3_PT|MAt1SyRH7K1mYk7OK5|z~ z&fZ_xK7sagom~%FPMYrOvPf?5+uNu9)G5U*l$Mn3ug4+u&GbGLZyMg(aS%<#&6CKD zN_99lWtPtvk3rBps$XI0G!{;I_`Q%J@}5q*bDT&w&&9&&zm#|OZ8`z{V%d?t_aRfKa**lL@n?iOpk)deRL2L-n1 z4ja;B)jb``w5`?dMrw{F6!*(c*PseI(lk_9tRxqv!12?$GE3l%5>gEP-+#GNlw_e) z^d{vL@c3!j|5$13q7lx5A9bAt^R=`zeoi*|759a#9$pd78ot}|_{jrh6+dD!dE)3~ z*ga;($G74A4_#bv?5cA=#}!Waa+i5*csOYL=YsI%$B+k@?jXSy)1LSIUn@BA>Q=#g zBGkQ+r25WpxTfGn`0+!h{;85VEDWcB_fWAxhLk((y?|ekB6qR{C{MQvwDq5kuj&qXs_RVUOkj52*3tsjD z^pGqD679SJfm?45J-&E}{`luL`f$GPG&XZ_&8o%_+)Ui683-7mpix$)t%qt;hNRI3 zkBr~v=6&W@<>p@2)Qs_NxF>aJGTYjQ>sdsH1Z*VP*1a%Jvcw1(1~)3a<*{bZNN!8+ zL;lQMsVd9@wBTmdPoB2*b|J%-m#pBqZQIc4H7TWONJvR!DqBh86%QYJe=+>=p;K|? zOP4QShD_$x?~sH!jKr^r(RMw0G~9>gOEGIGIf0L*F)?^;L4G{*A+=l%0 zo1EW4N>25#fC_Og-wJfJW@a#{Q4=t}5E9*?$C32m@C-oG?PcZyrMP+1ZPvE2v9SZU z$Csyipj~OtOJreb>4dy|YmP{PS1|b&cD{kUw!pwZ9;CFiGmTBUVL!LSB<{Azk5@%WP8J%K=fzQpw z)e@TlpPK|rxyQRI|E>o#!XRyU-9r3!T*Qi!0|~|lc}jm)7Q;wuxkvYCz!?6kLx0G) z|KP&(U%c24LukQ}2YgPVY&6MX_?qTkko-4LE@=ix(abO+BM zxx8@iXlT5r-G1b0OQcc49Q8yG&wW^L2kpatcPbc4xH2>OVNzb-p@XkCz7)^Fb+G+C z$}ES58t$uXr!Moszz5Y7mB=WqOc}s$_viX^UzDYawi{nKdPqs=<%q|TLvyJeGb0rr zY#a&YWiVmsi}g&GK}n+}l8?T?-UNFq)Pw3$waMj9sev3!jjJju4}yY&q4NPxiPwT| zH=(u`hSlJMPab-&PSqGgYZs1 z_;7!4LkI#PY~``&h1KJp*J98JJ;sKbTm4T4>Xye+mI>cl)R+TDh_yPVT_L{ z_5f(K(3a==-u(aFjI#e1UisJkY1-jSmL?8kv92m`eOG+$8~fN<``F9bdfCIj4&4;L zDJ3Efe@p7$kb-G8a<{~V#Kq;r#pzlcTmJvP!Og?Y$szFn`5SyQ@uBby0{_3?;N#?G k@9ksl=J|jBo}2LF|IaT`xpC*t!Ch0muXV5d&g1a^3)2G?-T(jq literal 0 HcmV?d00001 diff --git a/docs_new/cookbook/diffusion/Ideogram/Ideogram4.mdx b/docs_new/cookbook/diffusion/Ideogram/Ideogram4.mdx new file mode 100644 index 000000000..1432c4a77 --- /dev/null +++ b/docs_new/cookbook/diffusion/Ideogram/Ideogram4.mdx @@ -0,0 +1,82 @@ +--- +title: Ideogram 4 +metatags: + description: "Deploy Ideogram 4 with SGLang Diffusion for high-aesthetic text-to-image generation." +--- + +## 1. Model introduction + +[Ideogram 4](https://huggingface.co/ideogram-ai/ideogram-4-nf4) is Ideogram's text-to-image diffusion model. SGLang Diffusion supports the official NF4 checkpoint, the official FP8 checkpoint, and the Comfy-Org NVFP4 transformer checkpoint. + +Compared with previous open-source image models, Ideogram 4 provides a significant aesthetic lift, with stronger composition, more polished visual style, and better typography-aware generation. + +| Variant | Hugging Face model ID | Notes | +| --- | --- | --- | +| NF4 | `ideogram-ai/ideogram-4-nf4` | Official bitsandbytes NF4 checkpoint. Use this path first for low-memory deployment. | +| FP8 | `ideogram-ai/ideogram-4-fp8` | Official FP8 checkpoint. | +| NVFP4 | `Comfy-Org/Ideogram-4` | Comfy-Org NVFP4 transformer weights. SGLang loads non-transformer components from `ideogram-ai/ideogram-4-fp8`. | + +## 2. Prerequisites + +- NVIDIA CUDA GPU. +- SGLang installed with diffusion dependencies. +- `bitsandbytes>=0.46.1` for the NF4 checkpoint. +- `HF_TOKEN` with access to the Ideogram 4 gated repositories. + +## 3. Serve the model + +NF4: + +```bash Command +HF_TOKEN=$HF_TOKEN sglang serve \ + --model-path ideogram-ai/ideogram-4-nf4 \ + --num-gpus 1 \ + --performance-mode auto \ + --port 30010 +``` + +FP8: + +```bash Command +HF_TOKEN=$HF_TOKEN sglang serve \ + --model-path ideogram-ai/ideogram-4-fp8 \ + --num-gpus 1 \ + --performance-mode auto \ + --port 30010 +``` + +Comfy-Org NVFP4: + +```bash Command +HF_TOKEN=$HF_TOKEN sglang serve \ + --model-path Comfy-Org/Ideogram-4 \ + --num-gpus 1 \ + --performance-mode auto \ + --port 30010 +``` + +Use B200 or another Blackwell GPU for NVFP4. + +## 4. Generate an image + +```python Example +import base64 +from openai import OpenAI + +client = OpenAI(api_key="EMPTY", base_url="http://localhost:30010/v1") + +response = client.images.generate( + model="ideogram-ai/ideogram-4-nf4", + prompt="A cinematic poster of a quiet bookstore at dusk with elegant hand-lettered signage", + size="1024x1024", + n=1, + response_format="b64_json", + extra_body={"preset": "V4_QUALITY_48", "seed": 0}, +) + +image_bytes = base64.b64decode(response.data[0].b64_json) +with open("ideogram4.png", "wb") as f: + f.write(image_bytes) +``` + +Ideogram 4 presets are `V4_DEFAULT_20`, `V4_QUALITY_48`, and `V4_TURBO_12`. The preset controls both `num_inference_steps` and guidance, so do not set those fields directly. diff --git a/docs_new/cookbook/diffusion/intro.mdx b/docs_new/cookbook/diffusion/intro.mdx index a4235d52d..06475776d 100644 --- a/docs_new/cookbook/diffusion/intro.mdx +++ b/docs_new/cookbook/diffusion/intro.mdx @@ -23,6 +23,12 @@ Offline models generate each image or video request as a bounded denoising job. href="/cookbook/diffusion/FLUX/FLUX" img="/cards/logos/flux.png" /> + torch.distributed.ProcessGroup | None: - if config.parallel_folding: - if config.parallel_folding_mode == "sp": - return get_sp_group() - elif config.parallel_folding_mode == "ulysses": - return get_sp_group().ulysses_group - elif config.parallel_folding_mode == "ring": - return get_sp_group().ring_group - return get_tp_group() diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py index 1bce2a374..b37e02a8a 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py @@ -2,6 +2,9 @@ from typing import Literal, get_args +from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( + BitsAndBytesConfig, +) from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( QuantizationConfig, ) @@ -25,6 +28,7 @@ QuantizationMethods = Literal[ "modelopt", "modelopt_fp8", "modelopt_fp4", + "bitsandbytes", "modelslim", "mxfp8", "mxfp4", @@ -38,6 +42,7 @@ _CUSTOMIZED_METHOD_TO_QUANT_CONFIG = { "modelopt": ModelOptFp8DiffusionConfig, "modelopt_fp8": ModelOptFp8Config, "modelopt_fp4": ModelOptFp4Config, + "bitsandbytes": BitsAndBytesConfig, "modelslim": ModelSlimConfig, "fp8": Fp8Config, "mxfp4": Mxfp4Config, diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py b/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py new file mode 100644 index 000000000..f5726bbbd --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py @@ -0,0 +1,383 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Optional + +import torch +import torch.nn as nn +from packaging import version + +from sglang.multimodal_gen.runtime.layers.linear import ( + LinearBase, + LinearMethodBase, + UnquantizedLinearMethod, +) +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.multimodal_gen.runtime.models.utils import set_weight_attrs + + +def _require_bitsandbytes() -> None: + try: + import bitsandbytes + + if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"): + raise ImportError( + "bitsandbytes version is wrong. Please install bitsandbytes>=0.46.1." + ) + except ImportError as err: + raise ImportError( + "Please install bitsandbytes>=0.46.1 via " + "`pip install bitsandbytes>=0.46.1` to use bitsandbytes quantizer." + ) from err + + +def _calculate_quant_ratio(dtype: torch.dtype) -> int: + if dtype.is_floating_point: + return torch.finfo(dtype).bits // torch.iinfo(torch.uint8).bits + return torch.iinfo(dtype).bits // torch.iinfo(torch.uint8).bits + + +def _is_layer_skipped(prefix: str, skipped_modules: list[str]) -> bool: + components = prefix.split(".") + if any(module_name in components for module_name in skipped_modules): + return True + + prefixes = {".".join(components[: i + 1]) for i in range(len(components))} + return bool(set(skipped_modules) & prefixes) + + +class BitsAndBytesConfig(QuantizationConfig): + """Config class for pre-quantized bitsandbytes 4-bit checkpoints.""" + + def __init__( + self, + load_in_8bit: bool = False, + load_in_4bit: bool = True, + bnb_4bit_compute_dtype: str = "float32", + bnb_4bit_quant_storage: str = "uint8", + bnb_4bit_quant_type: str = "fp4", + bnb_4bit_use_double_quant: bool = False, + llm_int8_enable_fp32_cpu_offload: bool = False, + llm_int8_has_fp16_weight: bool = False, + llm_int8_skip_modules: list[str] | None = None, + llm_int8_threshold: float = 6.0, + ) -> None: + super().__init__() + self.load_in_8bit = load_in_8bit + self.load_in_4bit = load_in_4bit + self.bnb_4bit_compute_dtype = bnb_4bit_compute_dtype + self.bnb_4bit_quant_storage = bnb_4bit_quant_storage + self.bnb_4bit_quant_type = bnb_4bit_quant_type + self.bnb_4bit_use_double_quant = bnb_4bit_use_double_quant + self.llm_int8_enable_fp32_cpu_offload = llm_int8_enable_fp32_cpu_offload + self.llm_int8_has_fp16_weight = llm_int8_has_fp16_weight + self.llm_int8_skip_modules = llm_int8_skip_modules or [] + self.llm_int8_threshold = llm_int8_threshold + + if self.load_in_8bit or not self.load_in_4bit: + raise ValueError("SGLang diffusion only supports bitsandbytes 4-bit.") + if self.bnb_4bit_quant_storage != "uint8": + raise ValueError( + f"Unsupported bnb_4bit_quant_storage: {self.bnb_4bit_quant_storage}" + ) + + @classmethod + def get_name(cls) -> str: + return "bitsandbytes" + + def get_scaled_act_names(self) -> list[str]: + return [] + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.float32, torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + @staticmethod + def get_config_filenames() -> list[str]: + return [] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> "BitsAndBytesConfig": + def get_safe_value(keys, default_value=None): + try: + value = QuantizationConfig.get_from_keys(config, keys) + return value if value is not None else default_value + except ValueError: + return default_value + + return cls( + load_in_8bit=get_safe_value(["load_in_8bit"], False), + load_in_4bit=get_safe_value(["load_in_4bit"], True), + bnb_4bit_compute_dtype=get_safe_value( + ["bnb_4bit_compute_dtype"], "float32" + ), + bnb_4bit_quant_storage=get_safe_value(["bnb_4bit_quant_storage"], "uint8"), + bnb_4bit_quant_type=get_safe_value(["bnb_4bit_quant_type"], "fp4"), + bnb_4bit_use_double_quant=get_safe_value( + ["bnb_4bit_use_double_quant"], False + ), + llm_int8_enable_fp32_cpu_offload=get_safe_value( + ["llm_int8_enable_fp32_cpu_offload"], False + ), + llm_int8_has_fp16_weight=get_safe_value( + ["llm_int8_has_fp16_weight"], False + ), + llm_int8_skip_modules=get_safe_value(["llm_int8_skip_modules"], []), + llm_int8_threshold=get_safe_value(["llm_int8_threshold"], 6.0), + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + if isinstance(layer, LinearBase): + if _is_layer_skipped(prefix, self.llm_int8_skip_modules): + return UnquantizedLinearMethod() + return BitsAndBytesLinearMethod(self) + return None + + +class BitsAndBytesLinearMethod(LinearMethodBase): + """Linear method for pre-quantized bitsandbytes 4-bit weights.""" + + def __init__(self, quant_config: BitsAndBytesConfig): + _require_bitsandbytes() + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size + quant_ratio = _calculate_quant_ratio(params_dtype) + total_size = input_size_per_partition * sum(output_partition_sizes) + if total_size % quant_ratio != 0: + raise ValueError( + "The input size is not aligned with the quantized weight shape." + ) + + qweight = nn.Parameter( + torch.empty(total_size // quant_ratio, 1, dtype=torch.uint8), + requires_grad=False, + ) + set_weight_attrs( + qweight, + { + "input_dim": 0, + "output_dim": 0, + "pack_factor": quant_ratio, + "use_bitsandbytes_4bit": True, + }, + ) + layer.register_parameter("weight", qweight) + set_weight_attrs(qweight, extra_weight_attrs) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + original_type = x.dtype + original_shape = x.shape + if x.ndim > 2: + x = x.reshape(-1, x.size(-1)) + + out_dim = sum( + quant_state.shape[0] + for quant_state in layer.weight.bnb_quant_state.values() + ) + out = torch.empty(x.shape[0], out_dim, dtype=torch.bfloat16, device=x.device) + apply_bnb_4bit(x.to(torch.bfloat16), layer.weight, out) + out = out.to(original_type) + + if len(original_shape) > 2: + out = out.view(*original_shape[:-1], out.size(-1)) + + if bias is not None: + out = out + bias + return out + + +def apply_bnb_4bit( + x: torch.Tensor, + weight: torch.Tensor, + out: torch.Tensor, +) -> None: + from bitsandbytes import matmul_4bit + + offsets = weight.bnb_shard_offsets + quant_states = weight.bnb_quant_state + current_index = 0 + for i in range(len(quant_states)): + output_size = quant_states[i].shape[0] + out[:, current_index : current_index + output_size] = matmul_4bit( + x, + weight[offsets[i] : offsets[i + 1]].t(), + quant_states[i], + ) + current_index += output_size + + +class BitsAndBytes4BitLinear(nn.Module): + """Storage-only bitsandbytes 4-bit linear for nn.Linear-based encoders.""" + + def __init__( + self, + in_features: int, + out_features: int, + bias: bool = True, + compute_dtype: torch.dtype | None = None, + ) -> None: + super().__init__() + _require_bitsandbytes() + self.in_features = in_features + self.out_features = out_features + self.compute_dtype = compute_dtype + quant_ratio = _calculate_quant_ratio(compute_dtype or torch.get_default_dtype()) + total_size = in_features * out_features + if total_size % quant_ratio != 0: + raise ValueError( + "The input size is not aligned with the quantized weight shape." + ) + + self.weight = nn.Parameter( + torch.empty(total_size // quant_ratio, 1, dtype=torch.uint8), + requires_grad=False, + ) + set_weight_attrs( + self.weight, + { + "pack_factor": quant_ratio, + "use_bitsandbytes_4bit": True, + }, + ) + if bias: + self.bias = nn.Parameter( + torch.empty( + out_features, dtype=compute_dtype or torch.get_default_dtype() + ), + requires_grad=False, + ) + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + original_type = x.dtype + original_shape = x.shape + if x.ndim > 2: + x = x.reshape(-1, x.size(-1)) + + out = torch.empty( + x.shape[0], self.out_features, dtype=torch.bfloat16, device=x.device + ) + apply_bnb_4bit(x.to(torch.bfloat16), self.weight, out) + out = out.to(original_type) + + if len(original_shape) > 2: + out = out.view(*original_shape[:-1], out.size(-1)) + + if self.bias is not None: + out = out + self.bias + return out + + +def swap_linears_to_bitsandbytes_4bit(module: nn.Module) -> None: + for name, child in list(module.named_children()): + if isinstance(child, nn.Linear): + replacement = BitsAndBytes4BitLinear( + child.in_features, + child.out_features, + bias=child.bias is not None, + compute_dtype=child.weight.dtype, + ) + setattr(module, name, replacement) + else: + swap_linears_to_bitsandbytes_4bit(child) + + +_BNB_4BIT_STATE_SUFFIXES = { + "absmax", + "quant_map", + "nested_absmax", + "nested_quant_map", + "bitsandbytes", +} + + +def is_bitsandbytes_4bit_state_name(weight_name: str) -> bool: + suffix = weight_name.split(".")[-1] + return any(state_suffix in suffix for state_suffix in _BNB_4BIT_STATE_SUFFIXES) + + +def split_bitsandbytes_4bit_state( + weights: Any, +) -> tuple[list[tuple[str, torch.Tensor]], dict[str, torch.Tensor]]: + normal_weights: list[tuple[str, torch.Tensor]] = [] + quant_state_dict: dict[str, torch.Tensor] = {} + for name, tensor in weights: + if is_bitsandbytes_4bit_state_name(name): + if "quant_state.bitsandbytes" in name: + tensor = tensor.cpu().data + quant_state_dict[name] = tensor + continue + normal_weights.append((name, tensor)) + return normal_weights, quant_state_dict + + +def build_bitsandbytes_4bit_quant_states( + normal_weight_names: list[str], + quant_state_dict: dict[str, torch.Tensor], + device: torch.device, + param_names_mapping=None, +) -> dict[str, Any]: + from bitsandbytes.functional import QuantState + + quant_states: dict[str, Any] = {} + device_str = str(device) + for source_name in normal_weight_names: + if ( + f"{source_name}.quant_state.bitsandbytes__nf4" not in quant_state_dict + and f"{source_name}.quant_state.bitsandbytes__fp4" not in quant_state_dict + ): + continue + target_name = source_name + if param_names_mapping is not None: + target_name, _, _ = param_names_mapping(source_name) + state_tensors = { + name: tensor + for name, tensor in quant_state_dict.items() + if name.startswith(f"{source_name}.") + } + quant_states[target_name] = QuantState.from_dict( + state_tensors, device=device_str + ) + return quant_states + + +def attach_bitsandbytes_4bit_quant_states( + params_dict: dict[str, torch.nn.Parameter], + quant_states: dict[str, Any], +) -> None: + for param_name, quant_state in quant_states.items(): + param = params_dict.get(param_name) + if param is None: + raise ValueError(f"Parameter {param_name} not found in the model.") + + state_by_shard = {0: quant_state} + set_weight_attrs(param, {"bnb_quant_state": state_by_shard}) + offsets = torch.tensor([0, param.numel()]).cpu() + set_weight_attrs(param, {"bnb_shard_offsets": offsets}) diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py index f5a59a9d0..85c10eec4 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/text_encoder_loader.py @@ -306,6 +306,18 @@ class TextEncoderLoader(ComponentLoader): fsdp_cpu_offload = False should_offload = False + if ( + getattr( + model_config.arch_config, "requires_gpu_resident_text_encoder", False + ) + and should_offload + ): + logger.warning( + "Keeping bitsandbytes 4-bit text encoder GPU-resident; CUDA " + "weights and quant states are required for this checkpoint." + ) + should_offload = False + if should_offload and not current_platform.is_mps(): model_device = torch.device("cpu") else: diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py index 81cdf1187..3102bd4f6 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/transformer_loader.py @@ -34,6 +34,23 @@ def _server_args_for_transformer_component( if component_name not in ("transformer_2", "unconditional_transformer"): return server_args + # Some pipelines have secondary DiT components with their own quantized + # weight file. Keep the mapping model-owned and the loader generic. + component_weights_paths = getattr( + server_args, "component_transformer_weights_paths", {} + ) + component_weights_path = component_weights_paths.get(component_name) + if component_weights_path is not None: + component_server_args = copy.copy(server_args) + component_server_args.transformer_weights_path = component_weights_path + component_server_args.nunchaku_config = None + logger.info( + "Using transformer_weights_path override for %s: %s", + component_name, + component_weights_path, + ) + return component_server_args + if ( server_args.transformer_weights_path is None and server_args.nunchaku_config is None diff --git a/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py b/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py index efff17ba7..e660fcc39 100644 --- a/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py +++ b/python/sglang/multimodal_gen/runtime/loader/fsdp_load.py @@ -25,6 +25,11 @@ from torch.nn.modules.module import _IncompatibleKeys from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod +from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( + attach_bitsandbytes_4bit_quant_states, + build_bitsandbytes_4bit_quant_states, + split_bitsandbytes_4bit_state, +) from sglang.multimodal_gen.runtime.loader.utils import ( get_param_names_mapping, hf_to_custom_state_dict, @@ -51,6 +56,13 @@ _QUANTIZED_DTYPES = ( _DTYPE_MISMATCH_EXAMPLE_LIMIT = 3 +def _is_bitsandbytes_quant_config(quant_config: Any | None) -> bool: + if quant_config is None: + return False + quant_name_getter = getattr(type(quant_config), "get_name", None) + return bool(callable(quant_name_getter) and quant_name_getter() == "bitsandbytes") + + def _format_dtype_mismatch_summary( mismatch_counts: Counter[tuple[torch.dtype, torch.dtype]], mismatch_examples: dict[tuple[torch.dtype, torch.dtype], list[str]], @@ -244,11 +256,21 @@ def maybe_load_fsdp_model( pin_cpu_memory=pin_cpu_memory, ) + param_names_mapping_fn = get_param_names_mapping(model.param_names_mapping) weight_iterator = safetensors_weights_iterator(weight_dir_list) preprocess_loaded_state_dict = getattr(model, "preprocess_loaded_state_dict", None) if preprocess_loaded_state_dict is not None: weight_iterator = preprocess_loaded_state_dict(weight_iterator) - param_names_mapping_fn = get_param_names_mapping(model.param_names_mapping) + bnb_quant_states = None + if _is_bitsandbytes_quant_config(init_params.get("quant_config")): + normal_weights, raw_quant_state = split_bitsandbytes_4bit_state(weight_iterator) + bnb_quant_states = build_bitsandbytes_4bit_quant_states( + [name for name, _ in normal_weights], + raw_quant_state, + device, + param_names_mapping_fn, + ) + weight_iterator = iter(normal_weights) load_model_from_full_model_state_dict( model, weight_iterator, @@ -258,6 +280,10 @@ def maybe_load_fsdp_model( cpu_offload=cpu_offload, param_names_mapping=param_names_mapping_fn, ) + if bnb_quant_states: + attach_bitsandbytes_4bit_quant_states( + dict(model.named_parameters()), bnb_quant_states + ) for _, module in model.named_modules(): quant_method = getattr(module, "quant_method", None) diff --git a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py index fa3634727..21aced3f5 100644 --- a/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/transformer_load_utils.py @@ -270,6 +270,46 @@ class _ModelOptFp8OffloadAdapter(_TransformerQuantAdapter): ) +class _BitsAndBytes4BitAdapter(_TransformerQuantAdapter): + """Adapter for pre-quantized bitsandbytes 4-bit transformer checkpoints.""" + + def __init__( + self, + *, + server_args: ServerArgs, + quant_config: Optional[QuantizationConfig], + ) -> None: + self.server_args = server_args + self.quant_config = quant_config + + @staticmethod + def _maybe_disable_incompatible_offload_modes( + server_args: ServerArgs, + quant_config: Optional[QuantizationConfig], + ) -> None: + if _get_quant_config_name(quant_config) != "bitsandbytes": + return + + changed = [] + if server_args.dit_cpu_offload: + server_args.dit_cpu_offload = False + changed.append("dit_cpu_offload=False") + if server_args.use_fsdp_inference: + server_args.use_fsdp_inference = False + changed.append("use_fsdp_inference=False") + if changed: + logger.warning( + "Keeping bitsandbytes 4-bit transformer GPU-resident: %s", + ", ".join(changed), + ) + + def prepare(self) -> None: + _BitsAndBytes4BitAdapter._maybe_disable_incompatible_offload_modes( + server_args=self.server_args, + quant_config=self.quant_config, + ) + + def resolve_transformer_safetensors_to_load( server_args: ServerArgs, component_model_path: str ) -> list[str]: @@ -441,6 +481,10 @@ def _build_transformer_quant_adapters( server_args=server_args, quant_config=quant_config, ), + _BitsAndBytes4BitAdapter( + server_args=server_args, + quant_config=quant_config, + ), ] if nunchaku_config is not None: adapters.append( diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py index fd78b0a91..272196996 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py @@ -12,6 +12,10 @@ from sglang.multimodal_gen.runtime.layers.attention import ( USPAttention, build_varlen_mask_meta, ) +from sglang.multimodal_gen.runtime.layers.linear import ReplicatedLinear +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, +) from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( WeightOnlyFP8Linear, ) @@ -35,8 +39,27 @@ class Ideogram4RMSNorm(nn.Module): return F.rms_norm(x, self.weight.shape, self.weight, self.eps) -def _linear(in_features: int, out_features: int, bias: bool = True): - return WeightOnlyFP8Linear(in_features, out_features, bias=bias) +class Ideogram4QuantizedLinear(ReplicatedLinear): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return super().forward(x)[0] + + +def _linear( + in_features: int, + out_features: int, + bias: bool = True, + quant_config: QuantizationConfig | None = None, + prefix: str = "", +): + if quant_config is None: + return WeightOnlyFP8Linear(in_features, out_features, bias=bias) + return Ideogram4QuantizedLinear( + in_features, + out_features, + bias=bias, + quant_config=quant_config, + prefix=prefix, + ) class Ideogram4Attention(nn.Module): @@ -46,12 +69,20 @@ class Ideogram4Attention(nn.Module): num_heads: int, eps: float, supported_attention_backends, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: super().__init__() self.hidden_size = hidden_size self.num_heads = num_heads self.head_dim = hidden_size // num_heads - self.qkv = _linear(hidden_size, hidden_size * 3, bias=False) + self.qkv = _linear( + hidden_size, + hidden_size * 3, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv", + ) self.norm_q = Ideogram4RMSNorm(self.head_dim, eps=eps) self.norm_k = Ideogram4RMSNorm(self.head_dim, eps=eps) self.attn = USPAttention( @@ -62,7 +93,13 @@ class Ideogram4Attention(nn.Module): causal=False, supported_attention_backends=supported_attention_backends, ) - self.o = _linear(hidden_size, hidden_size, bias=False) + self.o = _linear( + hidden_size, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o", + ) def forward(self, x, cos, sin, attn_mask, attn_mask_meta): batch_size, seq_len, _ = x.shape @@ -77,11 +114,35 @@ class Ideogram4Attention(nn.Module): class Ideogram4MLP(nn.Module): - def __init__(self, dim: int, hidden_dim: int) -> None: + def __init__( + self, + dim: int, + hidden_dim: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() - self.w1 = _linear(dim, hidden_dim, bias=False) - self.w2 = _linear(hidden_dim, dim, bias=False) - self.w3 = _linear(dim, hidden_dim, bias=False) + self.w1 = _linear( + dim, + hidden_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.w1", + ) + self.w2 = _linear( + hidden_dim, + dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.w2", + ) + self.w3 = _linear( + dim, + hidden_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.w3", + ) def forward(self, x): return self.w2(F.silu(self.w1(x)) * self.w3(x)) @@ -96,6 +157,8 @@ class Ideogram4TransformerBlock(nn.Module): norm_eps, adaln_dim, supported_attention_backends, + quant_config: QuantizationConfig | None = None, + prefix: str = "", ): super().__init__() self.attention = Ideogram4Attention( @@ -103,13 +166,26 @@ class Ideogram4TransformerBlock(nn.Module): num_heads, eps=1e-5, supported_attention_backends=supported_attention_backends, + quant_config=quant_config, + prefix=f"{prefix}.attention", + ) + self.feed_forward = Ideogram4MLP( + hidden_size, + intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.feed_forward", ) - self.feed_forward = Ideogram4MLP(hidden_size, intermediate_size) self.attention_norm1 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) self.ffn_norm1 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) self.attention_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) self.ffn_norm2 = Ideogram4RMSNorm(hidden_size, eps=norm_eps) - self.adaln_modulation = _linear(adaln_dim, 4 * hidden_size, bias=True) + self.adaln_modulation = _linear( + adaln_dim, + 4 * hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.adaln_modulation", + ) def forward(self, x, cos, sin, adaln_input, attn_mask, attn_mask_meta): scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaln_modulation( @@ -144,12 +220,30 @@ def _sinusoidal_embedding(t: torch.Tensor, dim: int, scale: float = 1e4): class Ideogram4EmbedScalar(nn.Module): - def __init__(self, dim: int, input_range: tuple[float, float]) -> None: + def __init__( + self, + dim: int, + input_range: tuple[float, float], + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() self.dim = dim self.range_min, self.range_max = input_range - self.mlp_in = _linear(dim, dim, bias=True) - self.mlp_out = _linear(dim, dim, bias=True) + self.mlp_in = _linear( + dim, + dim, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.mlp_in", + ) + self.mlp_out = _linear( + dim, + dim, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.mlp_out", + ) def forward(self, x): compute_dtype = x.dtype @@ -160,11 +254,30 @@ class Ideogram4EmbedScalar(nn.Module): class Ideogram4FinalLayer(nn.Module): - def __init__(self, hidden_size: int, out_channels: int, adaln_dim: int) -> None: + def __init__( + self, + hidden_size: int, + out_channels: int, + adaln_dim: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: super().__init__() self.norm_final = nn.LayerNorm(hidden_size, eps=1e-6, elementwise_affine=False) - self.linear = _linear(hidden_size, out_channels, bias=True) - self.adaln_modulation = _linear(adaln_dim, hidden_size, bias=True) + self.linear = _linear( + hidden_size, + out_channels, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.linear", + ) + self.adaln_modulation = _linear( + adaln_dim, + hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.adaln_modulation", + ) def forward(self, x, c): scale = 1.0 + self.adaln_modulation(F.silu(c)) @@ -180,12 +293,12 @@ class Ideogram4Transformer2DModel(BaseDiT): ) param_names_mapping = {} reverse_param_names_mapping = {} - handles_checkpoint_quantization = True def __init__( self, config: Ideogram4DiTConfig, hf_config: dict[str, Any], + quant_config: QuantizationConfig | None = None, **kwargs, ) -> None: super().__init__(config, hf_config, **kwargs) @@ -195,11 +308,34 @@ class Ideogram4Transformer2DModel(BaseDiT): self.hidden_size = hidden_size self.num_attention_heads = cfg.num_attention_heads self.num_channels_latents = cfg.in_channels - self.input_proj = _linear(cfg.in_channels, hidden_size, bias=True) + self.input_proj = _linear( + cfg.in_channels, + hidden_size, + bias=True, + quant_config=quant_config, + prefix="input_proj", + ) self.llm_cond_norm = Ideogram4RMSNorm(cfg.llm_features_dim, eps=1e-6) - self.llm_cond_proj = _linear(cfg.llm_features_dim, hidden_size, bias=True) - self.t_embedding = Ideogram4EmbedScalar(hidden_size, input_range=(0.0, 1.0)) - self.adaln_proj = _linear(hidden_size, cfg.adaln_dim, bias=True) + self.llm_cond_proj = _linear( + cfg.llm_features_dim, + hidden_size, + bias=True, + quant_config=quant_config, + prefix="llm_cond_proj", + ) + self.t_embedding = Ideogram4EmbedScalar( + hidden_size, + input_range=(0.0, 1.0), + quant_config=quant_config, + prefix="t_embedding", + ) + self.adaln_proj = _linear( + hidden_size, + cfg.adaln_dim, + bias=True, + quant_config=quant_config, + prefix="adaln_proj", + ) self.embed_image_indicator = nn.Embedding(2, hidden_size) self.rotary_emb = Qwen3VLTextRotaryEmbedding( head_dim=cfg.attention_head_dim, @@ -215,14 +351,18 @@ class Ideogram4Transformer2DModel(BaseDiT): norm_eps=cfg.norm_eps, adaln_dim=cfg.adaln_dim, supported_attention_backends=self._supported_attention_backends, + quant_config=quant_config, + prefix=f"layers.{i}", ) - for _ in range(cfg.num_layers) + for i in range(cfg.num_layers) ] ) self.final_layer = Ideogram4FinalLayer( hidden_size=hidden_size, out_channels=cfg.in_channels, adaln_dim=cfg.adaln_dim, + quant_config=quant_config, + prefix="final_layer", ) def post_load_weights(self) -> None: diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/ideogram.py b/python/sglang/multimodal_gen/runtime/models/encoders/ideogram.py index d7df4c184..86a38475c 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/ideogram.py @@ -10,6 +10,12 @@ from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput from sglang.multimodal_gen.configs.models.encoders.ideogram import ( Ideogram4TextEncoderConfig, ) +from sglang.multimodal_gen.runtime.layers.quantization.bitsandbytes import ( + attach_bitsandbytes_4bit_quant_states, + build_bitsandbytes_4bit_quant_states, + is_bitsandbytes_4bit_state_name, + swap_linears_to_bitsandbytes_4bit, +) from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( swap_linears_to_weight_only_fp8, ) @@ -31,7 +37,12 @@ class IdeogramQwen3VLTextEncoder(TextEncoder): if isinstance(text_config, dict): text_config = Qwen3VLTextConfig(**text_config) self.language_model = Qwen3VLTextModel(text_config) - if getattr(arch_config, "ideogram_fp8_weight_only", False): + self._uses_bitsandbytes_4bit = getattr( + arch_config, "ideogram_bnb_4bit_weight_only", False + ) + if self._uses_bitsandbytes_4bit: + swap_linears_to_bitsandbytes_4bit(self.language_model) + elif getattr(arch_config, "ideogram_fp8_weight_only", False): swap_linears_to_weight_only_fp8(self.language_model) @torch.no_grad() @@ -100,6 +111,9 @@ class IdeogramQwen3VLTextEncoder(TextEncoder): return features def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + if self._uses_bitsandbytes_4bit: + return self._load_bitsandbytes_4bit_weights(weights) + loaded_params: set[str] = set() params_dict = dict(self.named_parameters(remove_duplicate=False)) for name, loaded_weight in weights: @@ -117,5 +131,51 @@ class IdeogramQwen3VLTextEncoder(TextEncoder): loaded_params.add(name) return loaded_params + def _load_bitsandbytes_4bit_weights( + self, weights: Iterable[Tuple[str, torch.Tensor]] + ): + params_dict = dict(self.named_parameters(remove_duplicate=False)) + raw_quant_state: dict[str, torch.Tensor] = {} + normal_weight_names: list[str] = [] + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if is_bitsandbytes_4bit_state_name(name): + if "quant_state.bitsandbytes" in name: + loaded_weight = loaded_weight.cpu().data + raw_quant_state[name] = loaded_weight + continue + if name.startswith("visual."): + continue + if "rotary_emb.inv_freq" in name: + continue + param = params_dict.get(name) + if param is None: + raise KeyError( + f"Unexpected weight name while loading Ideogram text encoder: {name}" + ) + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight.to(param.dtype)) + normal_weight_names.append(name) + loaded_params.add(name) + + quant_states = build_bitsandbytes_4bit_quant_states( + normal_weight_names, + raw_quant_state, + next(self.parameters()).device, + ) + attach_bitsandbytes_4bit_quant_states(params_dict, quant_states) + quantized_params_missing_state = [ + name + for name, param in params_dict.items() + if getattr(param, "use_bitsandbytes_4bit", False) + and name not in quant_states + ] + if quantized_params_missing_state: + raise ValueError( + "Missing bitsandbytes quant_state for Ideogram text encoder weights: " + f"{quantized_params_missing_state[:8]}" + ) + return loaded_params + EntryClass = IdeogramQwen3VLTextEncoder diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py index 5058de389..3a1b931ab 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/t5.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/t5.py @@ -30,7 +30,7 @@ import torch.nn.functional as F from torch import nn from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput, T5Config -from sglang.multimodal_gen.runtime.distributed import _get_folding_tp_group +from sglang.multimodal_gen.runtime.distributed import get_sp_group, get_tp_group from sglang.multimodal_gen.runtime.layers.activation import get_act_fn from sglang.multimodal_gen.runtime.layers.layernorm import RMSNorm from sglang.multimodal_gen.runtime.layers.linear import ( @@ -48,6 +48,19 @@ from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder from sglang.multimodal_gen.runtime.platforms import current_platform +def _get_folding_tp_group( + config: T5Config, +) -> torch.distributed.ProcessGroup | None: + if config.parallel_folding: + if config.parallel_folding_mode == "sp": + return get_sp_group() + elif config.parallel_folding_mode == "ulysses": + return get_sp_group().ulysses_group + elif config.parallel_folding_mode == "ring": + return get_sp_group().ring_group + return get_tp_group() + + class AttentionType: """ Attention type. diff --git a/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py b/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py index 796f1d5d9..92c7c369c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/ideogram.py @@ -1,5 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 +import os +from dataclasses import dataclass +from functools import lru_cache +from typing import Any, cast + from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.pipelines_core import LoRAPipeline from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( @@ -14,6 +19,88 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.i Ideogram4TextEncodingStage, ) from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( + maybe_download_model, + verify_model_config_and_directory, +) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +_IDEOGRAM4_BASE_MODEL = "ideogram-ai/ideogram-4-fp8" +_IDEOGRAM4_NVFP4_COND_FILE = "diffusion_models/ideogram4_nvfp4_mixed.safetensors" +_IDEOGRAM4_NVFP4_UNCOND_FILE = ( + "diffusion_models/ideogram4_unconditional_nvfp4_mixed.safetensors" +) + + +@dataclass(frozen=True) +class Ideogram4Nvfp4ModelResolution: + base_model_name: str + base_model_path: str + transformer_weights_path: str + unconditional_transformer_weights_path: str | None + + +@lru_cache(maxsize=1) +def _resolve_ideogram4_base_model_path() -> str: + return maybe_download_model(_IDEOGRAM4_BASE_MODEL, force_diffusers_model=True) + + +def _resolve_ideogram4_unconditional_transformer_weights_path( + transformer_weights_path: str, +) -> str | None: + if os.path.basename(transformer_weights_path) != os.path.basename( + _IDEOGRAM4_NVFP4_COND_FILE + ): + return None + return os.path.join( + os.path.dirname(transformer_weights_path), + os.path.basename(_IDEOGRAM4_NVFP4_UNCOND_FILE), + ) + + +def _resolve_ideogram4_nvfp4_transformer_weights_paths( + server_args: ServerArgs, model_path: str +) -> tuple[str, str | None]: + if server_args.transformer_weights_path is not None: + transformer_weights_path = server_args.transformer_weights_path + return ( + transformer_weights_path, + _resolve_ideogram4_unconditional_transformer_weights_path( + transformer_weights_path + ), + ) + + local_nvfp4_path = maybe_download_model( + model_path, + allow_patterns=[ + _IDEOGRAM4_NVFP4_COND_FILE, + _IDEOGRAM4_NVFP4_UNCOND_FILE, + ], + ) + return ( + os.path.join(local_nvfp4_path, _IDEOGRAM4_NVFP4_COND_FILE), + os.path.join(local_nvfp4_path, _IDEOGRAM4_NVFP4_UNCOND_FILE), + ) + + +def resolve_ideogram4_nvfp4_model( + server_args: ServerArgs, model_path: str +) -> Ideogram4Nvfp4ModelResolution: + ( + transformer_weights_path, + unconditional_transformer_weights_path, + ) = _resolve_ideogram4_nvfp4_transformer_weights_paths( + server_args, + model_path, + ) + return Ideogram4Nvfp4ModelResolution( + base_model_name=_IDEOGRAM4_BASE_MODEL, + base_model_path=_resolve_ideogram4_base_model_path(), + transformer_weights_path=transformer_weights_path, + unconditional_transformer_weights_path=unconditional_transformer_weights_path, + ) class Ideogram4Pipeline(LoRAPipeline, ComposedPipelineBase): @@ -55,4 +142,85 @@ class Ideogram4Pipeline(LoRAPipeline, ComposedPipelineBase): ) -EntryClass = Ideogram4Pipeline +class Ideogram4Nvfp4Pipeline(Ideogram4Pipeline): + pipeline_name = "Ideogram4Nvfp4Pipeline" + _model_resolution: Ideogram4Nvfp4ModelResolution | None = None + + def _get_model_resolution( + self, + server_args: ServerArgs | None = None, + ) -> Ideogram4Nvfp4ModelResolution: + if self._model_resolution is None: + if server_args is None: + raise ValueError( + "server_args is required to resolve Ideogram4 NVFP4 paths" + ) + self._model_resolution = resolve_ideogram4_nvfp4_model( + server_args, + self.model_path, + ) + return self._model_resolution + + def _load_config(self) -> dict[str, Any]: + model_resolution = self._get_model_resolution(self.server_args) + logger.info("Model path: %s", self.model_path) + logger.info( + "Using base model '%s' at %s for config and non-transformer components", + model_resolution.base_model_name, + model_resolution.base_model_path, + ) + config = verify_model_config_and_directory(model_resolution.base_model_path) + return cast(dict[str, Any], config) + + def _resolve_component_path( + self, + server_args: ServerArgs, + module_name: str, + load_module_name: str, + ) -> str: + override_path = server_args.component_paths.get(module_name) + if override_path is not None: + return maybe_download_model(override_path) + + component_model_path = os.path.join( + self._get_model_resolution(server_args).base_model_path, + load_module_name, + ) + logger.debug("Resolved component path: %s", component_model_path) + return component_model_path + + def load_modules( + self, + server_args: ServerArgs, + loaded_modules: dict | None = None, + ) -> dict: + model_resolution = self._get_model_resolution(server_args) + server_args.transformer_weights_path = model_resolution.transformer_weights_path + if model_resolution.unconditional_transformer_weights_path is not None: + # The loader treats transformer_weights_path as the base DiT override. + # Route the sibling unconditional DiT weights through the generic + # per-component override map instead of hard-coding Ideogram there. + component_transformer_weights_paths = dict( + getattr(server_args, "component_transformer_weights_paths", {}) + ) + component_transformer_weights_paths.setdefault( + "unconditional_transformer", + model_resolution.unconditional_transformer_weights_path, + ) + server_args.component_transformer_weights_paths = ( + component_transformer_weights_paths + ) + logger.info( + "NVFP4 transformer weights: %s", + model_resolution.transformer_weights_path, + ) + logger.info( + "NVFP4 unconditional transformer weights: %s", + server_args.component_transformer_weights_paths.get( + "unconditional_transformer" + ), + ) + return super().load_modules(server_args, loaded_modules) + + +EntryClass = [Ideogram4Pipeline, Ideogram4Nvfp4Pipeline] diff --git a/python/sglang/multimodal_gen/runtime/server_args.py b/python/sglang/multimodal_gen/runtime/server_args.py index fcda04afd..bfacbb9fd 100644 --- a/python/sglang/multimodal_gen/runtime/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args.py @@ -184,6 +184,11 @@ class ServerArgs(DisaggServerArgsMixin): # path to pre-quantized transformer weights (single .safetensors or directory). transformer_weights_path: str | None = None + # Per-component transformer weight overrides (key = model_index.json component name). + # Pipelines use this when a checkpoint ships separate quantized weights for + # secondary DiT components; the generic loader consumes it without model-specific + # filename logic. + component_transformer_weights_paths: dict[str, str] = field(default_factory=dict) # Quantization method for online quantization quantization: str | None = None diff --git a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py index 168227e7b..4e1c4bf22 100644 --- a/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py +++ b/python/sglang/multimodal_gen/runtime/utils/quantization_utils.py @@ -2,6 +2,7 @@ import glob import json import os import re +import struct from pathlib import Path from typing import Any, Dict, List, Optional @@ -38,8 +39,15 @@ def normalize_flat_modelopt_quant_config( def _infer_nvfp4_group_size_from_tensors(weight, scale) -> Optional[int]: """Infer NVFP4 group_size from serialized weight/scale tensor shapes.""" - weight_shape = tuple(getattr(weight, "shape", ())) - scale_shape = tuple(getattr(scale, "shape", ())) + return _infer_nvfp4_group_size_from_shapes( + getattr(weight, "shape", ()), + getattr(scale, "shape", ()), + ) + + +def _infer_nvfp4_group_size_from_shapes(weight_shape, scale_shape) -> Optional[int]: + weight_shape = tuple(weight_shape or ()) + scale_shape = tuple(scale_shape or ()) if len(weight_shape) < 2: return None @@ -67,9 +75,34 @@ def _infer_nvfp4_group_size_from_tensors(weight, scale) -> Optional[int]: return None +def _read_safetensors_tensor_metadata(file_path: str) -> dict[str, dict[str, Any]]: + with open(file_path, "rb") as f: + header_len = struct.unpack(" bool: + weight_metadata = tensor_metadata.get(f"{module_name}.weight") + scale_metadata = tensor_metadata.get(f"{module_name}.weight_scale") + if weight_metadata is None or scale_metadata is None: + return False + + weight_dtype = str(weight_metadata.get("dtype", "")).upper() + scale_dtype = str(scale_metadata.get("dtype", "")).upper() + scale_shape = scale_metadata.get("shape", []) + return weight_dtype == "U8" and "F8_E4M3" in scale_dtype and len(scale_shape) >= 2 + + def _resolve_quant_method_name(quant_cfg: dict) -> str: quant_cfg = normalize_flat_modelopt_quant_config(quant_cfg) or quant_cfg quant_method = quant_cfg.get("quant_method") + if quant_method == "bitsandbytes": + return "bitsandbytes" if quant_method != "modelopt": return quant_method @@ -285,6 +318,7 @@ def _build_nvfp4_config_from_safetensors_files( non_quantized_bfl_modules: set[str] = set() files_with_nvfp4_signal: list[str] = [] checkpoint_uses_packed_qkv = False + checkpoint_uses_comfy_quant = False packed_qkv_pattern = re.compile( r"^(double_blocks\.\d+\.(img|txt)_attn\.qkv|single_blocks\.\d+\.linear1)\." ) @@ -322,21 +356,26 @@ def _build_nvfp4_config_from_safetensors_files( if isinstance(layer_cfg, dict) and layer_cfg.get("format") == "nvfp4" ) + tensor_metadata = _read_safetensors_tensor_metadata(file_path) with safe_open(file_path, framework="pt", device="cpu") as f: all_keys = set(f.keys()) if any(packed_qkv_pattern.match(k) for k in all_keys): checkpoint_uses_packed_qkv = True + if any(k.endswith(".comfy_quant") for k in all_keys): + checkpoint_uses_comfy_quant = True # Some ModelOpt NVFP4 exports only store a flat config.json plus # per-file metadata without the diffusers `layers` section. Infer - # quantized modules directly from tensor families in that case: - # quantized modules ship `.weight` + `.weight_scale`, while BF16 - # fallbacks only ship `.weight`. + # quantized modules directly from tensor families in that case. + # Mixed checkpoints may also contain FP8 fallback layers with scalar + # `.weight_scale`, so require packed uint8 weights and block scales. file_quantized_modules.update( key[: -len(".weight_scale")] for key in all_keys if key.endswith(".weight_scale") - and f"{key[: -len('.weight_scale')]}.weight" in all_keys + and _is_nvfp4_tensor_family( + key[: -len(".weight_scale")], tensor_metadata + ) ) if file_quantized_modules or metadata_signals_nvfp4: @@ -347,10 +386,13 @@ def _build_nvfp4_config_from_safetensors_files( for layer_name in sorted(file_quantized_modules): weight_key = f"{layer_name}.weight" scale_key = f"{layer_name}.weight_scale" - if weight_key in all_keys and scale_key in all_keys: - w = f.get_tensor(weight_key) - s = f.get_tensor(scale_key) - group_size = _infer_nvfp4_group_size_from_tensors(w, s) + weight_metadata = tensor_metadata.get(weight_key) + scale_metadata = tensor_metadata.get(scale_key) + if weight_metadata is not None and scale_metadata is not None: + group_size = _infer_nvfp4_group_size_from_shapes( + weight_metadata.get("shape"), + scale_metadata.get("shape"), + ) if group_size is not None: break @@ -432,28 +474,32 @@ def _build_nvfp4_config_from_safetensors_files( try: quant_cls = get_quantization_config("modelopt_fp4") + checkpoint_uses_swizzled_scales = ( + checkpoint_uses_packed_qkv or checkpoint_uses_comfy_quant + ) result = quant_cls.from_config( { "quant_algo": "NVFP4", "group_size": group_size, "ignore": exclude_modules, "checkpoint_uses_packed_qkv": checkpoint_uses_packed_qkv, - # The official FLUX.2 mixed NVFP4 export is detected by its - # packed QKV tensors and stores block scales in the - # FlashInfer/CUTLASS-swizzled layout. SGLang-converted - # transformer repos keep the linear layout. + # packed-QKV and Comfy NVFP4 checkpoints store serialized + # weights/scales in the FlashInfer/CUTLASS checkpoint layout "checkpoint_weight_scale_layout": ( - "swizzled" if checkpoint_uses_packed_qkv else "linear" + "swizzled" if checkpoint_uses_swizzled_scales else "linear" ), + "swap_weight_nibbles": checkpoint_uses_swizzled_scales, } ) logger.info( - "Built NVFP4 quant config from %d safetensors: group_size=%d, %d excluded modules, packed_qkv=%s, scale_layout=%s", + "Built NVFP4 quant config from %d safetensors: group_size=%d, %d excluded modules, packed_qkv=%s, comfy_quant=%s, scale_layout=%s, swap_nibbles=%s", len(files_with_nvfp4_signal), group_size, len(exclude_modules), checkpoint_uses_packed_qkv, + checkpoint_uses_comfy_quant, getattr(result, "checkpoint_weight_scale_layout", "linear"), + getattr(result, "swap_weight_nibbles", False), ) return result except Exception as e: diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 652aa18ab..09dc41fbe 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -553,6 +553,15 @@ else: env_vars=MODELOPT_NVFP4_B200_ENV_VARS, run_consistency_check=True, ), + _make_modelopt_ci_case( + "ideogram4_nvfp4_t2i", + model_path="Comfy-Org/Ideogram-4", + modality="image", + sampling_params=IDEOGRAM4_CI_sampling_params, + extras=[], + env_vars=MODELOPT_NVFP4_B200_ENV_VARS, + run_consistency_check=True, + ), _make_modelopt_ci_case( "wan22_modelopt_nvfp4_t2v", model_path=MODELOPT_WAN22_NVFP4_MODEL, diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index 5f8b2492b..cb53038b6 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -33,7 +33,7 @@ if TYPE_CHECKING: logger = init_logger(__name__) -SGL_TEST_FILES_CI_DATA_REVISION = "50aa0d4d5d4d260302d74b80d97747efd0f0ae45" +SGL_TEST_FILES_CI_DATA_REVISION = "af6e712a2c49ab5fcd81dde58e2f54c78e77683b" SGL_TEST_FILES_CONSISTENCY_GT_ROOT = ( "https://raw.githubusercontent.com/" f"sgl-project/ci-data/{SGL_TEST_FILES_CI_DATA_REVISION}/" diff --git a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py index 020c49057..12e103e10 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ideogram4.py +++ b/python/sglang/multimodal_gen/test/unit/test_ideogram4.py @@ -24,6 +24,11 @@ from sglang.multimodal_gen.registry import _get_config_info, get_model_info from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType, get_module_role from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.layers.attention import USPAttention +from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod +from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( + ModelOptFp4Config, + ModelOptFp4LinearMethod, +) from sglang.multimodal_gen.runtime.layers.quantization.weight_only_fp8 import ( FP8_WEIGHT_DTYPE, WeightOnlyFP8Linear, @@ -47,6 +52,9 @@ from sglang.multimodal_gen.runtime.models.dits.ideogram import ( from sglang.multimodal_gen.runtime.models.encoders.ideogram import ( IdeogramQwen3VLTextEncoder, ) +from sglang.multimodal_gen.runtime.pipelines.ideogram import ( + _resolve_ideogram4_unconditional_transformer_weights_path, +) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ideogram import ( @@ -153,6 +161,33 @@ class TestIdeogram4(unittest.TestCase): self.assertIs(info.pipeline_config_cls, Ideogram4PipelineConfig) self.assertIs(info.sampling_param_cls, Ideogram4SamplingParams) + def test_registry_resolves_comfy_nvfp4_repo_to_native_pipeline(self): + get_model_info.cache_clear() + _get_config_info.cache_clear() + + info = get_model_info("Comfy-Org/Ideogram-4", backend="sglang") + + self.assertEqual(info.pipeline_cls.__name__, "Ideogram4Nvfp4Pipeline") + self.assertIs(info.pipeline_config_cls, Ideogram4PipelineConfig) + self.assertIs(info.sampling_param_cls, Ideogram4SamplingParams) + + def test_registry_resolves_official_nf4_repo_to_native_pipeline(self): + get_model_info.cache_clear() + _get_config_info.cache_clear() + + with patch( + "sglang.multimodal_gen.registry.maybe_download_model_index", + return_value={ + "_class_name": "Ideogram4Pipeline", + "_diffusers_version": "0.0.0", + }, + ): + info = get_model_info("ideogram-ai/ideogram-4-nf4", backend="sglang") + + self.assertEqual(info.pipeline_cls.__name__, "Ideogram4Pipeline") + self.assertIs(info.pipeline_config_cls, Ideogram4PipelineConfig) + self.assertIs(info.sampling_param_cls, Ideogram4SamplingParams) + def test_rowwise_fp8_dequant_uses_output_channel_scale(self): weight = torch.tensor( [[1.0, 2.0, -3.0], [4.0, -5.0, 6.0]], dtype=FP8_WEIGHT_DTYPE @@ -305,6 +340,7 @@ class TestIdeogram4(unittest.TestCase): server_args = SimpleNamespace( transformer_weights_path="/unused/override.safetensors", nunchaku_config={"enabled": True}, + component_transformer_weights_paths={}, ) component_args = _server_args_for_transformer_component( server_args, "unconditional_transformer" @@ -313,6 +349,45 @@ class TestIdeogram4(unittest.TestCase): self.assertIsNone(component_args.transformer_weights_path) self.assertIsNone(component_args.nunchaku_config) + def test_transformer_component_uses_per_component_weights_override(self): + server_args = SimpleNamespace( + transformer_weights_path=( + "/ckpt/diffusion_models/ideogram4_nvfp4_mixed.safetensors" + ), + nunchaku_config={"enabled": True}, + component_transformer_weights_paths={ + "unconditional_transformer": ( + "/ckpt/diffusion_models/" + "ideogram4_unconditional_nvfp4_mixed.safetensors" + ) + }, + ) + + component_args = _server_args_for_transformer_component( + server_args, + "unconditional_transformer", + ) + + self.assertIsNot(component_args, server_args) + self.assertEqual( + component_args.transformer_weights_path, + "/ckpt/diffusion_models/ideogram4_unconditional_nvfp4_mixed.safetensors", + ) + self.assertIsNone(component_args.nunchaku_config) + + def test_ideogram_nvfp4_unconditional_transformer_path_uses_sibling_file(self): + self.assertEqual( + _resolve_ideogram4_unconditional_transformer_weights_path( + "/ckpt/diffusion_models/ideogram4_nvfp4_mixed.safetensors" + ), + "/ckpt/diffusion_models/ideogram4_unconditional_nvfp4_mixed.safetensors", + ) + self.assertIsNone( + _resolve_ideogram4_unconditional_transformer_weights_path( + "/ckpt/custom_transformer.safetensors" + ) + ) + def test_ideogram_denoiser_does_not_request_dtype_cast(self): import sglang.multimodal_gen.runtime.server_args as server_args_module @@ -554,6 +629,69 @@ class TestIdeogram4(unittest.TestCase): ) self.assertEqual(state["layers.0.attention.qkv.weight"].dtype, FP8_WEIGHT_DTYPE) + def test_ideogram_dit_nvfp4_quant_config_uses_native_fp4_linears(self): + import sglang.multimodal_gen.runtime.server_args as server_args_module + + quant_config = ModelOptFp4Config( + is_checkpoint_nvfp4_serialized=True, + group_size=16, + exclude_modules=[ + "input_proj", + "llm_cond_proj", + "t_embedding.*", + "adaln_proj", + "layers.*.adaln_modulation", + "final_layer.*", + ], + ) + prev_args = server_args_module._global_server_args + try: + set_global_server_args( + SimpleNamespace(attention_backend="torch_sdpa", comfyui_mode=False) + ) + with patch( + "sglang.multimodal_gen.runtime.layers.attention.layer.get_ring_parallel_world_size", + return_value=1, + ): + with torch.device("meta"): + model = Ideogram4Transformer2DModel( + Ideogram4DiTConfig(), + {}, + quant_config=quant_config, + ) + finally: + set_global_server_args(prev_args) + + self.assertEqual(model.layers[0].attention.qkv.prefix, "layers.0.attention.qkv") + self.assertIsInstance( + model.layers[0].attention.qkv.quant_method, + ModelOptFp4LinearMethod, + ) + self.assertIsInstance(model.input_proj.quant_method, UnquantizedLinearMethod) + + state = model.state_dict() + self.assertEqual( + tuple(state["layers.0.attention.qkv.weight"].shape), + (13824, 2304), + ) + self.assertEqual(state["layers.0.attention.qkv.weight"].dtype, torch.uint8) + self.assertEqual( + tuple(state["layers.0.attention.qkv.weight_scale"].shape), + (13824, 288), + ) + self.assertEqual( + state["layers.0.attention.qkv.weight_scale"].dtype, + FP8_WEIGHT_DTYPE, + ) + self.assertEqual( + tuple(state["layers.0.attention.qkv.weight_scale_2"].shape), + (1,), + ) + self.assertEqual( + tuple(state["layers.0.attention.qkv.input_scale"].shape), + (1,), + ) + def test_missing_weight_only_fp8_scale_is_fatal(self): with torch.device("meta"): model = WeightOnlyFP8Linear(3, 2, bias=False) @@ -613,6 +751,27 @@ class TestIdeogram4(unittest.TestCase): config.arch_config.architectures, ["IdeogramQwen3VLTextEncoder"] ) self.assertTrue(config.arch_config.ideogram_fp8_weight_only) + self.assertFalse(config.arch_config.ideogram_bnb_4bit_weight_only) + self.assertFalse(config.arch_config.requires_gpu_resident_text_encoder) + + def test_ideogram_text_encoder_post_config_hook_uses_bnb_for_nf4(self): + config = Ideogram4TextEncoderConfig() + config.update_model_arch( + { + "quantization_config": { + "quant_method": "bitsandbytes", + "load_in_4bit": True, + "bnb_4bit_quant_type": "nf4", + } + } + ) + + self.assertEqual( + config.arch_config.architectures, ["IdeogramQwen3VLTextEncoder"] + ) + self.assertTrue(config.arch_config.ideogram_bnb_4bit_weight_only) + self.assertFalse(config.arch_config.ideogram_fp8_weight_only) + self.assertTrue(config.arch_config.requires_gpu_resident_text_encoder) def test_ideogram_text_encoder_swaps_linears_to_weight_only_fp8(self): config = Ideogram4TextEncoderConfig() diff --git a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py index 6d0ac4b9f..6e00922e0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py +++ b/python/sglang/multimodal_gen/test/unit/test_transformer_quant.py @@ -11,6 +11,7 @@ from types import SimpleNamespace from unittest.mock import patch import torch +from safetensors.torch import save_file partial_json_parser = types.ModuleType("partial_json_parser") partial_json_parser_core = types.ModuleType("partial_json_parser.core") @@ -58,6 +59,10 @@ from sglang.multimodal_gen.runtime.loader.transformer_load_utils import ( resolve_transformer_safetensors_to_load, ) from sglang.multimodal_gen.runtime.models.dits.flux import FluxSingleTransformerBlock +from sglang.multimodal_gen.runtime.utils.quantization_utils import ( + build_nvfp4_config_from_safetensors_list, + get_quant_config, +) from sglang.multimodal_gen.tools.build_modelopt_nvfp4_transformer import ( _updated_quant_config, ) @@ -260,6 +265,99 @@ class TestTransformerQuantHelpers(unittest.TestCase): self.assertFalse(config.swap_weight_nibbles) + def test_bitsandbytes_quant_config_resolves_from_hf_config(self): + config = get_quant_config( + { + "quantization_config": { + "quant_method": "bitsandbytes", + "load_in_4bit": True, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_quant_storage": "uint8", + } + }, + "/unused/component/path", + ) + + self.assertEqual(config.get_name(), "bitsandbytes") + self.assertTrue(config.load_in_4bit) + self.assertEqual(config.bnb_4bit_quant_type, "nf4") + + def test_nvfp4_safetensors_inference_ignores_fp8_fallback_scales(self): + with tempfile.NamedTemporaryFile(suffix=".safetensors") as f: + save_file( + { + "fallback.weight": torch.empty( + (4, 4), + dtype=torch.float8_e4m3fn, + ), + "fallback.weight_scale": torch.tensor(1.0, dtype=torch.float32), + "layers.0.attention.qkv.weight": torch.zeros( + (32, 8), + dtype=torch.uint8, + ), + "layers.0.attention.qkv.weight_scale": torch.empty( + (32, 1), + dtype=torch.float8_e4m3fn, + ), + "layers.0.attention.qkv.weight_scale_2": torch.tensor( + 1.0, + dtype=torch.float32, + ), + }, + f.name, + ) + + config = build_nvfp4_config_from_safetensors_list([f.name]) + + self.assertIsInstance(config, ModelOptFp4Config) + self.assertEqual(config.group_size, 16) + self.assertIn("fallback", config.exclude_modules) + self.assertNotIn("layers.0.attention.qkv", config.exclude_modules) + self.assertEqual(config.checkpoint_weight_scale_layout, "linear") + self.assertFalse(config.swap_weight_nibbles) + + def test_nvfp4_safetensors_inference_uses_comfy_checkpoint_layout(self): + with tempfile.NamedTemporaryFile(suffix=".safetensors") as f: + save_file( + { + "fallback.weight": torch.empty( + (4, 4), + dtype=torch.float8_e4m3fn, + ), + "fallback.weight_scale": torch.tensor(1.0, dtype=torch.float32), + "fallback.comfy_quant": torch.tensor( + list(b'{"format":"float8_e4m3fn"}'), + dtype=torch.uint8, + ), + "layers.0.attention.qkv.weight": torch.zeros( + (32, 8), + dtype=torch.uint8, + ), + "layers.0.attention.qkv.weight_scale": torch.empty( + (32, 1), + dtype=torch.float8_e4m3fn, + ), + "layers.0.attention.qkv.weight_scale_2": torch.tensor( + 1.0, + dtype=torch.float32, + ), + "layers.0.attention.qkv.comfy_quant": torch.tensor( + list(b'{"format":"nvfp4"}'), + dtype=torch.uint8, + ), + }, + f.name, + ) + + config = build_nvfp4_config_from_safetensors_list([f.name]) + + self.assertIsInstance(config, ModelOptFp4Config) + self.assertEqual(config.group_size, 16) + self.assertIn("fallback", config.exclude_modules) + self.assertNotIn("layers.0.attention.qkv", config.exclude_modules) + self.assertEqual(config.checkpoint_weight_scale_layout, "swizzled") + self.assertTrue(config.swap_weight_nibbles) + def test_builder_adds_diffusers_quant_type_for_nvfp4(self): updated = _updated_quant_config( { diff --git a/python/sglang/utils.py b/python/sglang/utils.py index 33bc175de..c7dc1ae78 100644 --- a/python/sglang/utils.py +++ b/python/sglang/utils.py @@ -34,6 +34,8 @@ logger = logging.getLogger(__name__) KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: dict[str, str] = { "hunyuan3d": "Hunyuan3D2Pipeline", "flux.2-dev-nvfp4": "Flux2NvfpPipeline", + "comfy-org/ideogram-4": "Ideogram4Nvfp4Pipeline", + "comfy-org--ideogram-4": "Ideogram4Nvfp4Pipeline", }