PR #41294: use the fused low-ratio compressor on Hopper (sm90 gate)
Source: https://github.com/sgl-project/sglang/pull/41294 (applied via GitHub .diff, 3way) Skipped: test_dsv41_fused_compress.py (file not in v0.5.21 base).
This commit is contained in:
@@ -87,12 +87,14 @@ def rope_tail(
|
||||
|
||||
|
||||
def fused_low_ratio_compress_supported() -> bool:
|
||||
"""The fused c1 / c2 / index-K decode kernels pack fp4 with
|
||||
`cvt.rn.satfinite.e2m1x2`, an sm100+ instruction; the answer also fixes the
|
||||
ratio-2 weight layout (`wkv_gate`, or `wkv` plus `wgate`)."""
|
||||
if not torch.cuda.is_available() or torch.version.hip is not None:
|
||||
"""The fused c1 / c2 / index-K decode kernels need sm90+ on CUDA or gfx950 on
|
||||
ROCm; the answer also fixes the ratio-2 weight layout (`wkv_gate`, or `wkv`
|
||||
plus `wgate`)."""
|
||||
if not torch.cuda.is_available():
|
||||
return False
|
||||
return torch.cuda.get_device_capability()[0] >= 10
|
||||
if torch.version.hip is not None:
|
||||
return is_gfx95_supported()
|
||||
return torch.cuda.get_device_capability()[0] >= 9
|
||||
|
||||
|
||||
class DeepseekV41Compressor(nn.Module):
|
||||
|
||||
Reference in New Issue
Block a user