From dbd4302bbb65ab8ecc56112ba78723d9e150e9e6 Mon Sep 17 00:00:00 2001 From: Ziang Li Date: Fri, 11 Sep 2026 22:18:02 -0700 Subject: [PATCH] Keep NVFP4 blockscale swizzle padding on the input device (#39141) --- python/sglang/srt/layers/quantization/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/layers/quantization/utils.py b/python/sglang/srt/layers/quantization/utils.py index 70c6d05a3..5cfc5d8b3 100644 --- a/python/sglang/srt/layers/quantization/utils.py +++ b/python/sglang/srt/layers/quantization/utils.py @@ -585,7 +585,10 @@ def swizzle_blockscale(scale: torch.Tensor): round_up_multiple = lambda x, m: (x + m - 1) // m * m M_padded = round_up_multiple(M, 128) K_padded = round_up_multiple(K, 4) - padded_scale = torch.zeros((B, M_padded, K_padded), dtype=scale.dtype) + # Online weight updates do not necessarily run under a CUDA device context. + padded_scale = torch.zeros( + (B, M_padded, K_padded), dtype=scale.dtype, device=scale.device + ) padded_scale[:B, :M, :K] = scale batches, rows, cols = padded_scale.shape assert rows % 128 == 0