chore: fix some typos (#18577)
Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Liangsheng Yin
parent
a2c38f7796
commit
8d2892330c
@@ -32,7 +32,7 @@ class DoubleSparseAttnBackend(AttentionBackend):
|
|||||||
self.heavy_token_num = model_runner.server_args.ds_heavy_token_num
|
self.heavy_token_num = model_runner.server_args.ds_heavy_token_num
|
||||||
|
|
||||||
self.sorted_channels = model_runner.sorted_channels
|
self.sorted_channels = model_runner.sorted_channels
|
||||||
self.sparse_decode_thresold = (
|
self.sparse_decode_threshold = (
|
||||||
model_runner.server_args.ds_sparse_decode_threshold
|
model_runner.server_args.ds_sparse_decode_threshold
|
||||||
)
|
)
|
||||||
self.att_out_approx: torch.Tensor = None
|
self.att_out_approx: torch.Tensor = None
|
||||||
@@ -210,7 +210,7 @@ class DoubleSparseAttnBackend(AttentionBackend):
|
|||||||
# and set a minimum value for sparse_decode
|
# and set a minimum value for sparse_decode
|
||||||
if (
|
if (
|
||||||
min_seq_len < self.heavy_token_num
|
min_seq_len < self.heavy_token_num
|
||||||
or max_seq_len < self.sparse_decode_thresold
|
or max_seq_len < self.sparse_decode_threshold
|
||||||
):
|
):
|
||||||
self.decode_attention_fwd(
|
self.decode_attention_fwd(
|
||||||
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
q.view(-1, layer.tp_q_head_num, layer.qk_head_dim),
|
||||||
|
|||||||
@@ -219,7 +219,7 @@ def _per_token_group_quant_8bit_raw(
|
|||||||
quantized tensor along with the scaling factor used for quantization.
|
quantized tensor along with the scaling factor used for quantization.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
x: The input tenosr with ndim >= 2.
|
x: The input tensor with ndim >= 2.
|
||||||
group_size: The group size used for quantization.
|
group_size: The group size used for quantization.
|
||||||
eps: The minimum to avoid dividing zero.
|
eps: The minimum to avoid dividing zero.
|
||||||
dtype: The dype of output tensor.
|
dtype: The dype of output tensor.
|
||||||
@@ -635,7 +635,7 @@ def static_quant_fp8(
|
|||||||
quantized tensor along with the scaling factor used for quantization.
|
quantized tensor along with the scaling factor used for quantization.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
x: The input tenosr with ndim >= 2.
|
x: The input tensor with ndim >= 2.
|
||||||
x_s: The quantization scale.
|
x_s: The quantization scale.
|
||||||
repeat_scale: Whether to broadcast per-tensor scale to per-channel scale.
|
repeat_scale: Whether to broadcast per-tensor scale to per-channel scale.
|
||||||
dtype: The dype of output tensor.
|
dtype: The dype of output tensor.
|
||||||
|
|||||||
@@ -143,7 +143,7 @@ def per_token_group_quant_int8(
|
|||||||
quantized tensor along with the scaling factor used for quantization.
|
quantized tensor along with the scaling factor used for quantization.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
x: The input tenosr with ndim >= 2.
|
x: The input tensor with ndim >= 2.
|
||||||
group_size: The group size used for quantization.
|
group_size: The group size used for quantization.
|
||||||
eps: The minimum to avoid dividing zero.
|
eps: The minimum to avoid dividing zero.
|
||||||
dtype: The dype of output tensor. Note that only `torch.int8` is supported for now.
|
dtype: The dype of output tensor. Note that only `torch.int8` is supported for now.
|
||||||
|
|||||||
@@ -128,10 +128,10 @@ def b_dynamic_mxfp4_quant(x):
|
|||||||
return x.view(h, b, d // 2), x_scales.view(h, b, d // 32)
|
return x.view(h, b, d // 2), x_scales.view(h, b, d // 32)
|
||||||
|
|
||||||
|
|
||||||
def mxfp4_to_f32(x, is_threed):
|
def mxfp4_to_f32(x, is_3d):
|
||||||
# 2 because we pack fp4 in uint8.
|
# 2 because we pack fp4 in uint8.
|
||||||
x = x.repeat_interleave(2, dim=-1)
|
x = x.repeat_interleave(2, dim=-1)
|
||||||
if is_threed:
|
if is_3d:
|
||||||
x[..., ::2] = x[..., ::2] & 0xF
|
x[..., ::2] = x[..., ::2] & 0xF
|
||||||
x[..., 1::2] = x[..., 1::2] >> 4
|
x[..., 1::2] = x[..., 1::2] >> 4
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -1271,7 +1271,7 @@ class UpdateWeightFromDiskReqInput(BaseReq):
|
|||||||
torch_empty_cache: bool = False
|
torch_empty_cache: bool = False
|
||||||
# Whether to keep the scheduler paused after weight update
|
# Whether to keep the scheduler paused after weight update
|
||||||
keep_pause: bool = False
|
keep_pause: bool = False
|
||||||
# Whether to recapture cuda graph after weight udpdate
|
# Whether to recapture cuda graph after weight update
|
||||||
recapture_cuda_graph: bool = False
|
recapture_cuda_graph: bool = False
|
||||||
# The trainer step id. Used to know which step's weights are used for sampling.
|
# The trainer step id. Used to know which step's weights are used for sampling.
|
||||||
token_step: int = 0
|
token_step: int = 0
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ TEST_SUITE = dict(
|
|||||||
|
|
||||||
class EngineWrapper:
|
class EngineWrapper:
|
||||||
"""
|
"""
|
||||||
A wrapper around Sglang engine to mock multi instance cases such as RL traing.
|
A wrapper around Sglang engine to mock multi instance cases such as RL training.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -198,17 +198,17 @@ def test_edge_case():
|
|||||||
assert result1 == result2
|
assert result1 == result2
|
||||||
print("Pass for normal test")
|
print("Pass for normal test")
|
||||||
|
|
||||||
class UnkownType:
|
class UnknownType:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
list_dispatcher(UnkownType())
|
list_dispatcher(UnknownType())
|
||||||
print("exception was thrown from list version as expected")
|
print("exception was thrown from list version as expected")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("exception thrown from list version was processed...")
|
print("exception thrown from list version was processed...")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
dict_dispatcher(UnkownType())
|
dict_dispatcher(UnknownType())
|
||||||
print("exception was thrown from dict version as expected")
|
print("exception was thrown from dict version as expected")
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print("exception thrown from dict version was processed...")
|
print("exception thrown from dict version was processed...")
|
||||||
|
|||||||
Reference in New Issue
Block a user