[AMD] [GLM-5.3-Flash] Enable FP8 and MXFP4 serving on gfx950 (#39273)
Co-authored-by: Byron Hsu <byron+per@periodiclabs.ai> Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Co-authored-by: long10024070 <long10024070@users.noreply.github.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: sunxxuns <126995791+sunxxuns@users.noreply.github.com> Co-authored-by: Long Luong <long.luong2+personal@onemount.com>
This commit is contained in:
co-authored by
Byron Hsu
Cheng Wan
long10024070
Kevin Mi
Kevin Mi
Claude Opus 5.5
sunxxuns
Long Luong
parent
80bb3fb651
commit
f17f7705a5
@@ -31,6 +31,7 @@ namespace {
|
||||
inline constexpr int kGroupTopK = SGL_GROUP_TOPK;
|
||||
inline constexpr int kThreadsPerBlock = 1024;
|
||||
|
||||
// Keep 4K threshold-bin candidates per round; rescan larger bins without clipping.
|
||||
inline constexpr std::size_t kSmem = 8 * 1024 * sizeof(uint32_t); // 32KB (bytes)
|
||||
|
||||
struct FastTopKParams {
|
||||
@@ -53,6 +54,11 @@ __device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto make_topk_key(float score, int index) -> uint64_t {
|
||||
// Unique keys resolve an overfull bin even when all scores are equal.
|
||||
return (static_cast<uint64_t>(convert_to_uint32(score)) << 32) | ~static_cast<uint32_t>(index);
|
||||
}
|
||||
|
||||
template <int K>
|
||||
__device__ void
|
||||
fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
|
||||
@@ -66,6 +72,7 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
|
||||
alignas(128) __shared__ int s_counter;
|
||||
alignas(128) __shared__ int s_threshold_bin_id;
|
||||
alignas(128) __shared__ int s_num_input[2];
|
||||
alignas(128) __shared__ uint64_t s_key_prefix;
|
||||
|
||||
auto& s_histogram = s_histogram_buf[0];
|
||||
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
|
||||
@@ -108,6 +115,7 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
|
||||
|
||||
const auto threshold_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[threshold_bin + 1];
|
||||
const auto num_threshold_candidates = s_histogram[threshold_bin] - s_histogram[threshold_bin + 1];
|
||||
|
||||
if (topk == 0) {
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
@@ -119,6 +127,63 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else if (num_threshold_candidates > int(SMEM_INPUT_SIZE)) {
|
||||
// Rescan the full row only on overflow. Preserve the unsorted output contract.
|
||||
if (tx == 0) s_key_prefix = 0;
|
||||
__syncthreads();
|
||||
|
||||
#pragma unroll 8
|
||||
for (int round = 0; round < 8; ++round) {
|
||||
if (tx < RADIX + 1) s_histogram[tx] = 0;
|
||||
__syncthreads();
|
||||
const auto prefix = s_key_prefix;
|
||||
const auto offset = 56 - round * 8;
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto raw_input = input[idx + row_start];
|
||||
if (convert_to_uint8(raw_input) != threshold_bin) continue;
|
||||
const auto key = make_topk_key(raw_input, idx);
|
||||
const bool prefix_matches = round == 0 || (key >> (64 - round * 8)) == prefix;
|
||||
if (prefix_matches) ::atomicAdd(&s_histogram[(key >> offset) & 0xFF], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
run_cumsum();
|
||||
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) s_threshold_bin_id = tx;
|
||||
__syncthreads();
|
||||
const auto key_bin = s_threshold_bin_id;
|
||||
topk -= s_histogram[key_bin + 1];
|
||||
if (tx == 0) s_key_prefix = (prefix << 8) | static_cast<uint64_t>(key_bin);
|
||||
__syncthreads();
|
||||
|
||||
if (topk == 0) {
|
||||
const auto selected_prefix = s_key_prefix;
|
||||
const auto prefix_bits = (round + 1) * 8;
|
||||
if (tx == 0) s_counter = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
const auto raw_input = input[idx + row_start];
|
||||
const auto coarse_bin = convert_to_uint8(raw_input);
|
||||
const auto key_prefix = make_topk_key(raw_input, idx) >> (64 - prefix_bits);
|
||||
if (coarse_bin > threshold_bin || (coarse_bin == threshold_bin && key_prefix > selected_prefix)) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const auto threshold_key = s_key_prefix;
|
||||
if (tx == 0) s_counter = 0;
|
||||
__syncthreads();
|
||||
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
|
||||
if (make_topk_key(input[idx + row_start], idx) >= threshold_key) {
|
||||
const auto pos = ::atomicAdd(&s_counter, 1);
|
||||
index[pos] = idx;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
return;
|
||||
} else {
|
||||
__syncthreads();
|
||||
if (tx < RADIX + 1) {
|
||||
@@ -294,7 +359,11 @@ void setup_kernel_smem_once(host::DebugInfo where = {}) {
|
||||
[[maybe_unused]]
|
||||
static const auto result = [] {
|
||||
const auto fptr = std::bit_cast<const void*>(f);
|
||||
#if defined(__HIP_PLATFORM_AMD__)
|
||||
return ::hipFuncSetAttribute(fptr, ::hipFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
|
||||
#else
|
||||
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
|
||||
#endif
|
||||
}();
|
||||
host::RuntimeDeviceCheck(result, where);
|
||||
}
|
||||
@@ -326,7 +395,7 @@ struct KpoolTopKTransformKernel {
|
||||
auto S = SymbolicSize{"score_stride"};
|
||||
auto out_cols_sym = SymbolicSize{"out_cols"};
|
||||
auto device = SymbolicDevice{};
|
||||
device.set_options<kDLCUDA>();
|
||||
device.set_options<kDLGPU>();
|
||||
|
||||
TensorMatcher({B, -1}).with_strides({S, 1}).with_dtype<float>().with_device(device).verify(score);
|
||||
TensorMatcher({B}).with_dtype<int32_t>().with_device(device).verify(lengths);
|
||||
|
||||
@@ -36,12 +36,33 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl, is_hip_runtime
|
||||
|
||||
# V-tile width of the fused verify kernel. Tuned on B200 at T=5 with
|
||||
# benchmark/kernels/bench_kda_verify_sweep.py; any power of two is
|
||||
# numerics-safe at num_warps=4 (bit-exact vs the BV=32 original).
|
||||
KDA_VERIFY_BLOCK_V = 4
|
||||
# gfx950 V-tile width for the measured GLM TP4 shapes: wider tiles share the q/k
|
||||
# convolution across more lanes
|
||||
KDA_VERIFY_BLOCK_V_HIP = 16
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _conv_product(x, weight, ROUND_PRODUCT: tl.constexpr):
|
||||
if ROUND_PRODUCT:
|
||||
# ROCm's unfused conv uses separately rounded fp32 products. Without
|
||||
# this boundary LLVM contracts the fused path into FMAs, which can
|
||||
# change the bf16 conv output before the recurrent update.
|
||||
return tl.inline_asm_elementwise(
|
||||
"v_mul_f32 $0, $1, $2",
|
||||
constraints="=v,v,v",
|
||||
args=[x.to(tl.float32), weight.to(tl.float32)],
|
||||
dtype=tl.float32,
|
||||
is_pure=True,
|
||||
pack=1,
|
||||
)
|
||||
else:
|
||||
return x * weight
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -89,6 +110,7 @@ def fused_kda_conv_gating_verify_kernel(
|
||||
USE_LOWER_BOUND: tl.constexpr,
|
||||
SAVE_INTERMEDIATE_WINDOW: tl.constexpr,
|
||||
CACHE_INTERMEDIATE_STATES: tl.constexpr,
|
||||
ROUND_CONV_PRODUCTS: tl.constexpr,
|
||||
USE_GDC: tl.constexpr = False,
|
||||
# ReplaySSM fused ring-write (spec verify): per-slot rings consumed by the
|
||||
# commit-time exact fold (kda_replayssm_spec_decode.py). Off -> dead code.
|
||||
@@ -211,18 +233,18 @@ def fused_kda_conv_gating_verify_kernel(
|
||||
acc_q = tl.zeros([BK], dtype=tl.float32)
|
||||
acc_k = tl.zeros([BK], dtype=tl.float32)
|
||||
acc_v = tl.zeros([BV], dtype=tl.float32)
|
||||
acc_q += q_c0 * wq0
|
||||
acc_q += q_c1 * wq1
|
||||
acc_q += q_c2 * wq2
|
||||
acc_q += x_q * wq3
|
||||
acc_k += k_c0 * wk0
|
||||
acc_k += k_c1 * wk1
|
||||
acc_k += k_c2 * wk2
|
||||
acc_k += x_k * wk3
|
||||
acc_v += v_c0 * wv0
|
||||
acc_v += v_c1 * wv1
|
||||
acc_v += v_c2 * wv2
|
||||
acc_v += x_v * wv3
|
||||
acc_q += _conv_product(q_c0, wq0, ROUND_CONV_PRODUCTS)
|
||||
acc_q += _conv_product(q_c1, wq1, ROUND_CONV_PRODUCTS)
|
||||
acc_q += _conv_product(q_c2, wq2, ROUND_CONV_PRODUCTS)
|
||||
acc_q += _conv_product(x_q, wq3, ROUND_CONV_PRODUCTS)
|
||||
acc_k += _conv_product(k_c0, wk0, ROUND_CONV_PRODUCTS)
|
||||
acc_k += _conv_product(k_c1, wk1, ROUND_CONV_PRODUCTS)
|
||||
acc_k += _conv_product(k_c2, wk2, ROUND_CONV_PRODUCTS)
|
||||
acc_k += _conv_product(x_k, wk3, ROUND_CONV_PRODUCTS)
|
||||
acc_v += _conv_product(v_c0, wv0, ROUND_CONV_PRODUCTS)
|
||||
acc_v += _conv_product(v_c1, wv1, ROUND_CONV_PRODUCTS)
|
||||
acc_v += _conv_product(v_c2, wv2, ROUND_CONV_PRODUCTS)
|
||||
acc_v += _conv_product(x_v, wv3, ROUND_CONV_PRODUCTS)
|
||||
|
||||
# Slide the window (reference: col0=col1; col1=col2; col2=x).
|
||||
q_c0 = q_c1
|
||||
@@ -420,7 +442,8 @@ def fused_kda_conv_gating_verify(
|
||||
# T=8 safe gate. conv_state is not comparable to the reference at all.
|
||||
# The ReplaySSM ring values are bit-exact at any num_warps: they are
|
||||
# elementwise (conv FMA chain, gate, sigmoid), upstream of every tl.sum.
|
||||
num_warps: int = 4,
|
||||
# None picks 1 on ROCm (faster on gfx950 and bit-exact with the reference), else 4
|
||||
num_warps: Optional[int] = None,
|
||||
# ReplaySSM fused ring-write; same parameter names as the unfused
|
||||
# fused_sigmoid_gating_delta_rule_update so ring_kwargs pass through both.
|
||||
cache_ring: bool = False,
|
||||
@@ -435,6 +458,8 @@ def fused_kda_conv_gating_verify(
|
||||
seq_len, dim = mixed_qkv.shape
|
||||
B = seq_len // T
|
||||
W = conv_weight.shape[1]
|
||||
if num_warps is None:
|
||||
num_warps = 1 if is_hip_runtime() else 4
|
||||
|
||||
assert mixed_qkv.stride(-1) == 1, "mixed_qkv must be contiguous in dim"
|
||||
assert dim == 2 * H * K + HV * V, f"packed dim mismatch: {dim}"
|
||||
@@ -458,6 +483,16 @@ def fused_kda_conv_gating_verify(
|
||||
# the gated RMSNorm into this kernel's epilogue is a dead end; it is
|
||||
# PDL-chained behind this kernel instead (see fused_norm_gate.py).
|
||||
BV = min(triton.next_power_of_2(V), KDA_VERIFY_BLOCK_V)
|
||||
if (
|
||||
is_hip_runtime()
|
||||
and num_warps == 1
|
||||
and T in (6, 8)
|
||||
and H == HV == 16
|
||||
and K == V == 128
|
||||
and conv_weight.dtype == torch.float32
|
||||
and 3 <= B <= 16
|
||||
):
|
||||
BV = KDA_VERIFY_BLOCK_V_HIP
|
||||
NV = triton.cdiv(V, BV)
|
||||
|
||||
a2 = a.reshape(seq_len, HV * K)
|
||||
@@ -572,6 +607,7 @@ def fused_kda_conv_gating_verify(
|
||||
USE_LOWER_BOUND=lower_bound is not None,
|
||||
SAVE_INTERMEDIATE_WINDOW=intermediate_conv_window is not None,
|
||||
CACHE_INTERMEDIATE_STATES=intermediate_states_buffer is not None,
|
||||
ROUND_CONV_PRODUCTS=is_hip_runtime() and conv_weight.dtype == torch.float32,
|
||||
replayssm_rawv=replayssm_rawv,
|
||||
replayssm_rawk=replayssm_rawk,
|
||||
replayssm_g=replayssm_g,
|
||||
|
||||
@@ -1166,6 +1166,30 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return "text", processed_messages.prompt_ids
|
||||
return "input_ids", processed_messages.prompt_ids
|
||||
|
||||
def _can_reuse_text_only_prompt_ids(
|
||||
self, processed_messages: MessageProcessingResult, is_multimodal: bool
|
||||
) -> bool:
|
||||
# Moss-VL invokes its processor for text-only requests, and that processor
|
||||
# requires the rendered text rather than pre-tokenized ids.
|
||||
is_moss_vl = (
|
||||
"MossVLForConditionalGeneration"
|
||||
in self.tokenizer_manager.model_config.hf_config.architectures
|
||||
)
|
||||
return (
|
||||
is_multimodal
|
||||
and not is_moss_vl
|
||||
and self.chat_encoding_spec is None
|
||||
and self.template_manager.chat_template_name is None
|
||||
and not self._prompt_text_round_trip_is_lossy
|
||||
and not self._tokenizer_auto_adds_specials
|
||||
and isinstance(processed_messages.prompt_ids, list)
|
||||
and bool(processed_messages.prompt_ids)
|
||||
and not processed_messages.image_data
|
||||
and not processed_messages.video_data
|
||||
and not processed_messages.audio_data
|
||||
and not processed_messages.modalities
|
||||
)
|
||||
|
||||
def _convert_to_internal_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -1233,7 +1257,9 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
|
||||
# Handle single vs multiple requests
|
||||
if request.input_ids is not None:
|
||||
if request.input_ids is not None or self._can_reuse_text_only_prompt_ids(
|
||||
processed_messages, is_multimodal
|
||||
):
|
||||
prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
|
||||
else:
|
||||
prompt_key, prompt_value = self._engine_prompt(
|
||||
@@ -1337,8 +1363,12 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tool_call_constraint = None
|
||||
|
||||
effective_tools = self._effective_tools(request)
|
||||
glm_constraint = self.tool_call_parser == "glm47" and not any(
|
||||
tool.function.strict for tool in effective_tools
|
||||
# Only tool-bearing requests get the full-assistant EBNF: its terminal
|
||||
# state finishes a request even under ignore_eos.
|
||||
glm_constraint = (
|
||||
self.tool_call_parser == "glm47"
|
||||
and bool(effective_tools)
|
||||
and not any(tool.function.strict for tool in effective_tools)
|
||||
)
|
||||
if glm_constraint:
|
||||
enable_thinking = (request.chat_template_kwargs or {}).get(
|
||||
|
||||
@@ -23,7 +23,6 @@ from sglang.srt.layers.attention.dsa.utils import (
|
||||
dsa_use_prefill_cp,
|
||||
)
|
||||
from sglang.srt.layers.attention.mqa_logits_utils import (
|
||||
MQA_LOGITS_BYTES_PER_ELEM,
|
||||
MQA_LOGITS_MAX_BYTES_ROCM,
|
||||
mqa_logits_budget_bytes,
|
||||
mqa_logits_row_bytes,
|
||||
@@ -901,7 +900,8 @@ class IndexerKPool(MultiPlatformOp):
|
||||
num_q, num_k = q_fp8.shape[0], k_fp8.shape[0]
|
||||
rows_per_call = mqa_logits_rows_per_chunk(
|
||||
num_rows=num_q,
|
||||
row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM,
|
||||
# aiter pads each output row to 256 columns
|
||||
row_bytes=mqa_logits_row_bytes(num_k),
|
||||
budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM,
|
||||
)
|
||||
if rows_per_call is None:
|
||||
|
||||
@@ -122,7 +122,7 @@ def _gather_index_k_scale_prefix_into_kernel(
|
||||
token_id = tl.program_id(0)
|
||||
page_idx = token_id // PAGE_SIZE
|
||||
token_offset_in_page = token_id % PAGE_SIZE
|
||||
page = tl.load(page_indices_ptr + page_idx)
|
||||
page = tl.load(page_indices_ptr + page_idx).to(tl.int64)
|
||||
|
||||
offs = tl.arange(0, BLOCK_D)
|
||||
mask = offs < HEAD_DIM
|
||||
|
||||
@@ -44,10 +44,19 @@ class DsaGraphVariants:
|
||||
|
||||
|
||||
def create_attention_graph_variants(hf_config) -> Optional[AttentionGraphVariants]:
|
||||
from sglang.srt.configs.model_config import get_dsa_index_topk, is_deepseek_dsa
|
||||
from sglang.srt.configs.model_config import (
|
||||
get_dsa_index_kpool,
|
||||
get_dsa_index_topk,
|
||||
is_deepseek_dsa,
|
||||
)
|
||||
from sglang.srt.utils import is_hip
|
||||
|
||||
if is_hip() and is_deepseek_dsa(hf_config):
|
||||
# KPool has no dense-skip path: both variants run the full indexer.
|
||||
# Capture it once, keeping the regular DSA dense/sparse split intact.
|
||||
if get_dsa_index_kpool(hf_config) > 1:
|
||||
return None
|
||||
|
||||
index_topk = get_dsa_index_topk(hf_config)
|
||||
logger.info(
|
||||
"[dense-decode] DSA dual-graph enabled: capturing "
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.srt.layers.attention.linear.utils import (
|
||||
select_verify_intermediate_state_indices,
|
||||
)
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_npu
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_npu
|
||||
from sglang.srt.utils.common import is_gfx95_supported, rank0_log
|
||||
|
||||
# KDA always uses the triton causal_conv1d_fn (no CUDA override).
|
||||
@@ -1285,6 +1285,16 @@ class KDAAttnBackend(MambaAttnBackendBase):
|
||||
# other batch/architecture combinations are measured. The snapshot
|
||||
# path and the separate CuTe path are unchanged.
|
||||
return False
|
||||
if is_hip() and not (
|
||||
is_gfx95_supported()
|
||||
and 1 <= batch_size <= 16
|
||||
and draft_token_num in (6, 8)
|
||||
and mixed_qkv.dtype == torch.bfloat16
|
||||
and layer.conv_weights.dtype == torch.float32
|
||||
):
|
||||
# measured gfx950 wins only: larger batches lose the launch saving to
|
||||
# duplicated convolution work
|
||||
return False
|
||||
expected_dim = (
|
||||
2 * layer.num_q_heads * layer.head_k_dim
|
||||
+ layer.num_v_heads * layer.head_v_dim
|
||||
|
||||
@@ -952,6 +952,9 @@ class ReqKvInfo:
|
||||
mamba_last_track_idx: Optional[int] = None # 0 or 1
|
||||
# Seq len of the last cached mamba state
|
||||
mamba_last_track_seqlen: Optional[int] = None
|
||||
# Seq len of the other ping-pong slot's state. None means what is in
|
||||
# that slot cannot be named (never written, donated, freed).
|
||||
mamba_prev_track_seqlen: Optional[int] = None
|
||||
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
|
||||
mamba_cow_src_index: Optional[torch.Tensor] = None
|
||||
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
|
||||
@@ -1475,7 +1478,14 @@ class Req(ReqDllmMixin):
|
||||
# overallocated range and are reclaimed by release_kv_cache. #22373.
|
||||
if get_serving().strip_thinking_cache and self.reasoning_tokens > 0:
|
||||
return min(self.kv.kv_committed_len, len(self.origin_input_ids))
|
||||
return self.kv.kv_committed_len
|
||||
if self.finished_len is None:
|
||||
return self.kv.kv_committed_len
|
||||
# A verify step commits a whole accepted chunk, so output_ids can run
|
||||
# past the stop; the client never sees those tokens, so nothing matches.
|
||||
return min(
|
||||
self.kv.kv_committed_len,
|
||||
len(self.origin_input_ids) + len(self.output_ids_through_stop),
|
||||
)
|
||||
|
||||
def update_spec_correct_drafts_histogram(self, num_correct_drafts: int):
|
||||
"""Record one step accepted draft count (excludes bonus token) into the histogram."""
|
||||
@@ -1988,6 +1998,7 @@ class Req(ReqDllmMixin):
|
||||
self.kv.mamba_next_track_idx = None
|
||||
self.kv.mamba_last_track_idx = None
|
||||
self.kv.mamba_last_track_seqlen = None
|
||||
self.kv.mamba_prev_track_seqlen = None
|
||||
self.mamba_branching_seqlen = None
|
||||
self.kv.mamba_cow_src_index = None
|
||||
self.kv.mamba_needs_clear = False
|
||||
@@ -2212,7 +2223,8 @@ def set_mamba_track_indices_from_reqs(
|
||||
if track_positions is None:
|
||||
# Guard: mamba_next_track_idx may be None for requests that haven't
|
||||
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
|
||||
# Default to 0 (first ping-pong slot) to avoid TypeError.
|
||||
# Keep the gather in bounds for those requests. Freed rows are
|
||||
# invalidated below so they cannot scatter into a recycled slot.
|
||||
track_positions = [
|
||||
(
|
||||
req.kv.mamba_next_track_idx
|
||||
@@ -2221,6 +2233,9 @@ def set_mamba_track_indices_from_reqs(
|
||||
)
|
||||
for req in batch.reqs
|
||||
]
|
||||
freed_rows = [
|
||||
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
|
||||
]
|
||||
batch.mamba_track_buffer_indices = list(track_positions)
|
||||
idx = (
|
||||
torch.tensor(
|
||||
@@ -2234,6 +2249,12 @@ def set_mamba_track_indices_from_reqs(
|
||||
batch.mamba_track_indices = (
|
||||
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
|
||||
)
|
||||
if freed_rows:
|
||||
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
|
||||
# been freed and reused by another request. The downstream scatter
|
||||
# treats a negative destination as a no-op, while the old fallback to
|
||||
# position 0 could overwrite the reused live slot.
|
||||
batch.mamba_track_indices[freed_rows] = -1
|
||||
|
||||
|
||||
def release_req(
|
||||
|
||||
@@ -648,7 +648,12 @@ class PrefillAdder:
|
||||
self.rem_chunk_tokens = rem_chunk_tokens
|
||||
self.chunked_req_limit: Optional[int] = None
|
||||
self.dllm_config = dllm_config
|
||||
self.exact_chunk_fill = _use_exact_chunk_fill() and dllm_config is None
|
||||
# Mamba checkpoints only land on page-aligned chunk ends.
|
||||
self.exact_chunk_fill = (
|
||||
_use_exact_chunk_fill()
|
||||
and dllm_config is None
|
||||
and not tree_cache.supports_mamba()
|
||||
)
|
||||
|
||||
if self.dllm_config is not None:
|
||||
self._init_dllm_meta(dllm_config)
|
||||
|
||||
@@ -1366,6 +1366,9 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
if completed_mamba_boundary and not lazy:
|
||||
req.kv.mamba_last_track_idx = batch.mamba_track_buffer_indices[i]
|
||||
# The slot that stops being the latest still holds its
|
||||
# checkpoint; name it so a short key can fall back to it.
|
||||
req.kv.mamba_prev_track_seqlen = req.kv.mamba_last_track_seqlen
|
||||
req.kv.mamba_last_track_seqlen = req.kv.kv_committed_len - lookahead
|
||||
elif (
|
||||
req.finished()
|
||||
@@ -1495,13 +1498,17 @@ class SchedulerBatchResultProcessor:
|
||||
track_idx = req.kv.mamba_next_track_idx
|
||||
if not known_boundary and batch.mamba_track_buffer_indices is not None:
|
||||
track_idx = batch.mamba_track_buffer_indices[i]
|
||||
previous_track_seqlen = req.kv.mamba_last_track_seqlen
|
||||
if not known_boundary:
|
||||
req.kv.mamba_last_track_seqlen = track_seqlen
|
||||
if lazy:
|
||||
# Lazy frees the slot it stops tracking, so nothing names the
|
||||
# previous checkpoint there; mamba_prev_track_seqlen stays None.
|
||||
self.mamba_lazy_post_decode_at_boundary(req, batch, track_idx)
|
||||
else:
|
||||
if not known_boundary:
|
||||
req.kv.mamba_last_track_idx = track_idx
|
||||
req.kv.mamba_prev_track_seqlen = previous_track_seqlen
|
||||
req.kv.mamba_next_track_idx = (
|
||||
batch.req_to_token_pool.get_mamba_ping_pong_other_idx(track_idx)
|
||||
)
|
||||
|
||||
@@ -81,6 +81,8 @@ class InsertParams:
|
||||
|
||||
# Mamba specific
|
||||
mamba_value: Optional[torch.Tensor] = None
|
||||
# The ping-pong slot prepare picked; cleanup keeps that same slot.
|
||||
mamba_keep_idx: Optional[int] = None
|
||||
|
||||
# DSV4 NPU C128 sidecar pages, one page id per physical C128 page group.
|
||||
c128_value: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -346,7 +346,12 @@ def _release_overallocated_kv_indices(
|
||||
# strip_thinking_cache intentionally reports output tokens as overallocated
|
||||
# so they fall into the free path below (#22373).
|
||||
if spec_algo is None and not get_serving().strip_thinking_cache:
|
||||
assert start_p == end_p, (
|
||||
# A stop landing before the last committed token does the same, via
|
||||
# effective_kv_committed_len().
|
||||
assert start_p == end_p or (
|
||||
req.finished_len is not None
|
||||
and len(req.origin_input_ids) + req.finished_len < req.kv.kv_committed_len
|
||||
), (
|
||||
f"Unexpected overallocated KV cache, {req.kv.kv_committed_len=}, {req.kv.kv_allocated_len=}"
|
||||
)
|
||||
|
||||
|
||||
@@ -1620,6 +1620,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
)
|
||||
buf[:n] = slots
|
||||
req.kv.mamba_ping_pong_track_buffer = buf
|
||||
req.kv.mamba_prev_track_seqlen = None
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = (
|
||||
0
|
||||
@@ -1720,6 +1721,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
req.kv.mamba_prev_track_seqlen = None
|
||||
req.kv.mamba_cow_src_index = None
|
||||
req.kv.mamba_needs_clear = False
|
||||
|
||||
|
||||
@@ -536,6 +536,28 @@ class MambaComponent(TreeComponent):
|
||||
else:
|
||||
self.cache.req_to_token_pool.mamba_allocator.free(mamba_value)
|
||||
|
||||
def _select_finished_checkpoint(
|
||||
self, req: Req, token_ids_len: int
|
||||
) -> Optional[tuple[int, int]]:
|
||||
# None means donate nothing, not "no slot found".
|
||||
pool = self.cache.req_to_token_pool
|
||||
keep_idx = pool.get_mamba_ping_pong_keep_idx(req)
|
||||
cache_len = req.kv.mamba_last_track_seqlen or 0
|
||||
|
||||
if cache_len <= token_ids_len:
|
||||
return cache_len, keep_idx
|
||||
|
||||
# Overshoot: the latest state ran past the key. The other slot always
|
||||
# holds a tensor; only this seqlen says whether a key can name it.
|
||||
previous_cache_len = req.kv.mamba_prev_track_seqlen
|
||||
if (
|
||||
pool.mamba_ping_pong_track_buffer_size != 2
|
||||
or previous_cache_len is None
|
||||
or previous_cache_len > token_ids_len
|
||||
):
|
||||
return None
|
||||
return previous_cache_len, pool.get_mamba_ping_pong_other_idx(keep_idx)
|
||||
|
||||
def prepare_for_caching_req(
|
||||
self,
|
||||
req: Req,
|
||||
@@ -564,9 +586,11 @@ class MambaComponent(TreeComponent):
|
||||
if cache_len is None:
|
||||
cache_len = 0
|
||||
if self.cache.enable_mamba_extra_buffer:
|
||||
keep_idx = self.cache.req_to_token_pool.get_mamba_ping_pong_keep_idx(
|
||||
req
|
||||
)
|
||||
checkpoint = self._select_finished_checkpoint(req, token_ids_len)
|
||||
if checkpoint is None:
|
||||
return 0
|
||||
cache_len, keep_idx = checkpoint
|
||||
insert_params.mamba_keep_idx = keep_idx
|
||||
active_value = (
|
||||
req.kv.mamba_ping_pong_track_buffer[keep_idx].unsqueeze(-1).clone()
|
||||
)
|
||||
@@ -639,11 +663,11 @@ class MambaComponent(TreeComponent):
|
||||
return
|
||||
|
||||
if self.cache.enable_mamba_extra_buffer:
|
||||
keep_idx = (
|
||||
pool.get_mamba_ping_pong_keep_idx(req)
|
||||
if mamba_value_inserted
|
||||
else None
|
||||
# Keep the slot prepare picked, so cleanup cannot pick another.
|
||||
prepared_keep_idx = (
|
||||
insert_params.mamba_keep_idx if insert_params is not None else None
|
||||
)
|
||||
keep_idx = prepared_keep_idx if mamba_value_inserted else None
|
||||
pool.free_mamba_cache(
|
||||
req, mamba_ping_pong_track_buffer_to_keep=keep_idx
|
||||
)
|
||||
@@ -657,6 +681,7 @@ class MambaComponent(TreeComponent):
|
||||
):
|
||||
self._free_mamba_value(insert_params.mamba_value)
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
req.kv.mamba_prev_track_seqlen = None
|
||||
|
||||
def build_external_linker_transfer(
|
||||
self,
|
||||
|
||||
@@ -526,8 +526,8 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
|
||||
declared is SharedReadEnds.IN_REPLAY
|
||||
and self.in_graph_metadata_prep_done is None
|
||||
):
|
||||
# TODO: this lands EARLIER than declared; POST_REPLAY is the sound one.
|
||||
return SharedReadEnds.PRE_REPLAY
|
||||
# no in-graph marker (e.g. HIP): only a post-replay event covers its reads
|
||||
return SharedReadEnds.POST_REPLAY
|
||||
return declared
|
||||
|
||||
def _publish_read_done(self, in_graph: bool):
|
||||
|
||||
@@ -638,9 +638,7 @@ class DeepseekMHAForwardMixin:
|
||||
|
||||
Returns: (kv_a, k_pe) both in BF16
|
||||
"""
|
||||
backend = get_attn_backend()
|
||||
if isinstance(backend, TboAttnBackend): # if enable tbo, get primary backend
|
||||
backend = backend.primary
|
||||
backend = resolve_attn_backend(forward_batch)
|
||||
kv_indices = backend.forward_metadata.page_table_1_flattened
|
||||
assert kv_indices is not None, (
|
||||
"page_table_1_flattened should have been generated for FP8 MHA path"
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
|
||||
from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
PLAIN_RESIDUAL_OPS,
|
||||
MHCState,
|
||||
@@ -317,11 +318,14 @@ class Glm5NextVisionModel(GlmOcrVisionModel):
|
||||
swiglu_limit=vision_config.swiglu_limit,
|
||||
)
|
||||
|
||||
self.downsample = nn.Conv2d(
|
||||
# These non-overlapping patches are equivalent to unfold + linear and
|
||||
# avoid MIOpen's expensive per-shape convolution search on ROCm.
|
||||
self.downsample = Conv2dLayer(
|
||||
in_channels=vision_config.hidden_size,
|
||||
out_channels=vision_config.out_hidden_size,
|
||||
kernel_size=vision_config.spatial_merge_size,
|
||||
stride=vision_config.spatial_merge_size,
|
||||
disable_linear=False,
|
||||
)
|
||||
self.post_layernorm = GlmOcrRMSNorm(
|
||||
vision_config.hidden_size, eps=vision_config.rms_norm_eps
|
||||
@@ -1596,8 +1600,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
def maybe_map_fp8_block_scale_name(name: str) -> str:
|
||||
if name.endswith("weight_scale"):
|
||||
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
|
||||
# Quark stores dequantization scales without native block-FP8's
|
||||
# "_inv" suffix, including fused w13/w2 expert parameters.
|
||||
if name not in params_dict and name.endswith("weight_scale"):
|
||||
candidate = name + "_inv"
|
||||
if candidate in params_dict:
|
||||
return candidate
|
||||
return name
|
||||
|
||||
@@ -59,6 +59,8 @@ from sglang.srt.utils.device_timer import device_timer_ctx
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker
|
||||
|
||||
_GREEDY_VARIANT = "greedy"
|
||||
|
||||
|
||||
@dataclass
|
||||
class EagleDraftInputBuffers(ForwardInputBuffers):
|
||||
@@ -307,8 +309,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
return torch.int64
|
||||
|
||||
def _make_graph_key(self, bs, stream_idx=None, variant_label=None):
|
||||
# EAGLE doesn't use stream_idx / lora variants.
|
||||
return ShapeKey(size=bs)
|
||||
# EAGLE doesn't use stream_idx; variant_label is _GREEDY_VARIANT or None.
|
||||
return ShapeKey(size=bs, variant_label=variant_label)
|
||||
|
||||
def _sampling_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
|
||||
# Under rejection sampling an all-greedy batch replays the graph that
|
||||
# skips the draft proposal distribution (see draft_forward).
|
||||
if (
|
||||
get_spec().speculative_use_rejection_sampling
|
||||
and forward_batch.sampling_info is not None
|
||||
and forward_batch.sampling_info.is_all_greedy
|
||||
):
|
||||
return _GREEDY_VARIANT
|
||||
return None
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# can_run_graph
|
||||
@@ -332,7 +345,13 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
cuda_graph_bs = forward_batch.batch_size
|
||||
|
||||
is_bs_supported = (
|
||||
self.backend.can_run(forward_batch, self._make_graph_key(cuda_graph_bs))
|
||||
self.backend.can_run(
|
||||
forward_batch,
|
||||
self._make_graph_key(
|
||||
cuda_graph_bs,
|
||||
variant_label=self._sampling_variant(forward_batch),
|
||||
),
|
||||
)
|
||||
if self.disable_padding
|
||||
else cuda_graph_bs <= self.max_bs
|
||||
)
|
||||
@@ -498,7 +517,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
# per-step forwards inside draft_forward must not re-plan.
|
||||
forward_batch.mark_forward_metadata_ready()
|
||||
self.deepep_adapter.capture(is_extend_in_batch=False)
|
||||
shape_key = self._make_graph_key(num_seqs)
|
||||
post_warmup_hook = getattr(
|
||||
self.draft_attn_backend, "on_after_cuda_graph_warmup", None
|
||||
)
|
||||
@@ -508,12 +526,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
post_warmup_hook=post_warmup_hook,
|
||||
run_lm_head=True,
|
||||
)
|
||||
self.backend.capture_one(
|
||||
shape_key,
|
||||
run_once,
|
||||
capture_inputs=None,
|
||||
post_warmup_hook=post_warmup_hook,
|
||||
variants = (
|
||||
(None, _GREEDY_VARIANT)
|
||||
if get_spec().speculative_use_rejection_sampling
|
||||
else (None,)
|
||||
)
|
||||
for variant in variants:
|
||||
sampling_info.is_all_greedy = variant == _GREEDY_VARIANT
|
||||
self.backend.capture_one(
|
||||
self._make_graph_key(num_seqs, variant_label=variant),
|
||||
run_once,
|
||||
capture_inputs=None,
|
||||
post_warmup_hook=post_warmup_hook,
|
||||
)
|
||||
|
||||
def _postprocess_output_to_raw_bs(self, out, raw_bs):
|
||||
parent_list, top_scores_index, draft_tokens, draft_probs = (
|
||||
@@ -694,7 +719,9 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
self.bs = bs
|
||||
|
||||
# Replay via backend
|
||||
shape_key = self._make_graph_key(bs)
|
||||
shape_key = self._make_graph_key(
|
||||
bs, variant_label=self._sampling_variant(forward_batch)
|
||||
)
|
||||
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft"):
|
||||
out = self._replay_graph(shape_key, forward_batch)
|
||||
if self.buffers.dsa_seed_topk is not None:
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Callable, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
|
||||
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
deployment_attn_dp_size,
|
||||
@@ -631,12 +632,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
)
|
||||
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
|
||||
|
||||
# Snapshot built -- the forward is done reading the shared pool. Publish
|
||||
# a read-done event the scheduler's WAR barrier waits on (draft extend
|
||||
# is the EAGLE-family last shared-read phase; last write wins the mailbox).
|
||||
read_done = self.device_module.Event()
|
||||
read_done.record()
|
||||
self.model_runner.shared_read_done_event = read_done
|
||||
# Draft extend is the last shared reader. UNKNOWN must discard any
|
||||
# earlier phase's event so the scheduler uses its whole-forward fence.
|
||||
self.model_runner.shared_read_done_event = None
|
||||
shared_read_ends = self.draft_extend_attn_backend.shared_read_ends(
|
||||
self.forward_mode
|
||||
)
|
||||
if shared_read_ends is SharedReadEnds.PRE_REPLAY:
|
||||
self._publish_read_done(in_graph=False)
|
||||
|
||||
self.raw_bs = raw_bs
|
||||
self.bs = bs
|
||||
@@ -644,6 +647,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft_extend"):
|
||||
out = self._replay_graph(shape_key, forward_batch)
|
||||
|
||||
# This runner has no in-graph read-done marker. A later event is safe;
|
||||
# an event before replay cannot cover metadata reads inside the graph.
|
||||
if shared_read_ends in (SharedReadEnds.IN_REPLAY, SharedReadEnds.POST_REPLAY):
|
||||
self._publish_read_done(in_graph=False)
|
||||
|
||||
out = LogitsProcessorOutput(
|
||||
next_token_logits=out.next_token_logits[:raw_bs],
|
||||
# CUDA graph replay reuses its captured output storage. These states
|
||||
|
||||
@@ -700,7 +700,13 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
score_list: List[torch.Tensor] = []
|
||||
token_list: List[torch.Tensor] = []
|
||||
parents_list: List[torch.Tensor] = []
|
||||
if get_spec().speculative_use_rejection_sampling:
|
||||
# Verify commits argmax for an all-greedy batch and never reads the
|
||||
# draft distribution, so skip building it.
|
||||
needs_draft_probs = (
|
||||
get_spec().speculative_use_rejection_sampling
|
||||
and not forward_batch.sampling_info.is_all_greedy
|
||||
)
|
||||
if needs_draft_probs:
|
||||
draft_probs_list: List[torch.Tensor] = [spec_info.draft_probs]
|
||||
|
||||
topk1_chain_fits = (
|
||||
@@ -715,7 +721,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
topk1_chain_fits
|
||||
and _is_cuda
|
||||
and self.hot_token_id is None
|
||||
and not get_spec().speculative_use_rejection_sampling
|
||||
and not needs_draft_probs
|
||||
):
|
||||
draft_tokens_topk1 = torch.empty(
|
||||
(topk_index.shape[0], self.speculative_num_steps),
|
||||
@@ -778,7 +784,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
maybe_detect_inf(
|
||||
logits_output.next_token_logits, f"draft_forward step {i}"
|
||||
)
|
||||
if get_spec().speculative_use_rejection_sampling:
|
||||
if needs_draft_probs:
|
||||
probs, topk_p, topk_index = sample_draft_proposal(
|
||||
logits_output.next_token_logits,
|
||||
forward_batch.sampling_info.temperatures,
|
||||
@@ -804,7 +810,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
probs = renorm_draft_probs(
|
||||
logits_output.next_token_logits,
|
||||
forward_batch.sampling_info,
|
||||
get_spec().speculative_use_rejection_sampling,
|
||||
needs_draft_probs,
|
||||
)
|
||||
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
|
||||
forward_batch.positions.add_(1)
|
||||
@@ -821,9 +827,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
hidden_states = logits_output.hidden_states
|
||||
|
||||
draft_probs = (
|
||||
torch.stack(draft_probs_list, dim=1)
|
||||
if get_spec().speculative_use_rejection_sampling
|
||||
else None
|
||||
torch.stack(draft_probs_list, dim=1) if needs_draft_probs else None
|
||||
)
|
||||
|
||||
# Organize the results
|
||||
|
||||
Reference in New Issue
Block a user