[AMD] [GLM-5.3-Flash] Enable FP8 and MXFP4 serving on gfx950 (#39273)

Co-authored-by: Byron Hsu <byron+per@periodiclabs.ai>
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
Co-authored-by: long10024070 <long10024070@users.noreply.github.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: sunxxuns <126995791+sunxxuns@users.noreply.github.com>
Co-authored-by: Long Luong <long.luong2+personal@onemount.com>
This commit is contained in:
sonle5
2026-10-01 15:59:44 -07:00
committed by GitHub
co-authored by Byron Hsu Cheng Wan long10024070 Kevin Mi Kevin Mi Claude Opus 5.5 sunxxuns Long Luong
parent 80bb3fb651
commit f17f7705a5
34 changed files with 1045 additions and 115 deletions
@@ -31,6 +31,7 @@ namespace {
inline constexpr int kGroupTopK = SGL_GROUP_TOPK;
inline constexpr int kThreadsPerBlock = 1024;
// Keep 4K threshold-bin candidates per round; rescan larger bins without clipping.
inline constexpr std::size_t kSmem = 8 * 1024 * sizeof(uint32_t); // 32KB (bytes)
struct FastTopKParams {
@@ -53,6 +54,11 @@ __device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ __forceinline__ auto make_topk_key(float score, int index) -> uint64_t {
// Unique keys resolve an overfull bin even when all scores are equal.
return (static_cast<uint64_t>(convert_to_uint32(score)) << 32) | ~static_cast<uint32_t>(index);
}
template <int K>
__device__ void
fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index, int row_start, int length) {
@@ -66,6 +72,7 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
alignas(128) __shared__ int s_counter;
alignas(128) __shared__ int s_threshold_bin_id;
alignas(128) __shared__ int s_num_input[2];
alignas(128) __shared__ uint64_t s_key_prefix;
auto& s_histogram = s_histogram_buf[0];
extern __shared__ int s_input_idx[][SMEM_INPUT_SIZE];
@@ -108,6 +115,7 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
const auto threshold_bin = s_threshold_bin_id;
topk -= s_histogram[threshold_bin + 1];
const auto num_threshold_candidates = s_histogram[threshold_bin] - s_histogram[threshold_bin + 1];
if (topk == 0) {
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
@@ -119,6 +127,63 @@ fast_topk_cuda_tl_impl(const float* __restrict__ input, int* __restrict__ index,
}
__syncthreads();
return;
} else if (num_threshold_candidates > int(SMEM_INPUT_SIZE)) {
// Rescan the full row only on overflow. Preserve the unsorted output contract.
if (tx == 0) s_key_prefix = 0;
__syncthreads();
#pragma unroll 8
for (int round = 0; round < 8; ++round) {
if (tx < RADIX + 1) s_histogram[tx] = 0;
__syncthreads();
const auto prefix = s_key_prefix;
const auto offset = 56 - round * 8;
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto raw_input = input[idx + row_start];
if (convert_to_uint8(raw_input) != threshold_bin) continue;
const auto key = make_topk_key(raw_input, idx);
const bool prefix_matches = round == 0 || (key >> (64 - round * 8)) == prefix;
if (prefix_matches) ::atomicAdd(&s_histogram[(key >> offset) & 0xFF], 1);
}
__syncthreads();
run_cumsum();
if (tx < RADIX && s_histogram[tx] > topk && s_histogram[tx + 1] <= topk) s_threshold_bin_id = tx;
__syncthreads();
const auto key_bin = s_threshold_bin_id;
topk -= s_histogram[key_bin + 1];
if (tx == 0) s_key_prefix = (prefix << 8) | static_cast<uint64_t>(key_bin);
__syncthreads();
if (topk == 0) {
const auto selected_prefix = s_key_prefix;
const auto prefix_bits = (round + 1) * 8;
if (tx == 0) s_counter = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
const auto raw_input = input[idx + row_start];
const auto coarse_bin = convert_to_uint8(raw_input);
const auto key_prefix = make_topk_key(raw_input, idx) >> (64 - prefix_bits);
if (coarse_bin > threshold_bin || (coarse_bin == threshold_bin && key_prefix > selected_prefix)) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
return;
}
}
const auto threshold_key = s_key_prefix;
if (tx == 0) s_counter = 0;
__syncthreads();
for (int idx = tx; idx < length; idx += BLOCK_SIZE) {
if (make_topk_key(input[idx + row_start], idx) >= threshold_key) {
const auto pos = ::atomicAdd(&s_counter, 1);
index[pos] = idx;
}
}
__syncthreads();
return;
} else {
__syncthreads();
if (tx < RADIX + 1) {
@@ -294,7 +359,11 @@ void setup_kernel_smem_once(host::DebugInfo where = {}) {
[[maybe_unused]]
static const auto result = [] {
const auto fptr = std::bit_cast<const void*>(f);
#if defined(__HIP_PLATFORM_AMD__)
return ::hipFuncSetAttribute(fptr, ::hipFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
#else
return ::cudaFuncSetAttribute(fptr, ::cudaFuncAttributeMaxDynamicSharedMemorySize, kMaxDynamicSMEM);
#endif
}();
host::RuntimeDeviceCheck(result, where);
}
@@ -326,7 +395,7 @@ struct KpoolTopKTransformKernel {
auto S = SymbolicSize{"score_stride"};
auto out_cols_sym = SymbolicSize{"out_cols"};
auto device = SymbolicDevice{};
device.set_options<kDLCUDA>();
device.set_options<kDLGPU>();
TensorMatcher({B, -1}).with_strides({S, 1}).with_dtype<float>().with_device(device).verify(score);
TensorMatcher({B}).with_dtype<int32_t>().with_device(device).verify(lengths);
@@ -36,12 +36,33 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.jit.utils import is_arch_support_pdl, is_hip_runtime
# V-tile width of the fused verify kernel. Tuned on B200 at T=5 with
# benchmark/kernels/bench_kda_verify_sweep.py; any power of two is
# numerics-safe at num_warps=4 (bit-exact vs the BV=32 original).
KDA_VERIFY_BLOCK_V = 4
# gfx950 V-tile width for the measured GLM TP4 shapes: wider tiles share the q/k
# convolution across more lanes
KDA_VERIFY_BLOCK_V_HIP = 16
@triton.jit
def _conv_product(x, weight, ROUND_PRODUCT: tl.constexpr):
if ROUND_PRODUCT:
# ROCm's unfused conv uses separately rounded fp32 products. Without
# this boundary LLVM contracts the fused path into FMAs, which can
# change the bf16 conv output before the recurrent update.
return tl.inline_asm_elementwise(
"v_mul_f32 $0, $1, $2",
constraints="=v,v,v",
args=[x.to(tl.float32), weight.to(tl.float32)],
dtype=tl.float32,
is_pure=True,
pack=1,
)
else:
return x * weight
@triton.jit
@@ -89,6 +110,7 @@ def fused_kda_conv_gating_verify_kernel(
USE_LOWER_BOUND: tl.constexpr,
SAVE_INTERMEDIATE_WINDOW: tl.constexpr,
CACHE_INTERMEDIATE_STATES: tl.constexpr,
ROUND_CONV_PRODUCTS: tl.constexpr,
USE_GDC: tl.constexpr = False,
# ReplaySSM fused ring-write (spec verify): per-slot rings consumed by the
# commit-time exact fold (kda_replayssm_spec_decode.py). Off -> dead code.
@@ -211,18 +233,18 @@ def fused_kda_conv_gating_verify_kernel(
acc_q = tl.zeros([BK], dtype=tl.float32)
acc_k = tl.zeros([BK], dtype=tl.float32)
acc_v = tl.zeros([BV], dtype=tl.float32)
acc_q += q_c0 * wq0
acc_q += q_c1 * wq1
acc_q += q_c2 * wq2
acc_q += x_q * wq3
acc_k += k_c0 * wk0
acc_k += k_c1 * wk1
acc_k += k_c2 * wk2
acc_k += x_k * wk3
acc_v += v_c0 * wv0
acc_v += v_c1 * wv1
acc_v += v_c2 * wv2
acc_v += x_v * wv3
acc_q += _conv_product(q_c0, wq0, ROUND_CONV_PRODUCTS)
acc_q += _conv_product(q_c1, wq1, ROUND_CONV_PRODUCTS)
acc_q += _conv_product(q_c2, wq2, ROUND_CONV_PRODUCTS)
acc_q += _conv_product(x_q, wq3, ROUND_CONV_PRODUCTS)
acc_k += _conv_product(k_c0, wk0, ROUND_CONV_PRODUCTS)
acc_k += _conv_product(k_c1, wk1, ROUND_CONV_PRODUCTS)
acc_k += _conv_product(k_c2, wk2, ROUND_CONV_PRODUCTS)
acc_k += _conv_product(x_k, wk3, ROUND_CONV_PRODUCTS)
acc_v += _conv_product(v_c0, wv0, ROUND_CONV_PRODUCTS)
acc_v += _conv_product(v_c1, wv1, ROUND_CONV_PRODUCTS)
acc_v += _conv_product(v_c2, wv2, ROUND_CONV_PRODUCTS)
acc_v += _conv_product(x_v, wv3, ROUND_CONV_PRODUCTS)
# Slide the window (reference: col0=col1; col1=col2; col2=x).
q_c0 = q_c1
@@ -420,7 +442,8 @@ def fused_kda_conv_gating_verify(
# T=8 safe gate. conv_state is not comparable to the reference at all.
# The ReplaySSM ring values are bit-exact at any num_warps: they are
# elementwise (conv FMA chain, gate, sigmoid), upstream of every tl.sum.
num_warps: int = 4,
# None picks 1 on ROCm (faster on gfx950 and bit-exact with the reference), else 4
num_warps: Optional[int] = None,
# ReplaySSM fused ring-write; same parameter names as the unfused
# fused_sigmoid_gating_delta_rule_update so ring_kwargs pass through both.
cache_ring: bool = False,
@@ -435,6 +458,8 @@ def fused_kda_conv_gating_verify(
seq_len, dim = mixed_qkv.shape
B = seq_len // T
W = conv_weight.shape[1]
if num_warps is None:
num_warps = 1 if is_hip_runtime() else 4
assert mixed_qkv.stride(-1) == 1, "mixed_qkv must be contiguous in dim"
assert dim == 2 * H * K + HV * V, f"packed dim mismatch: {dim}"
@@ -458,6 +483,16 @@ def fused_kda_conv_gating_verify(
# the gated RMSNorm into this kernel's epilogue is a dead end; it is
# PDL-chained behind this kernel instead (see fused_norm_gate.py).
BV = min(triton.next_power_of_2(V), KDA_VERIFY_BLOCK_V)
if (
is_hip_runtime()
and num_warps == 1
and T in (6, 8)
and H == HV == 16
and K == V == 128
and conv_weight.dtype == torch.float32
and 3 <= B <= 16
):
BV = KDA_VERIFY_BLOCK_V_HIP
NV = triton.cdiv(V, BV)
a2 = a.reshape(seq_len, HV * K)
@@ -572,6 +607,7 @@ def fused_kda_conv_gating_verify(
USE_LOWER_BOUND=lower_bound is not None,
SAVE_INTERMEDIATE_WINDOW=intermediate_conv_window is not None,
CACHE_INTERMEDIATE_STATES=intermediate_states_buffer is not None,
ROUND_CONV_PRODUCTS=is_hip_runtime() and conv_weight.dtype == torch.float32,
replayssm_rawv=replayssm_rawv,
replayssm_rawk=replayssm_rawk,
replayssm_g=replayssm_g,
@@ -1166,6 +1166,30 @@ class OpenAIServingChat(OpenAIServingBase):
return "text", processed_messages.prompt_ids
return "input_ids", processed_messages.prompt_ids
def _can_reuse_text_only_prompt_ids(
self, processed_messages: MessageProcessingResult, is_multimodal: bool
) -> bool:
# Moss-VL invokes its processor for text-only requests, and that processor
# requires the rendered text rather than pre-tokenized ids.
is_moss_vl = (
"MossVLForConditionalGeneration"
in self.tokenizer_manager.model_config.hf_config.architectures
)
return (
is_multimodal
and not is_moss_vl
and self.chat_encoding_spec is None
and self.template_manager.chat_template_name is None
and not self._prompt_text_round_trip_is_lossy
and not self._tokenizer_auto_adds_specials
and isinstance(processed_messages.prompt_ids, list)
and bool(processed_messages.prompt_ids)
and not processed_messages.image_data
and not processed_messages.video_data
and not processed_messages.audio_data
and not processed_messages.modalities
)
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
@@ -1233,7 +1257,9 @@ class OpenAIServingChat(OpenAIServingBase):
)
# Handle single vs multiple requests
if request.input_ids is not None:
if request.input_ids is not None or self._can_reuse_text_only_prompt_ids(
processed_messages, is_multimodal
):
prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
else:
prompt_key, prompt_value = self._engine_prompt(
@@ -1337,8 +1363,12 @@ class OpenAIServingChat(OpenAIServingBase):
tool_call_constraint = None
effective_tools = self._effective_tools(request)
glm_constraint = self.tool_call_parser == "glm47" and not any(
tool.function.strict for tool in effective_tools
# Only tool-bearing requests get the full-assistant EBNF: its terminal
# state finishes a request even under ignore_eos.
glm_constraint = (
self.tool_call_parser == "glm47"
and bool(effective_tools)
and not any(tool.function.strict for tool in effective_tools)
)
if glm_constraint:
enable_thinking = (request.chat_template_kwargs or {}).get(
@@ -23,7 +23,6 @@ from sglang.srt.layers.attention.dsa.utils import (
dsa_use_prefill_cp,
)
from sglang.srt.layers.attention.mqa_logits_utils import (
MQA_LOGITS_BYTES_PER_ELEM,
MQA_LOGITS_MAX_BYTES_ROCM,
mqa_logits_budget_bytes,
mqa_logits_row_bytes,
@@ -901,7 +900,8 @@ class IndexerKPool(MultiPlatformOp):
num_q, num_k = q_fp8.shape[0], k_fp8.shape[0]
rows_per_call = mqa_logits_rows_per_chunk(
num_rows=num_q,
row_bytes=num_k * MQA_LOGITS_BYTES_PER_ELEM,
# aiter pads each output row to 256 columns
row_bytes=mqa_logits_row_bytes(num_k),
budget_bytes=MQA_LOGITS_MAX_BYTES_ROCM,
)
if rows_per_call is None:
@@ -122,7 +122,7 @@ def _gather_index_k_scale_prefix_into_kernel(
token_id = tl.program_id(0)
page_idx = token_id // PAGE_SIZE
token_offset_in_page = token_id % PAGE_SIZE
page = tl.load(page_indices_ptr + page_idx)
page = tl.load(page_indices_ptr + page_idx).to(tl.int64)
offs = tl.arange(0, BLOCK_D)
mask = offs < HEAD_DIM
@@ -44,10 +44,19 @@ class DsaGraphVariants:
def create_attention_graph_variants(hf_config) -> Optional[AttentionGraphVariants]:
from sglang.srt.configs.model_config import get_dsa_index_topk, is_deepseek_dsa
from sglang.srt.configs.model_config import (
get_dsa_index_kpool,
get_dsa_index_topk,
is_deepseek_dsa,
)
from sglang.srt.utils import is_hip
if is_hip() and is_deepseek_dsa(hf_config):
# KPool has no dense-skip path: both variants run the full indexer.
# Capture it once, keeping the regular DSA dense/sparse split intact.
if get_dsa_index_kpool(hf_config) > 1:
return None
index_topk = get_dsa_index_topk(hf_config)
logger.info(
"[dense-decode] DSA dual-graph enabled: capturing "
@@ -20,7 +20,7 @@ from sglang.srt.layers.attention.linear.utils import (
select_verify_intermediate_state_indices,
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.utils import is_cpu, is_cuda, is_npu
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_npu
from sglang.srt.utils.common import is_gfx95_supported, rank0_log
# KDA always uses the triton causal_conv1d_fn (no CUDA override).
@@ -1285,6 +1285,16 @@ class KDAAttnBackend(MambaAttnBackendBase):
# other batch/architecture combinations are measured. The snapshot
# path and the separate CuTe path are unchanged.
return False
if is_hip() and not (
is_gfx95_supported()
and 1 <= batch_size <= 16
and draft_token_num in (6, 8)
and mixed_qkv.dtype == torch.bfloat16
and layer.conv_weights.dtype == torch.float32
):
# measured gfx950 wins only: larger batches lose the launch saving to
# duplicated convolution work
return False
expected_dim = (
2 * layer.num_q_heads * layer.head_k_dim
+ layer.num_v_heads * layer.head_v_dim
+23 -2
View File
@@ -952,6 +952,9 @@ class ReqKvInfo:
mamba_last_track_idx: Optional[int] = None # 0 or 1
# Seq len of the last cached mamba state
mamba_last_track_seqlen: Optional[int] = None
# Seq len of the other ping-pong slot's state. None means what is in
# that slot cannot be named (never written, donated, freed).
mamba_prev_track_seqlen: Optional[int] = None
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
mamba_cow_src_index: Optional[torch.Tensor] = None
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
@@ -1475,7 +1478,14 @@ class Req(ReqDllmMixin):
# overallocated range and are reclaimed by release_kv_cache. #22373.
if get_serving().strip_thinking_cache and self.reasoning_tokens > 0:
return min(self.kv.kv_committed_len, len(self.origin_input_ids))
return self.kv.kv_committed_len
if self.finished_len is None:
return self.kv.kv_committed_len
# A verify step commits a whole accepted chunk, so output_ids can run
# past the stop; the client never sees those tokens, so nothing matches.
return min(
self.kv.kv_committed_len,
len(self.origin_input_ids) + len(self.output_ids_through_stop),
)
def update_spec_correct_drafts_histogram(self, num_correct_drafts: int):
"""Record one step accepted draft count (excludes bonus token) into the histogram."""
@@ -1988,6 +1998,7 @@ class Req(ReqDllmMixin):
self.kv.mamba_next_track_idx = None
self.kv.mamba_last_track_idx = None
self.kv.mamba_last_track_seqlen = None
self.kv.mamba_prev_track_seqlen = None
self.mamba_branching_seqlen = None
self.kv.mamba_cow_src_index = None
self.kv.mamba_needs_clear = False
@@ -2212,7 +2223,8 @@ def set_mamba_track_indices_from_reqs(
if track_positions is None:
# Guard: mamba_next_track_idx may be None for requests that haven't
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
# Default to 0 (first ping-pong slot) to avoid TypeError.
# Keep the gather in bounds for those requests. Freed rows are
# invalidated below so they cannot scatter into a recycled slot.
track_positions = [
(
req.kv.mamba_next_track_idx
@@ -2221,6 +2233,9 @@ def set_mamba_track_indices_from_reqs(
)
for req in batch.reqs
]
freed_rows = [
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
]
batch.mamba_track_buffer_indices = list(track_positions)
idx = (
torch.tensor(
@@ -2234,6 +2249,12 @@ def set_mamba_track_indices_from_reqs(
batch.mamba_track_indices = (
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
)
if freed_rows:
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
# been freed and reused by another request. The downstream scatter
# treats a negative destination as a no-op, while the old fallback to
# position 0 could overwrite the reused live slot.
batch.mamba_track_indices[freed_rows] = -1
def release_req(
@@ -648,7 +648,12 @@ class PrefillAdder:
self.rem_chunk_tokens = rem_chunk_tokens
self.chunked_req_limit: Optional[int] = None
self.dllm_config = dllm_config
self.exact_chunk_fill = _use_exact_chunk_fill() and dllm_config is None
# Mamba checkpoints only land on page-aligned chunk ends.
self.exact_chunk_fill = (
_use_exact_chunk_fill()
and dllm_config is None
and not tree_cache.supports_mamba()
)
if self.dllm_config is not None:
self._init_dllm_meta(dllm_config)
@@ -1366,6 +1366,9 @@ class SchedulerBatchResultProcessor:
if completed_mamba_boundary and not lazy:
req.kv.mamba_last_track_idx = batch.mamba_track_buffer_indices[i]
# The slot that stops being the latest still holds its
# checkpoint; name it so a short key can fall back to it.
req.kv.mamba_prev_track_seqlen = req.kv.mamba_last_track_seqlen
req.kv.mamba_last_track_seqlen = req.kv.kv_committed_len - lookahead
elif (
req.finished()
@@ -1495,13 +1498,17 @@ class SchedulerBatchResultProcessor:
track_idx = req.kv.mamba_next_track_idx
if not known_boundary and batch.mamba_track_buffer_indices is not None:
track_idx = batch.mamba_track_buffer_indices[i]
previous_track_seqlen = req.kv.mamba_last_track_seqlen
if not known_boundary:
req.kv.mamba_last_track_seqlen = track_seqlen
if lazy:
# Lazy frees the slot it stops tracking, so nothing names the
# previous checkpoint there; mamba_prev_track_seqlen stays None.
self.mamba_lazy_post_decode_at_boundary(req, batch, track_idx)
else:
if not known_boundary:
req.kv.mamba_last_track_idx = track_idx
req.kv.mamba_prev_track_seqlen = previous_track_seqlen
req.kv.mamba_next_track_idx = (
batch.req_to_token_pool.get_mamba_ping_pong_other_idx(track_idx)
)
@@ -81,6 +81,8 @@ class InsertParams:
# Mamba specific
mamba_value: Optional[torch.Tensor] = None
# The ping-pong slot prepare picked; cleanup keeps that same slot.
mamba_keep_idx: Optional[int] = None
# DSV4 NPU C128 sidecar pages, one page id per physical C128 page group.
c128_value: Optional[torch.Tensor] = None
+6 -1
View File
@@ -346,7 +346,12 @@ def _release_overallocated_kv_indices(
# strip_thinking_cache intentionally reports output tokens as overallocated
# so they fall into the free path below (#22373).
if spec_algo is None and not get_serving().strip_thinking_cache:
assert start_p == end_p, (
# A stop landing before the last committed token does the same, via
# effective_kv_committed_len().
assert start_p == end_p or (
req.finished_len is not None
and len(req.origin_input_ids) + req.finished_len < req.kv.kv_committed_len
), (
f"Unexpected overallocated KV cache, {req.kv.kv_committed_len=}, {req.kv.kv_allocated_len=}"
)
@@ -1620,6 +1620,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
)
buf[:n] = slots
req.kv.mamba_ping_pong_track_buffer = buf
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = (
0
@@ -1720,6 +1721,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_cow_src_index = None
req.kv.mamba_needs_clear = False
@@ -536,6 +536,28 @@ class MambaComponent(TreeComponent):
else:
self.cache.req_to_token_pool.mamba_allocator.free(mamba_value)
def _select_finished_checkpoint(
self, req: Req, token_ids_len: int
) -> Optional[tuple[int, int]]:
# None means donate nothing, not "no slot found".
pool = self.cache.req_to_token_pool
keep_idx = pool.get_mamba_ping_pong_keep_idx(req)
cache_len = req.kv.mamba_last_track_seqlen or 0
if cache_len <= token_ids_len:
return cache_len, keep_idx
# Overshoot: the latest state ran past the key. The other slot always
# holds a tensor; only this seqlen says whether a key can name it.
previous_cache_len = req.kv.mamba_prev_track_seqlen
if (
pool.mamba_ping_pong_track_buffer_size != 2
or previous_cache_len is None
or previous_cache_len > token_ids_len
):
return None
return previous_cache_len, pool.get_mamba_ping_pong_other_idx(keep_idx)
def prepare_for_caching_req(
self,
req: Req,
@@ -564,9 +586,11 @@ class MambaComponent(TreeComponent):
if cache_len is None:
cache_len = 0
if self.cache.enable_mamba_extra_buffer:
keep_idx = self.cache.req_to_token_pool.get_mamba_ping_pong_keep_idx(
req
)
checkpoint = self._select_finished_checkpoint(req, token_ids_len)
if checkpoint is None:
return 0
cache_len, keep_idx = checkpoint
insert_params.mamba_keep_idx = keep_idx
active_value = (
req.kv.mamba_ping_pong_track_buffer[keep_idx].unsqueeze(-1).clone()
)
@@ -639,11 +663,11 @@ class MambaComponent(TreeComponent):
return
if self.cache.enable_mamba_extra_buffer:
keep_idx = (
pool.get_mamba_ping_pong_keep_idx(req)
if mamba_value_inserted
else None
# Keep the slot prepare picked, so cleanup cannot pick another.
prepared_keep_idx = (
insert_params.mamba_keep_idx if insert_params is not None else None
)
keep_idx = prepared_keep_idx if mamba_value_inserted else None
pool.free_mamba_cache(
req, mamba_ping_pong_track_buffer_to_keep=keep_idx
)
@@ -657,6 +681,7 @@ class MambaComponent(TreeComponent):
):
self._free_mamba_value(insert_params.mamba_value)
req.kv.mamba_last_track_seqlen = None
req.kv.mamba_prev_track_seqlen = None
def build_external_linker_transfer(
self,
@@ -526,8 +526,8 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
declared is SharedReadEnds.IN_REPLAY
and self.in_graph_metadata_prep_done is None
):
# TODO: this lands EARLIER than declared; POST_REPLAY is the sound one.
return SharedReadEnds.PRE_REPLAY
# no in-graph marker (e.g. HIP): only a post-replay event covers its reads
return SharedReadEnds.POST_REPLAY
return declared
def _publish_read_done(self, in_graph: bool):
@@ -638,9 +638,7 @@ class DeepseekMHAForwardMixin:
Returns: (kv_a, k_pe) both in BF16
"""
backend = get_attn_backend()
if isinstance(backend, TboAttnBackend): # if enable tbo, get primary backend
backend = backend.primary
backend = resolve_attn_backend(forward_batch)
kv_indices = backend.forward_metadata.page_table_1_flattened
assert kv_indices is not None, (
"page_table_1_flattened should have been generated for FP8 MHA path"
+9 -3
View File
@@ -25,6 +25,7 @@ from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.layers.attention import vision_utils
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.conv import Conv2dLayer
from sglang.srt.layers.layer_boundary import (
PLAIN_RESIDUAL_OPS,
MHCState,
@@ -317,11 +318,14 @@ class Glm5NextVisionModel(GlmOcrVisionModel):
swiglu_limit=vision_config.swiglu_limit,
)
self.downsample = nn.Conv2d(
# These non-overlapping patches are equivalent to unfold + linear and
# avoid MIOpen's expensive per-shape convolution search on ROCm.
self.downsample = Conv2dLayer(
in_channels=vision_config.hidden_size,
out_channels=vision_config.out_hidden_size,
kernel_size=vision_config.spatial_merge_size,
stride=vision_config.spatial_merge_size,
disable_linear=False,
)
self.post_layernorm = GlmOcrRMSNorm(
vision_config.hidden_size, eps=vision_config.rms_norm_eps
@@ -1596,8 +1600,10 @@ class Glm5NextForConditionalGeneration(nn.Module):
params_dict = dict(self.named_parameters())
def maybe_map_fp8_block_scale_name(name: str) -> str:
if name.endswith("weight_scale"):
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
# Quark stores dequantization scales without native block-FP8's
# "_inv" suffix, including fused w13/w2 expert parameters.
if name not in params_dict and name.endswith("weight_scale"):
candidate = name + "_inv"
if candidate in params_dict:
return candidate
return name
@@ -59,6 +59,8 @@ from sglang.srt.utils.device_timer import device_timer_ctx
if TYPE_CHECKING:
from sglang.srt.speculative.eagle_worker_v2 import EagleDraftWorker
_GREEDY_VARIANT = "greedy"
@dataclass
class EagleDraftInputBuffers(ForwardInputBuffers):
@@ -307,8 +309,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
return torch.int64
def _make_graph_key(self, bs, stream_idx=None, variant_label=None):
# EAGLE doesn't use stream_idx / lora variants.
return ShapeKey(size=bs)
# EAGLE doesn't use stream_idx; variant_label is _GREEDY_VARIANT or None.
return ShapeKey(size=bs, variant_label=variant_label)
def _sampling_variant(self, forward_batch: ForwardBatch) -> Optional[str]:
# Under rejection sampling an all-greedy batch replays the graph that
# skips the draft proposal distribution (see draft_forward).
if (
get_spec().speculative_use_rejection_sampling
and forward_batch.sampling_info is not None
and forward_batch.sampling_info.is_all_greedy
):
return _GREEDY_VARIANT
return None
# -----------------------------------------------------------------
# can_run_graph
@@ -332,7 +345,13 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
cuda_graph_bs = forward_batch.batch_size
is_bs_supported = (
self.backend.can_run(forward_batch, self._make_graph_key(cuda_graph_bs))
self.backend.can_run(
forward_batch,
self._make_graph_key(
cuda_graph_bs,
variant_label=self._sampling_variant(forward_batch),
),
)
if self.disable_padding
else cuda_graph_bs <= self.max_bs
)
@@ -498,7 +517,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
# per-step forwards inside draft_forward must not re-plan.
forward_batch.mark_forward_metadata_ready()
self.deepep_adapter.capture(is_extend_in_batch=False)
shape_key = self._make_graph_key(num_seqs)
post_warmup_hook = getattr(
self.draft_attn_backend, "on_after_cuda_graph_warmup", None
)
@@ -508,12 +526,19 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
post_warmup_hook=post_warmup_hook,
run_lm_head=True,
)
self.backend.capture_one(
shape_key,
run_once,
capture_inputs=None,
post_warmup_hook=post_warmup_hook,
variants = (
(None, _GREEDY_VARIANT)
if get_spec().speculative_use_rejection_sampling
else (None,)
)
for variant in variants:
sampling_info.is_all_greedy = variant == _GREEDY_VARIANT
self.backend.capture_one(
self._make_graph_key(num_seqs, variant_label=variant),
run_once,
capture_inputs=None,
post_warmup_hook=post_warmup_hook,
)
def _postprocess_output_to_raw_bs(self, out, raw_bs):
parent_list, top_scores_index, draft_tokens, draft_probs = (
@@ -694,7 +719,9 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
self.bs = bs
# Replay via backend
shape_key = self._make_graph_key(bs)
shape_key = self._make_graph_key(
bs, variant_label=self._sampling_variant(forward_batch)
)
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft"):
out = self._replay_graph(shape_key, forward_batch)
if self.buffers.dsa_seed_topk is not None:
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Callable, Optional
import torch
from sglang.srt.compilation.torch_compile_decoration import set_torch_compile_config
from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
deployment_attn_dp_size,
@@ -631,12 +632,14 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
)
self.draft_extend_attn_backend.init_forward_metadata_out_graph(fb_view)
# Snapshot built -- the forward is done reading the shared pool. Publish
# a read-done event the scheduler's WAR barrier waits on (draft extend
# is the EAGLE-family last shared-read phase; last write wins the mailbox).
read_done = self.device_module.Event()
read_done.record()
self.model_runner.shared_read_done_event = read_done
# Draft extend is the last shared reader. UNKNOWN must discard any
# earlier phase's event so the scheduler uses its whole-forward fence.
self.model_runner.shared_read_done_event = None
shared_read_ends = self.draft_extend_attn_backend.shared_read_ends(
self.forward_mode
)
if shared_read_ends is SharedReadEnds.PRE_REPLAY:
self._publish_read_done(in_graph=False)
self.raw_bs = raw_bs
self.bs = bs
@@ -644,6 +647,11 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
with device_timer_ctx(self.model_runner.device_timer, "eagle_draft_extend"):
out = self._replay_graph(shape_key, forward_batch)
# This runner has no in-graph read-done marker. A later event is safe;
# an event before replay cannot cover metadata reads inside the graph.
if shared_read_ends in (SharedReadEnds.IN_REPLAY, SharedReadEnds.POST_REPLAY):
self._publish_read_done(in_graph=False)
out = LogitsProcessorOutput(
next_token_logits=out.next_token_logits[:raw_bs],
# CUDA graph replay reuses its captured output storage. These states
@@ -700,7 +700,13 @@ class EagleDraftWorker(EagleDraftWorkerBase):
score_list: List[torch.Tensor] = []
token_list: List[torch.Tensor] = []
parents_list: List[torch.Tensor] = []
if get_spec().speculative_use_rejection_sampling:
# Verify commits argmax for an all-greedy batch and never reads the
# draft distribution, so skip building it.
needs_draft_probs = (
get_spec().speculative_use_rejection_sampling
and not forward_batch.sampling_info.is_all_greedy
)
if needs_draft_probs:
draft_probs_list: List[torch.Tensor] = [spec_info.draft_probs]
topk1_chain_fits = (
@@ -715,7 +721,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
topk1_chain_fits
and _is_cuda
and self.hot_token_id is None
and not get_spec().speculative_use_rejection_sampling
and not needs_draft_probs
):
draft_tokens_topk1 = torch.empty(
(topk_index.shape[0], self.speculative_num_steps),
@@ -778,7 +784,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
maybe_detect_inf(
logits_output.next_token_logits, f"draft_forward step {i}"
)
if get_spec().speculative_use_rejection_sampling:
if needs_draft_probs:
probs, topk_p, topk_index = sample_draft_proposal(
logits_output.next_token_logits,
forward_batch.sampling_info.temperatures,
@@ -804,7 +810,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
probs = renorm_draft_probs(
logits_output.next_token_logits,
forward_batch.sampling_info,
get_spec().speculative_use_rejection_sampling,
needs_draft_probs,
)
topk_p, topk_index = fast_topk(probs, self.topk, dim=-1)
forward_batch.positions.add_(1)
@@ -821,9 +827,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
hidden_states = logits_output.hidden_states
draft_probs = (
torch.stack(draft_probs_list, dim=1)
if get_spec().speculative_use_rejection_sampling
else None
torch.stack(draft_probs_list, dim=1) if needs_draft_probs else None
)
# Organize the results