[Perf] dsv41: Build the MegaGate row masks once per forward (#46071)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yuhao Yang
2026-10-23 14:15:01 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 27ac614f05
commit 5f7542d762
4 changed files with 83 additions and 15 deletions
+68 -10
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import functools
from contextlib import contextmanager
from typing import NamedTuple, Optional
import torch
@@ -20,6 +21,8 @@ from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
# DeepGEMM bf16_mega_gate scores one token block per launch.
_MEGA_GATE_MAX_ROWS = 2048
# Per-forward cache of the gate's row masks, installed by mega_gate_forward_scope.
_ROW_MASKS_ATTR = "_mega_gate_row_masks"
class MegaGateRouting(NamedTuple):
@@ -116,7 +119,57 @@ def _physical_dispatch_maps(info) -> tuple[torch.Tensor, torch.Tensor]:
)
def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateRouting]:
@contextmanager
def mega_gate_forward_scope(forward_batch):
"""One model forward: the gate's row masks are built once and shared by its
MoE layers. Each forward (eager, warmup or graph capture) builds its own."""
if forward_batch is None or not envs.SGLANG_OPT_DEEPGEMM_MEGA_GATE.get():
yield
return
try:
setattr(forward_batch, _ROW_MASKS_ATTR, {})
except (AttributeError, TypeError):
yield
return
try:
yield
finally:
try:
delattr(forward_batch, _ROW_MASKS_ATTR)
except AttributeError:
pass
def _gate_row_masks(moe, num_tokens, bound, image_input_ids, device, forward_batch):
"""(valid-row mask, image-token mask) for the gate, built once per forward."""
cache = getattr(forward_batch, _ROW_MASKS_ATTR, None)
key = (
num_tokens,
id(bound) if bound is not None else None,
id(image_input_ids) if image_input_ids is not None else None,
)
if isinstance(cache, dict):
hit = cache.get(key)
# The cached inputs are held, so their ids cannot be reused this forward.
if hit is not None and hit[0] is bound and hit[1] is image_input_ids:
return hit[2], hit[3]
valid = None
if bound is not None:
valid = (
torch.arange(num_tokens, device=device, dtype=torch.int32)
< bound.reshape(-1)[:1]
)
image = None
if image_input_ids is not None:
image = (image_input_ids[:num_tokens] == moe.config.image_token_id).contiguous()
if isinstance(cache, dict):
cache[key] = (bound, image_input_ids, valid, image)
return valid, image
def run_mega_gate(
moe, hidden_states, input_ids, buf=None, forward_batch=None
) -> Optional[MegaGateRouting]:
bound, dispatch_info = moe.topk.routing_inputs()
if dispatch_info is not None and dispatch_info.ep_dispatch_algorithm in (
"lp",
@@ -126,20 +179,25 @@ def run_mega_gate(moe, hidden_states, input_ids, buf=None) -> Optional[MegaGateR
if moe.is_hash and input_ids is None:
return None
num_tokens = hidden_states.shape[0]
valid = None
if bound is not None:
valid = (
torch.arange(num_tokens, device=hidden_states.device, dtype=torch.int32)
< bound.reshape(-1)[:1]
)
image = hash_ids = None
use_image = (
not moe.is_hash
and moe.gate.e_score_correction_bias_vl is not None
and input_ids is not None
)
valid, image = _gate_row_masks(
moe,
num_tokens,
bound,
input_ids if use_image else None,
hidden_states.device,
forward_batch,
)
hash_ids = None
if moe.is_hash:
token_ids = input_ids[:num_tokens]
if valid is not None:
token_ids = torch.where(valid, token_ids, 0)
hash_ids = moe.topk.tid2eid[token_ids].to(torch.int64)
elif moe.gate.e_score_correction_bias_vl is not None and input_ids is not None:
image = (input_ids[:num_tokens] == moe.config.image_token_id).contiguous()
routed_scaling_factor = float(moe.routed_scaling_factor)
fused_in_topk = moe.routed_scaling_factor_fused_in_topk
+3 -1
View File
@@ -290,7 +290,9 @@ def _run_mega_routed(
else None
)
with _configure_mega_moe_deep_gemm_num_sms(deep_gemm):
routing = run_mega_gate(moe, hidden_states, input_ids_global, buf)
routing = run_mega_gate(
moe, hidden_states, input_ids_global, buf, forward_batch
)
if routing is not None:
topk_ids = routing.topk_ids
+6 -3
View File
@@ -4293,10 +4293,13 @@ class DeepseekV4Model(nn.Module):
return self._forward_impl(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
)
from sglang.srt.layers.moe.mega_gate import mega_gate_forward_scope
try:
return self._forward_impl(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
)
with mega_gate_forward_scope(forward_batch):
return self._forward_impl(
input_ids, positions, forward_batch, input_embeds, pp_proxy_tensors
)
finally:
residual_batch.clear(forward_batch)
@@ -922,8 +922,13 @@ class DeepseekV4ForCausalLMDSpark(nn.Module):
) -> LogitsProcessorOutput:
if not _is_cuda:
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
from sglang.srt.layers.moe.mega_gate import mega_gate_forward_scope
try:
return self._forward_impl(input_ids, positions, forward_batch, input_embeds)
with mega_gate_forward_scope(forward_batch):
return self._forward_impl(
input_ids, positions, forward_batch, input_embeds
)
finally:
residual_batch.clear(forward_batch)