[Refactor][TCPCG] Relocate shared graph tensor and DSA head-gate helpers (1/9) (#42285)

This commit is contained in:
Yuwei An
2026-10-05 17:06:52 -07:00
committed by GitHub
parent 968726f3ce
commit 18ec2ecce3
7 changed files with 76 additions and 64 deletions
@@ -17,7 +17,9 @@ from sglang.srt.compilation.compile_phase import (
get_pcg_capture_stream,
is_in_torch_compile_warmup,
)
from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors
from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import (
weak_ref_tensors,
)
from sglang.srt.model_executor.runner_utils.pool import (
graph_pool_capture_scope,
graph_pool_replay_scope,
@@ -167,7 +167,7 @@ DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0
if _is_cuda or _is_hip:
# Plain-torch graph helpers: usable wherever the split-op surface is.
from sglang.srt.layers.attention.dsa.dsa_prefill_cuda_graph import (
from sglang.srt.layers.attention.dsa.head_gate import (
logits_head_gate_graph,
scale_head_gate_graph,
)
@@ -14,11 +14,10 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
get_tc_piecewise_forward_context,
is_in_tc_piecewise_cuda_graph,
)
from sglang.srt.utils import is_cuda, is_hip
from sglang.srt.utils import is_cuda
from sglang.srt.utils.custom_op import register_custom_op
_is_cuda = is_cuda()
_is_hip = is_hip()
GRAPH_WEIGHTS_PROJ_LORA_ERROR = (
"DSA indexer weights_proj LoRA is incompatible with "
@@ -32,60 +31,6 @@ def _is_in_piecewise_or_breakable_cuda_graph() -> bool:
return is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph()
if _is_cuda or _is_hip:
def _scale_head_gate_graph_fake_impl(
weights_raw: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
return torch.empty(
(weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]),
dtype=torch.float32,
device=weights_raw.device,
)
# In-graph (PCG/BCG) head gate for the fused path: weights_proj is folded
# into wk_weights_proj, so weights_raw is precomputed and there is no GEMM.
@register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl)
def scale_head_gate_graph(
weights_raw: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
weights = weights_raw * n_heads_inv_sqrt
return weights.unsqueeze(-1) * q_scale * softmax_scale
def _logits_head_gate_graph_fake_impl(
x: torch.Tensor,
weight: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
return torch.empty(
(x.shape[0], weight.shape[0], q_scale.shape[-1]),
dtype=torch.float32,
device=x.device,
)
# In-graph (PCG/BCG) head gate for the NON-prefill path
@register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl)
def logits_head_gate_graph(
x: torch.Tensor,
weight: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
out = torch.mm(x, weight.t(), out_dtype=torch.float32)
weights = out * n_heads_inv_sqrt
weights = weights.unsqueeze(-1) * q_scale * softmax_scale
return weights
@register_custom_op(mutates_args=["topk_result"])
@register_split_op()
def pcg_dsa_indexer_prefill_split(
@@ -0,0 +1,63 @@
from __future__ import annotations
import torch
from sglang.srt.utils import is_cuda, is_hip
from sglang.srt.utils.custom_op import register_custom_op
_is_cuda = is_cuda()
_is_hip = is_hip()
if _is_cuda or _is_hip:
def _scale_head_gate_graph_fake_impl(
weights_raw: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
return torch.empty(
(weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]),
dtype=torch.float32,
device=weights_raw.device,
)
# In-graph head gate for the fused path: weights_proj is folded
# into wk_weights_proj, so weights_raw is precomputed and there is no GEMM.
@register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl)
def scale_head_gate_graph(
weights_raw: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
weights = weights_raw * n_heads_inv_sqrt
return weights.unsqueeze(-1) * q_scale * softmax_scale
def _logits_head_gate_graph_fake_impl(
x: torch.Tensor,
weight: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
return torch.empty(
(x.shape[0], weight.shape[0], q_scale.shape[-1]),
dtype=torch.float32,
device=x.device,
)
# In-graph head gate for the NON-prefill path
@register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl)
def logits_head_gate_graph(
x: torch.Tensor,
weight: torch.Tensor,
n_heads_inv_sqrt: float,
softmax_scale: float,
q_scale: torch.Tensor,
) -> torch.Tensor:
out = torch.mm(x, weight.t(), out_dtype=torch.float32)
weights = out * n_heads_inv_sqrt
weights = weights.unsqueeze(-1) * q_scale * softmax_scale
return weights
@@ -166,7 +166,9 @@ def _weak_ref_if_tensor(x):
if torch.is_tensor(x):
if x.numel() == 0 or x.device.type == "cpu":
return x
from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors
from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import (
weak_ref_tensors,
)
return weak_ref_tensors(x)
if isinstance(x, tuple):