[Refactor][TCPCG] Relocate shared graph tensor and DSA head-gate helpers (1/9) (#42285)
This commit is contained in:
@@ -17,7 +17,9 @@ from sglang.srt.compilation.compile_phase import (
|
||||
get_pcg_capture_stream,
|
||||
is_in_torch_compile_warmup,
|
||||
)
|
||||
from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors
|
||||
from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import (
|
||||
weak_ref_tensors,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_utils.pool import (
|
||||
graph_pool_capture_scope,
|
||||
graph_pool_replay_scope,
|
||||
|
||||
@@ -167,7 +167,7 @@ DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0
|
||||
|
||||
if _is_cuda or _is_hip:
|
||||
# Plain-torch graph helpers: usable wherever the split-op surface is.
|
||||
from sglang.srt.layers.attention.dsa.dsa_prefill_cuda_graph import (
|
||||
from sglang.srt.layers.attention.dsa.head_gate import (
|
||||
logits_head_gate_graph,
|
||||
scale_head_gate_graph,
|
||||
)
|
||||
|
||||
@@ -14,11 +14,10 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
get_tc_piecewise_forward_context,
|
||||
is_in_tc_piecewise_cuda_graph,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip
|
||||
from sglang.srt.utils import is_cuda
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
|
||||
GRAPH_WEIGHTS_PROJ_LORA_ERROR = (
|
||||
"DSA indexer weights_proj LoRA is incompatible with "
|
||||
@@ -32,60 +31,6 @@ def _is_in_piecewise_or_breakable_cuda_graph() -> bool:
|
||||
return is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph()
|
||||
|
||||
|
||||
if _is_cuda or _is_hip:
|
||||
|
||||
def _scale_head_gate_graph_fake_impl(
|
||||
weights_raw: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]),
|
||||
dtype=torch.float32,
|
||||
device=weights_raw.device,
|
||||
)
|
||||
|
||||
# In-graph (PCG/BCG) head gate for the fused path: weights_proj is folded
|
||||
# into wk_weights_proj, so weights_raw is precomputed and there is no GEMM.
|
||||
@register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl)
|
||||
def scale_head_gate_graph(
|
||||
weights_raw: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
weights = weights_raw * n_heads_inv_sqrt
|
||||
return weights.unsqueeze(-1) * q_scale * softmax_scale
|
||||
|
||||
def _logits_head_gate_graph_fake_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(x.shape[0], weight.shape[0], q_scale.shape[-1]),
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
|
||||
# In-graph (PCG/BCG) head gate for the NON-prefill path
|
||||
@register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl)
|
||||
def logits_head_gate_graph(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
out = torch.mm(x, weight.t(), out_dtype=torch.float32)
|
||||
weights = out * n_heads_inv_sqrt
|
||||
weights = weights.unsqueeze(-1) * q_scale * softmax_scale
|
||||
return weights
|
||||
|
||||
|
||||
@register_custom_op(mutates_args=["topk_result"])
|
||||
@register_split_op()
|
||||
def pcg_dsa_indexer_prefill_split(
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils import is_cuda, is_hip
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
|
||||
|
||||
if _is_cuda or _is_hip:
|
||||
|
||||
def _scale_head_gate_graph_fake_impl(
|
||||
weights_raw: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]),
|
||||
dtype=torch.float32,
|
||||
device=weights_raw.device,
|
||||
)
|
||||
|
||||
# In-graph head gate for the fused path: weights_proj is folded
|
||||
# into wk_weights_proj, so weights_raw is precomputed and there is no GEMM.
|
||||
@register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl)
|
||||
def scale_head_gate_graph(
|
||||
weights_raw: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
weights = weights_raw * n_heads_inv_sqrt
|
||||
return weights.unsqueeze(-1) * q_scale * softmax_scale
|
||||
|
||||
def _logits_head_gate_graph_fake_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(
|
||||
(x.shape[0], weight.shape[0], q_scale.shape[-1]),
|
||||
dtype=torch.float32,
|
||||
device=x.device,
|
||||
)
|
||||
|
||||
# In-graph head gate for the NON-prefill path
|
||||
@register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl)
|
||||
def logits_head_gate_graph(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
n_heads_inv_sqrt: float,
|
||||
softmax_scale: float,
|
||||
q_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
out = torch.mm(x, weight.t(), out_dtype=torch.float32)
|
||||
weights = out * n_heads_inv_sqrt
|
||||
weights = weights.unsqueeze(-1) * q_scale * softmax_scale
|
||||
return weights
|
||||
+3
-1
@@ -166,7 +166,9 @@ def _weak_ref_if_tensor(x):
|
||||
if torch.is_tensor(x):
|
||||
if x.numel() == 0 or x.device.type == "cpu":
|
||||
return x
|
||||
from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors
|
||||
from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import (
|
||||
weak_ref_tensors,
|
||||
)
|
||||
|
||||
return weak_ref_tensors(x)
|
||||
if isinstance(x, tuple):
|
||||
|
||||
Reference in New Issue
Block a user