From 18ec2ecce399ad4b25b30a900de4ae428edbfb3a Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Mon, 5 Oct 2026 17:06:52 -0700 Subject: [PATCH] [Refactor][TCPCG] Relocate shared graph tensor and DSA head-gate helpers (1/9) (#42285) --- .../srt/compilation/cuda_piecewise_backend.py | 4 +- .../srt/layers/attention/dsa/dsa_indexer.py | 2 +- .../attention/dsa/dsa_prefill_cuda_graph.py | 57 +---------------- .../srt/layers/attention/dsa/head_gate.py | 63 +++++++++++++++++++ .../breakable_cuda_graph.py | 4 +- .../runner_backend_utils}/weak_ref_tensor.py | 0 .../attention/test_dsa_head_gate_guard.py | 10 +-- 7 files changed, 76 insertions(+), 64 deletions(-) create mode 100644 python/sglang/srt/layers/attention/dsa/head_gate.py rename python/sglang/srt/{compilation => model_executor/runner_backend_utils}/weak_ref_tensor.py (100%) diff --git a/python/sglang/srt/compilation/cuda_piecewise_backend.py b/python/sglang/srt/compilation/cuda_piecewise_backend.py index 75ce0e0dce..113f699a9d 100644 --- a/python/sglang/srt/compilation/cuda_piecewise_backend.py +++ b/python/sglang/srt/compilation/cuda_piecewise_backend.py @@ -17,7 +17,9 @@ from sglang.srt.compilation.compile_phase import ( get_pcg_capture_stream, is_in_torch_compile_warmup, ) -from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors +from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import ( + weak_ref_tensors, +) from sglang.srt.model_executor.runner_utils.pool import ( graph_pool_capture_scope, graph_pool_replay_scope, diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index aca0d26e55..bdee7d321e 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -167,7 +167,7 @@ DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0 if _is_cuda or _is_hip: # Plain-torch graph helpers: usable wherever the split-op surface is. - from sglang.srt.layers.attention.dsa.dsa_prefill_cuda_graph import ( + from sglang.srt.layers.attention.dsa.head_gate import ( logits_head_gate_graph, scale_head_gate_graph, ) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py index 92a661a186..0b97e5bdf3 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py @@ -14,11 +14,10 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo get_tc_piecewise_forward_context, is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.utils import is_cuda, is_hip +from sglang.srt.utils import is_cuda from sglang.srt.utils.custom_op import register_custom_op _is_cuda = is_cuda() -_is_hip = is_hip() GRAPH_WEIGHTS_PROJ_LORA_ERROR = ( "DSA indexer weights_proj LoRA is incompatible with " @@ -32,60 +31,6 @@ def _is_in_piecewise_or_breakable_cuda_graph() -> bool: return is_in_tc_piecewise_cuda_graph() or is_in_breakable_cuda_graph() -if _is_cuda or _is_hip: - - def _scale_head_gate_graph_fake_impl( - weights_raw: torch.Tensor, - n_heads_inv_sqrt: float, - softmax_scale: float, - q_scale: torch.Tensor, - ) -> torch.Tensor: - return torch.empty( - (weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]), - dtype=torch.float32, - device=weights_raw.device, - ) - - # In-graph (PCG/BCG) head gate for the fused path: weights_proj is folded - # into wk_weights_proj, so weights_raw is precomputed and there is no GEMM. - @register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl) - def scale_head_gate_graph( - weights_raw: torch.Tensor, - n_heads_inv_sqrt: float, - softmax_scale: float, - q_scale: torch.Tensor, - ) -> torch.Tensor: - weights = weights_raw * n_heads_inv_sqrt - return weights.unsqueeze(-1) * q_scale * softmax_scale - - def _logits_head_gate_graph_fake_impl( - x: torch.Tensor, - weight: torch.Tensor, - n_heads_inv_sqrt: float, - softmax_scale: float, - q_scale: torch.Tensor, - ) -> torch.Tensor: - return torch.empty( - (x.shape[0], weight.shape[0], q_scale.shape[-1]), - dtype=torch.float32, - device=x.device, - ) - - # In-graph (PCG/BCG) head gate for the NON-prefill path - @register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl) - def logits_head_gate_graph( - x: torch.Tensor, - weight: torch.Tensor, - n_heads_inv_sqrt: float, - softmax_scale: float, - q_scale: torch.Tensor, - ) -> torch.Tensor: - out = torch.mm(x, weight.t(), out_dtype=torch.float32) - weights = out * n_heads_inv_sqrt - weights = weights.unsqueeze(-1) * q_scale * softmax_scale - return weights - - @register_custom_op(mutates_args=["topk_result"]) @register_split_op() def pcg_dsa_indexer_prefill_split( diff --git a/python/sglang/srt/layers/attention/dsa/head_gate.py b/python/sglang/srt/layers/attention/dsa/head_gate.py new file mode 100644 index 0000000000..264a12a0f0 --- /dev/null +++ b/python/sglang/srt/layers/attention/dsa/head_gate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import torch + +from sglang.srt.utils import is_cuda, is_hip +from sglang.srt.utils.custom_op import register_custom_op + +_is_cuda = is_cuda() +_is_hip = is_hip() + + +if _is_cuda or _is_hip: + + def _scale_head_gate_graph_fake_impl( + weights_raw: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + return torch.empty( + (weights_raw.shape[0], weights_raw.shape[1], q_scale.shape[-1]), + dtype=torch.float32, + device=weights_raw.device, + ) + + # In-graph head gate for the fused path: weights_proj is folded + # into wk_weights_proj, so weights_raw is precomputed and there is no GEMM. + @register_custom_op(fake_impl=_scale_head_gate_graph_fake_impl) + def scale_head_gate_graph( + weights_raw: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + weights = weights_raw * n_heads_inv_sqrt + return weights.unsqueeze(-1) * q_scale * softmax_scale + + def _logits_head_gate_graph_fake_impl( + x: torch.Tensor, + weight: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + return torch.empty( + (x.shape[0], weight.shape[0], q_scale.shape[-1]), + dtype=torch.float32, + device=x.device, + ) + + # In-graph head gate for the NON-prefill path + @register_custom_op(fake_impl=_logits_head_gate_graph_fake_impl) + def logits_head_gate_graph( + x: torch.Tensor, + weight: torch.Tensor, + n_heads_inv_sqrt: float, + softmax_scale: float, + q_scale: torch.Tensor, + ) -> torch.Tensor: + out = torch.mm(x, weight.t(), out_dtype=torch.float32) + weights = out * n_heads_inv_sqrt + weights = weights.unsqueeze(-1) * q_scale * softmax_scale + return weights diff --git a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py index 0f4847ca21..bd4bf2ee8d 100644 --- a/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py +++ b/python/sglang/srt/model_executor/runner_backend_utils/breakable_cuda_graph/breakable_cuda_graph.py @@ -166,7 +166,9 @@ def _weak_ref_if_tensor(x): if torch.is_tensor(x): if x.numel() == 0 or x.device.type == "cpu": return x - from sglang.srt.compilation.weak_ref_tensor import weak_ref_tensors + from sglang.srt.model_executor.runner_backend_utils.weak_ref_tensor import ( + weak_ref_tensors, + ) return weak_ref_tensors(x) if isinstance(x, tuple): diff --git a/python/sglang/srt/compilation/weak_ref_tensor.py b/python/sglang/srt/model_executor/runner_backend_utils/weak_ref_tensor.py similarity index 100% rename from python/sglang/srt/compilation/weak_ref_tensor.py rename to python/sglang/srt/model_executor/runner_backend_utils/weak_ref_tensor.py diff --git a/test/registered/unit/layers/attention/test_dsa_head_gate_guard.py b/test/registered/unit/layers/attention/test_dsa_head_gate_guard.py index 246b24c4a6..7da0c6b884 100644 --- a/test/registered/unit/layers/attention/test_dsa_head_gate_guard.py +++ b/test/registered/unit/layers/attention/test_dsa_head_gate_guard.py @@ -19,23 +19,23 @@ class TestDsaHeadGateGuard(CustomTestCase): """ def test_definition_and_import_agree(self): - from sglang.srt.layers.attention.dsa import dsa_indexer, dsa_prefill_cuda_graph + from sglang.srt.layers.attention.dsa import dsa_indexer, head_gate for name in HELPERS: with self.subTest(helper=name): self.assertEqual( - hasattr(dsa_prefill_cuda_graph, name), - hasattr(dsa_indexer, name), + name in vars(head_gate), + name in vars(dsa_indexer), f"{name} is defined on one side of the guard but not the other", ) def test_helpers_exist_on_cuda_and_hip(self): - from sglang.srt.layers.attention.dsa import dsa_prefill_cuda_graph + from sglang.srt.layers.attention.dsa import head_gate expected = is_cuda() or is_hip() for name in HELPERS: with self.subTest(helper=name): - self.assertEqual(hasattr(dsa_prefill_cuda_graph, name), expected) + self.assertEqual(name in vars(head_gate), expected) if __name__ == "__main__":