[codex] Fix DSA indexer in prefill piecewise CUDA graph (#28644)
This commit is contained in:
@@ -99,6 +99,34 @@ if TYPE_CHECKING:
|
|||||||
DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0
|
DUAL_STREAM_TOKEN_THRESHOLD = 1024 if _is_cuda else 0
|
||||||
|
|
||||||
|
|
||||||
|
def _uses_dsa_attention_backend(forward_batch: ForwardBatch) -> bool:
|
||||||
|
attn_backend = get_attn_backend()
|
||||||
|
server_args = get_global_server_args()
|
||||||
|
prefill_backend, decode_backend = server_args.get_attention_backends()
|
||||||
|
prefill_backend = (
|
||||||
|
getattr(attn_backend, "prefill_attention_backend_str", None) or prefill_backend
|
||||||
|
)
|
||||||
|
decode_backend = (
|
||||||
|
getattr(attn_backend, "decode_attention_backend_str", None) or decode_backend
|
||||||
|
)
|
||||||
|
|
||||||
|
if forward_batch.forward_mode.is_decode_or_idle():
|
||||||
|
backend_name = decode_backend
|
||||||
|
elif (
|
||||||
|
forward_batch.forward_mode.is_target_verify()
|
||||||
|
or forward_batch.forward_mode.is_draft_extend_v2()
|
||||||
|
):
|
||||||
|
backend_name = (
|
||||||
|
decode_backend
|
||||||
|
if server_args.speculative_attention_mode == "decode"
|
||||||
|
else prefill_backend
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
backend_name = prefill_backend
|
||||||
|
|
||||||
|
return backend_name in ("dsa", "nsa")
|
||||||
|
|
||||||
|
|
||||||
if _is_cuda:
|
if _is_cuda:
|
||||||
from sglang.srt.compilation.compilation_config import register_split_op
|
from sglang.srt.compilation.compilation_config import register_split_op
|
||||||
from sglang.srt.utils.custom_op import register_custom_op
|
from sglang.srt.utils.custom_op import register_custom_op
|
||||||
@@ -120,6 +148,10 @@ if _is_cuda:
|
|||||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||||
indexer = get_tc_piecewise_forward_context().dsa_indexers[layer_id]
|
indexer = get_tc_piecewise_forward_context().dsa_indexers[layer_id]
|
||||||
metadata = get_attn_backend().get_indexer_metadata(layer_id, forward_batch)
|
metadata = get_attn_backend().get_indexer_metadata(layer_id, forward_batch)
|
||||||
|
assert metadata is not None, (
|
||||||
|
"DSA piecewise CUDA graph requires indexer metadata from the DSA "
|
||||||
|
"attention backend"
|
||||||
|
)
|
||||||
|
|
||||||
# slice off padding from piecewise CUDA graph
|
# slice off padding from piecewise CUDA graph
|
||||||
extend_num_tokens = forward_batch.extend_num_tokens
|
extend_num_tokens = forward_batch.extend_num_tokens
|
||||||
@@ -1575,6 +1607,9 @@ class Indexer(MultiPlatformOp):
|
|||||||
not enable_dual_stream
|
not enable_dual_stream
|
||||||
), "Internal error: piecewise CUDA graph should not be enabled with dual stream"
|
), "Internal error: piecewise CUDA graph should not be enabled with dual stream"
|
||||||
|
|
||||||
|
if not _uses_dsa_attention_backend(forward_batch):
|
||||||
|
return None
|
||||||
|
|
||||||
topk_result = torch.full(
|
topk_result = torch.full(
|
||||||
(q_fp8.shape[0], self.index_topk),
|
(q_fp8.shape[0], self.index_topk),
|
||||||
-1,
|
-1,
|
||||||
|
|||||||
Reference in New Issue
Block a user