[Perf] [AMD] monokernels: Let a plugin skip per-step attention metadata (#44601)

Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
Co-authored-by: kkHuang-amd <43161300+kkHuang-amd@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kevin Mi
2026-10-18 14:04:57 +00:00
committed by GitHub
co-authored by Kevin Mi kkHuang-amd Claude Opus 5.5
parent 57db5089c2
commit e9b22a0818
6 changed files with 55 additions and 6 deletions
+7 -4
View File
@@ -107,6 +107,11 @@ tracing = [
"opentelemetry-sdk",
]
# Monokernels for gfx950 (--monokernel-backend amd); the ROCm image checks out this pin too.
monokernels_hip = [
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@e07665c8b36f7d42a48f6615bacf911d0656f5c5",
]
# HIP (Heterogeneous-computing Interface for Portability) for AMD
# => base docker rocm/vllm-dev:20250114, not from public vllm whl
#
@@ -114,11 +119,10 @@ tracing = [
# in its compressed-tensors and torch pins. A dependency added here has to be
# added there too; nothing enforces that.
srt_hip = [
"sglang[monokernels_hip]",
"sglang[runtime_common]",
"torch",
"petit_kernel==0.0.2",
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"wave-lang==3.8.2",
]
@@ -128,8 +132,7 @@ srt_hip_rocm724 = [
# torch<2.11, while 0.16.0 supports torch>=2.10.
"compressed-tensors==0.16.0",
"petit_kernel==0.0.2",
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
"sglang[monokernels_hip]",
"sglang[runtime_common]",
"torch==2.11.0",
"wave-lang==3.8.2",
@@ -107,6 +107,8 @@ from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.dsv41_request_window import window_layout
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.monokernels.runtime import is_metadata_unread
from sglang.srt.monokernels.spec import StepMetadata
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
@@ -907,6 +909,7 @@ class DeepseekV4HipRadixBackend(
speculative_num_steps=0,
):
super().__init__()
self.model_runner = model_runner
self.device = torch.device(model_runner.device)
self.max_context_len = model_runner.model_config.context_len
head_dim = model_runner.model_config.head_dim
@@ -1768,6 +1771,11 @@ class DeepseekV4HipRadixBackend(
if not low_ratio_decode_rows_are_identity(
self, forward_batch, ratio
)
and not is_metadata_unread(
self.model_runner.model,
forward_batch,
StepMetadata.LOW_RATIO_DECODE_SCHEDULE,
)
},
max_context_len=self.max_context_len,
num_requests=forward_batch.batch_size,
+11
View File
@@ -21,6 +21,7 @@ from sglang.srt.monokernels.spec import (
MonoSpec,
Plan,
ShapeKey,
StepMetadata,
find_spec,
spec_registered,
)
@@ -100,6 +101,14 @@ def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]:
return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs)
def is_metadata_unread(
model: torch.nn.Module, forward_batch: ForwardBatch, metadata: StepMetadata
) -> bool:
"""Whether the monokernel forward that runs this step never reads the metadata."""
served, unread = model.__dict__.get("_monokernel_unread_metadata", ((), ()))
return metadata in unread and shape_key(forward_batch) in served
def install_monokernel_forward(
*,
model: torch.nn.Module,
@@ -340,6 +349,8 @@ def _install_forward(
return built.forward(Plan(key=key), *args, **kwargs)
model.forward = forward
# attention backends ask is_metadata_unread before building per-step metadata
model._monokernel_unread_metadata = (served, spec.unread_metadata)
def _format_steps(keys: Iterable[ShapeKey]) -> str:
+8
View File
@@ -40,6 +40,12 @@ class Feature(enum.Enum):
PARTITIONED_GPU = "partitioned GPUs"
class StepMetadata(enum.Enum):
"""Per-step attention metadata a backend can skip building."""
LOW_RATIO_DECODE_SCHEDULE = "the aiter decode schedule of each low-ratio indexer"
class DeviceLimits(msgspec.Struct, frozen=True):
"""What a spec's kernels support on one ISA, e.g. gfx950."""
@@ -153,6 +159,8 @@ class MonoSpec(msgspec.Struct, frozen=True):
] = None
# (model) -> tokens per request when different from the target, e.g. draft blocks
tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None
# per-step metadata the forward never reads; backends skip it on routed steps
unread_metadata: frozenset[StepMetadata] = frozenset()
def widths(self, isa: str) -> tuple[int, ...]:
return self.devices[isa].widths