[Perf] [AMD] monokernels: Let a plugin skip per-step attention metadata (#44601)
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai> Co-authored-by: kkHuang-amd <43161300+kkHuang-amd@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Kevin Mi
kkHuang-amd
Claude Opus 5.5
parent
57db5089c2
commit
e9b22a0818
@@ -440,7 +440,8 @@ ARG FHT_COMMIT="46efb7d776d38638fc39f3c803eaee3dd7016bd1"
|
||||
# gfx950); disable with --build-arg ENABLE_SGL_AMD_MONO=0.
|
||||
ARG ENABLE_SGL_AMD_MONO=1
|
||||
ARG SGL_AMD_MONO_REPO="https://github.com/sgl-project/sgl-amd-mono.git"
|
||||
ARG SGL_AMD_MONO_COMMIT="815b104f483d180027054a8f44bad0c2a3f895fd"
|
||||
# empty: the commit pip installed through the monokernels_hip extra
|
||||
ARG SGL_AMD_MONO_COMMIT=""
|
||||
|
||||
ARG ENABLE_MORI=0
|
||||
ARG NIC_BACKEND=none
|
||||
@@ -895,9 +896,11 @@ RUN /bin/bash -lc 'set -euo pipefail; \
|
||||
RUN /bin/bash -lc 'set -euo pipefail; \
|
||||
[ "${ENABLE_SGL_AMD_MONO}" = "1" ] || { echo "[sgl-amd-mono] skip (ENABLE_SGL_AMD_MONO=${ENABLE_SGL_AMD_MONO})"; exit 0; }; \
|
||||
case "${GPU_ARCH}" in gfx950*) ;; *) echo "[sgl-amd-mono] skip (GPU_ARCH=${GPU_ARCH})"; exit 0 ;; esac; \
|
||||
python3 -m pip show sgl-amd-mono > /dev/null || { echo "[sgl-amd-mono] skip (this SGLang does not install it)"; exit 0; }; \
|
||||
commit="${SGL_AMD_MONO_COMMIT:-$(python3 -c "import json, importlib.metadata as md; print(json.loads(md.distribution(\"sgl-amd-mono\").read_text(\"direct_url.json\"))[\"vcs_info\"][\"commit_id\"])")}"; \
|
||||
git clone "${SGL_AMD_MONO_REPO}" /sgl-workspace/sgl-amd-mono; \
|
||||
cd /sgl-workspace/sgl-amd-mono; \
|
||||
git checkout -f "${SGL_AMD_MONO_COMMIT}"; \
|
||||
git checkout -f "${commit}"; \
|
||||
python3 -m pip install --no-cache-dir --no-deps -e .; \
|
||||
python3 -c "from importlib.metadata import entry_points; \
|
||||
assert any(e.name == \"sgl_amd_mono\" for e in entry_points(group=\"sglang.srt.plugins\")), \"sgl-amd-mono plugin not registered\""'
|
||||
|
||||
@@ -107,6 +107,11 @@ tracing = [
|
||||
"opentelemetry-sdk",
|
||||
]
|
||||
|
||||
# Monokernels for gfx950 (--monokernel-backend amd); the ROCm image checks out this pin too.
|
||||
monokernels_hip = [
|
||||
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@e07665c8b36f7d42a48f6615bacf911d0656f5c5",
|
||||
]
|
||||
|
||||
# HIP (Heterogeneous-computing Interface for Portability) for AMD
|
||||
# => base docker rocm/vllm-dev:20250114, not from public vllm whl
|
||||
#
|
||||
@@ -114,11 +119,10 @@ tracing = [
|
||||
# in its compressed-tensors and torch pins. A dependency added here has to be
|
||||
# added there too; nothing enforces that.
|
||||
srt_hip = [
|
||||
"sglang[monokernels_hip]",
|
||||
"sglang[runtime_common]",
|
||||
"torch",
|
||||
"petit_kernel==0.0.2",
|
||||
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
|
||||
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
|
||||
"wave-lang==3.8.2",
|
||||
]
|
||||
|
||||
@@ -128,8 +132,7 @@ srt_hip_rocm724 = [
|
||||
# torch<2.11, while 0.16.0 supports torch>=2.10.
|
||||
"compressed-tensors==0.16.0",
|
||||
"petit_kernel==0.0.2",
|
||||
# Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono.
|
||||
"sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca",
|
||||
"sglang[monokernels_hip]",
|
||||
"sglang[runtime_common]",
|
||||
"torch==2.11.0",
|
||||
"wave-lang==3.8.2",
|
||||
|
||||
@@ -107,6 +107,8 @@ from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.dsv41_request_window import window_layout
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.monokernels.runtime import is_metadata_unread
|
||||
from sglang.srt.monokernels.spec import StepMetadata
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_parallel,
|
||||
@@ -907,6 +909,7 @@ class DeepseekV4HipRadixBackend(
|
||||
speculative_num_steps=0,
|
||||
):
|
||||
super().__init__()
|
||||
self.model_runner = model_runner
|
||||
self.device = torch.device(model_runner.device)
|
||||
self.max_context_len = model_runner.model_config.context_len
|
||||
head_dim = model_runner.model_config.head_dim
|
||||
@@ -1768,6 +1771,11 @@ class DeepseekV4HipRadixBackend(
|
||||
if not low_ratio_decode_rows_are_identity(
|
||||
self, forward_batch, ratio
|
||||
)
|
||||
and not is_metadata_unread(
|
||||
self.model_runner.model,
|
||||
forward_batch,
|
||||
StepMetadata.LOW_RATIO_DECODE_SCHEDULE,
|
||||
)
|
||||
},
|
||||
max_context_len=self.max_context_len,
|
||||
num_requests=forward_batch.batch_size,
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.monokernels.spec import (
|
||||
MonoSpec,
|
||||
Plan,
|
||||
ShapeKey,
|
||||
StepMetadata,
|
||||
find_spec,
|
||||
spec_registered,
|
||||
)
|
||||
@@ -100,6 +101,14 @@ def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]:
|
||||
return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs)
|
||||
|
||||
|
||||
def is_metadata_unread(
|
||||
model: torch.nn.Module, forward_batch: ForwardBatch, metadata: StepMetadata
|
||||
) -> bool:
|
||||
"""Whether the monokernel forward that runs this step never reads the metadata."""
|
||||
served, unread = model.__dict__.get("_monokernel_unread_metadata", ((), ()))
|
||||
return metadata in unread and shape_key(forward_batch) in served
|
||||
|
||||
|
||||
def install_monokernel_forward(
|
||||
*,
|
||||
model: torch.nn.Module,
|
||||
@@ -340,6 +349,8 @@ def _install_forward(
|
||||
return built.forward(Plan(key=key), *args, **kwargs)
|
||||
|
||||
model.forward = forward
|
||||
# attention backends ask is_metadata_unread before building per-step metadata
|
||||
model._monokernel_unread_metadata = (served, spec.unread_metadata)
|
||||
|
||||
|
||||
def _format_steps(keys: Iterable[ShapeKey]) -> str:
|
||||
|
||||
@@ -40,6 +40,12 @@ class Feature(enum.Enum):
|
||||
PARTITIONED_GPU = "partitioned GPUs"
|
||||
|
||||
|
||||
class StepMetadata(enum.Enum):
|
||||
"""Per-step attention metadata a backend can skip building."""
|
||||
|
||||
LOW_RATIO_DECODE_SCHEDULE = "the aiter decode schedule of each low-ratio indexer"
|
||||
|
||||
|
||||
class DeviceLimits(msgspec.Struct, frozen=True):
|
||||
"""What a spec's kernels support on one ISA, e.g. gfx950."""
|
||||
|
||||
@@ -153,6 +159,8 @@ class MonoSpec(msgspec.Struct, frozen=True):
|
||||
] = None
|
||||
# (model) -> tokens per request when different from the target, e.g. draft blocks
|
||||
tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None
|
||||
# per-step metadata the forward never reads; backends skip it on routed steps
|
||||
unread_metadata: frozenset[StepMetadata] = frozenset()
|
||||
|
||||
def widths(self, isa: str) -> tuple[int, ...]:
|
||||
return self.devices[isa].widths
|
||||
|
||||
@@ -33,6 +33,7 @@ from sglang.srt.monokernels.spec import (
|
||||
NumericsKind,
|
||||
Requirements,
|
||||
ShapeKey,
|
||||
StepMetadata,
|
||||
)
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
@@ -291,6 +292,21 @@ class TestMonokernelFramework(CustomTestCase):
|
||||
lm(torch.zeros(1), None, batch("decode", 2, 2))
|
||||
self.assertEqual(built.steps, [(2, {"layers": 1}), (2, {"layers": 1})])
|
||||
|
||||
def test_only_routed_steps_skip_the_metadata_a_spec_never_reads(self):
|
||||
schedule = StepMetadata.LOW_RATIO_DECODE_SCHEDULE
|
||||
lm, _ = switched(spec=replace(SPEC, unread_metadata=frozenset({schedule})))
|
||||
self.assertTrue(runtime.is_metadata_unread(lm, batch("decode", 2, 2), schedule))
|
||||
# a width not built runs the standard forward, which reads it
|
||||
self.assertFalse(
|
||||
runtime.is_metadata_unread(lm, batch("decode", 3, 3), schedule)
|
||||
)
|
||||
reads_all, _ = switched()
|
||||
without_monokernels = Lm()
|
||||
for model in (reads_all, without_monokernels):
|
||||
self.assertFalse(
|
||||
runtime.is_metadata_unread(model, batch("decode", 2, 2), schedule)
|
||||
)
|
||||
|
||||
def test_the_weights_iterator_passes_weights_through_and_keeps_copies(self):
|
||||
iterator = runtime.monokernel_weights_iterator
|
||||
model, other = torch.nn.Module(), torch.nn.Module()
|
||||
|
||||
Reference in New Issue
Block a user