diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index afbe0ca503..83b62f9396 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -440,7 +440,8 @@ ARG FHT_COMMIT="46efb7d776d38638fc39f3c803eaee3dd7016bd1" # gfx950); disable with --build-arg ENABLE_SGL_AMD_MONO=0. ARG ENABLE_SGL_AMD_MONO=1 ARG SGL_AMD_MONO_REPO="https://github.com/sgl-project/sgl-amd-mono.git" -ARG SGL_AMD_MONO_COMMIT="815b104f483d180027054a8f44bad0c2a3f895fd" +# empty: the commit pip installed through the monokernels_hip extra +ARG SGL_AMD_MONO_COMMIT="" ARG ENABLE_MORI=0 ARG NIC_BACKEND=none @@ -895,9 +896,11 @@ RUN /bin/bash -lc 'set -euo pipefail; \ RUN /bin/bash -lc 'set -euo pipefail; \ [ "${ENABLE_SGL_AMD_MONO}" = "1" ] || { echo "[sgl-amd-mono] skip (ENABLE_SGL_AMD_MONO=${ENABLE_SGL_AMD_MONO})"; exit 0; }; \ case "${GPU_ARCH}" in gfx950*) ;; *) echo "[sgl-amd-mono] skip (GPU_ARCH=${GPU_ARCH})"; exit 0 ;; esac; \ + python3 -m pip show sgl-amd-mono > /dev/null || { echo "[sgl-amd-mono] skip (this SGLang does not install it)"; exit 0; }; \ + commit="${SGL_AMD_MONO_COMMIT:-$(python3 -c "import json, importlib.metadata as md; print(json.loads(md.distribution(\"sgl-amd-mono\").read_text(\"direct_url.json\"))[\"vcs_info\"][\"commit_id\"])")}"; \ git clone "${SGL_AMD_MONO_REPO}" /sgl-workspace/sgl-amd-mono; \ cd /sgl-workspace/sgl-amd-mono; \ - git checkout -f "${SGL_AMD_MONO_COMMIT}"; \ + git checkout -f "${commit}"; \ python3 -m pip install --no-cache-dir --no-deps -e .; \ python3 -c "from importlib.metadata import entry_points; \ assert any(e.name == \"sgl_amd_mono\" for e in entry_points(group=\"sglang.srt.plugins\")), \"sgl-amd-mono plugin not registered\""' diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 41b0beccb5..203ca3d864 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -107,6 +107,11 @@ tracing = [ "opentelemetry-sdk", ] +# Monokernels for gfx950 (--monokernel-backend amd); the ROCm image checks out this pin too. +monokernels_hip = [ + "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@e07665c8b36f7d42a48f6615bacf911d0656f5c5", +] + # HIP (Heterogeneous-computing Interface for Portability) for AMD # => base docker rocm/vllm-dev:20250114, not from public vllm whl # @@ -114,11 +119,10 @@ tracing = [ # in its compressed-tensors and torch pins. A dependency added here has to be # added there too; nothing enforces that. srt_hip = [ + "sglang[monokernels_hip]", "sglang[runtime_common]", "torch", "petit_kernel==0.0.2", - # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono. - "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca", "wave-lang==3.8.2", ] @@ -128,8 +132,7 @@ srt_hip_rocm724 = [ # torch<2.11, while 0.16.0 supports torch>=2.10. "compressed-tensors==0.16.0", "petit_kernel==0.0.2", - # Monokernels for gfx950 (--monokernel-backend amd); bump with sgl-amd-mono. - "sgl-amd-mono @ git+https://github.com/sgl-project/sgl-amd-mono.git@8f1bff3c06f8467cda1512122844d58d389ad7ca", + "sglang[monokernels_hip]", "sglang[runtime_common]", "torch==2.11.0", "wave-lang==3.8.2", diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index ee2083cb97..275b5125f7 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -107,6 +107,8 @@ from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.dsv41_request_window import window_layout from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.monokernels.runtime import is_metadata_unread +from sglang.srt.monokernels.spec import StepMetadata from sglang.srt.runtime_context import ( get_exec, get_parallel, @@ -907,6 +909,7 @@ class DeepseekV4HipRadixBackend( speculative_num_steps=0, ): super().__init__() + self.model_runner = model_runner self.device = torch.device(model_runner.device) self.max_context_len = model_runner.model_config.context_len head_dim = model_runner.model_config.head_dim @@ -1768,6 +1771,11 @@ class DeepseekV4HipRadixBackend( if not low_ratio_decode_rows_are_identity( self, forward_batch, ratio ) + and not is_metadata_unread( + self.model_runner.model, + forward_batch, + StepMetadata.LOW_RATIO_DECODE_SCHEDULE, + ) }, max_context_len=self.max_context_len, num_requests=forward_batch.batch_size, diff --git a/python/sglang/srt/monokernels/runtime.py b/python/sglang/srt/monokernels/runtime.py index dcededa8f3..3272aa0e90 100644 --- a/python/sglang/srt/monokernels/runtime.py +++ b/python/sglang/srt/monokernels/runtime.py @@ -21,6 +21,7 @@ from sglang.srt.monokernels.spec import ( MonoSpec, Plan, ShapeKey, + StepMetadata, find_spec, spec_registered, ) @@ -100,6 +101,14 @@ def shape_key(forward_batch: ForwardBatch) -> Optional[ShapeKey]: return ShapeKey(forward_mode=name, batch_size=bs, tokens_per_request=rows // bs) +def is_metadata_unread( + model: torch.nn.Module, forward_batch: ForwardBatch, metadata: StepMetadata +) -> bool: + """Whether the monokernel forward that runs this step never reads the metadata.""" + served, unread = model.__dict__.get("_monokernel_unread_metadata", ((), ())) + return metadata in unread and shape_key(forward_batch) in served + + def install_monokernel_forward( *, model: torch.nn.Module, @@ -340,6 +349,8 @@ def _install_forward( return built.forward(Plan(key=key), *args, **kwargs) model.forward = forward + # attention backends ask is_metadata_unread before building per-step metadata + model._monokernel_unread_metadata = (served, spec.unread_metadata) def _format_steps(keys: Iterable[ShapeKey]) -> str: diff --git a/python/sglang/srt/monokernels/spec.py b/python/sglang/srt/monokernels/spec.py index 985159dc0c..d48dd02053 100644 --- a/python/sglang/srt/monokernels/spec.py +++ b/python/sglang/srt/monokernels/spec.py @@ -40,6 +40,12 @@ class Feature(enum.Enum): PARTITIONED_GPU = "partitioned GPUs" +class StepMetadata(enum.Enum): + """Per-step attention metadata a backend can skip building.""" + + LOW_RATIO_DECODE_SCHEDULE = "the aiter decode schedule of each low-ratio indexer" + + class DeviceLimits(msgspec.Struct, frozen=True): """What a spec's kernels support on one ISA, e.g. gfx950.""" @@ -153,6 +159,8 @@ class MonoSpec(msgspec.Struct, frozen=True): ] = None # (model) -> tokens per request when different from the target, e.g. draft blocks tokens_per_request: Optional[Callable[[torch.nn.Module], int]] = None + # per-step metadata the forward never reads; backends skip it on routed steps + unread_metadata: frozenset[StepMetadata] = frozenset() def widths(self, isa: str) -> tuple[int, ...]: return self.devices[isa].widths diff --git a/test/registered/unit/monokernels/test_mono_framework.py b/test/registered/unit/monokernels/test_mono_framework.py index 2d4a3a9b1f..dd9697ebf1 100644 --- a/test/registered/unit/monokernels/test_mono_framework.py +++ b/test/registered/unit/monokernels/test_mono_framework.py @@ -33,6 +33,7 @@ from sglang.srt.monokernels.spec import ( NumericsKind, Requirements, ShapeKey, + StepMetadata, ) from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs @@ -291,6 +292,21 @@ class TestMonokernelFramework(CustomTestCase): lm(torch.zeros(1), None, batch("decode", 2, 2)) self.assertEqual(built.steps, [(2, {"layers": 1}), (2, {"layers": 1})]) + def test_only_routed_steps_skip_the_metadata_a_spec_never_reads(self): + schedule = StepMetadata.LOW_RATIO_DECODE_SCHEDULE + lm, _ = switched(spec=replace(SPEC, unread_metadata=frozenset({schedule}))) + self.assertTrue(runtime.is_metadata_unread(lm, batch("decode", 2, 2), schedule)) + # a width not built runs the standard forward, which reads it + self.assertFalse( + runtime.is_metadata_unread(lm, batch("decode", 3, 3), schedule) + ) + reads_all, _ = switched() + without_monokernels = Lm() + for model in (reads_all, without_monokernels): + self.assertFalse( + runtime.is_metadata_unread(model, batch("decode", 2, 2), schedule) + ) + def test_the_weights_iterator_passes_weights_through_and_keeps_copies(self): iterator = runtime.monokernel_weights_iterator model, other = torch.nn.Module(), torch.nn.Module()