From cfec1b91bb33f3267d1811680b51fd581e0df458 Mon Sep 17 00:00:00 2001 From: Baizhou Zhang Date: Tue, 20 Oct 2026 12:54:53 -0700 Subject: [PATCH] [Fix] muse_spark: support unified KV memory with TRT-LLM (#45356) --- .github/workflows/_pr-test-stage.yml | 2 +- .github/workflows/rerun-test.yml | 2 +- .../srt/mem_cache/kv_cache_configurator.py | 8 +- python/sglang/srt/models/muse_spark.py | 6 +- .../models/test_muse_spark_unified_memory.py | 110 ++++++++++++++++++ 5 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 test/registered/e2e/models/test_muse_spark_unified_memory.py diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 9ee7bf4532..bc358290ad 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -216,7 +216,7 @@ jobs: CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }} RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }} GPU_CONFIG: ${{ inputs.runner_config }} - HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }} + HF_MUSE_SPARK_TOKEN: ${{ (inputs.self_name == 'base-c-test-8-gpu-b300' || inputs.self_name == 'extra-b-test-4-gpu-b200') && secrets.HF_MUSE_SPARK_TOKEN || '' }} # Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere. # Left unconditional because an `env:` key cannot be omitted, and an empty # value would reach the runai streamer as a malformed limit. diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index 73af90f156..c79603f050 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -183,7 +183,7 @@ jobs: env: TEST_COMMAND: ${{ matrix.test_command }} REQUESTED_TEST_COMMAND: ${{ inputs.test_command }} - HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }} + HF_MUSE_SPARK_TOKEN: ${{ inputs.pr_head_sha == '' && ((inputs.runs_on == '8-gpu-b300' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py')) || (inputs.runs_on == '4-gpu-b200' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_unified_memory.py'))) && secrets.HF_MUSE_SPARK_TOKEN || '' }} SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }} run: | if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index c4e87f316e..660941cbc6 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -1156,9 +1156,13 @@ class KVCacheConfigurator: # A fused draft stores its rows in the host's KV dtype. An explicit # draft dtype resolves here as the draft runner will; `auto` follows # the draft's quant config, which only the draft runner knows, so it - # is checked when the draft binds. + # is checked when the draft binds. DFLASH-family FA4 is already known: + # it always uses the draft's compute dtype, including with default/auto + # KV settings. Decline fusion now if that differs from the host. draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype - if draft_kv_cache_dtype not in (None, "auto"): + if draft_kv_cache_dtype not in (None, "auto") or ( + self.spec_algorithm.is_dflash_family() and draft_backend == "fa4" + ): from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype _, draft_kv_dtype = configure_kv_cache_dtype( diff --git a/python/sglang/srt/models/muse_spark.py b/python/sglang/srt/models/muse_spark.py index 8442ae4251..35a458bedf 100644 --- a/python/sglang/srt/models/muse_spark.py +++ b/python/sglang/srt/models/muse_spark.py @@ -91,7 +91,7 @@ from sglang.srt.model_executor.forward_context import ( get_token_to_kv_pool, ) from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_forward, get_parallel +from sglang.srt.runtime_context import get_forward, get_memory, get_parallel from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend from sglang.srt.utils.common import get_device_capability, is_cuda @@ -353,6 +353,10 @@ class MuseSpark1xAttention(nn.Module): def _fused_kv_store( self, forward_batch: ForwardBatch ) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: + # Unified pools interleave layers in each token row. Reuse the prefill + # path, whose attention-backend cache writer supports these strides. + if get_memory().enable_unified_memory: + return None backend = get_attn_backend() pool = get_token_to_kv_pool() if not ( diff --git a/test/registered/e2e/models/test_muse_spark_unified_memory.py b/test/registered/e2e/models/test_muse_spark_unified_memory.py new file mode 100644 index 0000000000..d6d89f7295 --- /dev/null +++ b/test/registered/e2e/models/test_muse_spark_unified_memory.py @@ -0,0 +1,110 @@ +"""Muse Spark unified KV cache coverage with TRT-LLM attention and DFlash2.""" + +import json +import os +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.sgl_eval_utils import run_sgl_eval +from sglang.test.test_utils import ( + _wait_for_gpu_idle_in_ci, + is_in_ci, + try_cached_model, + write_github_step_summary, +) + +register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-b200") + +MODEL_PATH = "meta-models/Muse-Spark-Open-816B-A42B-NVFP4" +DRAFT_MODEL_PATH = "RadixArk/Muse-Spark-Open-DFlash2" + + +class TestMuseSparkUnifiedMemory(DefaultServerBase): + timeout = 3600 + gsm8k_accuracy_thres = 0.93 + + @classmethod + def setUpClass(cls): + # Use the gated-model credential only in this test and its server children. + if token := os.environ.get("HF_MUSE_SPARK_TOKEN"): + token_env = patch.dict(os.environ, {"HF_TOKEN": token}) + token_env.start() + cls.addClassCleanup(token_env.stop) + cls.model = try_cached_model(MODEL_PATH) + cls.other_args = [ + "--tp-size", + "4", + "--ep-size", + "4", + "--quantization", + "modelopt_fp4", + "--moe-runner-backend", + "flashinfer_trtllm", + "--attention-backend", + "trtllm_mha", + "--page-size", + "128", + "--mem-fraction-static", + "0.8", + "--max-running-requests", + "128", + "--enable-layernorm-sp", + "--enable-megamoe-for-layernorm-sp", + "--speculative-algorithm", + "DFLASH", + "--speculative-draft-model-path", + try_cached_model(DRAFT_MODEL_PATH), + "--speculative-draft-attention-backend", + "fa4", + "--speculative-num-draft-tokens", + "8", + "--enable-unified-memory", + "--mm-feature-transport", + "cpu", + "--reasoning-parser", + "muse", + "--tool-call-parser", + "muse", + ] + super().setUpClass() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + _wait_for_gpu_idle_in_ci(timeout=120) + + def test_gsm8k(self): + # Exercise decode/verify writes into the shared GQA/SWA pool with all + # 1,319 questions, including the FA4 draft's separate BF16 KV cache. + metrics = run_sgl_eval( + SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + num_examples=None, + num_threads=64, + sgl_eval_thinking=True, + reasoning_effort="xhigh", + max_tokens=32768, + temperature=1.0, + top_p=0.95, + ) + ) + result = json.loads(Path(metrics["sgl_eval_metrics_path"]).read_text()) + self.assertEqual(result["num_examples"], 1319) + self.assertEqual(result["n_repeats"], 1) + self.assertEqual(metrics["error_rate"], 0.0) + if is_in_ci(): + write_github_step_summary( + f"### Muse Spark NVFP4 unified memory (TP4/EP4)\n" + f"GSM8K (1319): {metrics['score']:.4f}\n" + ) + self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres) + + +if __name__ == "__main__": + unittest.main()