[Fix] muse_spark: support unified KV memory with TRT-LLM (#45356)

This commit is contained in:
Baizhou Zhang
2026-10-20 12:54:53 -07:00
committed by GitHub
parent 29274b41c6
commit cfec1b91bb
5 changed files with 123 additions and 5 deletions
+1 -1
View File
@@ -216,7 +216,7 @@ jobs:
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
GPU_CONFIG: ${{ inputs.runner_config }}
HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
HF_MUSE_SPARK_TOKEN: ${{ (inputs.self_name == 'base-c-test-8-gpu-b300' || inputs.self_name == 'extra-b-test-4-gpu-b200') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
# Left unconditional because an `env:` key cannot be omitted, and an empty
# value would reach the runai streamer as a malformed limit.
+1 -1
View File
@@ -183,7 +183,7 @@ jobs:
env:
TEST_COMMAND: ${{ matrix.test_command }}
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
HF_MUSE_SPARK_TOKEN: ${{ inputs.pr_head_sha == '' && ((inputs.runs_on == '8-gpu-b300' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py')) || (inputs.runs_on == '4-gpu-b200' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_unified_memory.py'))) && secrets.HF_MUSE_SPARK_TOKEN || '' }}
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
run: |
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
@@ -1156,9 +1156,13 @@ class KVCacheConfigurator:
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves here as the draft runner will; `auto` follows
# the draft's quant config, which only the draft runner knows, so it
# is checked when the draft binds.
# is checked when the draft binds. DFLASH-family FA4 is already known:
# it always uses the draft's compute dtype, including with default/auto
# KV settings. Decline fusion now if that differs from the host.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
if draft_kv_cache_dtype not in (None, "auto") or (
self.spec_algorithm.is_dflash_family() and draft_backend == "fa4"
):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
_, draft_kv_dtype = configure_kv_cache_dtype(
+5 -1
View File
@@ -91,7 +91,7 @@ from sglang.srt.model_executor.forward_context import (
get_token_to_kv_pool,
)
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_forward, get_parallel
from sglang.srt.runtime_context import get_forward, get_memory, get_parallel
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend
from sglang.srt.utils.common import get_device_capability, is_cuda
@@ -353,6 +353,10 @@ class MuseSpark1xAttention(nn.Module):
def _fused_kv_store(
self, forward_batch: ForwardBatch
) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
# Unified pools interleave layers in each token row. Reuse the prefill
# path, whose attention-backend cache writer supports these strides.
if get_memory().enable_unified_memory:
return None
backend = get_attn_backend()
pool = get_token_to_kv_pool()
if not (
@@ -0,0 +1,110 @@
"""Muse Spark unified KV cache coverage with TRT-LLM attention and DFlash2."""
import json
import os
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
from sglang.test.sgl_eval_utils import run_sgl_eval
from sglang.test.test_utils import (
_wait_for_gpu_idle_in_ci,
is_in_ci,
try_cached_model,
write_github_step_summary,
)
register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-b200")
MODEL_PATH = "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"
DRAFT_MODEL_PATH = "RadixArk/Muse-Spark-Open-DFlash2"
class TestMuseSparkUnifiedMemory(DefaultServerBase):
timeout = 3600
gsm8k_accuracy_thres = 0.93
@classmethod
def setUpClass(cls):
# Use the gated-model credential only in this test and its server children.
if token := os.environ.get("HF_MUSE_SPARK_TOKEN"):
token_env = patch.dict(os.environ, {"HF_TOKEN": token})
token_env.start()
cls.addClassCleanup(token_env.stop)
cls.model = try_cached_model(MODEL_PATH)
cls.other_args = [
"--tp-size",
"4",
"--ep-size",
"4",
"--quantization",
"modelopt_fp4",
"--moe-runner-backend",
"flashinfer_trtllm",
"--attention-backend",
"trtllm_mha",
"--page-size",
"128",
"--mem-fraction-static",
"0.8",
"--max-running-requests",
"128",
"--enable-layernorm-sp",
"--enable-megamoe-for-layernorm-sp",
"--speculative-algorithm",
"DFLASH",
"--speculative-draft-model-path",
try_cached_model(DRAFT_MODEL_PATH),
"--speculative-draft-attention-backend",
"fa4",
"--speculative-num-draft-tokens",
"8",
"--enable-unified-memory",
"--mm-feature-transport",
"cpu",
"--reasoning-parser",
"muse",
"--tool-call-parser",
"muse",
]
super().setUpClass()
@classmethod
def tearDownClass(cls):
super().tearDownClass()
_wait_for_gpu_idle_in_ci(timeout=120)
def test_gsm8k(self):
# Exercise decode/verify writes into the shared GQA/SWA pool with all
# 1,319 questions, including the FA4 draft's separate BF16 KV cache.
metrics = run_sgl_eval(
SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
num_examples=None,
num_threads=64,
sgl_eval_thinking=True,
reasoning_effort="xhigh",
max_tokens=32768,
temperature=1.0,
top_p=0.95,
)
)
result = json.loads(Path(metrics["sgl_eval_metrics_path"]).read_text())
self.assertEqual(result["num_examples"], 1319)
self.assertEqual(result["n_repeats"], 1)
self.assertEqual(metrics["error_rate"], 0.0)
if is_in_ci():
write_github_step_summary(
f"### Muse Spark NVFP4 unified memory (TP4/EP4)\n"
f"GSM8K (1319): {metrics['score']:.4f}\n"
)
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
if __name__ == "__main__":
unittest.main()