[Fix] muse_spark: support unified KV memory with TRT-LLM (#45356)
This commit is contained in:
@@ -216,7 +216,7 @@ jobs:
|
||||
CONTINUE_ON_ERROR_FLAG: ${{ fromJson(inputs.check_changes).continue_on_error == 'true' && '--continue-on-error' || '' }}
|
||||
RUNNER_LABELS: ${{ fromJson(fromJson(inputs.check_changes).runs_on_map)[inputs.runner_config] }}
|
||||
GPU_CONFIG: ${{ inputs.runner_config }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.self_name == 'base-c-test-8-gpu-b300' && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ (inputs.self_name == 'base-c-test-8-gpu-b300' || inputs.self_name == 'extra-b-test-4-gpu-b200') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
# Read only by test_runai_model_loader.py (nightly 1-gpu); inert elsewhere.
|
||||
# Left unconditional because an `env:` key cannot be omitted, and an empty
|
||||
# value would reach the runai streamer as a malformed limit.
|
||||
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
env:
|
||||
TEST_COMMAND: ${{ matrix.test_command }}
|
||||
REQUESTED_TEST_COMMAND: ${{ inputs.test_command }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.runs_on == '8-gpu-b300' && inputs.pr_head_sha == '' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py') && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
HF_MUSE_SPARK_TOKEN: ${{ inputs.pr_head_sha == '' && ((inputs.runs_on == '8-gpu-b300' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_12_b300_pd.py')) || (inputs.runs_on == '4-gpu-b200' && contains(matrix.test_command, 'registered/e2e/models/test_muse_spark_unified_memory.py'))) && secrets.HF_MUSE_SPARK_TOKEN || '' }}
|
||||
SGLANG_PRECISION_HF_TOKEN: ${{ inputs.refresh_precision_baseline && inputs.mode == 'cuda' && inputs.runs_on == '8-gpu-h200' && inputs.test_command == 'registered/debug_utils/test_nightly_precision_regression.py' && inputs.pr_head_sha == '' && secrets.HF_TOKEN_PRECISION_STORE || '' }}
|
||||
run: |
|
||||
if [[ "${{ inputs.refresh_precision_baseline }}" == "true" ]]; then
|
||||
|
||||
@@ -1156,9 +1156,13 @@ class KVCacheConfigurator:
|
||||
# A fused draft stores its rows in the host's KV dtype. An explicit
|
||||
# draft dtype resolves here as the draft runner will; `auto` follows
|
||||
# the draft's quant config, which only the draft runner knows, so it
|
||||
# is checked when the draft binds.
|
||||
# is checked when the draft binds. DFLASH-family FA4 is already known:
|
||||
# it always uses the draft's compute dtype, including with default/auto
|
||||
# KV settings. Decline fusion now if that differs from the host.
|
||||
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
|
||||
if draft_kv_cache_dtype not in (None, "auto"):
|
||||
if draft_kv_cache_dtype not in (None, "auto") or (
|
||||
self.spec_algorithm.is_dflash_family() and draft_backend == "fa4"
|
||||
):
|
||||
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
|
||||
|
||||
_, draft_kv_dtype = configure_kv_cache_dtype(
|
||||
|
||||
@@ -91,7 +91,7 @@ from sglang.srt.model_executor.forward_context import (
|
||||
get_token_to_kv_pool,
|
||||
)
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel
|
||||
from sglang.srt.runtime_context import get_forward, get_memory, get_parallel
|
||||
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
|
||||
from sglang.srt.utils import add_prefix, is_cpu, use_intel_amx_backend
|
||||
from sglang.srt.utils.common import get_device_capability, is_cuda
|
||||
@@ -353,6 +353,10 @@ class MuseSpark1xAttention(nn.Module):
|
||||
def _fused_kv_store(
|
||||
self, forward_batch: ForwardBatch
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
|
||||
# Unified pools interleave layers in each token row. Reuse the prefill
|
||||
# path, whose attention-backend cache writer supports these strides.
|
||||
if get_memory().enable_unified_memory:
|
||||
return None
|
||||
backend = get_attn_backend()
|
||||
pool = get_token_to_kv_pool()
|
||||
if not (
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Muse Spark unified KV cache coverage with TRT-LLM attention and DFlash2."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.server_fixtures.default_fixture import DefaultServerBase
|
||||
from sglang.test.sgl_eval_utils import run_sgl_eval
|
||||
from sglang.test.test_utils import (
|
||||
_wait_for_gpu_idle_in_ci,
|
||||
is_in_ci,
|
||||
try_cached_model,
|
||||
write_github_step_summary,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=800, stage="extra-b", runner_config="4-gpu-b200")
|
||||
|
||||
MODEL_PATH = "meta-models/Muse-Spark-Open-816B-A42B-NVFP4"
|
||||
DRAFT_MODEL_PATH = "RadixArk/Muse-Spark-Open-DFlash2"
|
||||
|
||||
|
||||
class TestMuseSparkUnifiedMemory(DefaultServerBase):
|
||||
timeout = 3600
|
||||
gsm8k_accuracy_thres = 0.93
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Use the gated-model credential only in this test and its server children.
|
||||
if token := os.environ.get("HF_MUSE_SPARK_TOKEN"):
|
||||
token_env = patch.dict(os.environ, {"HF_TOKEN": token})
|
||||
token_env.start()
|
||||
cls.addClassCleanup(token_env.stop)
|
||||
cls.model = try_cached_model(MODEL_PATH)
|
||||
cls.other_args = [
|
||||
"--tp-size",
|
||||
"4",
|
||||
"--ep-size",
|
||||
"4",
|
||||
"--quantization",
|
||||
"modelopt_fp4",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--attention-backend",
|
||||
"trtllm_mha",
|
||||
"--page-size",
|
||||
"128",
|
||||
"--mem-fraction-static",
|
||||
"0.8",
|
||||
"--max-running-requests",
|
||||
"128",
|
||||
"--enable-layernorm-sp",
|
||||
"--enable-megamoe-for-layernorm-sp",
|
||||
"--speculative-algorithm",
|
||||
"DFLASH",
|
||||
"--speculative-draft-model-path",
|
||||
try_cached_model(DRAFT_MODEL_PATH),
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
"--speculative-num-draft-tokens",
|
||||
"8",
|
||||
"--enable-unified-memory",
|
||||
"--mm-feature-transport",
|
||||
"cpu",
|
||||
"--reasoning-parser",
|
||||
"muse",
|
||||
"--tool-call-parser",
|
||||
"muse",
|
||||
]
|
||||
super().setUpClass()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
super().tearDownClass()
|
||||
_wait_for_gpu_idle_in_ci(timeout=120)
|
||||
|
||||
def test_gsm8k(self):
|
||||
# Exercise decode/verify writes into the shared GQA/SWA pool with all
|
||||
# 1,319 questions, including the FA4 draft's separate BF16 KV cache.
|
||||
metrics = run_sgl_eval(
|
||||
SimpleNamespace(
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
num_examples=None,
|
||||
num_threads=64,
|
||||
sgl_eval_thinking=True,
|
||||
reasoning_effort="xhigh",
|
||||
max_tokens=32768,
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
)
|
||||
)
|
||||
result = json.loads(Path(metrics["sgl_eval_metrics_path"]).read_text())
|
||||
self.assertEqual(result["num_examples"], 1319)
|
||||
self.assertEqual(result["n_repeats"], 1)
|
||||
self.assertEqual(metrics["error_rate"], 0.0)
|
||||
if is_in_ci():
|
||||
write_github_step_summary(
|
||||
f"### Muse Spark NVFP4 unified memory (TP4/EP4)\n"
|
||||
f"GSM8K (1319): {metrics['score']:.4f}\n"
|
||||
)
|
||||
self.assertGreaterEqual(metrics["score"], self.gsm8k_accuracy_thres)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user