diff --git a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py index d5bdc80dae..7bcc016724 100644 --- a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py +++ b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py @@ -81,8 +81,19 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: # Prefill only: a decode node serves plain TP. "disaggregation-mode decode": cfg.disaggregation_mode == "decode", } - else: - unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null" + elif model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration": + if cfg.disaggregation_mode != "null": + unsupported |= { + "disaggregation-mode decode": cfg.disaggregation_mode == "decode", + "disaggregation-transfer-backend != mooncake": ( + cfg.disaggregation_transfer_backend != "mooncake" + ), + "speculative-algorithm != DFLASH": cfg.speculative_algorithm + != "DFLASH", + "pp-size > 1": cfg.pp_size > 1, + "dp-size > 1": cfg.dp_size > 1, + "ep-size > 1": view.ep_size > 1, + } for flag, enabled in unsupported.items(): if enabled: raise ValueError(f"--{flag} is not supported with CP-TP group sharing.") diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index e40bf982a2..29e06e6763 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -77,6 +77,7 @@ from sglang.srt.runtime_context import ( get_memory, get_observability, get_schedule, + get_spec, ) from sglang.srt.server_args import ServerArgs from sglang.srt.utils.network import NetworkAddress @@ -951,6 +952,20 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): a reason rejects the peer, ``fully_checked`` retires the per-transfer check. """ + if ( + get_spec().speculative_algorithm == "DFLASH" + # Bootstrap runs concurrently with temporary draft TP scopes. + # Use the manager's captured target layout, not scoped CP flags. + and self.state_tp_size != self.attn_tp_size + and registration_info.dst_attn_tp_size != self.state_tp_size + ): + # Appended draft MHA KV uses physical TP shards, while target MLA + # KV is replicated. The draft transfer cannot reslice those shards. + return False, ( + "CP/TP-sharing DFLASH PD requires the same physical TP size: " + f"prefill={self.state_tp_size}, " + f"decode={registration_info.dst_attn_tp_size}" + ) state_types = getattr(self.kv_args, "state_types", []) fully_checked = True for i, st in enumerate(state_types): diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index bcd8173c5e..aa6c4977b7 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -3958,6 +3958,10 @@ class DeepseekSparseAttnBackend( kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim) block_tables = page_table_1.unsqueeze(1) seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens + if is_prefill: + # CP graph metadata includes the physical shard's padding; its + # eager attention bridge can narrow Q to the live local rows. + seq_lens = seq_lens[:batch_size] attention_kwargs = dict( kv_cache=kv, diff --git a/python/sglang/srt/layers/attention/graph_utils.py b/python/sglang/srt/layers/attention/graph_utils.py index d1bd074670..1f94dc8db2 100644 --- a/python/sglang/srt/layers/attention/graph_utils.py +++ b/python/sglang/srt/layers/attention/graph_utils.py @@ -32,8 +32,23 @@ def _zero_padded_tokens(output, actual_tokens): output[actual_tokens:].zero_() +def attention_query_tokens(forward_batch, tokens): + # Import at call time: the CP package also imports RadixAttention. + from sglang.srt.layers.cp.padding import attention_query_tokens as cp_query_tokens + + return cp_query_tokens(forward_batch, tokens) + + +def attention_output_tokens(forward_batch, tokens): + from sglang.srt.layers.cp.padding import attention_output_tokens as cp_output_tokens + + return cp_output_tokens(forward_batch, tokens) + + @contextmanager -def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs): +def attention_input_scope( + forward_batch, output, query_tokens, kv_tokens, kwargs, *, cache_tokens=None +): """Narrow per-token operands and restore the caller's batch even on failure.""" kwargs = dict(kwargs) for name in ( @@ -56,9 +71,11 @@ def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs ] cache_loc, positions = forward_batch.out_cache_loc, forward_batch.positions previous_output = forward_batch._attn_output - forward_batch.out_cache_loc = cache_loc[:query_tokens] + if cache_tokens is None: + cache_tokens = query_tokens + forward_batch.out_cache_loc = cache_loc[:cache_tokens] if positions is not None: - forward_batch.positions = positions[:query_tokens] + forward_batch.positions = positions[:cache_tokens] forward_batch._attn_output = output[:query_tokens] try: yield kwargs diff --git a/python/sglang/srt/layers/cp/base.py b/python/sglang/srt/layers/cp/base.py index b195b84685..42124386cd 100644 --- a/python/sglang/srt/layers/cp/base.py +++ b/python/sglang/srt/layers/cp/base.py @@ -25,6 +25,7 @@ The strategy implementation is split across: from __future__ import annotations from abc import ABC, abstractmethod +from contextlib import contextmanager from dataclasses import dataclass from enum import IntEnum from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple @@ -247,6 +248,22 @@ def _is_dsa_active() -> bool: _STRATEGY: Optional[ContextParallelStrategy] = None +@contextmanager +def disable_cp_strategy(): + """Suspend the cached target strategy while a TP-only draft runs. + + Pair with a runtime topology scope that disables prefill CP so lazy lookup + cannot reconstruct the target strategy. Restore it even if draft work fails. + """ + global _STRATEGY + strategy = _STRATEGY + _STRATEGY = None + try: + yield + finally: + _STRATEGY = strategy + + def init_cp_strategy( *, enable_prefill_cp: bool, cp_size: int, cp_strategy: str ) -> None: diff --git a/python/sglang/srt/layers/cp/bcg/interleave_bcg.py b/python/sglang/srt/layers/cp/bcg/interleave_bcg.py index d90299cfcd..d708299ca8 100644 --- a/python/sglang/srt/layers/cp/bcg/interleave_bcg.py +++ b/python/sglang/srt/layers/cp/bcg/interleave_bcg.py @@ -1,8 +1,7 @@ -"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash. +"""Interleave BCG adapter for GLM-5.3 Flash text prefills. Keep full-sequence KPool positions separate from the local DSA positions. -Prefix-free text batches replay when their total token count matches a capture, -independently of request boundaries. Other batches retain eager CP, including MM. +Graph row capacity is fixed while live token counts and prefix state can change. """ import torch @@ -27,16 +26,14 @@ class InterleaveCPBCGInput(PrefillCPBCGInput): result.input_ids_global = torch.zeros( capacity, dtype=torch.int64, device=runner.device ) + # Graphs must read the live count from the same address on each replay. + result.moe_local_token_count = torch.zeros( + (), dtype=torch.int32, device=runner.device + ) return result def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs): - return ( - batch_size > 0 - and num_tokens in self.bucket_local_tokens - and not contains_mm_inputs - and prefix_lens is not None - and not any(prefix_lens) - ) + return batch_size > 0 and not contains_mm_inputs and prefix_lens is not None @staticmethod def supports(server_args): @@ -63,17 +60,21 @@ class InterleaveCPBCGInput(PrefillCPBCGInput): runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture ) metadata = forward_batch.attn_cp_metadata - cp_size = len(metadata.per_rank_actual_token) - base, remainder = divmod(static_num_tokens, cp_size) - metadata.total_seq_lens = static_num_tokens - metadata.per_rank_logical_token = [ - base + int(rank < remainder) for rank in range(cp_size) - ] + # KDA projections/reorders retain the captured shape. Eager KPool + # compression, KV writes and the output gather use the live lengths + # produced by prepare_cp_forward, never the bucket's padded tail. + metadata.graph_num_tokens = static_num_tokens + self.moe_local_token_count.fill_( + metadata.per_rank_logical_token[get_parallel().attn_cp_rank] + ) + metadata.moe_local_token_count = self.moe_local_token_count forward_batch.out_cache_loc = out_cache_loc # KPool compresses consecutive global tokens; only the model argument # (DSA RoPE) is local. Do not replace ForwardBatch's global positions. forward_batch.positions = global_positions - ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch) + ids = cp_interleave_input_ids( + forward_batch.input_ids[: forward_batch.extend_num_tokens], forward_batch + ) self.input_ids_global[: ids.shape[0]].copy_(ids) forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]] diff --git a/python/sglang/srt/layers/cp/interleave.py b/python/sglang/srt/layers/cp/interleave.py index 1cf64e5381..5e013b566e 100644 --- a/python/sglang/srt/layers/cp/interleave.py +++ b/python/sglang/srt/layers/cp/interleave.py @@ -76,6 +76,9 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata): gather_index: Optional[torch.Tensor] = None local_index: Optional[torch.Tensor] = None moe_local_token_count: Optional[torch.Tensor] = None + # Only sequence-dependent graph segments use this capacity. Eager cache + # operations and gathers must continue using total_seq_lens (live tokens). + graph_num_tokens: Optional[int] = None def is_interleave_extend(forward_batch): @@ -92,7 +95,9 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch): ``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute. The residual and mHC coefficients do not move: only the already-read mixer - input is reordered. Remove physical CP padding before the recurrent kernel. + input is reordered. Eager execution removes physical CP padding. Graph + execution keeps the capture shape; KDA's refreshed sequence table masks + padded rows out of its recurrent state updates. """ if not is_interleave_extend(forward_batch): return hidden_states @@ -101,10 +106,15 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch): return hidden_states.index_select(0, metadata.gather_index) size = get_parallel().attn_cp_size rows = hidden_states.shape[0] // size + num_tokens = ( + metadata.graph_num_tokens + if metadata.graph_num_tokens is not None + else metadata.total_seq_lens + ) return ( hidden_states.reshape(size, rows, *hidden_states.shape[1:]) .transpose(0, 1) - .flatten(0, 1)[: metadata.total_seq_lens] + .flatten(0, 1)[:num_tokens] .contiguous() ) diff --git a/python/sglang/srt/layers/cp/padding.py b/python/sglang/srt/layers/cp/padding.py index 8c2921506f..1de1842751 100644 --- a/python/sglang/srt/layers/cp/padding.py +++ b/python/sglang/srt/layers/cp/padding.py @@ -60,3 +60,22 @@ def pad_local_rows(x: torch.Tensor, metadata: Any, dim: int) -> torch.Tensor: pad_shape = list(x.shape) pad_shape[dim] = pad_size return torch.cat([x, x.new_zeros(pad_shape)], dim=dim) + + +def attention_query_tokens(forward_batch, n): + cp_metadata = getattr(forward_batch, "attn_cp_metadata", None) + if getattr(cp_metadata, "graph_num_tokens", None) is not None: + # Interleave BCG gathers KV in global sequence order but Q stays + # local. Empty padded queries must not enter sparse attention. + n = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank] + return n + + +def attention_output_tokens(forward_batch, raw): + cp_metadata = getattr(forward_batch, "attn_cp_metadata", None) + if cp_metadata is not None and cp_metadata.per_rank_logical_token is not None: + # Q/output rows are CP-local; cache locations and positions + # above remain global. Zero every invalid local row before + # captured projections/MoE can consume an empty-attention NaN. + raw = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank] + return raw diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py index 9d0b67ae02..09ffa41d1b 100644 --- a/python/sglang/srt/layers/radix_attention.py +++ b/python/sglang/srt/layers/radix_attention.py @@ -26,6 +26,8 @@ from sglang.srt.layers.attention.graph_utils import ( _zero_padded_tokens, allocate_attention_outputs, attention_input_scope, + attention_output_tokens, + attention_query_tokens, padded_extend_real_tokens, ) from sglang.srt.model_executor.forward_context import ( @@ -190,6 +192,8 @@ class RadixAttention(nn.Module): rows = output.shape[0] n = forward_batch.global_num_token_non_padded_cpu kv_n = n if key_value_num_tokens is None else key_value_num_tokens + cache_n = n + n = attention_query_tokens(forward_batch, n) return_lse = bool(kwargs.get("return_lse") or forward_batch.mha_return_lse) lse = None if n == 0: @@ -200,7 +204,7 @@ class RadixAttention(nn.Module): lse = q.new_zeros((rows, q.shape[1]), dtype=torch.float32) else: with attention_input_scope( - forward_batch, output, n, kv_n, kwargs + forward_batch, output, n, kv_n, kwargs, cache_tokens=cache_n ) as kwargs: result = get_attn_backend().forward( q[:n], @@ -220,6 +224,7 @@ class RadixAttention(nn.Module): if result.data_ptr() != output.data_ptr(): output[:n].view(result.shape).copy_(result) raw = get_prefill_raw_num_tokens() + raw = attention_output_tokens(forward_batch, raw) _zero_padded_tokens(output, raw) if idx_output is not None: _zero_padded_tokens(idx_output, raw) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 7122fe86f7..54b7e9f230 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1985,6 +1985,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): mamba_track_indices=mamba_track_indices, mamba_track_mask=mamba_track_mask, mamba_track_seqlens=mamba_track_seqlens, + # CP replay plans attention against this static view, including + # KDA's host-side checkpoint plan for the current request. + mamba_prefill_track_mask_cpu=forward_batch.mamba_prefill_track_mask_cpu, + mamba_track_seqlens_cpu=forward_batch.mamba_track_seqlens_cpu, encoder_lens=forward_batch.encoder_lens, return_logprob=( forward_batch.return_logprob diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 9ca5aa30cd..3b3b0c4dba 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -103,7 +103,7 @@ from sglang.srt.speculative.spec_utils import ( draft_pp_context, draft_tp_context, ) -from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu +from sglang.srt.utils import async_h2d, is_cuda, is_hip, is_npu, is_xpu _is_npu = is_npu() @@ -2375,8 +2375,10 @@ class DFlashWorkerV2(BaseSpecWorker): # Materialize prompt tokens into the draft KV cache immediately. This is required # for radix cache safety (the scheduler may update radix after prefill returns). device = next_token_ids.device - ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device) - draft_seq_lens = torch.tensor( + # Enqueue metadata copies without waiting for the target prefill's + # outstanding GPU work; the same stream orders their consumers. + ctx_lens = async_h2d(batch.extend_lens, dtype=torch.int32, device=device) + draft_seq_lens = async_h2d( batch.prefix_lens, dtype=torch.int32, device=device ) diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 3b4685664e..ea92a4beea 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -34,6 +34,7 @@ from sglang.srt.distributed.parallel_state import ( ) from sglang.srt.environ import envs from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled +from sglang.srt.layers.cp.base import disable_cp_strategy from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs from sglang.srt.managers.utils import _async_d2h from sglang.srt.mem_cache.allocation import ( @@ -819,8 +820,24 @@ def draft_tp_context(owns_attention: bool): """ if not owns_attention: return contextlib.nullcontext() - group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group") - return patch_tensor_parallel_group(group, owns_attention=True) + parallel = get_parallel() + group = getattr(parallel, f"{draft_attn_parallel_group()}_group") + tp_context = patch_tensor_parallel_group(group, owns_attention=True) + if parallel.enable_prefill_cp: + return _draft_without_cp(tp_context) + return tp_context + + +@contextmanager +def _draft_without_cp(tp_context): + with ( + tp_context, + get_parallel().override( + enable_prefill_cp=False, enable_cp_tp_group_sharing=False + ), + disable_cp_strategy(), + ): + yield def spec_stage_span(name: str): diff --git a/test/registered/cp/test_cp_strategy_unit.py b/test/registered/cp/test_cp_strategy_unit.py index 02f268fe0d..a76b4ac983 100644 --- a/test/registered/cp/test_cp_strategy_unit.py +++ b/test/registered/cp/test_cp_strategy_unit.py @@ -183,8 +183,9 @@ class TestPrefillCPBCGReplay(CustomTestCase): 512, ) self.assertTrue(runner.can_run_graph(batch)) - batch.extend_prefix_lens_cpu = [0, 1, 0] - self.assertFalse(runner.can_run_graph(batch)) + with get_parallel().override(attn_cp_rank=0, attn_cp_size=4): + batch.extend_prefix_lens_cpu = [0, 1, 0] + self.assertTrue(runner.can_run_graph(batch)) def test_local_capacity_overflow_uses_next_capture_bucket(self): runner = self._make_runner() diff --git a/test/registered/cp/test_glm53_flash_cp.py b/test/registered/cp/test_glm53_flash_cp.py deleted file mode 100644 index 5016ef9804..0000000000 --- a/test/registered/cp/test_glm53_flash_cp.py +++ /dev/null @@ -1,137 +0,0 @@ -"""GLM-5.3-Flash interleave CP4 + DFlash2 on four Blackwell GPUs.""" - -import base64 -import io -import unittest - -import requests -from PIL import Image - -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - _wait_for_gpu_idle_in_ci, - popen_launch_server, - terminate_and_kill_process_tree, - try_cached_model, -) - -register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200") - - -class TestGLM53FlashB200ContextParallel( - GSM8KMixin, - CustomTestCase, -): - gsm8k_score_threshold = 0.93 - gsm8k_num_examples = 500 - gsm8k_num_shots = 20 - server_args = [ - "--tp-size", - "4", - "--dsa-prefill-backend", - "trtllm", - "--dsa-decode-backend", - "trtllm", - "--kv-cache-dtype", - "fp8_e4m3", - "--moe-runner-backend", - "flashinfer_trtllm", - "--reasoning-parser", - "auto", - "--tool-call-parser", - "auto", - "--speculative-algorithm", - "DFLASH", - "--speculative-draft-model-path", - "incoai/GLM-5.3-Flash-DFlash2", - "--speculative-draft-attention-backend", - "fa4", - "--mem-fraction-static", - "0.85", - "--enable-prefill-cp", - "--cp-strategy", - "interleave", - "--attn-cp-size", - "4", - # KDA partitions heads over TP4, sharing the CP group. EP1 keeps every MoE - # on TP4; dense FFNs also use TP4 rather than per-rank computation. - "--ep-size", - "1", - "--moe-dense-tp-size", - "4", - ] - - @classmethod - def setUpClass(cls): - cls.model = try_cached_model("zai-org/GLM-5.3-Flash") - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = None - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=3600, - other_args=cls.server_args, - ) - - @classmethod - def tearDownClass(cls): - if process := getattr(cls, "process", None): - terminate_and_kill_process_tree(process) - _wait_for_gpu_idle_in_ci(timeout=120) - - def test_gsm8k(self): - super().test_gsm8k() - # Require the metric as well as the floor: an accuracy pass alone must - # not hide disabled speculation or a draft whose tokens are mostly rejected. - response = requests.get(self.base_url + "/server_info", timeout=30) - response.raise_for_status() - self.assertGreater( - response.json()["internal_states"][0]["avg_spec_accept_length"], 4.0 - ) - - def test_image_input(self): - # Color is supplied only by the image: the prompt cannot substitute for - # the encoder. Exercise two images to catch missing/cached embeddings. - for color in ("red", "blue"): - with self.subTest(color=color): - png = io.BytesIO() - Image.new("RGB", (256, 256), color=color).save(png, format="PNG") - image_url = ( - "data:image/png;base64," + base64.b64encode(png.getvalue()).decode() - ) - response = requests.post( - self.base_url + "/v1/chat/completions", - json={ - "model": self.model, - "messages": [ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": {"url": image_url}, - }, - { - "type": "text", - "text": "What is the dominant color of this image? Answer with one English color word.", - }, - ], - } - ], - "temperature": 0, - # The checkpoint template always opens a thinking block. - # Allow it to finish before checking the final answer. - "max_tokens": 1024, - }, - timeout=180, - ) - self.assertEqual(response.status_code, 200, response.text) - answer = response.json()["choices"][0]["message"]["content"] - self.assertIn(color, answer.lower(), response.text) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/e2e/disaggregation/test_glm53_cp_bcg_dflash.py b/test/registered/e2e/disaggregation/test_glm53_cp_bcg_dflash.py new file mode 100644 index 0000000000..a24e560aa4 --- /dev/null +++ b/test/registered/e2e/disaggregation/test_glm53_cp_bcg_dflash.py @@ -0,0 +1,223 @@ +"""GLM CP4/BCG prefill -> TP4 DFlash2 decode, with padding and cached prefixes.""" + +import unittest + +import requests +from transformers import AutoTokenizer + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) +from sglang.test.test_utils import popen_with_error_check, try_cached_model + +register_cuda_ci(est_time=900, stage="extra-b", runner_config="8-gpu-b300") + +COMMON_ARGS = [ + "--dsa-prefill-backend", + "trtllm", + "--dsa-decode-backend", + "trtllm", + "--kv-cache-dtype", + "fp8_e4m3", + "--moe-runner-backend", + "flashinfer_trtllm", + "--speculative-algorithm", + "DFLASH", + "--speculative-draft-attention-backend", + "fa4", + "--reasoning-parser", + "auto", + "--tool-call-parser", + "auto", + "--max-running-requests", + "16", + "--cuda-graph-max-bs-decode", + "16", + "--chunked-prefill-size", + "8192", + "--mem-fraction-static", + "0.9", + "--enable-metrics", + "--random-seed", + "42", +] + + +class TestGlm53CpBcgDflashPD(PDDisaggregationServerBase, GSM8KMixin): + gsm8k_score_threshold = 0.96 + gsm8k_num_examples = 1319 + gsm8k_num_threads = 256 + gsm8k_backend = "sgl_eval" + gsm8k_thinking = True + gsm8k_max_tokens = 8192 + gsm8k_temperature = 1.0 + gsm8k_top_p = 0.95 + + prefill_tp_size = 4 + decode_tp_size = 4 + decode_base_gpu_id = 4 + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model = try_cached_model("zai-org/GLM-5.3-Flash") + draft = try_cached_model("incoai/GLM-5.3-Flash-DFlash2") + common = COMMON_ARGS + ["--speculative-draft-model-path", draft] + cls.extra_prefill_args = common + [ + "--attn-cp-size", + str(cls.prefill_tp_size), + "--enable-prefill-cp", + "--cp-strategy", + "interleave", + "--cuda-graph-backend-prefill", + "breakable", + "--cuda-graph-bs-prefill", + "2", + "4", + "8", + "16", + "32", + "64", + "128", + "256", + "512", + "1024", + "2048", + "4096", + "8192", + ] + cls.extra_decode_args = common + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, trust_remote_code=True) + cls.launch_all() + + @classmethod + def launch_lb(cls): + cls.process_lb = popen_with_error_check( + [ + "python3", + "-m", + "sglang_router.launch_router", + "--pd-disaggregation", + "--prefill", + cls.prefill_url, + cls.bootstrap_port, + "--decode", + cls.decode_url, + "--host", + cls.base_host, + "--port", + cls.lb_port, + "--request-timeout-secs", + "7200", + "--worker-startup-timeout-secs", + "7200", + "--retry-max-retries", + "1", + "--health-check-timeout-secs", + "60", + ] + ) + cls.wait_server_ready(cls.lb_url + "/readiness", process=cls.process_lb) + + def _prompt(self, length): + suffix = self.tokenizer.encode( + "\nThe access code is 739391. Copy the access code: 739391\n" + "The access code is 739391. Copy the access code:", + add_special_tokens=False, + ) + filler = self.tokenizer.encode( + "These are background notes. ", add_special_tokens=False + ) + count = length - len(suffix) + self.assertGreater(count, 0) + return (filler * ((count + len(filler) - 1) // len(filler)))[:count] + suffix + + def test_padding_chunk_and_prefix_replay(self): + outputs = [] + cases = ( + (257, True), + (257, False), + (512, True), + (513, True), + (513, False), + (16421, True), + (16421, False), + (16423, False), + ) + for length, cold_cache in cases: + if cold_cache: + for url in (self.prefill_url, self.decode_url): + requests.post( + url + "/flush_cache", params={"timeout": 30}, timeout=60 + ).raise_for_status() + before = self._prefill_graph_passes() + response = requests.post( + self.lb_url + "/generate", + json={ + "input_ids": self._prompt(length), + "sampling_params": {"temperature": 0, "max_new_tokens": 16}, + "return_logprob": True, + }, + timeout=600, + ) + self.assertEqual(response.status_code, 200, response.text) + output = response.json() + self.assertIn("739391", output["text"]) + self.assertEqual(output["meta_info"]["prompt_tokens"], length) + self.assertGreater(output["meta_info"]["spec_verify_ct"], 0) + graph_passes = self._prefill_graph_passes() - before + cached_tokens = output["meta_info"]["cached_tokens"] + print( + f"PD smoke: length={length}, cached_tokens={cached_tokens}, " + f"graph_passes={graph_passes}" + ) + # Fewer tokens than the CP size cannot activate CP. Other text + # shapes fit a bucket, including prefills with cached prefixes. + if length - cached_tokens >= self.prefill_tp_size: + self.assertGreater(graph_passes, 0, f"length={length}") + else: + self.assertEqual(graph_passes, 0, f"length={length}") + if not cold_cache and length >= 16421: + self.assertGreater(cached_tokens, 0) + outputs.append(output) + self.assertEqual(outputs[0]["text"], outputs[1]["text"]) + for process in (self.process_prefill, self.process_decode): + self.assertIsNone(process.poll()) + + def test_uneven_batched_padding_replay(self): + for url in (self.prefill_url, self.decode_url): + requests.post( + url + "/flush_cache", params={"timeout": 30}, timeout=60 + ).raise_for_status() + before = self._prefill_graph_passes() + lengths = [257, 513, 1027] + response = requests.post( + self.lb_url + "/generate", + json={ + "input_ids": [self._prompt(length) for length in lengths], + "sampling_params": {"temperature": 0, "max_new_tokens": 16}, + }, + timeout=600, + ) + self.assertEqual(response.status_code, 200, response.text) + for length, output in zip(lengths, response.json(), strict=True): + self.assertIn("739391", output["text"]) + self.assertEqual(output["meta_info"]["prompt_tokens"], length) + self.assertGreater(output["meta_info"]["spec_verify_ct"], 0) + self.assertGreater(self._prefill_graph_passes(), before) + + def _prefill_graph_passes(self): + response = requests.get(self.prefill_url + "/metrics", timeout=30) + response.raise_for_status() + return sum( + float(line.rsplit(" ", 1)[1]) + for line in response.text.splitlines() + if line.startswith("sglang:cuda_graph_passes_total{") + and 'mode="prefill_cuda_graph"' in line + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/e2e/models/test_glm53_flash_b200.py b/test/registered/e2e/models/test_glm53_flash_b200.py index 0084843c3c..fc9b6e1707 100644 --- a/test/registered/e2e/models/test_glm53_flash_b200.py +++ b/test/registered/e2e/models/test_glm53_flash_b200.py @@ -1,6 +1,6 @@ """B200 per-commit coverage for the GLM-5.3-Flash serving recipes. -Runs the Low Latency, DFlash2, and High Throughput TP4 recipes on four B200 +Runs the Low Latency and High Throughput TP4 recipes on four B200 GPUs. All recipes must retain GSM8K accuracy; the Low Latency recipe also checks EAGLE speculative acceptance and single-request decode performance. """ @@ -22,7 +22,6 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=2400, stage="base-c", runner_config="4-gpu-b200") MODEL_PATH = "zai-org/GLM-5.3-Flash" -DFLASH2_DRAFT_MODEL_PATH = "incoai/GLM-5.3-Flash-DFlash2" SERVER_LAUNCH_TIMEOUT = 3600 GPU_IDLE_TIMEOUT = 120 @@ -118,28 +117,5 @@ class TestGLM53FlashB200HighThroughput( ] -class TestGLM53FlashB200DFlash2( - GSM8KMixin, - _GLM53FlashB200Base, -): - gsm8k_score_threshold = 0.96 - gsm8k_num_examples = 1319 - gsm8k_num_threads = 256 - gsm8k_backend = "sgl_eval" - gsm8k_thinking = True - gsm8k_max_tokens = 8192 - gsm8k_temperature = 1.0 - gsm8k_top_p = 0.95 - server_args = [ - *COMMON_SERVER_ARGS, - "--speculative-algorithm", - "DFLASH", - "--speculative-draft-model-path", - DFLASH2_DRAFT_MODEL_PATH, - "--speculative-draft-attention-backend", - "fa4", - ] - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py index 8980d366fb..1d64a5eaec 100644 --- a/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py +++ b/test/registered/unit/layers/attention/test_dsa_backend_dp_padding.py @@ -65,6 +65,12 @@ class TestDSABackendDPPadding(unittest.TestCase): ) def test_trtllm_decode_runs_real_rows_then_restores_physical_batch(self): + self._check_trtllm_padding(is_prefill=False) + + def test_trtllm_prefill_narrows_metadata_to_live_cp_queries(self): + self._check_trtllm_padding(is_prefill=True) + + def _check_trtllm_padding(self, *, is_prefill): metadata = SimpleNamespace( cache_seqlens_int32=torch.tensor([8, 12], dtype=torch.int32), page_table_1=torch.zeros((2, 12), dtype=torch.int32), @@ -110,6 +116,16 @@ class TestDSABackendDPPadding(unittest.TestCase): forward_batch = SimpleNamespace() q = torch.arange(4 * 2 * 3, dtype=torch.float32).view(4, 2, 3) topk_indices = torch.arange(4 * 2, dtype=torch.int32).view(4, 2) + if is_prefill: + # The CP graph bridge has removed padded Q rows; metadata was + # planned for the physical shard before entering the eager break. + q = q[:2] + topk_indices = topk_indices[:2] + metadata.cache_seqlens_int32 = torch.tensor( + [8, 12, 0, 0], dtype=torch.int32 + ) + backend.use_fused_topk = True + backend._get_fused_topk_page_table = lambda indices: indices flashinfer = ModuleType("flashinfer") flashinfer_decode = ModuleType("flashinfer.decode") @@ -159,10 +175,12 @@ class TestDSABackendDPPadding(unittest.TestCase): seq_lens=metadata.cache_seqlens_int32, save_kv_cache=False, topk_indices=topk_indices, + is_prefill=is_prefill, ) self.assertEqual(captured["query"].shape, (2, 1, 2, 3)) - self.assertEqual(output.shape, (4, 1, 2, 2)) + self.assertEqual(captured["seq_lens"].tolist(), [8, 12]) + self.assertEqual(output.shape, (2 if is_prefill else 4, 1, 2, 2)) self.assertTrue(torch.all(output[:2] == 1)) self.assertTrue(torch.all(output[2:] == 0))