[Feat] glm5.3flash: Support padded CP BCG and prefix replay with PD/DFlash2 (#43774)

This commit is contained in:
Baizhou Zhang
2026-10-23 14:21:31 -07:00
committed by GitHub
parent 5f7542d762
commit ea226cf782
17 changed files with 398 additions and 195 deletions
@@ -81,8 +81,19 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
# Prefill only: a decode node serves plain TP.
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
}
else:
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
elif model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration":
if cfg.disaggregation_mode != "null":
unsupported |= {
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
"disaggregation-transfer-backend != mooncake": (
cfg.disaggregation_transfer_backend != "mooncake"
),
"speculative-algorithm != DFLASH": cfg.speculative_algorithm
!= "DFLASH",
"pp-size > 1": cfg.pp_size > 1,
"dp-size > 1": cfg.dp_size > 1,
"ep-size > 1": view.ep_size > 1,
}
for flag, enabled in unsupported.items():
if enabled:
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
@@ -77,6 +77,7 @@ from sglang.srt.runtime_context import (
get_memory,
get_observability,
get_schedule,
get_spec,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.network import NetworkAddress
@@ -951,6 +952,20 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
a reason rejects the peer, ``fully_checked`` retires the per-transfer
check.
"""
if (
get_spec().speculative_algorithm == "DFLASH"
# Bootstrap runs concurrently with temporary draft TP scopes.
# Use the manager's captured target layout, not scoped CP flags.
and self.state_tp_size != self.attn_tp_size
and registration_info.dst_attn_tp_size != self.state_tp_size
):
# Appended draft MHA KV uses physical TP shards, while target MLA
# KV is replicated. The draft transfer cannot reslice those shards.
return False, (
"CP/TP-sharing DFLASH PD requires the same physical TP size: "
f"prefill={self.state_tp_size}, "
f"decode={registration_info.dst_attn_tp_size}"
)
state_types = getattr(self.kv_args, "state_types", [])
fully_checked = True
for i, st in enumerate(state_types):
@@ -3958,6 +3958,10 @@ class DeepseekSparseAttnBackend(
kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim)
block_tables = page_table_1.unsqueeze(1)
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
if is_prefill:
# CP graph metadata includes the physical shard's padding; its
# eager attention bridge can narrow Q to the live local rows.
seq_lens = seq_lens[:batch_size]
attention_kwargs = dict(
kv_cache=kv,
@@ -32,8 +32,23 @@ def _zero_padded_tokens(output, actual_tokens):
output[actual_tokens:].zero_()
def attention_query_tokens(forward_batch, tokens):
# Import at call time: the CP package also imports RadixAttention.
from sglang.srt.layers.cp.padding import attention_query_tokens as cp_query_tokens
return cp_query_tokens(forward_batch, tokens)
def attention_output_tokens(forward_batch, tokens):
from sglang.srt.layers.cp.padding import attention_output_tokens as cp_output_tokens
return cp_output_tokens(forward_batch, tokens)
@contextmanager
def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs):
def attention_input_scope(
forward_batch, output, query_tokens, kv_tokens, kwargs, *, cache_tokens=None
):
"""Narrow per-token operands and restore the caller's batch even on failure."""
kwargs = dict(kwargs)
for name in (
@@ -56,9 +71,11 @@ def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs
]
cache_loc, positions = forward_batch.out_cache_loc, forward_batch.positions
previous_output = forward_batch._attn_output
forward_batch.out_cache_loc = cache_loc[:query_tokens]
if cache_tokens is None:
cache_tokens = query_tokens
forward_batch.out_cache_loc = cache_loc[:cache_tokens]
if positions is not None:
forward_batch.positions = positions[:query_tokens]
forward_batch.positions = positions[:cache_tokens]
forward_batch._attn_output = output[:query_tokens]
try:
yield kwargs
+17
View File
@@ -25,6 +25,7 @@ The strategy implementation is split across:
from __future__ import annotations
from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass
from enum import IntEnum
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
@@ -247,6 +248,22 @@ def _is_dsa_active() -> bool:
_STRATEGY: Optional[ContextParallelStrategy] = None
@contextmanager
def disable_cp_strategy():
"""Suspend the cached target strategy while a TP-only draft runs.
Pair with a runtime topology scope that disables prefill CP so lazy lookup
cannot reconstruct the target strategy. Restore it even if draft work fails.
"""
global _STRATEGY
strategy = _STRATEGY
_STRATEGY = None
try:
yield
finally:
_STRATEGY = strategy
def init_cp_strategy(
*, enable_prefill_cp: bool, cp_size: int, cp_strategy: str
) -> None:
@@ -1,8 +1,7 @@
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
"""Interleave BCG adapter for GLM-5.3 Flash text prefills.
Keep full-sequence KPool positions separate from the local DSA positions.
Prefix-free text batches replay when their total token count matches a capture,
independently of request boundaries. Other batches retain eager CP, including MM.
Graph row capacity is fixed while live token counts and prefix state can change.
"""
import torch
@@ -27,16 +26,14 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
result.input_ids_global = torch.zeros(
capacity, dtype=torch.int64, device=runner.device
)
# Graphs must read the live count from the same address on each replay.
result.moe_local_token_count = torch.zeros(
(), dtype=torch.int32, device=runner.device
)
return result
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
return (
batch_size > 0
and num_tokens in self.bucket_local_tokens
and not contains_mm_inputs
and prefix_lens is not None
and not any(prefix_lens)
)
return batch_size > 0 and not contains_mm_inputs and prefix_lens is not None
@staticmethod
def supports(server_args):
@@ -63,17 +60,21 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
)
metadata = forward_batch.attn_cp_metadata
cp_size = len(metadata.per_rank_actual_token)
base, remainder = divmod(static_num_tokens, cp_size)
metadata.total_seq_lens = static_num_tokens
metadata.per_rank_logical_token = [
base + int(rank < remainder) for rank in range(cp_size)
]
# KDA projections/reorders retain the captured shape. Eager KPool
# compression, KV writes and the output gather use the live lengths
# produced by prepare_cp_forward, never the bucket's padded tail.
metadata.graph_num_tokens = static_num_tokens
self.moe_local_token_count.fill_(
metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
)
metadata.moe_local_token_count = self.moe_local_token_count
forward_batch.out_cache_loc = out_cache_loc
# KPool compresses consecutive global tokens; only the model argument
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
forward_batch.positions = global_positions
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
ids = cp_interleave_input_ids(
forward_batch.input_ids[: forward_batch.extend_num_tokens], forward_batch
)
self.input_ids_global[: ids.shape[0]].copy_(ids)
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
+12 -2
View File
@@ -76,6 +76,9 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
gather_index: Optional[torch.Tensor] = None
local_index: Optional[torch.Tensor] = None
moe_local_token_count: Optional[torch.Tensor] = None
# Only sequence-dependent graph segments use this capacity. Eager cache
# operations and gathers must continue using total_seq_lens (live tokens).
graph_num_tokens: Optional[int] = None
def is_interleave_extend(forward_batch):
@@ -92,7 +95,9 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
The residual and mHC coefficients do not move: only the already-read mixer
input is reordered. Remove physical CP padding before the recurrent kernel.
input is reordered. Eager execution removes physical CP padding. Graph
execution keeps the capture shape; KDA's refreshed sequence table masks
padded rows out of its recurrent state updates.
"""
if not is_interleave_extend(forward_batch):
return hidden_states
@@ -101,10 +106,15 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
return hidden_states.index_select(0, metadata.gather_index)
size = get_parallel().attn_cp_size
rows = hidden_states.shape[0] // size
num_tokens = (
metadata.graph_num_tokens
if metadata.graph_num_tokens is not None
else metadata.total_seq_lens
)
return (
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
.transpose(0, 1)
.flatten(0, 1)[: metadata.total_seq_lens]
.flatten(0, 1)[:num_tokens]
.contiguous()
)
+19
View File
@@ -60,3 +60,22 @@ def pad_local_rows(x: torch.Tensor, metadata: Any, dim: int) -> torch.Tensor:
pad_shape = list(x.shape)
pad_shape[dim] = pad_size
return torch.cat([x, x.new_zeros(pad_shape)], dim=dim)
def attention_query_tokens(forward_batch, n):
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
if getattr(cp_metadata, "graph_num_tokens", None) is not None:
# Interleave BCG gathers KV in global sequence order but Q stays
# local. Empty padded queries must not enter sparse attention.
n = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
return n
def attention_output_tokens(forward_batch, raw):
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
if cp_metadata is not None and cp_metadata.per_rank_logical_token is not None:
# Q/output rows are CP-local; cache locations and positions
# above remain global. Zero every invalid local row before
# captured projections/MoE can consume an empty-attention NaN.
raw = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
return raw
+6 -1
View File
@@ -26,6 +26,8 @@ from sglang.srt.layers.attention.graph_utils import (
_zero_padded_tokens,
allocate_attention_outputs,
attention_input_scope,
attention_output_tokens,
attention_query_tokens,
padded_extend_real_tokens,
)
from sglang.srt.model_executor.forward_context import (
@@ -190,6 +192,8 @@ class RadixAttention(nn.Module):
rows = output.shape[0]
n = forward_batch.global_num_token_non_padded_cpu
kv_n = n if key_value_num_tokens is None else key_value_num_tokens
cache_n = n
n = attention_query_tokens(forward_batch, n)
return_lse = bool(kwargs.get("return_lse") or forward_batch.mha_return_lse)
lse = None
if n == 0:
@@ -200,7 +204,7 @@ class RadixAttention(nn.Module):
lse = q.new_zeros((rows, q.shape[1]), dtype=torch.float32)
else:
with attention_input_scope(
forward_batch, output, n, kv_n, kwargs
forward_batch, output, n, kv_n, kwargs, cache_tokens=cache_n
) as kwargs:
result = get_attn_backend().forward(
q[:n],
@@ -220,6 +224,7 @@ class RadixAttention(nn.Module):
if result.data_ptr() != output.data_ptr():
output[:n].view(result.shape).copy_(result)
raw = get_prefill_raw_num_tokens()
raw = attention_output_tokens(forward_batch, raw)
_zero_padded_tokens(output, raw)
if idx_output is not None:
_zero_padded_tokens(idx_output, raw)
@@ -1985,6 +1985,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
mamba_track_indices=mamba_track_indices,
mamba_track_mask=mamba_track_mask,
mamba_track_seqlens=mamba_track_seqlens,
# CP replay plans attention against this static view, including
# KDA's host-side checkpoint plan for the current request.
mamba_prefill_track_mask_cpu=forward_batch.mamba_prefill_track_mask_cpu,
mamba_track_seqlens_cpu=forward_batch.mamba_track_seqlens_cpu,
encoder_lens=forward_batch.encoder_lens,
return_logprob=(
forward_batch.return_logprob
@@ -103,7 +103,7 @@ from sglang.srt.speculative.spec_utils import (
draft_pp_context,
draft_tp_context,
)
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
from sglang.srt.utils import async_h2d, is_cuda, is_hip, is_npu, is_xpu
_is_npu = is_npu()
@@ -2375,8 +2375,10 @@ class DFlashWorkerV2(BaseSpecWorker):
# Materialize prompt tokens into the draft KV cache immediately. This is required
# for radix cache safety (the scheduler may update radix after prefill returns).
device = next_token_ids.device
ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device)
draft_seq_lens = torch.tensor(
# Enqueue metadata copies without waiting for the target prefill's
# outstanding GPU work; the same stream orders their consumers.
ctx_lens = async_h2d(batch.extend_lens, dtype=torch.int32, device=device)
draft_seq_lens = async_h2d(
batch.prefix_lens, dtype=torch.int32, device=device
)
+19 -2
View File
@@ -34,6 +34,7 @@ from sglang.srt.distributed.parallel_state import (
)
from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.layers.cp.base import disable_cp_strategy
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
from sglang.srt.managers.utils import _async_d2h
from sglang.srt.mem_cache.allocation import (
@@ -819,8 +820,24 @@ def draft_tp_context(owns_attention: bool):
"""
if not owns_attention:
return contextlib.nullcontext()
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
return patch_tensor_parallel_group(group, owns_attention=True)
parallel = get_parallel()
group = getattr(parallel, f"{draft_attn_parallel_group()}_group")
tp_context = patch_tensor_parallel_group(group, owns_attention=True)
if parallel.enable_prefill_cp:
return _draft_without_cp(tp_context)
return tp_context
@contextmanager
def _draft_without_cp(tp_context):
with (
tp_context,
get_parallel().override(
enable_prefill_cp=False, enable_cp_tp_group_sharing=False
),
disable_cp_strategy(),
):
yield
def spec_stage_span(name: str):
+3 -2
View File
@@ -183,8 +183,9 @@ class TestPrefillCPBCGReplay(CustomTestCase):
512,
)
self.assertTrue(runner.can_run_graph(batch))
batch.extend_prefix_lens_cpu = [0, 1, 0]
self.assertFalse(runner.can_run_graph(batch))
with get_parallel().override(attn_cp_rank=0, attn_cp_size=4):
batch.extend_prefix_lens_cpu = [0, 1, 0]
self.assertTrue(runner.can_run_graph(batch))
def test_local_capacity_overflow_uses_next_capture_bucket(self):
runner = self._make_runner()
-137
View File
@@ -1,137 +0,0 @@
"""GLM-5.3-Flash interleave CP4 + DFlash2 on four Blackwell GPUs."""
import base64
import io
import unittest
import requests
from PIL import Image
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
_wait_for_gpu_idle_in_ci,
popen_launch_server,
terminate_and_kill_process_tree,
try_cached_model,
)
register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200")
class TestGLM53FlashB200ContextParallel(
GSM8KMixin,
CustomTestCase,
):
gsm8k_score_threshold = 0.93
gsm8k_num_examples = 500
gsm8k_num_shots = 20
server_args = [
"--tp-size",
"4",
"--dsa-prefill-backend",
"trtllm",
"--dsa-decode-backend",
"trtllm",
"--kv-cache-dtype",
"fp8_e4m3",
"--moe-runner-backend",
"flashinfer_trtllm",
"--reasoning-parser",
"auto",
"--tool-call-parser",
"auto",
"--speculative-algorithm",
"DFLASH",
"--speculative-draft-model-path",
"incoai/GLM-5.3-Flash-DFlash2",
"--speculative-draft-attention-backend",
"fa4",
"--mem-fraction-static",
"0.85",
"--enable-prefill-cp",
"--cp-strategy",
"interleave",
"--attn-cp-size",
"4",
# KDA partitions heads over TP4, sharing the CP group. EP1 keeps every MoE
# on TP4; dense FFNs also use TP4 rather than per-rank computation.
"--ep-size",
"1",
"--moe-dense-tp-size",
"4",
]
@classmethod
def setUpClass(cls):
cls.model = try_cached_model("zai-org/GLM-5.3-Flash")
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = None
cls.process = popen_launch_server(
cls.model,
cls.base_url,
timeout=3600,
other_args=cls.server_args,
)
@classmethod
def tearDownClass(cls):
if process := getattr(cls, "process", None):
terminate_and_kill_process_tree(process)
_wait_for_gpu_idle_in_ci(timeout=120)
def test_gsm8k(self):
super().test_gsm8k()
# Require the metric as well as the floor: an accuracy pass alone must
# not hide disabled speculation or a draft whose tokens are mostly rejected.
response = requests.get(self.base_url + "/server_info", timeout=30)
response.raise_for_status()
self.assertGreater(
response.json()["internal_states"][0]["avg_spec_accept_length"], 4.0
)
def test_image_input(self):
# Color is supplied only by the image: the prompt cannot substitute for
# the encoder. Exercise two images to catch missing/cached embeddings.
for color in ("red", "blue"):
with self.subTest(color=color):
png = io.BytesIO()
Image.new("RGB", (256, 256), color=color).save(png, format="PNG")
image_url = (
"data:image/png;base64," + base64.b64encode(png.getvalue()).decode()
)
response = requests.post(
self.base_url + "/v1/chat/completions",
json={
"model": self.model,
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": image_url},
},
{
"type": "text",
"text": "What is the dominant color of this image? Answer with one English color word.",
},
],
}
],
"temperature": 0,
# The checkpoint template always opens a thinking block.
# Allow it to finish before checking the final answer.
"max_tokens": 1024,
},
timeout=180,
)
self.assertEqual(response.status_code, 200, response.text)
answer = response.json()["choices"][0]["message"]["content"]
self.assertIn(color, answer.lower(), response.text)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,223 @@
"""GLM CP4/BCG prefill -> TP4 DFlash2 decode, with padding and cached prefixes."""
import unittest
import requests
from transformers import AutoTokenizer
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.disaggregation_fixture import (
PDDisaggregationServerBase,
)
from sglang.test.test_utils import popen_with_error_check, try_cached_model
register_cuda_ci(est_time=900, stage="extra-b", runner_config="8-gpu-b300")
COMMON_ARGS = [
"--dsa-prefill-backend",
"trtllm",
"--dsa-decode-backend",
"trtllm",
"--kv-cache-dtype",
"fp8_e4m3",
"--moe-runner-backend",
"flashinfer_trtllm",
"--speculative-algorithm",
"DFLASH",
"--speculative-draft-attention-backend",
"fa4",
"--reasoning-parser",
"auto",
"--tool-call-parser",
"auto",
"--max-running-requests",
"16",
"--cuda-graph-max-bs-decode",
"16",
"--chunked-prefill-size",
"8192",
"--mem-fraction-static",
"0.9",
"--enable-metrics",
"--random-seed",
"42",
]
class TestGlm53CpBcgDflashPD(PDDisaggregationServerBase, GSM8KMixin):
gsm8k_score_threshold = 0.96
gsm8k_num_examples = 1319
gsm8k_num_threads = 256
gsm8k_backend = "sgl_eval"
gsm8k_thinking = True
gsm8k_max_tokens = 8192
gsm8k_temperature = 1.0
gsm8k_top_p = 0.95
prefill_tp_size = 4
decode_tp_size = 4
decode_base_gpu_id = 4
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.model = try_cached_model("zai-org/GLM-5.3-Flash")
draft = try_cached_model("incoai/GLM-5.3-Flash-DFlash2")
common = COMMON_ARGS + ["--speculative-draft-model-path", draft]
cls.extra_prefill_args = common + [
"--attn-cp-size",
str(cls.prefill_tp_size),
"--enable-prefill-cp",
"--cp-strategy",
"interleave",
"--cuda-graph-backend-prefill",
"breakable",
"--cuda-graph-bs-prefill",
"2",
"4",
"8",
"16",
"32",
"64",
"128",
"256",
"512",
"1024",
"2048",
"4096",
"8192",
]
cls.extra_decode_args = common
cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, trust_remote_code=True)
cls.launch_all()
@classmethod
def launch_lb(cls):
cls.process_lb = popen_with_error_check(
[
"python3",
"-m",
"sglang_router.launch_router",
"--pd-disaggregation",
"--prefill",
cls.prefill_url,
cls.bootstrap_port,
"--decode",
cls.decode_url,
"--host",
cls.base_host,
"--port",
cls.lb_port,
"--request-timeout-secs",
"7200",
"--worker-startup-timeout-secs",
"7200",
"--retry-max-retries",
"1",
"--health-check-timeout-secs",
"60",
]
)
cls.wait_server_ready(cls.lb_url + "/readiness", process=cls.process_lb)
def _prompt(self, length):
suffix = self.tokenizer.encode(
"\nThe access code is 739391. Copy the access code: 739391\n"
"The access code is 739391. Copy the access code:",
add_special_tokens=False,
)
filler = self.tokenizer.encode(
"These are background notes. ", add_special_tokens=False
)
count = length - len(suffix)
self.assertGreater(count, 0)
return (filler * ((count + len(filler) - 1) // len(filler)))[:count] + suffix
def test_padding_chunk_and_prefix_replay(self):
outputs = []
cases = (
(257, True),
(257, False),
(512, True),
(513, True),
(513, False),
(16421, True),
(16421, False),
(16423, False),
)
for length, cold_cache in cases:
if cold_cache:
for url in (self.prefill_url, self.decode_url):
requests.post(
url + "/flush_cache", params={"timeout": 30}, timeout=60
).raise_for_status()
before = self._prefill_graph_passes()
response = requests.post(
self.lb_url + "/generate",
json={
"input_ids": self._prompt(length),
"sampling_params": {"temperature": 0, "max_new_tokens": 16},
"return_logprob": True,
},
timeout=600,
)
self.assertEqual(response.status_code, 200, response.text)
output = response.json()
self.assertIn("739391", output["text"])
self.assertEqual(output["meta_info"]["prompt_tokens"], length)
self.assertGreater(output["meta_info"]["spec_verify_ct"], 0)
graph_passes = self._prefill_graph_passes() - before
cached_tokens = output["meta_info"]["cached_tokens"]
print(
f"PD smoke: length={length}, cached_tokens={cached_tokens}, "
f"graph_passes={graph_passes}"
)
# Fewer tokens than the CP size cannot activate CP. Other text
# shapes fit a bucket, including prefills with cached prefixes.
if length - cached_tokens >= self.prefill_tp_size:
self.assertGreater(graph_passes, 0, f"length={length}")
else:
self.assertEqual(graph_passes, 0, f"length={length}")
if not cold_cache and length >= 16421:
self.assertGreater(cached_tokens, 0)
outputs.append(output)
self.assertEqual(outputs[0]["text"], outputs[1]["text"])
for process in (self.process_prefill, self.process_decode):
self.assertIsNone(process.poll())
def test_uneven_batched_padding_replay(self):
for url in (self.prefill_url, self.decode_url):
requests.post(
url + "/flush_cache", params={"timeout": 30}, timeout=60
).raise_for_status()
before = self._prefill_graph_passes()
lengths = [257, 513, 1027]
response = requests.post(
self.lb_url + "/generate",
json={
"input_ids": [self._prompt(length) for length in lengths],
"sampling_params": {"temperature": 0, "max_new_tokens": 16},
},
timeout=600,
)
self.assertEqual(response.status_code, 200, response.text)
for length, output in zip(lengths, response.json(), strict=True):
self.assertIn("739391", output["text"])
self.assertEqual(output["meta_info"]["prompt_tokens"], length)
self.assertGreater(output["meta_info"]["spec_verify_ct"], 0)
self.assertGreater(self._prefill_graph_passes(), before)
def _prefill_graph_passes(self):
response = requests.get(self.prefill_url + "/metrics", timeout=30)
response.raise_for_status()
return sum(
float(line.rsplit(" ", 1)[1])
for line in response.text.splitlines()
if line.startswith("sglang:cuda_graph_passes_total{")
and 'mode="prefill_cuda_graph"' in line
)
if __name__ == "__main__":
unittest.main()
@@ -1,6 +1,6 @@
"""B200 per-commit coverage for the GLM-5.3-Flash serving recipes.
Runs the Low Latency, DFlash2, and High Throughput TP4 recipes on four B200
Runs the Low Latency and High Throughput TP4 recipes on four B200
GPUs. All recipes must retain GSM8K accuracy; the Low Latency recipe also
checks EAGLE speculative acceptance and single-request decode performance.
"""
@@ -22,7 +22,6 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=2400, stage="base-c", runner_config="4-gpu-b200")
MODEL_PATH = "zai-org/GLM-5.3-Flash"
DFLASH2_DRAFT_MODEL_PATH = "incoai/GLM-5.3-Flash-DFlash2"
SERVER_LAUNCH_TIMEOUT = 3600
GPU_IDLE_TIMEOUT = 120
@@ -118,28 +117,5 @@ class TestGLM53FlashB200HighThroughput(
]
class TestGLM53FlashB200DFlash2(
GSM8KMixin,
_GLM53FlashB200Base,
):
gsm8k_score_threshold = 0.96
gsm8k_num_examples = 1319
gsm8k_num_threads = 256
gsm8k_backend = "sgl_eval"
gsm8k_thinking = True
gsm8k_max_tokens = 8192
gsm8k_temperature = 1.0
gsm8k_top_p = 0.95
server_args = [
*COMMON_SERVER_ARGS,
"--speculative-algorithm",
"DFLASH",
"--speculative-draft-model-path",
DFLASH2_DRAFT_MODEL_PATH,
"--speculative-draft-attention-backend",
"fa4",
]
if __name__ == "__main__":
unittest.main()
@@ -65,6 +65,12 @@ class TestDSABackendDPPadding(unittest.TestCase):
)
def test_trtllm_decode_runs_real_rows_then_restores_physical_batch(self):
self._check_trtllm_padding(is_prefill=False)
def test_trtllm_prefill_narrows_metadata_to_live_cp_queries(self):
self._check_trtllm_padding(is_prefill=True)
def _check_trtllm_padding(self, *, is_prefill):
metadata = SimpleNamespace(
cache_seqlens_int32=torch.tensor([8, 12], dtype=torch.int32),
page_table_1=torch.zeros((2, 12), dtype=torch.int32),
@@ -110,6 +116,16 @@ class TestDSABackendDPPadding(unittest.TestCase):
forward_batch = SimpleNamespace()
q = torch.arange(4 * 2 * 3, dtype=torch.float32).view(4, 2, 3)
topk_indices = torch.arange(4 * 2, dtype=torch.int32).view(4, 2)
if is_prefill:
# The CP graph bridge has removed padded Q rows; metadata was
# planned for the physical shard before entering the eager break.
q = q[:2]
topk_indices = topk_indices[:2]
metadata.cache_seqlens_int32 = torch.tensor(
[8, 12, 0, 0], dtype=torch.int32
)
backend.use_fused_topk = True
backend._get_fused_topk_page_table = lambda indices: indices
flashinfer = ModuleType("flashinfer")
flashinfer_decode = ModuleType("flashinfer.decode")
@@ -159,10 +175,12 @@ class TestDSABackendDPPadding(unittest.TestCase):
seq_lens=metadata.cache_seqlens_int32,
save_kv_cache=False,
topk_indices=topk_indices,
is_prefill=is_prefill,
)
self.assertEqual(captured["query"].shape, (2, 1, 2, 3))
self.assertEqual(output.shape, (4, 1, 2, 2))
self.assertEqual(captured["seq_lens"].tolist(), [8, 12])
self.assertEqual(output.shape, (2 if is_prefill else 4, 1, 2, 2))
self.assertTrue(torch.all(output[:2] == 1))
self.assertTrue(torch.all(output[2:] == 0))