[Feat] glm5.3flash: Support padded CP BCG and prefix replay with PD/DFlash2 (#43774)
This commit is contained in:
@@ -81,8 +81,19 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None:
|
||||
# Prefill only: a decode node serves plain TP.
|
||||
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
|
||||
}
|
||||
else:
|
||||
unsupported["disaggregation-mode"] = cfg.disaggregation_mode != "null"
|
||||
elif model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration":
|
||||
if cfg.disaggregation_mode != "null":
|
||||
unsupported |= {
|
||||
"disaggregation-mode decode": cfg.disaggregation_mode == "decode",
|
||||
"disaggregation-transfer-backend != mooncake": (
|
||||
cfg.disaggregation_transfer_backend != "mooncake"
|
||||
),
|
||||
"speculative-algorithm != DFLASH": cfg.speculative_algorithm
|
||||
!= "DFLASH",
|
||||
"pp-size > 1": cfg.pp_size > 1,
|
||||
"dp-size > 1": cfg.dp_size > 1,
|
||||
"ep-size > 1": view.ep_size > 1,
|
||||
}
|
||||
for flag, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(f"--{flag} is not supported with CP-TP group sharing.")
|
||||
|
||||
@@ -77,6 +77,7 @@ from sglang.srt.runtime_context import (
|
||||
get_memory,
|
||||
get_observability,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.network import NetworkAddress
|
||||
@@ -951,6 +952,20 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
a reason rejects the peer, ``fully_checked`` retires the per-transfer
|
||||
check.
|
||||
"""
|
||||
if (
|
||||
get_spec().speculative_algorithm == "DFLASH"
|
||||
# Bootstrap runs concurrently with temporary draft TP scopes.
|
||||
# Use the manager's captured target layout, not scoped CP flags.
|
||||
and self.state_tp_size != self.attn_tp_size
|
||||
and registration_info.dst_attn_tp_size != self.state_tp_size
|
||||
):
|
||||
# Appended draft MHA KV uses physical TP shards, while target MLA
|
||||
# KV is replicated. The draft transfer cannot reslice those shards.
|
||||
return False, (
|
||||
"CP/TP-sharing DFLASH PD requires the same physical TP size: "
|
||||
f"prefill={self.state_tp_size}, "
|
||||
f"decode={registration_info.dst_attn_tp_size}"
|
||||
)
|
||||
state_types = getattr(self.kv_args, "state_types", [])
|
||||
fully_checked = True
|
||||
for i, st in enumerate(state_types):
|
||||
|
||||
@@ -3958,6 +3958,10 @@ class DeepseekSparseAttnBackend(
|
||||
kv = kv_cache.view(-1, 1, self.physical_page_size, self.kv_cache_dim)
|
||||
block_tables = page_table_1.unsqueeze(1)
|
||||
seq_lens = metadata.cache_seqlens_int32 if seq_lens is None else seq_lens
|
||||
if is_prefill:
|
||||
# CP graph metadata includes the physical shard's padding; its
|
||||
# eager attention bridge can narrow Q to the live local rows.
|
||||
seq_lens = seq_lens[:batch_size]
|
||||
|
||||
attention_kwargs = dict(
|
||||
kv_cache=kv,
|
||||
|
||||
@@ -32,8 +32,23 @@ def _zero_padded_tokens(output, actual_tokens):
|
||||
output[actual_tokens:].zero_()
|
||||
|
||||
|
||||
def attention_query_tokens(forward_batch, tokens):
|
||||
# Import at call time: the CP package also imports RadixAttention.
|
||||
from sglang.srt.layers.cp.padding import attention_query_tokens as cp_query_tokens
|
||||
|
||||
return cp_query_tokens(forward_batch, tokens)
|
||||
|
||||
|
||||
def attention_output_tokens(forward_batch, tokens):
|
||||
from sglang.srt.layers.cp.padding import attention_output_tokens as cp_output_tokens
|
||||
|
||||
return cp_output_tokens(forward_batch, tokens)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs):
|
||||
def attention_input_scope(
|
||||
forward_batch, output, query_tokens, kv_tokens, kwargs, *, cache_tokens=None
|
||||
):
|
||||
"""Narrow per-token operands and restore the caller's batch even on failure."""
|
||||
kwargs = dict(kwargs)
|
||||
for name in (
|
||||
@@ -56,9 +71,11 @@ def attention_input_scope(forward_batch, output, query_tokens, kv_tokens, kwargs
|
||||
]
|
||||
cache_loc, positions = forward_batch.out_cache_loc, forward_batch.positions
|
||||
previous_output = forward_batch._attn_output
|
||||
forward_batch.out_cache_loc = cache_loc[:query_tokens]
|
||||
if cache_tokens is None:
|
||||
cache_tokens = query_tokens
|
||||
forward_batch.out_cache_loc = cache_loc[:cache_tokens]
|
||||
if positions is not None:
|
||||
forward_batch.positions = positions[:query_tokens]
|
||||
forward_batch.positions = positions[:cache_tokens]
|
||||
forward_batch._attn_output = output[:query_tokens]
|
||||
try:
|
||||
yield kwargs
|
||||
|
||||
@@ -25,6 +25,7 @@ The strategy implementation is split across:
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
|
||||
@@ -247,6 +248,22 @@ def _is_dsa_active() -> bool:
|
||||
_STRATEGY: Optional[ContextParallelStrategy] = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def disable_cp_strategy():
|
||||
"""Suspend the cached target strategy while a TP-only draft runs.
|
||||
|
||||
Pair with a runtime topology scope that disables prefill CP so lazy lookup
|
||||
cannot reconstruct the target strategy. Restore it even if draft work fails.
|
||||
"""
|
||||
global _STRATEGY
|
||||
strategy = _STRATEGY
|
||||
_STRATEGY = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_STRATEGY = strategy
|
||||
|
||||
|
||||
def init_cp_strategy(
|
||||
*, enable_prefill_cp: bool, cp_size: int, cp_strategy: str
|
||||
) -> None:
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
"""Experimental, exact-shape interleave BCG adapter for GLM-5.3 Flash.
|
||||
"""Interleave BCG adapter for GLM-5.3 Flash text prefills.
|
||||
|
||||
Keep full-sequence KPool positions separate from the local DSA positions.
|
||||
Prefix-free text batches replay when their total token count matches a capture,
|
||||
independently of request boundaries. Other batches retain eager CP, including MM.
|
||||
Graph row capacity is fixed while live token counts and prefix state can change.
|
||||
"""
|
||||
|
||||
import torch
|
||||
@@ -27,16 +26,14 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
|
||||
result.input_ids_global = torch.zeros(
|
||||
capacity, dtype=torch.int64, device=runner.device
|
||||
)
|
||||
# Graphs must read the live count from the same address on each replay.
|
||||
result.moe_local_token_count = torch.zeros(
|
||||
(), dtype=torch.int32, device=runner.device
|
||||
)
|
||||
return result
|
||||
|
||||
def allows_replay(self, batch_size, num_tokens, prefix_lens, contains_mm_inputs):
|
||||
return (
|
||||
batch_size > 0
|
||||
and num_tokens in self.bucket_local_tokens
|
||||
and not contains_mm_inputs
|
||||
and prefix_lens is not None
|
||||
and not any(prefix_lens)
|
||||
)
|
||||
return batch_size > 0 and not contains_mm_inputs and prefix_lens is not None
|
||||
|
||||
@staticmethod
|
||||
def supports(server_args):
|
||||
@@ -63,17 +60,21 @@ class InterleaveCPBCGInput(PrefillCPBCGInput):
|
||||
runner, forward_batch, static_num_tokens=static_num_tokens, capture=capture
|
||||
)
|
||||
metadata = forward_batch.attn_cp_metadata
|
||||
cp_size = len(metadata.per_rank_actual_token)
|
||||
base, remainder = divmod(static_num_tokens, cp_size)
|
||||
metadata.total_seq_lens = static_num_tokens
|
||||
metadata.per_rank_logical_token = [
|
||||
base + int(rank < remainder) for rank in range(cp_size)
|
||||
]
|
||||
# KDA projections/reorders retain the captured shape. Eager KPool
|
||||
# compression, KV writes and the output gather use the live lengths
|
||||
# produced by prepare_cp_forward, never the bucket's padded tail.
|
||||
metadata.graph_num_tokens = static_num_tokens
|
||||
self.moe_local_token_count.fill_(
|
||||
metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
)
|
||||
metadata.moe_local_token_count = self.moe_local_token_count
|
||||
forward_batch.out_cache_loc = out_cache_loc
|
||||
# KPool compresses consecutive global tokens; only the model argument
|
||||
# (DSA RoPE) is local. Do not replace ForwardBatch's global positions.
|
||||
forward_batch.positions = global_positions
|
||||
ids = cp_interleave_input_ids(forward_batch.input_ids, forward_batch)
|
||||
ids = cp_interleave_input_ids(
|
||||
forward_batch.input_ids[: forward_batch.extend_num_tokens], forward_batch
|
||||
)
|
||||
self.input_ids_global[: ids.shape[0]].copy_(ids)
|
||||
forward_batch.input_ids_global = self.input_ids_global[: ids.shape[0]]
|
||||
|
||||
|
||||
@@ -76,6 +76,9 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata):
|
||||
gather_index: Optional[torch.Tensor] = None
|
||||
local_index: Optional[torch.Tensor] = None
|
||||
moe_local_token_count: Optional[torch.Tensor] = None
|
||||
# Only sequence-dependent graph segments use this capacity. Eager cache
|
||||
# operations and gathers must continue using total_seq_lens (live tokens).
|
||||
graph_num_tokens: Optional[int] = None
|
||||
|
||||
|
||||
def is_interleave_extend(forward_batch):
|
||||
@@ -92,7 +95,9 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
|
||||
``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute.
|
||||
|
||||
The residual and mHC coefficients do not move: only the already-read mixer
|
||||
input is reordered. Remove physical CP padding before the recurrent kernel.
|
||||
input is reordered. Eager execution removes physical CP padding. Graph
|
||||
execution keeps the capture shape; KDA's refreshed sequence table masks
|
||||
padded rows out of its recurrent state updates.
|
||||
"""
|
||||
if not is_interleave_extend(forward_batch):
|
||||
return hidden_states
|
||||
@@ -101,10 +106,15 @@ def cp_interleave_to_sequence_order(hidden_states, forward_batch):
|
||||
return hidden_states.index_select(0, metadata.gather_index)
|
||||
size = get_parallel().attn_cp_size
|
||||
rows = hidden_states.shape[0] // size
|
||||
num_tokens = (
|
||||
metadata.graph_num_tokens
|
||||
if metadata.graph_num_tokens is not None
|
||||
else metadata.total_seq_lens
|
||||
)
|
||||
return (
|
||||
hidden_states.reshape(size, rows, *hidden_states.shape[1:])
|
||||
.transpose(0, 1)
|
||||
.flatten(0, 1)[: metadata.total_seq_lens]
|
||||
.flatten(0, 1)[:num_tokens]
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
@@ -60,3 +60,22 @@ def pad_local_rows(x: torch.Tensor, metadata: Any, dim: int) -> torch.Tensor:
|
||||
pad_shape = list(x.shape)
|
||||
pad_shape[dim] = pad_size
|
||||
return torch.cat([x, x.new_zeros(pad_shape)], dim=dim)
|
||||
|
||||
|
||||
def attention_query_tokens(forward_batch, n):
|
||||
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
|
||||
if getattr(cp_metadata, "graph_num_tokens", None) is not None:
|
||||
# Interleave BCG gathers KV in global sequence order but Q stays
|
||||
# local. Empty padded queries must not enter sparse attention.
|
||||
n = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
return n
|
||||
|
||||
|
||||
def attention_output_tokens(forward_batch, raw):
|
||||
cp_metadata = getattr(forward_batch, "attn_cp_metadata", None)
|
||||
if cp_metadata is not None and cp_metadata.per_rank_logical_token is not None:
|
||||
# Q/output rows are CP-local; cache locations and positions
|
||||
# above remain global. Zero every invalid local row before
|
||||
# captured projections/MoE can consume an empty-attention NaN.
|
||||
raw = cp_metadata.per_rank_logical_token[get_parallel().attn_cp_rank]
|
||||
return raw
|
||||
|
||||
@@ -26,6 +26,8 @@ from sglang.srt.layers.attention.graph_utils import (
|
||||
_zero_padded_tokens,
|
||||
allocate_attention_outputs,
|
||||
attention_input_scope,
|
||||
attention_output_tokens,
|
||||
attention_query_tokens,
|
||||
padded_extend_real_tokens,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import (
|
||||
@@ -190,6 +192,8 @@ class RadixAttention(nn.Module):
|
||||
rows = output.shape[0]
|
||||
n = forward_batch.global_num_token_non_padded_cpu
|
||||
kv_n = n if key_value_num_tokens is None else key_value_num_tokens
|
||||
cache_n = n
|
||||
n = attention_query_tokens(forward_batch, n)
|
||||
return_lse = bool(kwargs.get("return_lse") or forward_batch.mha_return_lse)
|
||||
lse = None
|
||||
if n == 0:
|
||||
@@ -200,7 +204,7 @@ class RadixAttention(nn.Module):
|
||||
lse = q.new_zeros((rows, q.shape[1]), dtype=torch.float32)
|
||||
else:
|
||||
with attention_input_scope(
|
||||
forward_batch, output, n, kv_n, kwargs
|
||||
forward_batch, output, n, kv_n, kwargs, cache_tokens=cache_n
|
||||
) as kwargs:
|
||||
result = get_attn_backend().forward(
|
||||
q[:n],
|
||||
@@ -220,6 +224,7 @@ class RadixAttention(nn.Module):
|
||||
if result.data_ptr() != output.data_ptr():
|
||||
output[:n].view(result.shape).copy_(result)
|
||||
raw = get_prefill_raw_num_tokens()
|
||||
raw = attention_output_tokens(forward_batch, raw)
|
||||
_zero_padded_tokens(output, raw)
|
||||
if idx_output is not None:
|
||||
_zero_padded_tokens(idx_output, raw)
|
||||
|
||||
@@ -1985,6 +1985,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
mamba_track_indices=mamba_track_indices,
|
||||
mamba_track_mask=mamba_track_mask,
|
||||
mamba_track_seqlens=mamba_track_seqlens,
|
||||
# CP replay plans attention against this static view, including
|
||||
# KDA's host-side checkpoint plan for the current request.
|
||||
mamba_prefill_track_mask_cpu=forward_batch.mamba_prefill_track_mask_cpu,
|
||||
mamba_track_seqlens_cpu=forward_batch.mamba_track_seqlens_cpu,
|
||||
encoder_lens=forward_batch.encoder_lens,
|
||||
return_logprob=(
|
||||
forward_batch.return_logprob
|
||||
|
||||
@@ -103,7 +103,7 @@ from sglang.srt.speculative.spec_utils import (
|
||||
draft_pp_context,
|
||||
draft_tp_context,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu
|
||||
from sglang.srt.utils import async_h2d, is_cuda, is_hip, is_npu, is_xpu
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
@@ -2375,8 +2375,10 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
# Materialize prompt tokens into the draft KV cache immediately. This is required
|
||||
# for radix cache safety (the scheduler may update radix after prefill returns).
|
||||
device = next_token_ids.device
|
||||
ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device)
|
||||
draft_seq_lens = torch.tensor(
|
||||
# Enqueue metadata copies without waiting for the target prefill's
|
||||
# outstanding GPU work; the same stream orders their consumers.
|
||||
ctx_lens = async_h2d(batch.extend_lens, dtype=torch.int32, device=device)
|
||||
draft_seq_lens = async_h2d(
|
||||
batch.prefix_lens, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.distributed.parallel_state import (
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.layers.cp.base import disable_cp_strategy
|
||||
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
|
||||
from sglang.srt.managers.utils import _async_d2h
|
||||
from sglang.srt.mem_cache.allocation import (
|
||||
@@ -819,8 +820,24 @@ def draft_tp_context(owns_attention: bool):
|
||||
"""
|
||||
if not owns_attention:
|
||||
return contextlib.nullcontext()
|
||||
group = getattr(get_parallel(), f"{draft_attn_parallel_group()}_group")
|
||||
return patch_tensor_parallel_group(group, owns_attention=True)
|
||||
parallel = get_parallel()
|
||||
group = getattr(parallel, f"{draft_attn_parallel_group()}_group")
|
||||
tp_context = patch_tensor_parallel_group(group, owns_attention=True)
|
||||
if parallel.enable_prefill_cp:
|
||||
return _draft_without_cp(tp_context)
|
||||
return tp_context
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _draft_without_cp(tp_context):
|
||||
with (
|
||||
tp_context,
|
||||
get_parallel().override(
|
||||
enable_prefill_cp=False, enable_cp_tp_group_sharing=False
|
||||
),
|
||||
disable_cp_strategy(),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def spec_stage_span(name: str):
|
||||
|
||||
@@ -183,8 +183,9 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
||||
512,
|
||||
)
|
||||
self.assertTrue(runner.can_run_graph(batch))
|
||||
batch.extend_prefix_lens_cpu = [0, 1, 0]
|
||||
self.assertFalse(runner.can_run_graph(batch))
|
||||
with get_parallel().override(attn_cp_rank=0, attn_cp_size=4):
|
||||
batch.extend_prefix_lens_cpu = [0, 1, 0]
|
||||
self.assertTrue(runner.can_run_graph(batch))
|
||||
|
||||
def test_local_capacity_overflow_uses_next_capture_bucket(self):
|
||||
runner = self._make_runner()
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
"""GLM-5.3-Flash interleave CP4 + DFlash2 on four Blackwell GPUs."""
|
||||
|
||||
import base64
|
||||
import io
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
_wait_for_gpu_idle_in_ci,
|
||||
popen_launch_server,
|
||||
terminate_and_kill_process_tree,
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200")
|
||||
|
||||
|
||||
class TestGLM53FlashB200ContextParallel(
|
||||
GSM8KMixin,
|
||||
CustomTestCase,
|
||||
):
|
||||
gsm8k_score_threshold = 0.93
|
||||
gsm8k_num_examples = 500
|
||||
gsm8k_num_shots = 20
|
||||
server_args = [
|
||||
"--tp-size",
|
||||
"4",
|
||||
"--dsa-prefill-backend",
|
||||
"trtllm",
|
||||
"--dsa-decode-backend",
|
||||
"trtllm",
|
||||
"--kv-cache-dtype",
|
||||
"fp8_e4m3",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--reasoning-parser",
|
||||
"auto",
|
||||
"--tool-call-parser",
|
||||
"auto",
|
||||
"--speculative-algorithm",
|
||||
"DFLASH",
|
||||
"--speculative-draft-model-path",
|
||||
"incoai/GLM-5.3-Flash-DFlash2",
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
"--mem-fraction-static",
|
||||
"0.85",
|
||||
"--enable-prefill-cp",
|
||||
"--cp-strategy",
|
||||
"interleave",
|
||||
"--attn-cp-size",
|
||||
"4",
|
||||
# KDA partitions heads over TP4, sharing the CP group. EP1 keeps every MoE
|
||||
# on TP4; dense FFNs also use TP4 rather than per-rank computation.
|
||||
"--ep-size",
|
||||
"1",
|
||||
"--moe-dense-tp-size",
|
||||
"4",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = try_cached_model("zai-org/GLM-5.3-Flash")
|
||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||
cls.process = None
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=3600,
|
||||
other_args=cls.server_args,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if process := getattr(cls, "process", None):
|
||||
terminate_and_kill_process_tree(process)
|
||||
_wait_for_gpu_idle_in_ci(timeout=120)
|
||||
|
||||
def test_gsm8k(self):
|
||||
super().test_gsm8k()
|
||||
# Require the metric as well as the floor: an accuracy pass alone must
|
||||
# not hide disabled speculation or a draft whose tokens are mostly rejected.
|
||||
response = requests.get(self.base_url + "/server_info", timeout=30)
|
||||
response.raise_for_status()
|
||||
self.assertGreater(
|
||||
response.json()["internal_states"][0]["avg_spec_accept_length"], 4.0
|
||||
)
|
||||
|
||||
def test_image_input(self):
|
||||
# Color is supplied only by the image: the prompt cannot substitute for
|
||||
# the encoder. Exercise two images to catch missing/cached embeddings.
|
||||
for color in ("red", "blue"):
|
||||
with self.subTest(color=color):
|
||||
png = io.BytesIO()
|
||||
Image.new("RGB", (256, 256), color=color).save(png, format="PNG")
|
||||
image_url = (
|
||||
"data:image/png;base64," + base64.b64encode(png.getvalue()).decode()
|
||||
)
|
||||
response = requests.post(
|
||||
self.base_url + "/v1/chat/completions",
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": image_url},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is the dominant color of this image? Answer with one English color word.",
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0,
|
||||
# The checkpoint template always opens a thinking block.
|
||||
# Allow it to finish before checking the final answer.
|
||||
"max_tokens": 1024,
|
||||
},
|
||||
timeout=180,
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
answer = response.json()["choices"][0]["message"]["content"]
|
||||
self.assertIn(color, answer.lower(), response.text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""GLM CP4/BCG prefill -> TP4 DFlash2 decode, with padding and cached prefixes."""
|
||||
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
from sglang.test.test_utils import popen_with_error_check, try_cached_model
|
||||
|
||||
register_cuda_ci(est_time=900, stage="extra-b", runner_config="8-gpu-b300")
|
||||
|
||||
COMMON_ARGS = [
|
||||
"--dsa-prefill-backend",
|
||||
"trtllm",
|
||||
"--dsa-decode-backend",
|
||||
"trtllm",
|
||||
"--kv-cache-dtype",
|
||||
"fp8_e4m3",
|
||||
"--moe-runner-backend",
|
||||
"flashinfer_trtllm",
|
||||
"--speculative-algorithm",
|
||||
"DFLASH",
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
"--reasoning-parser",
|
||||
"auto",
|
||||
"--tool-call-parser",
|
||||
"auto",
|
||||
"--max-running-requests",
|
||||
"16",
|
||||
"--cuda-graph-max-bs-decode",
|
||||
"16",
|
||||
"--chunked-prefill-size",
|
||||
"8192",
|
||||
"--mem-fraction-static",
|
||||
"0.9",
|
||||
"--enable-metrics",
|
||||
"--random-seed",
|
||||
"42",
|
||||
]
|
||||
|
||||
|
||||
class TestGlm53CpBcgDflashPD(PDDisaggregationServerBase, GSM8KMixin):
|
||||
gsm8k_score_threshold = 0.96
|
||||
gsm8k_num_examples = 1319
|
||||
gsm8k_num_threads = 256
|
||||
gsm8k_backend = "sgl_eval"
|
||||
gsm8k_thinking = True
|
||||
gsm8k_max_tokens = 8192
|
||||
gsm8k_temperature = 1.0
|
||||
gsm8k_top_p = 0.95
|
||||
|
||||
prefill_tp_size = 4
|
||||
decode_tp_size = 4
|
||||
decode_base_gpu_id = 4
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.model = try_cached_model("zai-org/GLM-5.3-Flash")
|
||||
draft = try_cached_model("incoai/GLM-5.3-Flash-DFlash2")
|
||||
common = COMMON_ARGS + ["--speculative-draft-model-path", draft]
|
||||
cls.extra_prefill_args = common + [
|
||||
"--attn-cp-size",
|
||||
str(cls.prefill_tp_size),
|
||||
"--enable-prefill-cp",
|
||||
"--cp-strategy",
|
||||
"interleave",
|
||||
"--cuda-graph-backend-prefill",
|
||||
"breakable",
|
||||
"--cuda-graph-bs-prefill",
|
||||
"2",
|
||||
"4",
|
||||
"8",
|
||||
"16",
|
||||
"32",
|
||||
"64",
|
||||
"128",
|
||||
"256",
|
||||
"512",
|
||||
"1024",
|
||||
"2048",
|
||||
"4096",
|
||||
"8192",
|
||||
]
|
||||
cls.extra_decode_args = common
|
||||
cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, trust_remote_code=True)
|
||||
cls.launch_all()
|
||||
|
||||
@classmethod
|
||||
def launch_lb(cls):
|
||||
cls.process_lb = popen_with_error_check(
|
||||
[
|
||||
"python3",
|
||||
"-m",
|
||||
"sglang_router.launch_router",
|
||||
"--pd-disaggregation",
|
||||
"--prefill",
|
||||
cls.prefill_url,
|
||||
cls.bootstrap_port,
|
||||
"--decode",
|
||||
cls.decode_url,
|
||||
"--host",
|
||||
cls.base_host,
|
||||
"--port",
|
||||
cls.lb_port,
|
||||
"--request-timeout-secs",
|
||||
"7200",
|
||||
"--worker-startup-timeout-secs",
|
||||
"7200",
|
||||
"--retry-max-retries",
|
||||
"1",
|
||||
"--health-check-timeout-secs",
|
||||
"60",
|
||||
]
|
||||
)
|
||||
cls.wait_server_ready(cls.lb_url + "/readiness", process=cls.process_lb)
|
||||
|
||||
def _prompt(self, length):
|
||||
suffix = self.tokenizer.encode(
|
||||
"\nThe access code is 739391. Copy the access code: 739391\n"
|
||||
"The access code is 739391. Copy the access code:",
|
||||
add_special_tokens=False,
|
||||
)
|
||||
filler = self.tokenizer.encode(
|
||||
"These are background notes. ", add_special_tokens=False
|
||||
)
|
||||
count = length - len(suffix)
|
||||
self.assertGreater(count, 0)
|
||||
return (filler * ((count + len(filler) - 1) // len(filler)))[:count] + suffix
|
||||
|
||||
def test_padding_chunk_and_prefix_replay(self):
|
||||
outputs = []
|
||||
cases = (
|
||||
(257, True),
|
||||
(257, False),
|
||||
(512, True),
|
||||
(513, True),
|
||||
(513, False),
|
||||
(16421, True),
|
||||
(16421, False),
|
||||
(16423, False),
|
||||
)
|
||||
for length, cold_cache in cases:
|
||||
if cold_cache:
|
||||
for url in (self.prefill_url, self.decode_url):
|
||||
requests.post(
|
||||
url + "/flush_cache", params={"timeout": 30}, timeout=60
|
||||
).raise_for_status()
|
||||
before = self._prefill_graph_passes()
|
||||
response = requests.post(
|
||||
self.lb_url + "/generate",
|
||||
json={
|
||||
"input_ids": self._prompt(length),
|
||||
"sampling_params": {"temperature": 0, "max_new_tokens": 16},
|
||||
"return_logprob": True,
|
||||
},
|
||||
timeout=600,
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
output = response.json()
|
||||
self.assertIn("739391", output["text"])
|
||||
self.assertEqual(output["meta_info"]["prompt_tokens"], length)
|
||||
self.assertGreater(output["meta_info"]["spec_verify_ct"], 0)
|
||||
graph_passes = self._prefill_graph_passes() - before
|
||||
cached_tokens = output["meta_info"]["cached_tokens"]
|
||||
print(
|
||||
f"PD smoke: length={length}, cached_tokens={cached_tokens}, "
|
||||
f"graph_passes={graph_passes}"
|
||||
)
|
||||
# Fewer tokens than the CP size cannot activate CP. Other text
|
||||
# shapes fit a bucket, including prefills with cached prefixes.
|
||||
if length - cached_tokens >= self.prefill_tp_size:
|
||||
self.assertGreater(graph_passes, 0, f"length={length}")
|
||||
else:
|
||||
self.assertEqual(graph_passes, 0, f"length={length}")
|
||||
if not cold_cache and length >= 16421:
|
||||
self.assertGreater(cached_tokens, 0)
|
||||
outputs.append(output)
|
||||
self.assertEqual(outputs[0]["text"], outputs[1]["text"])
|
||||
for process in (self.process_prefill, self.process_decode):
|
||||
self.assertIsNone(process.poll())
|
||||
|
||||
def test_uneven_batched_padding_replay(self):
|
||||
for url in (self.prefill_url, self.decode_url):
|
||||
requests.post(
|
||||
url + "/flush_cache", params={"timeout": 30}, timeout=60
|
||||
).raise_for_status()
|
||||
before = self._prefill_graph_passes()
|
||||
lengths = [257, 513, 1027]
|
||||
response = requests.post(
|
||||
self.lb_url + "/generate",
|
||||
json={
|
||||
"input_ids": [self._prompt(length) for length in lengths],
|
||||
"sampling_params": {"temperature": 0, "max_new_tokens": 16},
|
||||
},
|
||||
timeout=600,
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
for length, output in zip(lengths, response.json(), strict=True):
|
||||
self.assertIn("739391", output["text"])
|
||||
self.assertEqual(output["meta_info"]["prompt_tokens"], length)
|
||||
self.assertGreater(output["meta_info"]["spec_verify_ct"], 0)
|
||||
self.assertGreater(self._prefill_graph_passes(), before)
|
||||
|
||||
def _prefill_graph_passes(self):
|
||||
response = requests.get(self.prefill_url + "/metrics", timeout=30)
|
||||
response.raise_for_status()
|
||||
return sum(
|
||||
float(line.rsplit(" ", 1)[1])
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("sglang:cuda_graph_passes_total{")
|
||||
and 'mode="prefill_cuda_graph"' in line
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,6 +1,6 @@
|
||||
"""B200 per-commit coverage for the GLM-5.3-Flash serving recipes.
|
||||
|
||||
Runs the Low Latency, DFlash2, and High Throughput TP4 recipes on four B200
|
||||
Runs the Low Latency and High Throughput TP4 recipes on four B200
|
||||
GPUs. All recipes must retain GSM8K accuracy; the Low Latency recipe also
|
||||
checks EAGLE speculative acceptance and single-request decode performance.
|
||||
"""
|
||||
@@ -22,7 +22,6 @@ from sglang.test.test_utils import (
|
||||
register_cuda_ci(est_time=2400, stage="base-c", runner_config="4-gpu-b200")
|
||||
|
||||
MODEL_PATH = "zai-org/GLM-5.3-Flash"
|
||||
DFLASH2_DRAFT_MODEL_PATH = "incoai/GLM-5.3-Flash-DFlash2"
|
||||
SERVER_LAUNCH_TIMEOUT = 3600
|
||||
GPU_IDLE_TIMEOUT = 120
|
||||
|
||||
@@ -118,28 +117,5 @@ class TestGLM53FlashB200HighThroughput(
|
||||
]
|
||||
|
||||
|
||||
class TestGLM53FlashB200DFlash2(
|
||||
GSM8KMixin,
|
||||
_GLM53FlashB200Base,
|
||||
):
|
||||
gsm8k_score_threshold = 0.96
|
||||
gsm8k_num_examples = 1319
|
||||
gsm8k_num_threads = 256
|
||||
gsm8k_backend = "sgl_eval"
|
||||
gsm8k_thinking = True
|
||||
gsm8k_max_tokens = 8192
|
||||
gsm8k_temperature = 1.0
|
||||
gsm8k_top_p = 0.95
|
||||
server_args = [
|
||||
*COMMON_SERVER_ARGS,
|
||||
"--speculative-algorithm",
|
||||
"DFLASH",
|
||||
"--speculative-draft-model-path",
|
||||
DFLASH2_DRAFT_MODEL_PATH,
|
||||
"--speculative-draft-attention-backend",
|
||||
"fa4",
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -65,6 +65,12 @@ class TestDSABackendDPPadding(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_trtllm_decode_runs_real_rows_then_restores_physical_batch(self):
|
||||
self._check_trtllm_padding(is_prefill=False)
|
||||
|
||||
def test_trtllm_prefill_narrows_metadata_to_live_cp_queries(self):
|
||||
self._check_trtllm_padding(is_prefill=True)
|
||||
|
||||
def _check_trtllm_padding(self, *, is_prefill):
|
||||
metadata = SimpleNamespace(
|
||||
cache_seqlens_int32=torch.tensor([8, 12], dtype=torch.int32),
|
||||
page_table_1=torch.zeros((2, 12), dtype=torch.int32),
|
||||
@@ -110,6 +116,16 @@ class TestDSABackendDPPadding(unittest.TestCase):
|
||||
forward_batch = SimpleNamespace()
|
||||
q = torch.arange(4 * 2 * 3, dtype=torch.float32).view(4, 2, 3)
|
||||
topk_indices = torch.arange(4 * 2, dtype=torch.int32).view(4, 2)
|
||||
if is_prefill:
|
||||
# The CP graph bridge has removed padded Q rows; metadata was
|
||||
# planned for the physical shard before entering the eager break.
|
||||
q = q[:2]
|
||||
topk_indices = topk_indices[:2]
|
||||
metadata.cache_seqlens_int32 = torch.tensor(
|
||||
[8, 12, 0, 0], dtype=torch.int32
|
||||
)
|
||||
backend.use_fused_topk = True
|
||||
backend._get_fused_topk_page_table = lambda indices: indices
|
||||
|
||||
flashinfer = ModuleType("flashinfer")
|
||||
flashinfer_decode = ModuleType("flashinfer.decode")
|
||||
@@ -159,10 +175,12 @@ class TestDSABackendDPPadding(unittest.TestCase):
|
||||
seq_lens=metadata.cache_seqlens_int32,
|
||||
save_kv_cache=False,
|
||||
topk_indices=topk_indices,
|
||||
is_prefill=is_prefill,
|
||||
)
|
||||
|
||||
self.assertEqual(captured["query"].shape, (2, 1, 2, 3))
|
||||
self.assertEqual(output.shape, (4, 1, 2, 2))
|
||||
self.assertEqual(captured["seq_lens"].tolist(), [8, 12])
|
||||
self.assertEqual(output.shape, (2 if is_prefill else 4, 1, 2, 2))
|
||||
self.assertTrue(torch.all(output[:2] == 1))
|
||||
self.assertTrue(torch.all(output[2:] == 0))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user