diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 4445e13803..11f75bae24 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -130,7 +130,7 @@ For agentic MXFP4 serving on MI355X, select **Low Latency** with **L1 + L2**. Th ### Multimodal memory -All strategies enable multimodal serving. Image and video input need a Transformers build that ships `Glm5NextProcessor` (the MI355X checks used commit `e4052f55`, which also requires `tokenizers>=0.23.1`). With the pinned `transformers==5.12.1`, the server loads only the tokenizer and silently answers image requests from the text alone. On MI355X, text, image, and an image-bearing follow-up were qualified with that build; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. +All strategies enable multimodal serving. Image and video input need a Transformers build that ships `Glm5NextProcessor` (the current `transformers==5.17.0` dependency includes it and requires `tokenizers>=0.23.1`). With older `transformers==5.12.1`, the server loads only the tokenizer and silently answers image requests from the text alone. On MI355X, text, image, and an image-bearing follow-up were qualified with Transformers commit `e4052f55`; video was not evaluated. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. The default multimodal feature transport is automatic, and on a single CUDA node auto resolves to CPU transport. The MI355X selection explicitly emits CPU transport, matching its image-serving check. CUDA IPC is opt-in: pass `--mm-feature-transport cuda_ipc` when lower transfer latency matters more than the GPU memory the IPC pool reserves. CUDA VMM transport applies only to multi-node GB200/GB300 systems on the MNNVL fabric, where auto selects it. @@ -202,7 +202,38 @@ This topology served image requests and videos up to 238,080 visual tokens. In a Install `torchcodec` for video and see the [encoder disaggregation guide](/docs/advanced_features/epd_disaggregation) for the generic architecture and operational model. -### 3.5 PD disaggregation (preview) +### 3.5 Prefill Context Parallelism + +Use prefill context parallelism (CP) to distribute DSA prefill computation for long prompts, including multimodal inputs. This hybrid model uses **interleave CP for DSA layers** and **head-wise tensor parallelism for KDA layers**, sharing the same communication group. Layer boundaries all-gather the context shards before KDA and reduce-scatter its output back onto those shards. The mHC residual streams and their coefficients stay local, with each output's reduction completed before the residual update. + + + +```bash Command +sglang serve \ + --model-path zai-org/GLM-5.3-Flash \ + --tp-size 4 \ + --attn-cp-size 4 \ + --enable-prefill-cp \ + --cp-strategy interleave \ + --moe-dense-tp-size 4 \ + --ep-size 1 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 5 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 6 \ + --dsa-prefill-backend trtllm \ + --dsa-decode-backend trtllm \ + --kv-cache-dtype fp8_e4m3 \ + --moe-runner-backend flashinfer_trtllm \ + --reasoning-parser auto \ + --tool-call-parser auto \ + --host 0.0.0.0 \ + --port 30000 +``` + + + +### 3.6 PD disaggregation (preview) PD splits prefill and decode into separate server groups behind a router. For this hybrid model, the transfer moves both the paged DSA KV and the KDA recurrent state. diff --git a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py index a0afd7633e..e76963b36e 100644 --- a/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py +++ b/python/sglang/srt/arg_groups/cp_tp_group_sharing_hook.py @@ -17,7 +17,7 @@ from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_pha # MoE and linear attention keep their TP partition (the linear attention # gathers the sequence where its recurrence needs it). Model ports register # here; every other model keeps the ordinary prefill CP path. -_SUPPORTED_MODELS: set[str] = set() +_SUPPORTED_MODELS: set[str] = {"Glm5NextForConditionalGeneration"} def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: @@ -34,11 +34,27 @@ def resolve_cp_tp_group_sharing(server_args: Any, model: Any) -> None: "CP-TP group sharing requires --attn-cp-size == --tp-size, got " f"attn_cp_size={view.attn_cp_size}, tp_size={cfg.tp_size}." ) - if cfg.cp_strategy != "zigzag": - raise ValueError("CP-TP group sharing requires --cp-strategy zigzag.") + strategy = ( + "interleave" + if model.hf_config.architectures[0] == "Glm5NextForConditionalGeneration" + else "zigzag" + ) + if cfg.cp_strategy != strategy: + raise ValueError(f"CP-TP group sharing requires --cp-strategy {strategy}.") + + # This hook runs before speculative defaults and NEXTN alias resolution. + # Native MTP keeps the target's CP/TP placement; other draft architectures + # have their own topology and model-boundary contracts. + if cfg.speculative_algorithm is not None and ( + model.hf_config.architectures[0] != "Glm5NextForConditionalGeneration" + or cfg.speculative_algorithm.upper() not in ("EAGLE", "NEXTN") + or cfg.speculative_draft_model_path not in (None, cfg.model_path) + or cfg.speculative_eagle_topk not in (None, 1) + ): + raise ValueError("CP/TP group sharing only supports native MTP with top-k 1.") unsupported = { - "speculative-algorithm": cfg.speculative_algorithm is not None, + "enable-dsa-cache-layer-split": cfg.enable_dsa_cache_layer_split, "dcp-size > 1": cfg.dcp_size > 1, "enable-mixed-chunk": cfg.enable_mixed_chunk, "moe-dp-size > 1": cfg.moe_dp_size > 1, diff --git a/python/sglang/srt/configs/glm5_next.py b/python/sglang/srt/configs/glm5_next.py index 17c149ff02..e908152447 100644 --- a/python/sglang/srt/configs/glm5_next.py +++ b/python/sglang/srt/configs/glm5_next.py @@ -4,7 +4,6 @@ from transformers.configuration_utils import PretrainedConfig from transformers.models.glm_ocr.configuration_glm_ocr import GlmOcrVisionConfig from sglang.srt.configs.mamba_utils import KimiLinearCacheParams, KimiLinearStateShape -from sglang.srt.runtime_context import get_parallel _GLM5_NEXT_TOP_LEVEL_CONFIG_KEYS = ( "architectures", @@ -262,8 +261,10 @@ class Glm5NextTextConfig(PretrainedConfig): @property def mamba2_cache_params(self) -> KimiLinearCacheParams: + from sglang.srt.runtime_context import linear_attn_tp_size + shape = KimiLinearStateShape.create( - tp_world_size=get_parallel().attn_tp_size, + tp_world_size=linear_attn_tp_size(), num_heads=self.linear_attn_config["num_heads"], head_dim=self.linear_attn_config["head_dim"], conv_kernel_size=self.linear_attn_config["short_conv_kernel_size"], diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py index 15fe941ec6..23896054ad 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_kpool.py @@ -13,6 +13,7 @@ from sglang.srt.layers.attention.dsa.kpool_plan import ( update_kpool_write_plan, update_pooled_paged_mqa_metadata, ) +from sglang.srt.layers.cp.interleave_kpool import local_query_inputs if TYPE_CHECKING: from sglang.srt.layers.attention.dsa.dsa_backend_mtp_precompute import ( @@ -100,6 +101,7 @@ class DeepseekSparseAttnBackendKPoolMixin: topk_transform_method=topk_transform_method, full_real_page_table=kpool_inputs.full_real_page_table, full_seqlens_expanded=kpool_inputs.full_seqlens_expanded, + **local_query_inputs(forward_batch, metadata), ) metadata = init_pooled_paged_mqa_metadata( diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index cc4baf5419..8aae41a421 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -600,12 +600,17 @@ class IndexerKPool(MultiPlatformOp): return_compressed: bool = False, write_cache: bool = True, ): - if key.shape[0] == 0: - return None - if gate_score is None: gate_score = F.linear(x, self.index_kpool_compress_gate) + from sglang.srt.layers.cp.interleave_kpool import materialize_compression_inputs + + key, gate_score, positions = materialize_compression_inputs( + key, gate_score, positions, forward_batch + ) + if key.shape[0] == 0: + return None + if forward_batch.forward_mode.is_decode_or_idle(): self._compress_write_decode( key=key, diff --git a/python/sglang/srt/layers/cp/interleave.py b/python/sglang/srt/layers/cp/interleave.py index 55b5af282b..0f1a7e1ca9 100644 --- a/python/sglang/srt/layers/cp/interleave.py +++ b/python/sglang/srt/layers/cp/interleave.py @@ -78,6 +78,61 @@ class InterleaveContextParallelMetadata(BaseContextParallelMetadata): moe_local_token_count: Optional[torch.Tensor] = None +def is_interleave_extend(forward_batch): + return forward_batch.forward_mode.is_context_parallel_extend() and isinstance( + forward_batch.attn_cp_metadata, InterleaveContextParallelMetadata + ) + + +def cp_interleave_to_sequence_order(hidden_states, forward_batch): + """Restore sequence order after CP attention and before linear attention. + + With CP2 and seven tokens, the boundary all-gathers the interleaved shards + as ``[t0, t2, t4, t6, t1, t3, t5, pad]``. This returns + ``[t0, t1, t2, t3, t4, t5, t6]`` for sequence-dependent compute. + + The residual and mHC coefficients do not move: only the already-read mixer + input is reordered. Remove physical CP padding before the recurrent kernel. + """ + if not is_interleave_extend(forward_batch): + return hidden_states + metadata = forward_batch.attn_cp_metadata + if metadata.gather_index is not None: + return hidden_states.index_select(0, metadata.gather_index) + size = get_parallel().attn_cp_size + rows = hidden_states.shape[0] // size + return ( + hidden_states.reshape(size, rows, *hidden_states.shape[1:]) + .transpose(0, 1) + .flatten(0, 1)[: metadata.total_seq_lens] + .contiguous() + ) + + +def cp_sequence_to_interleave_order(hidden_states, forward_batch, gathered_rows): + """Restore interleave order after linear attention and before CP attention. + + With CP2 and seven tokens, ``[t0, t1, t2, t3, t4, t5, t6]`` becomes + ``[t0, t2, t4, t6, t1, t3, t5, 0]``. These padded rank-major rows match + the boundary's reduce-scatter and the local residual/mHC rows. The reorder + itself performs no communication or reduction. + """ + if not is_interleave_extend(forward_batch): + return hidden_states + metadata = forward_batch.attn_cp_metadata + padded = hidden_states.new_zeros((gathered_rows, *hidden_states.shape[1:])) + if metadata.gather_index is not None: + return padded.index_copy_(0, metadata.gather_index, hidden_states) + padded[: hidden_states.shape[0]] = hidden_states + size = get_parallel().attn_cp_size + return ( + padded.reshape(-1, size, *hidden_states.shape[1:]) + .transpose(0, 1) + .flatten(0, 1) + .contiguous() + ) + + class InterleaveCPStrategy(ContextParallelStrategy): name = "interleave" kind = ContextParallelStrategyKind.INTERLEAVE diff --git a/python/sglang/srt/layers/cp/interleave_kpool.py b/python/sglang/srt/layers/cp/interleave_kpool.py new file mode 100644 index 0000000000..429f6c465e --- /dev/null +++ b/python/sglang/srt/layers/cp/interleave_kpool.py @@ -0,0 +1,50 @@ +"""KPool compression and query metadata for interleave context parallelism.""" + +import torch + +from sglang.srt.layers.cp.base import get_cp_strategy +from sglang.srt.layers.cp.interleave import ( + interleave_rows_per_request, + is_interleave_extend, +) +from sglang.srt.runtime_context import get_parallel + + +def local_query_inputs(forward_batch, metadata): + """Compression covers full requests; logits cover only this rank's queries.""" + if not is_interleave_extend(forward_batch): + return {} + parallel = get_parallel() + counts = interleave_rows_per_request( + forward_batch.extend_seq_lens_cpu, parallel.attn_cp_rank, parallel.attn_cp_size + ) + indices = torch.tensor( + [i for i, count in enumerate(counts) if count], + dtype=torch.long, + device=forward_batch.req_pool_indices.device, + ) + return dict( + local_real_page_table=metadata.real_page_table, + local_seqlens_expanded=metadata.dsa_seqlens_expanded, + local_extend_seq_lens_cpu=metadata.dsa_extend_seq_lens_list, + local_seq_lens_cpu=metadata.indexer_seq_lens_cpu.tolist(), + local_req_pool_indices=forward_batch.req_pool_indices.index_select(0, indices), + ) + + +def materialize_compression_inputs(key, score, positions, forward_batch): + """Replicate ordered keys/scores before pooling across adjacent tokens. + + Queries and head gates stay local. Every rank writes its replicated index + cache and tail state, so subsequent decode needs no layer-owner protocol. + """ + if not is_interleave_extend(forward_batch): + return key, score, positions + width = key.shape[-1] + packed = torch.cat((key, score), dim=-1) + packed = get_cp_strategy().gather_kv_cache(packed, forward_batch) + return ( + packed[..., :width].contiguous(), + packed[..., width:].contiguous(), + forward_batch.positions, + ) diff --git a/python/sglang/srt/layers/cp/utils.py b/python/sglang/srt/layers/cp/utils.py index 00ba6eb962..039f646550 100644 --- a/python/sglang/srt/layers/cp/utils.py +++ b/python/sglang/srt/layers/cp/utils.py @@ -278,8 +278,10 @@ def cp_shard_model_inputs( ): """Restore the shared batch so logits processing keeps full-batch metadata.""" assert is_cp_active(forward_batch) - sharded_hidden_states = cp_shard_hidden_states( - complete_hidden_states, forward_batch + sharded_hidden_states = ( + cp_shard_hidden_states(complete_hidden_states, forward_batch) + if complete_hidden_states is not None + else None ) sharded_positions = cp_shard_position_ids(complete_position_ids, forward_batch) model_input_ids = ( @@ -300,6 +302,7 @@ def cp_shard_model_inputs( spec_hidden_states_backup = None if ( spec_hidden_states is not None + and complete_hidden_states is not None and spec_hidden_states.shape[0] == complete_hidden_states.shape[0] ): spec_hidden_states_backup = spec_hidden_states diff --git a/python/sglang/srt/layers/layer_boundary/boundary.py b/python/sglang/srt/layers/layer_boundary/boundary.py index 7dd22f76a7..449ba718f3 100644 --- a/python/sglang/srt/layers/layer_boundary/boundary.py +++ b/python/sglang/srt/layers/layer_boundary/boundary.py @@ -57,6 +57,7 @@ from sglang.srt.layers.layer_boundary.prepare import ( _attn_tp_slice_update_read, _dispatch_by_update, _dp_gather_sum_read, + _move_before_read, _reduce_update_read, _reduce_update_read_dp_gather, _run_entry, @@ -404,6 +405,36 @@ def _select_entry_step( raise NotImplementedError(f"{produced=}") gathered = produced.layout.sharded - need.layout.sharded - need.gathered_by_compute sliced = need.layout.sharded - produced.layout.sharded + if residual_to.sharded - produced.layout.sharded == {TokenAxis.ATTN_CP}: + # A head-parallel mixer consumes full context, but its residual and + # readout coefficients remain on the CP shard. Complete the producer's + # sum onto that shard before the (possibly nonlinear) update/read. + if ( + residual != residual_to + or cp_moves is None + or owes not in (None, SumGroup.ATTN_CP) + or (owes is not None and cp_moves.reduce_scatter is None) + ): + raise NotImplementedError(f"{produced=} {residual=} {need=}") + local = msgspec.structs.replace( + produced, layout=residual_to, group=None, always_partial=False + ) + read_local, input_move = _select_entry_step( + local, + residual=residual, + residual_to=residual_to, + need=need, + is_plain_add=is_plain_add, + fusions=(), + residual_joins_sum=False, + cp_moves=cp_moves, + enters_stack=enters_stack, + ) + return partial( + _move_before_read, + move=cp_moves.reduce_scatter if owes is not None else cp_moves.take_back, + read=read_local, + ), input_move if sliced: # Each attention-TP rank takes its own slice: the reduce-scatter # completes the attention-TP sum and slices in one collective; a @@ -536,7 +567,7 @@ def _select_entry_step( return (partial(_tp_sum_with_residual_read, read=read), None) # A sum over TP completes only on rows every TP rank holds: the TP group # spans attention DP and CP. - if owes is not SumGroup.ATTN_TP and ( + if owes not in (SumGroup.ATTN_TP, SumGroup.ATTN_CP) and ( owes is not SumGroup.TP or produced.layout.sharded ): raise NotImplementedError(f"{produced=} {need=}") @@ -551,7 +582,7 @@ def _select_entry_step( ), None, ) - if owes not in (None, SumGroup.ATTN_TP): + if owes not in (None, SumGroup.ATTN_TP, SumGroup.ATTN_CP): raise NotImplementedError(f"{produced=} {need=}") owes_attention_tp = owes is SumGroup.ATTN_TP # The partial order adds the residual on attention-TP rank 0 before the DP @@ -579,7 +610,8 @@ def _select_entry_step( partial( _reduce_update_read_dp_gather, gathers_residual=gathers_residual, - reduces_attention_tp=owes_attention_tp, + reduces_attention_tp=owes is not None, + group=owes, places_cp_shards=places_cp_shards, read=read, ), diff --git a/python/sglang/srt/layers/layer_boundary/construction.py b/python/sglang/srt/layers/layer_boundary/construction.py index 460d80d71c..6a9180f4de 100644 --- a/python/sglang/srt/layers/layer_boundary/construction.py +++ b/python/sglang/srt/layers/layer_boundary/construction.py @@ -282,6 +282,7 @@ def _bind_stage(declaration, norm, incoming, outgoing, **options): declaration.update.applied_at_exit and TokenAxis.ATTN_CP in edge.produced.layout.sharded - edge.need.layout.sharded + and not _cp_gathers_over_attn_cp() ): raise NotImplementedError("MHC with a gather over attention CP") attn_input_adapter = None diff --git a/python/sglang/srt/layers/layer_boundary/factories.py b/python/sglang/srt/layers/layer_boundary/factories.py index 73c08eeda6..9b8e4cfa4c 100644 --- a/python/sglang/srt/layers/layer_boundary/factories.py +++ b/python/sglang/srt/layers/layer_boundary/factories.py @@ -209,6 +209,8 @@ class StageDeclaration: reduction: Whether the next stage's input always completes the sum (an attention's), or the exit decides (an FFN's or a mixer's). gathers_attn_tp_input: Whether attention gathers TP-sharded input itself. + tp_group: Group partitioning attention heads. Full TP consumers require + full token rows, including when TP shares the prefill CP group. dense_tp_size: Dense FFN compute width: None uses the configured width, 1 means local compute, and the full TP size means TP compute. exit_rows: Required FFN output rows at the layer or branch exit. @@ -228,6 +230,7 @@ class StageDeclaration: output_transform: Optional[OutputTransform] = None reduction: ProducerReduction = ProducerReduction.EXIT_SCOPED gathers_attn_tp_input: bool = False + tp_group: SumGroup = SumGroup.ATTN_TP dense_tp_size: Optional[int] = None exit_rows: Optional[ExitRows] = None # Only declarations participate in construction, never executable stages. @@ -273,6 +276,7 @@ def declare_attn( reduction=ProducerReduction.ALWAYS_PARTIAL, gathers_attn_tp_input=True, output_transform=None, + tp_group=SumGroup.ATTN_TP, ): """Declare attention or a mixer; construct its executable boundary later. @@ -286,6 +290,7 @@ def declare_attn( complete, before the residual update (a sandwich norm). The next stage's input runs it, so no fused add + norm takes that input. Requires ALWAYS_PARTIAL. + tp_group: Head partition; ATTN_TP by default, TP for full-TP consumers. Returns: A StageDeclaration with no norm, tensors or execution plan. @@ -302,6 +307,7 @@ def declare_attn( output_transform=output_transform, reduction=reduction, gathers_attn_tp_input=gathers_attn_tp_input, + tp_group=tp_group, ) @@ -375,11 +381,30 @@ def _resolve_stage(stage, variant, following=None): else frozenset() ) owes = not sp and axes[TokenAxis.ATTN_TP] > 1 + group = SumGroup.ATTN_TP + compute_rows = attention + if stage.tp_group is SumGroup.TP: + parallel = get_parallel() + if parallel.attn_dp_size != 1 or sp or scattered: + raise NotImplementedError( + "full-TP attention requires unscattered input without attention DP" + ) + compute_rows = full + owes = parallel.tp_size > 1 + # Canonicalize equal groups to the CP transport's reduction contract. + # initialize_model_parallel aliases ATTN_CP to TP at this topology. + group = ( + SumGroup.ATTN_CP + if parallel.attn_cp_size == parallel.tp_size and parallel.attn_cp_size > 1 + else SumGroup.TP + ) + elif stage.tp_group is not SumGroup.ATTN_TP: + raise ValueError("attention heads must be partitioned over ATTN_TP or TP") declaration = StageContract( - InputContract(attention, gathered_by_compute=gathers, read=stage.read), + InputContract(compute_rows, gathered_by_compute=gathers, read=stage.read), OutputContract( - local if sp else attention, - group=SumGroup.ATTN_TP if owes else None, + local if sp else compute_rows, + group=group if owes else None, always_partial=owes and ( stage.reduction is ProducerReduction.ALWAYS_PARTIAL diff --git a/python/sglang/srt/layers/layer_boundary/layout.py b/python/sglang/srt/layers/layer_boundary/layout.py index d98f8e9074..5b3c491db0 100644 --- a/python/sglang/srt/layers/layer_boundary/layout.py +++ b/python/sglang/srt/layers/layer_boundary/layout.py @@ -73,6 +73,7 @@ class SumGroup(Enum): from get_parallel() when the sum runs.""" ATTN_TP = auto() + ATTN_CP = auto() TP = auto() # The group one all-reduce of a MoE output runs over. MOE_OUTPUT = auto() @@ -145,6 +146,8 @@ def _sum_group(group: SumGroup) -> GroupCoordinator: parallel = get_parallel() if group is SumGroup.ATTN_TP: return parallel.attn_tp_group + if group is SumGroup.ATTN_CP: + return parallel.attn_cp_group if group is SumGroup.TP: return parallel.tp_group if group is SumGroup.MOE_OUTPUT: diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 1376c2e315..605df1bea4 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -43,6 +43,7 @@ from sglang.srt.layers.layer_boundary.adapters.attention import ( from sglang.srt.layers.layer_boundary.layout import ( SumGroup, _cp_shard_token_rows, + _sum_group, ) from sglang.srt.layers.layer_boundary.output import ( DeferredFinalize, @@ -105,6 +106,8 @@ def _reduce_update_read( ) elif group is SumGroup.TP: hidden_states = tensor_model_parallel_all_reduce(hidden_states) + elif group is not None: + hidden_states = _sum_group(group).all_reduce(hidden_states) if _is_npu and cache is not None: _ = prepare_weight_cache(hidden_states, cache) return read.update_and_read( @@ -127,6 +130,7 @@ def _reduce_update_read_dp_gather( gathers_residual: bool, reduces_attention_tp: bool, places_cp_shards: bool = False, + group: SumGroup = SumGroup.ATTN_TP, read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, ): @@ -138,7 +142,11 @@ def _reduce_update_read_dp_gather( residual = update.gather_residual_attn_tp(residual) if hidden_states.shape[0] != 0: if reduces_attention_tp: - hidden_states = attention_tensor_model_parallel_all_reduce(hidden_states) + hidden_states = ( + attention_tensor_model_parallel_all_reduce(hidden_states) + if group is SumGroup.ATTN_TP + else _sum_group(group).all_reduce(hidden_states) + ) with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric(), @@ -417,16 +425,25 @@ def _then_attn_cp_gather( cache=None, gather: Callable, update: ResidualUpdate = PLAIN_ADD, + **call, ): """DSA and MLA CP: complete this rank's shard, then gather the shards, of equal length, over the attention-CP group. The residual stays on the shard.""" hidden_states, residual = gather( - hidden_states, residual, forward_batch, norm, update=update, cache=cache + hidden_states, residual, forward_batch, norm, update=update, cache=cache, **call ) return attn_cp_interleave_gather(hidden_states), residual +def _move_before_read( + hidden_states, residual, forward_batch, norm, *, move, read, **call +): + """Complete and place a producer contribution before its residual update.""" + hidden_states, residual = move(hidden_states, residual, forward_batch) + return read(hidden_states, residual, forward_batch, norm, **call) + + def _then_moe_cp_gather( hidden_states: torch.Tensor, residual: torch.Tensor, diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 7b17e75c94..2958e22f34 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -654,32 +654,22 @@ def _embed_mm_inputs_with_split( return input_embeds, other_info -def general_mm_embed_routine( +def prepare_mm_inputs( input_ids: torch.Tensor, forward_batch: ForwardBatch, language_model: nn.Module, + kwargs: dict, multimodal_model: Optional[nn.Module] = None, data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None, placeholder_tokens: Optional[dict[Modality, List[int]]] = None, use_deepstack: Dict[Modality, bool] = {}, feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None, - **kwargs, -) -> torch.Tensor: - """ - Process multimodal inputs and forward through language model. +) -> Optional[torch.Tensor]: + """Prepare full-sequence embeddings without running the language model. - Args: - input_ids: Input token IDs tensor - forward_batch: Batch information for model forward pass - language_model: Base language model to use - data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function. - placeholder_tokens: Token IDs for multimodal placeholders - use_deepstack: Whether to use deepstack embeddings for each modality, default False - feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup. - **kwargs: Additional arguments passed to language model - - Returns: - Hidden states from language model forward pass + Preserve multimodal cache/offload bookkeeping and the MTP embedding copy. + Additional language-model inputs, such as DeepStack features, update kwargs. + Non-first pipeline ranks consume proxy tensors and return no embeddings. """ assert hasattr(language_model, "get_input_embeddings") embed_tokens = language_model.get_input_embeddings() @@ -808,6 +798,47 @@ def general_mm_embed_routine( input_embeds = forward_batch.input_embeds else: input_embeds = None + return input_embeds + + +def general_mm_embed_routine( + input_ids: torch.Tensor, + forward_batch: ForwardBatch, + language_model: nn.Module, + multimodal_model: Optional[nn.Module] = None, + data_embedding_funcs: Dict[Modality, DataEmbeddingFunc] = None, + placeholder_tokens: Optional[dict[Modality, List[int]]] = None, + use_deepstack: Dict[Modality, bool] = {}, + feature_dtypes: Optional[Dict[Modality, torch.dtype]] = None, + **kwargs, +) -> torch.Tensor: + """ + Process multimodal inputs and forward through language model. + + Args: + input_ids: Input token IDs tensor + forward_batch: Batch information for model forward pass + language_model: Base language model to use + data_embedding_funcs: A dictionary mapping from modality type to the corresponding embedding function. + placeholder_tokens: Token IDs for multimodal placeholders + use_deepstack: Whether to use deepstack embeddings for each modality, default False + feature_dtypes: Optional per-modality dtypes for preparing CUDA features before cache lookup. + **kwargs: Additional arguments passed to language model + + Returns: + Hidden states from language model forward pass + """ + input_embeds = prepare_mm_inputs( + input_ids, + forward_batch, + language_model, + kwargs, + multimodal_model, + data_embedding_funcs, + placeholder_tokens, + use_deepstack, + feature_dtypes, + ) with torch.profiler.record_function("sglang.vlm.language_model_prefill"): hidden_states = language_model( diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index c5ac346686..5115e4c90c 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -370,8 +370,12 @@ class EagerRunner(BaseRunner): forward_batch, **kwargs, ) - # Under CP-TP group sharing the model owns the CP row layout. - elif cp_active and not get_parallel().enable_cp_tp_group_sharing: + elif cp_active and ( + not get_parallel().enable_cp_tp_group_sharing + or getattr(model_runner.model, "prepare_cp_inputs", None) is not None + ): + # GLM delegates its CP boundary to this runner via prepare_cp_inputs. + # Other group-sharing models keep owning the layout in forward(). ret = self._execute_extend_cp(forward_batch, kwargs) else: ret = model_runner.model.forward( @@ -390,18 +394,25 @@ class EagerRunner(BaseRunner): """ model = self.model_runner.model - input_embeds = kwargs.get("input_embeds") - if input_embeds is None: - input_embeds = model.get_input_embeddings()(forward_batch.input_ids) + if prepare_inputs := getattr(model, "prepare_cp_inputs", None): + input_embeds, positions, model_kwargs = prepare_inputs( + forward_batch, **kwargs + ) + else: + input_embeds = kwargs.get("input_embeds") + if input_embeds is None: + input_embeds = model.get_input_embeddings()(forward_batch.input_ids) + positions = forward_batch.positions + model_kwargs = {} + if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None: + model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors with cp_shard_model_inputs( input_embeds, - forward_batch.positions, + positions, forward_batch, forward_batch.input_ids, ) as (sharded_input_embeds, sharded_positions, model_input_ids): - model_kwargs = {"input_embeds": sharded_input_embeds} - if (pp_proxy_tensors := kwargs.get("pp_proxy_tensors")) is not None: - model_kwargs["pp_proxy_tensors"] = pp_proxy_tensors + model_kwargs["input_embeds"] = sharded_input_embeds hidden_states = model.model( model_input_ids, sharded_positions, diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index 46b7f708e4..462b2039c7 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -137,6 +137,28 @@ class DeepseekModelNextN(nn.Module): self.shared_head = nn.Module() self.shared_head.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + def embed_input_ids( + self, input_ids: torch.Tensor, forward_batch: ForwardBatch + ) -> torch.Tensor: + # MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels + # (far above vocab_size). Use target-produced mm_input_embeds for + # these positions and only call embed_tokens on the appended + # next-token to avoid embed OOB. + input_embeds = forward_batch.mm_input_embeds + if ( + forward_batch.forward_mode.is_extend() + and forward_batch.contains_mm_inputs() + and not forward_batch.forward_mode.is_draft_extend_v2() + ): + assert input_embeds is not None + last_indices = ( + forward_batch.extend_start_loc + forward_batch.extend_seq_lens - 1 + ).long() + input_embeds[last_indices] = self.embed_tokens(input_ids[last_indices]) + if input_embeds is None: + input_embeds = self.embed_tokens(input_ids) + return input_embeds + def forward( self, input_ids: torch.Tensor, @@ -168,27 +190,7 @@ class DeepseekModelNextN(nn.Module): ) if input_embeds is None: - # MM positions in input_ids hold MM_PAD_SHIFT_VALUE+hash sentinels - # (far above vocab_size). Use target-produced mm_input_embeds for - # these positions and only call embed_tokens on the appended - # next-token to avoid embed OOB. - input_embeds = forward_batch.mm_input_embeds - if ( - forward_batch.forward_mode.is_extend() - and forward_batch.contains_mm_inputs() - and not forward_batch.forward_mode.is_draft_extend_v2() - ): - assert input_embeds is not None - last_indices = ( - forward_batch.extend_start_loc - + forward_batch.extend_seq_lens - - 1 - ).long() - input_embeds[last_indices] = self.embed_tokens( - input_ids[last_indices] - ) - if input_embeds is None: - input_embeds = self.embed_tokens(input_ids) + input_embeds = self.embed_input_ids(input_ids, forward_batch) hidden_states = input_embeds if hidden_states.shape[0] > 0: diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 0ad1aa0f04..7fce8b7393 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -30,6 +30,7 @@ from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import ( PLAIN_RESIDUAL_OPS, MHCState, + SumGroup, append_stages, declare_attn, declare_ffn, @@ -68,6 +69,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, general_mm_embed_routine, + prepare_mm_inputs, ) from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs from sglang.srt.model_executor.cuda_graph_config import ( @@ -395,7 +397,13 @@ class Glm5NextLinearAttention(nn.Module): **kwargs, ) -> None: super().__init__() - head_shard_size = get_parallel().attn_tp_size + from sglang.srt.runtime_context import ( + linear_attn_parallel_group, + linear_attn_tp_size, + ) + + head_shard_size = linear_attn_tp_size() + head_parallel_group = linear_attn_parallel_group() self.hidden_size = hidden_size self.config = config @@ -434,7 +442,7 @@ class Glm5NextLinearAttention(nn.Module): self.fg_sizes, quant_config=None, prefix=f"{prefix}.fused_qkvbfg_a_proj", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) self.split_sizes = [ 3 * projection_size // head_shard_size, @@ -446,7 +454,7 @@ class Glm5NextLinearAttention(nn.Module): self.head_dim, projection_size, dtype=self.fused_qkvbfg_a_proj.params_dtype, - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) else: self.qkv_proj = QKVParallelLinear( @@ -456,7 +464,7 @@ class Glm5NextLinearAttention(nn.Module): self.num_k_heads, bias=False, quant_config=quant_config, - parallel_group="attn_tp", + parallel_group=head_parallel_group, prefix=f"{prefix}.qkv_proj", ) @@ -467,7 +475,7 @@ class Glm5NextLinearAttention(nn.Module): [self.head_dim, self.head_dim], quant_config=None, prefix=f"{prefix}.fused_bfg_a_proj", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) self.bfg_split_sizes = [self.local_num_heads, 2 * self.head_dim] self.fused_fg_b_proj = ColumnParallelBatchedLinear( @@ -475,7 +483,7 @@ class Glm5NextLinearAttention(nn.Module): self.head_dim, projection_size, dtype=self.fused_bfg_a_proj.params_dtype, - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) else: self.f_a_proj = ReplicatedLinear( @@ -492,7 +500,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.f_b_proj", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) self.b_proj = ColumnParallelLinear( @@ -501,7 +509,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.b_proj", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) self.g_a_proj = ReplicatedLinear( @@ -517,7 +525,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.g_b_proj", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) self.dt_bias = nn.Parameter( @@ -526,7 +534,11 @@ class Glm5NextLinearAttention(nn.Module): set_weight_attrs( self.dt_bias, - {"weight_loader": sharded_weight_loader(0, parallel_group="attn_tp")}, + { + "weight_loader": sharded_weight_loader( + 0, parallel_group=head_parallel_group + ) + }, ) self.qkv_conv1d = MergedColumnParallelLinear( @@ -535,7 +547,7 @@ class Glm5NextLinearAttention(nn.Module): bias=False, params_dtype=torch.float32, prefix=f"{prefix}.qkv_conv1d", - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) # ColumnParallelLinear's loader cannot reshape conv1d weights, so add the # singleton dimension after construction. @@ -546,7 +558,11 @@ class Glm5NextLinearAttention(nn.Module): ) set_weight_attrs( self.A_log, - {"weight_loader": sharded_weight_loader(2, parallel_group="attn_tp")}, + { + "weight_loader": sharded_weight_loader( + 2, parallel_group=head_parallel_group + ) + }, ) self.o_norm = FusedRMSNormGated( @@ -559,7 +575,7 @@ class Glm5NextLinearAttention(nn.Module): quant_config=quant_config, prefix=f"{prefix}.o_proj", reduce_results=reduce_results, - parallel_group="attn_tp", + parallel_group=head_parallel_group, ) conv_weights = self.qkv_conv1d.weight.squeeze(1) @@ -629,6 +645,14 @@ class Glm5NextLinearAttention(nn.Module): if forward_batch.forward_mode.is_idle(): return hidden_states + from sglang.srt.layers.cp.interleave import ( + cp_interleave_to_sequence_order, + cp_sequence_to_interleave_order, + ) + + gathered_rows = hidden_states.shape[0] + hidden_states = cp_interleave_to_sequence_order(hidden_states, forward_batch) + if self.do_fuse_qkvbfg: mixed_qkv, beta, forget_gate, g_proj_states = self.forward_qkvbfg_fused( hidden_states, forward_batch @@ -653,7 +677,9 @@ class Glm5NextLinearAttention(nn.Module): core_attn_out = self.o_norm(core_attn_out, norm_gate) core_attn_out = core_attn_out.squeeze(0).flatten(-2) - return self.o_proj(core_attn_out)[0] + return cp_sequence_to_interleave_order( + self.o_proj(core_attn_out)[0], forward_batch, gathered_rows + ) class Glm5NextDecoderLayer(nn.Module): @@ -785,7 +811,13 @@ class Glm5NextDecoderLayer(nn.Module): ).residual_ops() self.attn_boundary, self.ffn_boundary = append_stages( ( - declare_attn(read=residual.attn_readout, update=residual.attn_update), + declare_attn( + read=residual.attn_readout, + update=residual.attn_update, + tp_group=SumGroup.TP + if self.is_linear_attn and get_parallel().enable_cp_tp_group_sharing + else SumGroup.ATTN_TP, + ), self.input_layernorm, { "qkv_latent_func": self.self_attn.prepare_qkv_latent @@ -939,6 +971,7 @@ class Glm5NextDecoderLayer(nn.Module): isinstance(self.mlp, Glm5NextMoE) and not self.mlp.experts.moe_runner_config.inplace and not torch.compiler.is_compiling() + and hidden_states_orig.shape[0] == hidden_states.shape[0] ): from sglang.srt.layers.moe.moe_runner.base import moe_output_buffer_ctx @@ -1564,6 +1597,24 @@ class Glm5NextForConditionalGeneration(nn.Module): video_embeds = self.visual(pixel_values, grid_thw=flattened_video_grid_thw) return video_embeds + def prepare_cp_inputs(self, forward_batch, **kwargs): + """Finish multimodal embedding injection before the runner shards tokens.""" + input_embeds = kwargs.pop("input_embeds", None) + positions = ( + forward_batch.mrope_positions + if self.is_mrope_enabled + else forward_batch.positions + ) + if input_embeds is None or forward_batch.contains_mm_inputs(): + input_embeds = prepare_mm_inputs( + forward_batch.input_ids, + forward_batch, + self.model, + kwargs, + multimodal_model=self, + ) + return input_embeds, positions, kwargs + @torch.no_grad() def forward( self, @@ -1577,14 +1628,25 @@ class Glm5NextForConditionalGeneration(nn.Module): positions = forward_batch.mrope_positions with get_attn_tp_context().maybe_input_scattered(forward_batch): - hidden_states = general_mm_embed_routine( - input_ids=input_ids, - forward_batch=forward_batch, - language_model=self.model, - multimodal_model=self, - positions=positions, - pp_proxy_tensors=pp_proxy_tensors, - ) + if input_embeds is not None and not forward_batch.contains_mm_inputs(): + # Preserve input_embeds / replace_embeds supplied by the runner. + # The MM routine otherwise performs a fresh token lookup. + hidden_states = self.model( + input_ids=input_ids, + positions=positions, + forward_batch=forward_batch, + input_embeds=input_embeds, + pp_proxy_tensors=pp_proxy_tensors, + ) + else: + hidden_states = general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.model, + multimodal_model=self, + positions=positions, + pp_proxy_tensors=pp_proxy_tensors, + ) aux_hidden_states = None if self.capture_aux_hidden_states: diff --git a/python/sglang/srt/models/glm5_next_nextn.py b/python/sglang/srt/models/glm5_next_nextn.py index 2b768b128c..d32162c997 100644 --- a/python/sglang/srt/models/glm5_next_nextn.py +++ b/python/sglang/srt/models/glm5_next_nextn.py @@ -22,6 +22,15 @@ logger = logging.getLogger(__name__) class Glm5NextForConditionalGenerationNextN(DeepseekV3ForCausalLMNextN): + def prepare_cp_inputs(self, forward_batch, **kwargs): + # The worker has rotated the target's multimodal embeddings per request. + # Fill each appended token while indices still address full sequences, + # then let the runner shard embeddings and target states together. + input_embeds = self.model.embed_input_ids( + forward_batch.input_ids, forward_batch + ) + return input_embeds, forward_batch.positions, {} + @classmethod def get_hf_to_sglang_mapper(cls, config) -> WeightsMapper: text_config = getattr(config, "text_config", config) diff --git a/test/registered/cp/test_glm53_flash_cp.py b/test/registered/cp/test_glm53_flash_cp.py new file mode 100644 index 0000000000..1f232facda --- /dev/null +++ b/test/registered/cp/test_glm53_flash_cp.py @@ -0,0 +1,137 @@ +"""GLM-5.3-Flash interleave CP4 + native MTP on four Blackwell GPUs.""" + +import base64 +import io +import unittest + +import requests +from PIL import Image + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + _wait_for_gpu_idle_in_ci, + popen_launch_server, + terminate_and_kill_process_tree, + try_cached_model, +) + +register_cuda_ci(est_time=600, stage="extra-b", runner_config="4-gpu-b200") + + +class TestGLM53FlashB200ContextParallel( + GSM8KMixin, + CustomTestCase, +): + gsm8k_score_threshold = 0.93 + gsm8k_num_examples = 500 + gsm8k_num_shots = 20 + server_args = [ + "--tp-size", + "4", + "--dsa-prefill-backend", + "trtllm", + "--dsa-decode-backend", + "trtllm", + "--kv-cache-dtype", + "fp8_e4m3", + "--moe-runner-backend", + "flashinfer_trtllm", + "--reasoning-parser", + "auto", + "--tool-call-parser", + "auto", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "6", + "--enable-prefill-cp", + "--cp-strategy", + "interleave", + "--attn-cp-size", + "4", + # KDA partitions heads over TP4, sharing the CP group. EP1 keeps every MoE + # on TP4; dense FFNs also use TP4 rather than per-rank computation. + "--ep-size", + "1", + "--moe-dense-tp-size", + "4", + ] + + @classmethod + def setUpClass(cls): + cls.model = try_cached_model("zai-org/GLM-5.3-Flash") + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = None + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=3600, + other_args=cls.server_args, + ) + + @classmethod + def tearDownClass(cls): + if process := getattr(cls, "process", None): + terminate_and_kill_process_tree(process) + _wait_for_gpu_idle_in_ci(timeout=120) + + def test_gsm8k(self): + super().test_gsm8k() + # Require the metric as well as the floor: an accuracy pass alone must + # not hide disabled MTP or a draft whose tokens are mostly rejected. + response = requests.get(self.base_url + "/server_info", timeout=30) + response.raise_for_status() + self.assertGreater( + response.json()["internal_states"][0]["avg_spec_accept_length"], 4.0 + ) + + def test_image_input(self): + # Color is supplied only by the image: the prompt cannot substitute for + # the encoder. Exercise two images to catch missing/cached embeddings. + for color in ("red", "blue"): + with self.subTest(color=color): + png = io.BytesIO() + Image.new("RGB", (256, 256), color=color).save(png, format="PNG") + image_url = ( + "data:image/png;base64," + base64.b64encode(png.getvalue()).decode() + ) + response = requests.post( + self.base_url + "/v1/chat/completions", + json={ + "model": self.model, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": image_url}, + }, + { + "type": "text", + "text": "What is the dominant color of this image? Answer with one English color word.", + }, + ], + } + ], + "temperature": 0, + # The checkpoint template always opens a thinking block. + # Allow it to finish before checking the final answer. + "max_tokens": 1024, + }, + timeout=180, + ) + self.assertEqual(response.status_code, 200, response.text) + answer = response.json()["choices"][0]["message"]["content"] + self.assertIn(color, answer.lower(), response.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index f3c5b0c77a..776ab71160 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -574,15 +574,14 @@ class TestMhcOnTheDeclarations(CustomTestCase): # The write-back consumes this layer's coefficients. self.assertIsNone(communicator.mhc.h_res) - def test_a_gather_over_attention_cp_is_rejected(self): + def test_mhc_can_keep_its_residual_on_attention_cp_shards(self): # A MoE on the TP group under DSA prefill CP gathers its input over - # attention CP, which MHC has not been run with. + # attention CP while MHC retains its residual and coefficients locally. parallel = parallel_of( attn_dp=1, attn_tp=1, attn_cp=2, enable_prefill_cp=True, moe_dense_tp_size=1 ) facts = layer_case(1, 3, sparse=True, previous_sparse=False) - with self.assertRaises(NotImplementedError): - build_mhc(facts, parallel, dsa_cp=True) + build_mhc(facts, parallel, dsa_cp=True) # A dense layer on every rank computes on its own shard: nothing moves. facts = layer_case(1, 3, sparse=False, previous_sparse=False) build_mhc(facts, parallel, dsa_cp=True)