diff --git a/docs/docs/developer_guide/layer_boundary.mdx b/docs/docs/developer_guide/layer_boundary.mdx index 326443b6e1..1a5b494cd7 100644 --- a/docs/docs/developer_guide/layer_boundary.mdx +++ b/docs/docs/developer_guide/layer_boundary.mdx @@ -195,7 +195,7 @@ hidden_states = residual_batch.final_norm( `skip_empty=True` still completes owed work on an empty batch, which may be a collective that other ranks join, and skips only the norm kernel. `final_norm()` and `to_pp()` release the batch's stream after they hand on its output. -This is a non-pipeline stack fragment. Pipeline reception uses the first boundary's `from_pp(tensors, forward_batch)` to reconstruct a stream; the sender uses `residual_batch.to_pp()`. Dynamic completion work is resolved before transport. By default, `to_pp()` preserves a statically declared partial sum for the receiver's incoming contract to reconstruct and complete. Do not call `complete_output()` before that handoff: it would complete a sum that the receiver still expects to perform. +This is a non-pipeline stack fragment. Pipeline reception uses the first boundary's `from_pp(tensors, forward_batch)` to reconstruct a stream; the sender uses `residual_batch.to_pp()`. A stream the producer already wrote, such as MHC's expanded streams, travels as `hidden_states` alone and arrives written. Dynamic completion work is resolved before transport. By default, `to_pp()` preserves a statically declared partial sum for the receiver's incoming contract to reconstruct and complete. Do not call `complete_output()` before that handoff: it would complete a sum that the receiver still expects to perform. A terminal FFN normally completes its output before the final norm. The exception is a MoE finalize handoff. When the stage's fusion provider can defer the finalize at the terminal layer (for CuteDSL, `install_cutedsl_fusion(..., terminal_finalize=True)`), the exit may leave a `DeferredFinalize` for the final norm. Pass the handle straight to `residual_batch.final_norm(hidden_states, forward_batch, self.norm, finalize_norm=service)`, where `service` is the object `install_cutedsl_fusion()` returned; its `finalize(handoff, residual, gamma)` fuses finalize, all-reduce, residual add, and norm. The fused path reads `layernorm.gemma_weight`, so it currently expects a `GemmaRMSNorm` final norm. Without `finalize_norm`, `final_norm()` first completes the handoff unfused, and `capture` is then allowed. With `finalize_norm`, `final_norm()` rejects `capture` when a handoff arrives. Calling `complete_output()` first runs the producer's unfused finalize, and `snapshot()` raises on a handoff. See `python/sglang/srt/models/qwen3_5.py`. diff --git a/python/sglang/srt/layers/layer_boundary/residual/batch.py b/python/sglang/srt/layers/layer_boundary/residual/batch.py index 83929dcb64..3bd56baab5 100644 --- a/python/sglang/srt/layers/layer_boundary/residual/batch.py +++ b/python/sglang/srt/layers/layer_boundary/residual/batch.py @@ -115,13 +115,19 @@ def to_pp(hidden_states, forward_batch, *, preserve_declared=True): partial sum unreduced for the receiver's from_pp to complete; pass False only when the receiver does not declare that sum. - Runtime-selected completion work is finished before transport. + Runtime-selected completion work is finished before transport. A stream + the producer already wrote (MHC writes its streams at the FFN exit) has no + separate residual and is sent as hidden_states alone; the receiver's + from_pp reconstructs it as written. """ hidden_states, residual = stream_of(forward_batch).export( hidden_states, preserve_declared=preserve_declared ) forward_batch.residual_stream = None - return PPProxyTensors({"hidden_states": hidden_states, "residual": residual}) + tensors = {"hidden_states": hidden_states} + if residual is not None: + tensors["residual"] = residual + return PPProxyTensors(tensors) def snapshot(hidden_states, forward_batch): diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index 4a9f605c18..ca7c54ee93 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -1185,8 +1185,6 @@ class Glm5NextModel(nn.Module): ) if not self.pp_group.is_last_rank: - if self.config.mhc: - return PPProxyTensors({"hidden_states": hidden_states}) return residual_batch.to_pp(hidden_states, forward_batch) else: hidden_states = residual_batch.complete_output(hidden_states, forward_batch)