diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d999206cf7..2383c0196e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1078,6 +1078,8 @@ class Envs: DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False) SGLANG_ZBAL_LOCAL_MEM_SIZE = EnvInt(0) SGLANG_ZBAL_BOOTSTRAP_URL = EnvStr("") + # Set to 0 to allow AscendAttnBackend to set needs_cpu_seq_lens to False for dsa models. + SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS = EnvBool(True) # =================================================================== # MUSA diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index bf63aa1079..e608474493 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -12,6 +12,7 @@ from sgl_kernel_npu.attention.sinks_attention import ( from sglang.srt.configs.model_config import AttentionArch from sglang.srt.dllm.config import DllmConfig +from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import ( AscendTorchNativeAttnBackend, ) @@ -417,6 +418,12 @@ class AscendAttnBackend(AttentionBackend): and self.token_to_kv_pool.swa_layer_nums > 0 ) + if not envs.SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS.get(): + assert not self.is_hybrid_swa, ( + "Hybrid swa is not supply with `needs_cpu_seq_lens=False`." + ) + self.needs_cpu_seq_lens = False + # head num padding self.padding_size_list = [1, 2, 4, 8, 16, 32, 64, 128] self.q_head_num_padding = None @@ -523,51 +530,59 @@ class AscendAttnBackend(AttentionBackend): def init_forward_metadata(self, forward_batch: ForwardBatch): """Init the metadata for a forward pass.""" self.forward_metadata = ForwardMetadata() - seq_lens_max = forward_batch.seq_lens.max() - if forward_batch.forward_mode.is_target_verify(): - if ( - forward_batch.spec_algorithm is not None - and forward_batch.spec_algorithm.is_dflash() + if self.needs_cpu_seq_lens: + seq_lens_max = forward_batch.seq_lens.max() + if forward_batch.forward_mode.is_target_verify(): + if ( + forward_batch.spec_algorithm is not None + and forward_batch.spec_algorithm.is_dflash() + ): + # dflash_worker_v2 already publishes seq_lens_cpu as prefix + + # one verify block, which already covers the draft block. + seq_lens_max = forward_batch.seq_lens_cpu.max().item() + else: + # Overlap scheduling can publish the CPU sequence length one + # step ahead of the device tensor. FIA consumes seq_lens_cpu + # below, so derive the block-table width from the same source. + # Otherwise a page-aligned request can expose KV_S=N while + # asking FIA for N+1. + spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num) + seq_lens_max = ( + forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req + ) + elif ( + forward_batch.forward_mode.is_decode_or_idle() + and forward_batch.spec_info is not None ): - # dflash_worker_v2 already publishes seq_lens_cpu as prefix + - # one verify block, which already covers the draft block. - seq_lens_max = forward_batch.seq_lens_cpu.max().item() + seq_lens_max = forward_batch.seq_lens.max() + seq_lens_max += self.speculative_step_id + 1 else: - # Overlap scheduling can publish the CPU sequence length one - # step ahead of the device tensor. FIA consumes seq_lens_cpu - # below, so derive the block-table width from the same source. - # Otherwise a page-aligned request can expose KV_S=N while - # asking FIA for N+1. - spec_tokens_per_req = int(forward_batch.spec_info.draft_token_num) - seq_lens_max = ( - forward_batch.seq_lens_cpu.max().item() + spec_tokens_per_req + seq_lens_max = forward_batch.seq_lens.max() + self.forward_metadata.block_tables = ( + self.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, :seq_lens_max + ][:, :: self.page_size] + // self.page_size + ) + if self.is_hybrid_swa: + self.forward_metadata.block_tables_swa = ( + ( + self.full_to_swa_index_mapping[ + self.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, :seq_lens_max + ] + ][:, :: self.page_size] + // self.page_size + ) + .to(torch.int32) + .contiguous() ) - elif ( - forward_batch.forward_mode.is_decode_or_idle() - and forward_batch.spec_info is not None - ): - seq_lens_max = forward_batch.seq_lens.max() - seq_lens_max += self.speculative_step_id + 1 else: - seq_lens_max = forward_batch.seq_lens.max() - self.forward_metadata.block_tables = ( - self.req_to_token_pool.req_to_token[ - forward_batch.req_pool_indices, :seq_lens_max - ][:, :: self.page_size] - // self.page_size - ) - if self.is_hybrid_swa: - self.forward_metadata.block_tables_swa = ( - ( - self.full_to_swa_index_mapping[ - self.req_to_token_pool.req_to_token[ - forward_batch.req_pool_indices, :seq_lens_max - ] - ][:, :: self.page_size] - // self.page_size - ) - .to(torch.int32) - .contiguous() + self.forward_metadata.block_tables = ( + self.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, :: self.page_size + ] + // self.page_size ) if forward_batch.extend_seq_lens is not None: self.forward_metadata.extend_seq_lens = forward_batch.extend_seq_lens @@ -581,7 +596,8 @@ class AscendAttnBackend(AttentionBackend): self.device ).int() - self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int() + if self.needs_cpu_seq_lens: + self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int() # In graph mode (see _init_cuda_graph_metadata) seq_lens_cpu_int stays # None so forward_mtp binds seq_lens_cpu_list instead: graph.update can # only rebind the Host-side IntArray when captured as a Python list. @@ -893,7 +909,7 @@ class AscendAttnBackend(AttentionBackend): bs: int, req_pool_indices: torch.Tensor, seq_lens: torch.Tensor, - seq_lens_cpu: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], forward_mode: ForwardMode, spec_info: Optional[SpecInput], out_cache_loc: Optional[torch.Tensor] = None, @@ -912,59 +928,75 @@ class AscendAttnBackend(AttentionBackend): self.cuda_graph_swa_out_cache_loc[:n].copy_( self.token_to_kv_pool.translate_loc_from_full_to_swa(out_cache_loc) ) - # Compute the host-side KV lengths visible to this attention step. Target - # verify adds the draft block except for DFlash, whose seq_lens_cpu already - # includes it; speculative decode adds tokens through the current draft step. - attention_kv_lens_cpu = seq_lens_cpu[:bs].int() - if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info): - attention_kv_lens_cpu = ( - attention_kv_lens_cpu + self.speculative_num_draft_tokens - ) - elif forward_mode.is_decode_or_idle() and spec_info is not None: - attention_kv_lens_cpu = attention_kv_lens_cpu + self.speculative_step_id + 1 - max_len = attention_kv_lens_cpu.max().item() - max_seq_pages = (max_len + self.page_size - 1) // self.page_size + if self.needs_cpu_seq_lens: + # Compute the host-side KV lengths visible to this attention step. Target + # verify adds the draft block except for DFlash, whose seq_lens_cpu already + # includes it; speculative decode adds tokens through the current draft step. + attention_kv_lens_cpu = seq_lens_cpu[:bs].int() + if forward_mode.is_target_verify() and not _is_dflash_verify(spec_info): + attention_kv_lens_cpu = ( + attention_kv_lens_cpu + self.speculative_num_draft_tokens + ) + elif forward_mode.is_decode_or_idle() and spec_info is not None: + attention_kv_lens_cpu = ( + attention_kv_lens_cpu + self.speculative_step_id + 1 + ) + max_len = attention_kv_lens_cpu.max().item() + max_seq_pages = (max_len + self.page_size - 1) // self.page_size - if self.is_hybrid_swa: - full_page_locs = self.req_to_token[ - req_pool_indices[:bs], - 0 : max_len : self.page_size, - ] - swa_page_table = ( - self.full_to_swa_index_mapping[full_page_locs] // self.page_size + if self.is_hybrid_swa: + full_page_locs = self.req_to_token[ + req_pool_indices[:bs], + 0 : max_len : self.page_size, + ] + swa_page_table = ( + self.full_to_swa_index_mapping[full_page_locs] // self.page_size + ) + + metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table) + metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0) + metadata.block_tables_swa[bs:, :].fill_(0) + + # Update SWA mask: True = masked out (don't attend), False = attend. + # DFlash verify seq_lens is prefix-only, so use seq_lens_cpu + # (= prefix + block_size) to keep draft KV inside the mask window. + if ( + forward_mode.is_target_verify() + and _is_dflash_verify(spec_info) + and seq_lens_cpu is not None + ): + # seq_lens_cpu may be a CPU tensor; swa_indices is on-device. + seq_lens_int = seq_lens_cpu[:bs].int().to(self.device) + else: + seq_lens_int = seq_lens[:bs].int() + starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0) + indices = self.graph_metadata["swa_indices"] + start_exp = starts.unsqueeze(1) + seq_exp = seq_lens_int.unsqueeze(1) + mask = (indices.unsqueeze(0) < start_exp) | ( + indices.unsqueeze(0) >= seq_exp + ) + metadata.swa_mask[:bs, 0, :].copy_(mask) + metadata.swa_mask[bs:, :, :].fill_(True) + metadata.block_tables[:bs, :max_seq_pages].copy_( + self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size] + // self.page_size ) - metadata.block_tables_swa[:bs, :max_seq_pages].copy_(swa_page_table) - metadata.block_tables_swa[:bs, max_seq_pages:].fill_(0) - metadata.block_tables_swa[bs:, :].fill_(0) - - # Update SWA mask: True = masked out (don't attend), False = attend. - # DFlash verify seq_lens is prefix-only, so use seq_lens_cpu - # (= prefix + block_size) to keep draft KV inside the mask window. - if ( - forward_mode.is_target_verify() - and _is_dflash_verify(spec_info) - and seq_lens_cpu is not None - ): - # seq_lens_cpu may be a CPU tensor; swa_indices is on-device. - seq_lens_int = seq_lens_cpu[:bs].int().to(self.device) - else: - seq_lens_int = seq_lens[:bs].int() - starts = torch.clamp(seq_lens_int - self.sliding_window_size, min=0) - indices = self.graph_metadata["swa_indices"] - start_exp = starts.unsqueeze(1) - seq_exp = seq_lens_int.unsqueeze(1) - mask = (indices.unsqueeze(0) < start_exp) | ( - indices.unsqueeze(0) >= seq_exp + metadata.block_tables[:bs, max_seq_pages:].fill_(0) + else: + total_pages = min( + metadata.block_tables.shape[1], + (self.req_to_token.shape[1] + self.page_size - 1) // self.page_size, ) - metadata.swa_mask[:bs, 0, :].copy_(mask) - metadata.swa_mask[bs:, :, :].fill_(True) - metadata.block_tables[:bs, :max_seq_pages].copy_( - self.req_to_token[req_pool_indices[:bs], 0 : max_len : self.page_size] - // self.page_size - ) - - metadata.block_tables[:bs, max_seq_pages:].fill_(0) + metadata.block_tables[:bs, :total_pages].copy_( + self.req_to_token[ + req_pool_indices[:bs], + 0 : total_pages * self.page_size : self.page_size, + ] + // self.page_size + ) + metadata.block_tables[:bs, total_pages:].fill_(0) if get_parallel().dcp_enabled and not self.is_draft_worker: if "dcp_origin_out_cache_loc" in self.graph_metadata: @@ -3430,6 +3462,7 @@ class AscendAttnMultiStepDraftBackend: self.attn_backends.append( AscendAttnBackend(model_runner, speculative_step_id=step_id) ) + self.needs_cpu_seq_lens = any(b.needs_cpu_seq_lens for b in self.attn_backends) def common_template(self, forward_batch: ForwardBatch, call_fn: int): assert forward_batch.spec_info is not None diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py index 82531343e8..8233ccd9d0 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/eagle_draft_npu_graph_runner.py @@ -78,12 +78,12 @@ class EAGLEDraftNpuGraphRunner(EAGLEDraftCudaGraphRunner): decision = torch.tensor( int(can_run_graph and seed_ready), dtype=torch.int32, - device=self.device, + device="cpu", ) torch.distributed.all_reduce( decision, op=torch.distributed.ReduceOp.MIN, - group=get_parallel().tp_group.device_group, + group=get_parallel().tp_group.cpu_group, ) return bool(decision.item()) diff --git a/test/manual/ascend/llm_models/test_npu_glm_5_1_w4a8_draft_prefetch.py b/test/manual/ascend/llm_models/test_npu_glm_5_1_w4a8_draft_prefetch.py new file mode 100644 index 0000000000..8b79e06ecc --- /dev/null +++ b/test/manual/ascend/llm_models/test_npu_glm_5_1_w4a8_draft_prefetch.py @@ -0,0 +1,68 @@ +import os +import unittest + +from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin +from sglang.test.ascend.test_ascend_utils import GLM_5_1_W4A8_MODEL_PATH +from sglang.test.test_utils import CustomTestCase + + +class TestGLM51W4A8GraphWithDraftPrefetch(GSM8KAscendMixin, CustomTestCase): + """Testcase: Verify the inference accuracy of GLM-5.1-W4A8 on GSM8K with DraftPrefetch feature. + + [Test Category] Model + [Test Target] GLM-5.1-W4A8 + [Test Config] Prefill+Decode, npu graph enabled, NextN speculative decoding, w4a8 quantization, dp attention, draft prefetch, pin memory + """ + + model = GLM_5_1_W4A8_MODEL_PATH + accuracy = 0.8 + other_args = [ + "--trust-remote-code", + "--mem-fraction-static", + "0.87", + "--attention-backend", + "ascend", + "--quantization", + "modelslim", + "--tp-size", + "16", + "--dp-size", + "4", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "4", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "5", + "--speculative-draft-model-quantization", + "unquant", + ] + + env = { + # copy from GSM8KAscendMixin + **os.environ, + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "ASCEND_MF_STORE_URL": "tcp://127.0.0.1:24666", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "24", + "USE_VLLM_CUSTOM_ALLREDUCE": "1", + "HCCL_EXEC_TIMEOUT": "200", + "STREAMS_PER_DEVICE": "32", + "SGLANG_ENBLE_TORCH_COMILE": "1", + "AUTO_USE_UC_MEMORY": "0", + "P2P_HCCL_BUFFSIZE": "20", + # feature test + "HCCL_BUFFSIZE": "1000", + "SGLANG_NPU_ATTN_BACKEND_NEEDS_CPU_SEQ_LENS": "0", + "SGLANG_NPU_ENABLE_PIN_MEMORY": "1", + } + + +if __name__ == "__main__": + unittest.main()