diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py index ef6fdcc839..26158b08be 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/npu_cudagraph_backend.py @@ -70,6 +70,9 @@ class NPUCudaGraphBackend(BaseCudaGraphBackend): initializer=self._device_module.set_device, initargs=(self._device_id,), ) + # Event of the last replay; the next rebind waits on it so the device is + # done reading the host-side seq_lens array before we rewrite it. + self._rebind_fence = None @contextmanager def capture_session(self, stream): @@ -175,11 +178,18 @@ class NPUCudaGraphBackend(BaseCudaGraphBackend): graph = self._graphs[shape_key] + if self._rebind_fence is not None: + self._rebind_fence.synchronize() + update_future = self._update_executor.submit( graph.update, cpu_update_input=cpu_update_input ) graph.replay() update_future.result() + + fence = self._device_module.Event() + fence.record() + self._rebind_fence = fence return self._outputs[shape_key] def cleanup(self) -> None: diff --git a/python/sglang/srt/platforms/npu.py b/python/sglang/srt/platforms/npu.py index d01220efb7..c1382ff768 100644 --- a/python/sglang/srt/platforms/npu.py +++ b/python/sglang/srt/platforms/npu.py @@ -56,7 +56,9 @@ class NPUDeviceMixin(DeviceMixin): return torch.npu.mem_get_info(device_id) def is_pin_memory_available(self, device=None) -> bool: - return False + if device is not None and str(device) == "cpu": + return False + return True @classmethod def seed_everything(cls, seed: int | None = None) -> None: diff --git a/test/registered/unit/platforms/test_platform_interface.py b/test/registered/unit/platforms/test_platform_interface.py index 163bb6cc33..17d6eca01b 100644 --- a/test/registered/unit/platforms/test_platform_interface.py +++ b/test/registered/unit/platforms/test_platform_interface.py @@ -280,13 +280,10 @@ class TestNpuDeviceMixin(CustomTestCase): mock_npu.get_device_capability.assert_not_called() def test_pin_memory_available_for_npu_targets(self): - # Pinned memory stays disabled on NPU: torch_npu's pinned-memory + - # non_blocking H2D path is not verified against CANN (see - # NPUSRTPlatform.is_pin_memory_available). base = NPUSRTPlatform() - self.assertFalse(base.is_pin_memory_available()) - self.assertFalse(base.is_pin_memory_available(device="npu")) - self.assertFalse(base.is_pin_memory_available(device=torch.device("npu", 0))) + self.assertTrue(base.is_pin_memory_available()) + self.assertTrue(base.is_pin_memory_available(device="npu")) + self.assertTrue(base.is_pin_memory_available(device=torch.device("npu", 0))) self.assertFalse(base.is_pin_memory_available(device="cpu")) self.assertFalse(base.is_pin_memory_available(device=torch.device("cpu")))