fix(dp-attn): consistent overlap disable decision across DP ranks (#20853)
This commit is contained in:
@@ -1326,12 +1326,21 @@ class Scheduler(
|
|||||||
def is_disable_overlap_for_batch(self, batch: ScheduleBatch) -> bool:
|
def is_disable_overlap_for_batch(self, batch: ScheduleBatch) -> bool:
|
||||||
# For two consecutive prefill batches, we disable overlap to improve the TTFT of the first batch.
|
# For two consecutive prefill batches, we disable overlap to improve the TTFT of the first batch.
|
||||||
# This might slightly hurt the throughput, so we use an environment variable to control it.
|
# This might slightly hurt the throughput, so we use an environment variable to control it.
|
||||||
|
# In DP attention mode, use the globally synchronized is_extend_in_batch
|
||||||
|
# so all DP ranks make the same overlap decision (avoiding deadlock).
|
||||||
|
# In non-DP mode, use the local forward_mode directly.
|
||||||
|
if self.require_mlp_sync:
|
||||||
|
is_extend = lambda b: b and b.is_extend_in_batch
|
||||||
|
else:
|
||||||
|
is_extend = lambda b: b and b.forward_mode.is_extend()
|
||||||
|
|
||||||
|
batch_is_extend = is_extend(batch)
|
||||||
|
last_batch_is_extend = is_extend(self.last_batch)
|
||||||
|
|
||||||
disable_overlap_for_batch = (
|
disable_overlap_for_batch = (
|
||||||
envs.SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP.get()
|
envs.SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP.get()
|
||||||
and batch
|
and batch_is_extend
|
||||||
and batch.forward_mode.is_extend()
|
and last_batch_is_extend
|
||||||
and self.last_batch
|
|
||||||
and self.last_batch.forward_mode.is_extend()
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# We do not support overlap + spec + grammar yet,
|
# We do not support overlap + spec + grammar yet,
|
||||||
|
|||||||
@@ -38,6 +38,10 @@ class TestDPAttentionDP2TP2(
|
|||||||
def setUpClass(cls):
|
def setUpClass(cls):
|
||||||
cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST
|
||||||
cls.base_url = DEFAULT_URL_FOR_TEST
|
cls.base_url = DEFAULT_URL_FOR_TEST
|
||||||
|
cls._env_override = envs.SGLANG_DISABLE_CONSECUTIVE_PREFILL_OVERLAP.override(
|
||||||
|
True
|
||||||
|
)
|
||||||
|
cls._env_override.__enter__()
|
||||||
cls.process = popen_launch_server(
|
cls.process = popen_launch_server(
|
||||||
cls.model,
|
cls.model,
|
||||||
cls.base_url,
|
cls.base_url,
|
||||||
@@ -58,6 +62,7 @@ class TestDPAttentionDP2TP2(
|
|||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls):
|
def tearDownClass(cls):
|
||||||
kill_process_tree(cls.process.pid)
|
kill_process_tree(cls.process.pid)
|
||||||
|
cls._env_override.__exit__(None, None, None)
|
||||||
|
|
||||||
def test_mgsm_en(self):
|
def test_mgsm_en(self):
|
||||||
args = SimpleNamespace(
|
args = SimpleNamespace(
|
||||||
|
|||||||
Reference in New Issue
Block a user