Deprecate the parallel getters the context answers, and ratchet them shut (#40342)
This commit is contained in:
@@ -11,7 +11,7 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch.cuda.streams import ExternalStream
|
||||
|
||||
from sglang.srt.distributed.parallel_state import set_pdmux_status
|
||||
from sglang.srt.distributed.parallel_state import pdmux_prefill_tp_group
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.multiplex.pdmux_context import (
|
||||
get_current_stream_idx,
|
||||
@@ -21,7 +21,7 @@ from sglang.srt.multiplex.pdmux_context import (
|
||||
load_pdmux_config,
|
||||
set_current_stream_idx,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_disagg
|
||||
from sglang.srt.runtime_context import get_device, get_disagg, get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
@@ -37,7 +37,7 @@ class SchedulerMultiplexMixin:
|
||||
|
||||
# for pd_multiplexing, Init stream_groups, exclude normal stream for prefill only and decode only
|
||||
self.pdmux_config = load_pdmux_config(get_disagg().pdmux_config_path)
|
||||
initialize_stream_groups(self.gpu_id, self.pdmux_config)
|
||||
initialize_stream_groups(get_device().gpu_id, self.pdmux_config)
|
||||
self.stream_groups = get_stream_groups()
|
||||
self.sm_counts = get_sm_counts()
|
||||
self.real_sm_group_num = len(self.stream_groups)
|
||||
@@ -113,12 +113,10 @@ class SchedulerMultiplexMixin:
|
||||
|
||||
while True:
|
||||
with torch.cuda.stream(decode_stream):
|
||||
set_pdmux_status(False)
|
||||
self.ingest_requests()
|
||||
running_batch = self.running_batch
|
||||
|
||||
with torch.cuda.stream(prefill_stream):
|
||||
set_pdmux_status(True)
|
||||
with torch.cuda.stream(prefill_stream), pdmux_prefill_tp_group():
|
||||
sm_count = self.sm_counts[stream_idx][0]
|
||||
if not wait_prefill_kernel_done:
|
||||
created, running_batch = self.update_split_prefill_batch(
|
||||
@@ -128,7 +126,6 @@ class SchedulerMultiplexMixin:
|
||||
adjust_stream_group = created or adjust_stream_group
|
||||
|
||||
with torch.cuda.stream(decode_stream):
|
||||
set_pdmux_status(False)
|
||||
running_batch = self.update_running_batch(running_batch)
|
||||
self.running_batch = running_batch
|
||||
adjust_stream_group = adjust_stream_group or (
|
||||
@@ -152,15 +149,13 @@ class SchedulerMultiplexMixin:
|
||||
)
|
||||
|
||||
with torch.cuda.stream(decode_stream):
|
||||
set_pdmux_status(False)
|
||||
# process decode batch
|
||||
if running_batch and not running_batch.is_empty():
|
||||
decode_result = self.run_batch(running_batch)
|
||||
decode_done = True
|
||||
else:
|
||||
decode_done = False
|
||||
with torch.cuda.stream(prefill_stream):
|
||||
set_pdmux_status(True)
|
||||
with torch.cuda.stream(prefill_stream), pdmux_prefill_tp_group():
|
||||
if (
|
||||
self.split_prefill_batch
|
||||
and not self.split_prefill_batch.is_empty()
|
||||
@@ -197,13 +192,11 @@ class SchedulerMultiplexMixin:
|
||||
prefill_done = False
|
||||
|
||||
with torch.cuda.stream(decode_stream):
|
||||
set_pdmux_status(False)
|
||||
decode_stream.synchronize()
|
||||
if decode_done:
|
||||
self.process_batch_result(running_batch, decode_result)
|
||||
|
||||
with torch.cuda.stream(prefill_stream):
|
||||
set_pdmux_status(True)
|
||||
with torch.cuda.stream(prefill_stream), pdmux_prefill_tp_group():
|
||||
if prefill_done and self.split_prefill_batch.split_prefill_finished:
|
||||
wait_prefill_kernel_done = True
|
||||
prefill_exe_done_flag = prefill_exe_done.query()
|
||||
@@ -214,7 +207,7 @@ class SchedulerMultiplexMixin:
|
||||
)
|
||||
|
||||
self.tp_cpu_group.allreduce(flags, dist.ReduceOp.SUM).wait()
|
||||
if flags.item() == self.tp_size:
|
||||
if flags.item() == get_parallel().tp_size:
|
||||
self.process_batch_result(
|
||||
self.split_prefill_batch, prefill_result
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user