[DSpark] Fold draft sampling for DeepSeek-V4.1 heads under AUTO (#42987)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
This commit is contained in:
@@ -195,9 +195,6 @@ def _resolve_folded_sampling(
|
||||
return False
|
||||
if mode == DsparkFoldedSampling.FORCE:
|
||||
return True
|
||||
# The V4.1 TP head reduces compact argmax summaries in the greedy graph.
|
||||
if getattr(model.markov_head, "supports_sharded_greedy", False):
|
||||
return False
|
||||
vocab = int(model.lm_head.org_vocab_size)
|
||||
noise_bytes = max_bs * vocab * 4
|
||||
logits_bytes = max_bs * gamma * vocab * _base_logits_dtype(model).itemsize
|
||||
|
||||
@@ -122,7 +122,7 @@ class TestDsparkDpAttentionMoeA2aGate(CustomTestCase):
|
||||
|
||||
|
||||
class TestDsparkFoldedSamplingDefault(CustomTestCase):
|
||||
def test_sharded_greedy_default_and_sampling_override(self):
|
||||
def test_auto_folds_sampling_when_buffers_fit(self):
|
||||
from sglang.srt.environ import DsparkFoldedSampling, envs
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft_sampler import (
|
||||
_resolve_folded_sampling,
|
||||
@@ -140,17 +140,22 @@ class TestDsparkFoldedSamplingDefault(CustomTestCase):
|
||||
tp_rank=0,
|
||||
available_memory_gb=16,
|
||||
)
|
||||
with envs.SGLANG_DSPARK_FOLDED_SAMPLING.override(
|
||||
DsparkFoldedSampling.AUTO.value
|
||||
):
|
||||
self.assertFalse(_resolve_folded_sampling(**args))
|
||||
model.markov_head.supports_sharded_greedy = False
|
||||
self.assertTrue(_resolve_folded_sampling(**args))
|
||||
model.markov_head.supports_sharded_greedy = True
|
||||
# AUTO decides on memory alone, also for sharded-greedy (V4.1 TP) heads.
|
||||
for sharded_greedy in (True, False):
|
||||
model.markov_head.supports_sharded_greedy = sharded_greedy
|
||||
with envs.SGLANG_DSPARK_FOLDED_SAMPLING.override(
|
||||
DsparkFoldedSampling.AUTO.value
|
||||
):
|
||||
self.assertTrue(_resolve_folded_sampling(**args))
|
||||
self.assertFalse(
|
||||
_resolve_folded_sampling(**{**args, "available_memory_gb": 0.5})
|
||||
)
|
||||
with envs.SGLANG_DSPARK_FOLDED_SAMPLING.override(
|
||||
DsparkFoldedSampling.FORCE.value
|
||||
):
|
||||
self.assertTrue(_resolve_folded_sampling(**args))
|
||||
self.assertTrue(
|
||||
_resolve_folded_sampling(**{**args, "available_memory_gb": 0.5})
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user