[Cleanup] Move mamba-max-states-per-path validation into _handle_mamba_backend (#32502)
This commit is contained in:
@@ -36,7 +36,7 @@ If you do not want to use a Docker image, reproduce the dependency installation
|
|||||||
|
|
||||||
Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8, GB300 2×4, MI350X/MI355X 1×8), so it is not a separate choice.
|
Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8, GB300 2×4, MI350X/MI355X 1×8), so it is not a separate choice.
|
||||||
|
|
||||||
**PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#pd-disaggregation)); `Prefill` ships two strategies on the TP8 platforms, both chunked at 16k: `Default` (TP8) and `Long-Context` (`--pp-size 8 --tp-size 1`, see [Deep PP](#deep-pp-for-long-context-prefill)).
|
**PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#3-4-pd-disaggregation)); `Prefill` ships two strategies on the TP8 platforms, both chunked at 16k: `Default` (TP8) and `Long-Context` (`--pp-size 8 --tp-size 1`, see [Deep PP](#deep-pp-for-long-context-prefill)).
|
||||||
|
|
||||||
**Strategy** — the operating point within that shape:
|
**Strategy** — the operating point within that shape:
|
||||||
|
|
||||||
|
|||||||
@@ -3409,8 +3409,6 @@ class ServerArgs:
|
|||||||
# _handle_model_specific_adjustments never runs.
|
# _handle_model_specific_adjustments never runs.
|
||||||
self._resolved_overrides = []
|
self._resolved_overrides = []
|
||||||
|
|
||||||
self._validate_mamba_max_states_per_path()
|
|
||||||
|
|
||||||
if self.model_path.lower() in ["none", "dummy"]:
|
if self.model_path.lower() in ["none", "dummy"]:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -3588,14 +3586,6 @@ class ServerArgs:
|
|||||||
|
|
||||||
materialize_declarations(self)
|
materialize_declarations(self)
|
||||||
|
|
||||||
def _validate_mamba_max_states_per_path(self):
|
|
||||||
value = self.mamba_max_states_per_path
|
|
||||||
if value == 0 or value < -1:
|
|
||||||
raise ValueError(
|
|
||||||
"--mamba-max-states-per-path must be -1 (unlimited) or a positive "
|
|
||||||
f"integer, got {value}."
|
|
||||||
)
|
|
||||||
|
|
||||||
def _handle_model_capability_adjustments(self):
|
def _handle_model_capability_adjustments(self):
|
||||||
if parse_connector_type(self.model_path) == ConnectorType.INSTANCE:
|
if parse_connector_type(self.model_path) == ConnectorType.INSTANCE:
|
||||||
return
|
return
|
||||||
@@ -5812,6 +5802,12 @@ class ServerArgs:
|
|||||||
if self.mamba_cache_philox_rounds < 0:
|
if self.mamba_cache_philox_rounds < 0:
|
||||||
raise ValueError("--mamba-cache-philox-rounds must be non-negative.")
|
raise ValueError("--mamba-cache-philox-rounds must be non-negative.")
|
||||||
|
|
||||||
|
if self.mamba_max_states_per_path == 0 or self.mamba_max_states_per_path < -1:
|
||||||
|
raise ValueError(
|
||||||
|
"--mamba-max-states-per-path must be -1 (unlimited) or a positive "
|
||||||
|
f"integer, got {self.mamba_max_states_per_path}."
|
||||||
|
)
|
||||||
|
|
||||||
if self.enable_mamba_cache_stochastic_rounding:
|
if self.enable_mamba_cache_stochastic_rounding:
|
||||||
if self.mamba_ssm_dtype != "float16":
|
if self.mamba_ssm_dtype != "float16":
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
|
|||||||
@@ -103,14 +103,12 @@ class TestMambaPathStateCap(unittest.TestCase):
|
|||||||
|
|
||||||
def test_server_arg_rejects_zero_and_values_below_negative_one(self):
|
def test_server_arg_rejects_zero_and_values_below_negative_one(self):
|
||||||
for value in (0, -2):
|
for value in (0, -2):
|
||||||
|
args = ServerArgs(model_path="dummy", mamba_max_states_per_path=value)
|
||||||
with self.subTest(value=value), self.assertRaisesRegex(
|
with self.subTest(value=value), self.assertRaisesRegex(
|
||||||
ValueError,
|
ValueError,
|
||||||
"must be -1 \\(unlimited\\) or a positive integer",
|
"must be -1 \\(unlimited\\) or a positive integer",
|
||||||
):
|
):
|
||||||
ServerArgs(
|
args._handle_mamba_backend()
|
||||||
model_path="dummy",
|
|
||||||
mamba_max_states_per_path=value,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_unified_cache_removes_only_shallow_mamba_state(self):
|
def test_unified_cache_removes_only_shallow_mamba_state(self):
|
||||||
component, nodes, core, cache = _build_unified_chain(cap=2)
|
component, nodes, core, cache = _build_unified_chain(cap=2)
|
||||||
|
|||||||
Reference in New Issue
Block a user