[bug fix] use npu phy id in container env (#14266)
Co-authored-by: jinke15 <jinke15@jd.com>
This commit is contained in:
@@ -276,6 +276,10 @@ Use mooncake backend, more details can be found in mooncake section.
|
|||||||
```bash
|
```bash
|
||||||
export ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE=true
|
export ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE=true
|
||||||
```
|
```
|
||||||
|
ASCEND_NPU_PHY_ID need to be set in container env
|
||||||
|
```bash
|
||||||
|
export ASCEND_NPU_PHY_ID=xxx
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
### Llama Single Node
|
### Llama Single Node
|
||||||
|
|||||||
@@ -3,7 +3,8 @@ import logging
|
|||||||
import os
|
import os
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
|
|
||||||
from sglang.srt.utils import get_bool_env_var, get_free_port, maybe_wrap_ipv6_address
|
from sglang.srt.environ import envs
|
||||||
|
from sglang.srt.utils import get_free_port, maybe_wrap_ipv6_address
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -164,8 +165,12 @@ class MooncakeTransferEngine:
|
|||||||
device_name: Optional[str],
|
device_name: Optional[str],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Initialize the mooncake instance."""
|
"""Initialize the mooncake instance."""
|
||||||
if get_bool_env_var("ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE", "false"):
|
if envs.ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE.get():
|
||||||
|
npu_phy_id = envs.ASCEND_NPU_PHY_ID.get()
|
||||||
|
if npu_phy_id == -1:
|
||||||
hostname += f":{get_free_port()}:npu_{self.gpu_id}"
|
hostname += f":{get_free_port()}:npu_{self.gpu_id}"
|
||||||
|
else:
|
||||||
|
hostname += f":{get_free_port()}:npu_{npu_phy_id}"
|
||||||
ret_value = self.engine.initialize(
|
ret_value = self.engine.initialize(
|
||||||
hostname,
|
hostname,
|
||||||
"P2PHANDSHAKE",
|
"P2PHANDSHAKE",
|
||||||
|
|||||||
@@ -214,6 +214,7 @@ class Envs:
|
|||||||
# Mooncake KV Transfer
|
# Mooncake KV Transfer
|
||||||
SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)
|
SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)
|
||||||
ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False)
|
ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE = EnvBool(False)
|
||||||
|
ASCEND_NPU_PHY_ID = EnvInt(-1)
|
||||||
|
|
||||||
# Mooncake Store
|
# Mooncake Store
|
||||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH = EnvStr(None)
|
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH = EnvStr(None)
|
||||||
|
|||||||
Reference in New Issue
Block a user