[NPU] upgrade npu mf_apater plugin (#15853)
This commit is contained in:
@@ -57,7 +57,7 @@ ENV LC_ALL=en_US.UTF-8
|
|||||||
|
|
||||||
|
|
||||||
### Install MemFabric
|
### Install MemFabric
|
||||||
RUN ${PIP_INSTALL} mf-adapter==1.0.0
|
RUN ${PIP_INSTALL} memfabric-hybrid==1.0.0
|
||||||
### Install SGLang Model Gateway
|
### Install SGLang Model Gateway
|
||||||
RUN ${PIP_INSTALL} sglang-router
|
RUN ${PIP_INSTALL} sglang-router
|
||||||
|
|
||||||
|
|||||||
@@ -266,10 +266,10 @@ python -m sglang.launch_server \
|
|||||||
|
|
||||||
### Usage
|
### Usage
|
||||||
|
|
||||||
Use ascend backend with [mf_adapter(download link)](https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com:443/sglang/mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl?AccessKeyId=HPUAXT4YM0U8JNTERLST&Expires=1783151861&Signature=3j10QDUjqk70enaq8lostYV2bEA%3D) and ASCEND_MF_STORE_URL being set
|
Use ascend backend with [memfabric_hybrid](https://gitcode.com/Ascend/memfabric_hybrid) and ASCEND_MF_STORE_URL being set
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl --force-reinstall
|
pip install memfabric-hybrid==1.0.0
|
||||||
export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx"
|
export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx"
|
||||||
```
|
```
|
||||||
Use mooncake backend, more details can be found in mooncake section.
|
Use mooncake backend, more details can be found in mooncake section.
|
||||||
|
|||||||
@@ -41,12 +41,12 @@ conda activate sglang_npu
|
|||||||
|
|
||||||
Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.3.RC2 or higher, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1/softwareinst/instg/instg_0008.html?Mode=PmIns&InstallType=local&OS=openEuler&Software=cannToolKit)
|
Prior to start work with SGLang on Ascend you need to install CANN Toolkit, Kernels operator package and NNAL version 8.3.RC2 or higher, check the [installation guide](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1/softwareinst/instg/instg_0008.html?Mode=PmIns&InstallType=local&OS=openEuler&Software=cannToolKit)
|
||||||
|
|
||||||
#### MemFabric Adaptor
|
#### MemFabric-Hybrid
|
||||||
|
|
||||||
If you want to use PD disaggregation mode, you need to install MemFabric Adaptor. MemFabric Adaptor is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters.
|
If you want to use PD disaggregation mode, you need to install MemFabric-Hybrid. MemFabric-Hybrid is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters.
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
pip install mf-adapter==1.0.0
|
pip install memfabric-hybrid==1.0.0
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Pytorch and Pytorch Framework Adaptor on Ascend
|
#### Pytorch and Pytorch Framework Adaptor on Ascend
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferE
|
|||||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from mf_adapter import TransferEngine
|
from memfabric_hybrid import TransferEngine
|
||||||
|
|
||||||
import_error = None
|
import_error = None
|
||||||
except ImportError as e:
|
except ImportError as e:
|
||||||
@@ -25,7 +25,7 @@ class AscendTransferEngine(MooncakeTransferEngine):
|
|||||||
):
|
):
|
||||||
if import_error is not None:
|
if import_error is not None:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Please install mf_adapter, for details, see docs/backend/pd_disaggregation.md"
|
"Please install memfabric_hybrid, for details, see docs/backend/pd_disaggregation.md"
|
||||||
)
|
)
|
||||||
raise import_error
|
raise import_error
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ def start_disagg_service(
|
|||||||
)
|
)
|
||||||
if is_create_store:
|
if is_create_store:
|
||||||
try:
|
try:
|
||||||
from mf_adapter import create_config_store
|
from memfabric_hybrid import create_config_store
|
||||||
|
|
||||||
ascend_url = os.getenv("ASCEND_MF_STORE_URL")
|
ascend_url = os.getenv("ASCEND_MF_STORE_URL")
|
||||||
create_config_store(ascend_url)
|
create_config_store(ascend_url)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ ${PIP_INSTALL} wheel==0.45.1 pybind11
|
|||||||
|
|
||||||
|
|
||||||
### Install MemFabric
|
### Install MemFabric
|
||||||
${PIP_INSTALL} mf-adapter==1.0.0
|
${PIP_INSTALL} memfabric-hybrid==1.0.0
|
||||||
|
|
||||||
|
|
||||||
### Install PyTorch and PTA
|
### Install PyTorch and PTA
|
||||||
|
|||||||
Reference in New Issue
Block a user