diff --git a/.github/workflows/pr-test-amd-rocm720.yml b/.github/workflows/pr-test-amd-rocm720.yml index 1dae73e7b..2a6b4b128 100644 --- a/.github/workflows/pr-test-amd-rocm720.yml +++ b/.github/workflows/pr-test-amd-rocm720.yml @@ -1060,7 +1060,8 @@ jobs: echo "" echo "=== 5. Detect RDMA Devices for test environment ===" if [ -d "/sys/class/infiniband" ]; then - RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) + # RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) + RDMA_DEVS=rdma0 echo "Detected RDMA Devices: $RDMA_DEVS" echo "SGLANG_TEST_RDMA_DEVICE=$RDMA_DEVS" >> $GITHUB_ENV else @@ -1130,8 +1131,12 @@ jobs: - name: Run test_disaggregation timeout-minutes: 60 run: | + docker exec ci_sglang rocm-smi --showtopotype || true + docker exec ci_sglang bash -c 'strings $(python3 -c "import mooncake,os;print(os.path.dirname(mooncake.__file__))")/engine*.so | grep -iE "rdma,hip|Unsupported multi-protocol"' || true bash scripts/ci/amd/amd_ci_exec.sh \ -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ + -e MC_LOG_LEVEL=TRACE \ + -e GLOG_logtostderr=1 \ -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} # =============================================== DeepSeek-V4 (MI35x, 8-GPU) ==================================================== diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml index 31d233525..c757a983c 100644 --- a/.github/workflows/pr-test-amd.yml +++ b/.github/workflows/pr-test-amd.yml @@ -1115,7 +1115,8 @@ jobs: echo "" echo "=== 5. Detect RDMA Devices for test environment ===" if [ -d "/sys/class/infiniband" ]; then - RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) + # RDMA_DEVS=$(ls /sys/class/infiniband | paste -sd "," -) + RDMA_DEVS=rdma0 echo "Detected RDMA Devices: $RDMA_DEVS" echo "SGLANG_TEST_RDMA_DEVICE=$RDMA_DEVS" >> $GITHUB_ENV else @@ -1185,8 +1186,12 @@ jobs: - name: Run test_disaggregation timeout-minutes: 60 run: | + docker exec ci_sglang rocm-smi --showtopotype || true + docker exec ci_sglang bash -c 'strings $(python3 -c "import mooncake,os;print(os.path.dirname(mooncake.__file__))")/engine*.so | grep -iE "rdma,hip|Unsupported multi-protocol"' || true bash scripts/ci/amd/amd_ci_exec.sh \ -e SGLANG_TEST_RDMA_DEVICE="${{ env.SGLANG_TEST_RDMA_DEVICE }}" \ + -e MC_LOG_LEVEL=TRACE \ + -e GLOG_logtostderr=1 \ -w "/sglang-checkout/test" python3 run_suite.py --hw amd --suite stage-b-test-large-8-gpu-mi35x-disaggregation-amd --timeout-per-file 1800 ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} pr-test-amd-finish: diff --git a/docker/rocm.Dockerfile b/docker/rocm.Dockerfile index cb01a75c8..79a93f197 100644 --- a/docker/rocm.Dockerfile +++ b/docker/rocm.Dockerfile @@ -97,7 +97,7 @@ ARG LLVM_BRANCH="MainOpSelV2" ARG LLVM_COMMIT="6520ace8227ffe2728148d5f3b9872a870b0a560" ARG MOONCAKE_REPO="https://github.com/kvcache-ai/Mooncake.git" -ARG MOONCAKE_COMMIT="b6a841dc78c707ec655a563453277d969fb8f38d" +ARG MOONCAKE_COMMIT="01d1eb2a7ec37fd5e20a88573e9b4956e7846e9a" ARG TILELANG_REPO="https://github.com/tile-ai/tilelang.git" ARG TILELANG_COMMIT="a55a82302bf7f3c5af635b5c9146f728185cc900" @@ -264,7 +264,7 @@ RUN if [ "$BUILD_MOONCAKE" = "1" ]; then \ rm go1.22.2.linux-amd64.tar.gz && \ mkdir -p build && \ cd build && \ - cmake .. -DUSE_HIP=ON -DUSE_ETCD=ON && \ + cmake .. -DUSE_HIP=ON -DUSE_ETCD=ON -DENABLE_MULTI_PROTOCOL=ON -DWITH_STORE=ON -DBUILD_UNIT_TESTS=OFF && \ make -j "$(nproc)" && make install; \ fi