diff --git a/.github/workflows/nightly-test-amd-rocm720.yml b/.github/workflows/nightly-test-amd-rocm720.yml index bf5273d08..a59532244 100644 --- a/.github/workflows/nightly-test-amd-rocm720.yml +++ b/.github/workflows/nightly-test-amd-rocm720.yml @@ -1162,52 +1162,20 @@ jobs: with: ref: ${{ inputs.ref || github.sha }} - - name: Resolve DSv4 image tag - id: dsv4_image - run: | - # Pick the latest Docker Hub tag matching rocm720-mi35x---DSv4. - # Docker Hub returns results sorted by last_updated DESC by default, so the - # first regex match is the most recent daily build. - AUTH_HEADER=() - if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then - TOKEN=$(curl -s -H "Content-Type: application/json" \ - -X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \ - https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))") - if [[ -n "$TOKEN" ]]; then - AUTH_HEADER=(-H "Authorization: JWT $TOKEN") - fi - fi - TAG=$(curl -s "${AUTH_HEADER[@]}" \ - "https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \ - | grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \ - | head -n 1 | cut -d'"' -f4) - if [ -z "$TAG" ]; then - echo "::error::No DSv4 image found matching rocm720-mi35x---DSv4 on Docker Hub" - exit 1 - fi - echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT" - echo "Resolved DSv4 image: rocm/sgl-dev:$TAG" - - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2 DSv4) + - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }} + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 env: GITHUB_WORKSPACE: ${{ github.workspace }} - - name: Install dependencies (preserve DSv4 sglang/aiter from image) + - name: Install dependencies run: | - # --skip-sglang-build: keep the image's pre-installed DSv4 sglang - # (default would `pip install -e /sglang-checkout/python` and clobber it with main's source). - # --skip-aiter-build: keep the image's DSv4-tuned aiter - # (default reads /sglang-checkout/docker/rocm.Dockerfile from main and rebuilds aiter to that commit). # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. - bash scripts/ci/amd/amd_ci_install_dependency.sh \ - --skip-sglang-build --skip-aiter-build --skip-test-time-deps - # tabulate is the only thing run_suite.py imports that may not be in the DSv4 image. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Flash FP8 + FP4) @@ -1229,43 +1197,20 @@ jobs: with: ref: ${{ inputs.ref || github.sha }} - - name: Resolve DSv4 image tag - id: dsv4_image - run: | - AUTH_HEADER=() - if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then - TOKEN=$(curl -s -H "Content-Type: application/json" \ - -X POST -d "{\"username\":\"${DOCKERHUB_AMD_USERNAME}\",\"password\":\"${DOCKERHUB_AMD_TOKEN}\"}" \ - https://hub.docker.com/v2/users/login/ | python3 -c "import json,sys; print(json.load(sys.stdin).get('token',''))") - if [[ -n "$TOKEN" ]]; then - AUTH_HEADER=(-H "Authorization: JWT $TOKEN") - fi - fi - TAG=$(curl -s "${AUTH_HEADER[@]}" \ - "https://hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=DSv4" \ - | grep -oE '"name":"rocm720-mi35x-[a-f0-9]{7}-[0-9]{8}-DSv4"' \ - | head -n 1 | cut -d'"' -f4) - if [ -z "$TAG" ]; then - echo "::error::No DSv4 image found matching rocm720-mi35x---DSv4 on Docker Hub" - exit 1 - fi - echo "image=rocm/sgl-dev:$TAG" >> "$GITHUB_OUTPUT" - echo "Resolved DSv4 image: rocm/sgl-dev:$TAG" - - name: Ensure VRAM is clear run: bash scripts/ci/amd/ensure_vram_clear.sh rocm - - name: Setup docker (ROCm 7.2 DSv4) + - name: Setup docker (ROCm 7.2) run: | touch github_summary.md - bash scripts/ci/amd/amd_ci_start_container.sh --custom-image ${{ steps.dsv4_image.outputs.image }} + bash scripts/ci/amd/amd_ci_start_container.sh --rocm-version rocm720 env: GITHUB_WORKSPACE: ${{ github.workspace }} - - name: Install dependencies (preserve DSv4 sglang/aiter from image) + - name: Install dependencies run: | - bash scripts/ci/amd/amd_ci_install_dependency.sh \ - --skip-sglang-build --skip-aiter-build --skip-test-time-deps + # --skip-test-time-deps: GSM8K + bench_one_batch_server don't need lmms-eval / human-eval. + bash scripts/ci/amd/amd_ci_install_dependency.sh --skip-test-time-deps bash scripts/ci/amd/amd_ci_exec.sh pip install tabulate - name: Accuracy + Performance Test MI35x ROCm 7.2 (8-GPU DeepSeek-V4-Pro FP8 + FP4) diff --git a/test/registered/amd/test_deepseek_v4_flash_fp4.py b/test/registered/amd/test_deepseek_v4_flash_fp4.py index 28764b5f6..a407ba870 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp4.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp4.py @@ -35,38 +35,32 @@ DEEPSEEK_V4_FP4_MODEL_PATH = os.environ.get( ) SERVER_LAUNCH_TIMEOUT = 3600 -# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). -# Source of truth: python/run_dsv4.sh. +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: AITER indexer + triton attn + ROCm700A). COMMON_ENV_VARS = { - "SGLANG_OPT_USE_FUSED_COMPRESS": "true", - "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", - "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", - "SGLANG_OPT_USE_TRITON_SWA_PREPARE": "true", - "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", - "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", - "SGLANG_OPT_USE_AITER_MHC_PRE": "true", - "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", - "SGLANG_OPT_USE_AITER_MHC_POST": "true", - "SGLANG_ENABLE_THINKING": "1", - "SGLANG_USE_AITER": "1", - "AITER_BF16_FP8_MOE_BOUND": "1", - "SGLANG_USE_ROCM700A": "1", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", - "SGLANG_OPT_DPSK_V4_RADIX": "0", - "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", - "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", - "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", - "SGLANG_OPT_USE_TILELANG_INDEXER": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_OPT_USE_FUSED_COMPRESS": "true", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_OPT_FP8_WO_A_GEMM": "false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", + "SGLANG_OPT_USE_TOPK_V2": "false", + "SGLANG_OPT_USE_AITER_INDEXER": "true", + "SGLANG_OPT_USE_TILELANG_INDEXER": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP": "false", + "SGLANG_ROCM_USE_MULTI_STREAM": "false", + "AITER_BF16_FP8_MOE_BOUND": "0", } -# FP4 variant: FP4 mixed-precision experts. +# FP4 variant FP4_ENV_VARS = { "SGLANG_DSV4_FP4_EXPERTS": "true", - "SGLANG_FORCE_TRITON_MOE_FP8": "0", } @@ -86,11 +80,15 @@ class TestDeepseekV4Fp4(CustomTestCase): "8", "--disable-radix-cache", "--attention-backend", - "compressed", + "dsv4", "--max-running-requests", "256", "--page-size", "256", + "--mem-fraction-static", + "0.90", + "--swa-full-tokens-ratio", + "0.1", "--chunked-prefill-size", "8192", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_flash_fp8.py b/test/registered/amd/test_deepseek_v4_flash_fp8.py index 9d56ee22b..53a51bc7d 100644 --- a/test/registered/amd/test_deepseek_v4_flash_fp8.py +++ b/test/registered/amd/test_deepseek_v4_flash_fp8.py @@ -35,38 +35,32 @@ DEEPSEEK_V4_FP8_MODEL_PATH = os.environ.get( ) SERVER_LAUNCH_TIMEOUT = 3600 -# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). -# Source of truth: python/run_dsv4.sh. +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: AITER indexer + triton attn + ROCm700A). COMMON_ENV_VARS = { - "SGLANG_OPT_USE_FUSED_COMPRESS": "true", - "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", - "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", - "SGLANG_OPT_USE_TRITON_SWA_PREPARE": "true", - "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", - "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", - "SGLANG_OPT_USE_AITER_MHC_PRE": "true", - "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", - "SGLANG_OPT_USE_AITER_MHC_POST": "true", - "SGLANG_ENABLE_THINKING": "1", - "SGLANG_USE_AITER": "1", - "AITER_BF16_FP8_MOE_BOUND": "1", - "SGLANG_USE_ROCM700A": "1", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", - "SGLANG_OPT_DPSK_V4_RADIX": "0", - "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", - "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", - "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", - "SGLANG_OPT_USE_TILELANG_INDEXER": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_OPT_USE_FUSED_COMPRESS": "true", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_OPT_FP8_WO_A_GEMM": "false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", + "SGLANG_OPT_USE_TOPK_V2": "false", + "SGLANG_OPT_USE_AITER_INDEXER": "true", + "SGLANG_OPT_USE_TILELANG_INDEXER": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP": "false", + "SGLANG_ROCM_USE_MULTI_STREAM": "false", + "AITER_BF16_FP8_MOE_BOUND": "0", } -# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path. +# FP8 variant FP8_ENV_VARS = { "SGLANG_DSV4_FP4_EXPERTS": "false", - "SGLANG_FORCE_TRITON_MOE_FP8": "1", } @@ -86,11 +80,15 @@ class TestDeepseekV4Fp8(CustomTestCase): "8", "--disable-radix-cache", "--attention-backend", - "compressed", + "dsv4", "--max-running-requests", "256", "--page-size", "256", + "--mem-fraction-static", + "0.90", + "--swa-full-tokens-ratio", + "0.1", "--chunked-prefill-size", "8192", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4.py b/test/registered/amd/test_deepseek_v4_pro_fp4.py index 784143a11..b1d91e66e 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4.py @@ -37,38 +37,32 @@ DEEPSEEK_V4_PRO_FP4_MODEL_PATH = os.environ.get( # Pro is 1.6T; weight load + warmup is much longer than Flash 285B. SERVER_LAUNCH_TIMEOUT = 5400 -# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). -# Source of truth: python/run_dsv4.sh. +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: AITER indexer + triton attn + ROCm700A). COMMON_ENV_VARS = { - "SGLANG_OPT_USE_FUSED_COMPRESS": "true", - "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", - "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", - "SGLANG_OPT_USE_TRITON_SWA_PREPARE": "true", - "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", - "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", - "SGLANG_OPT_USE_AITER_MHC_PRE": "true", - "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", - "SGLANG_OPT_USE_AITER_MHC_POST": "true", - "SGLANG_ENABLE_THINKING": "1", - "SGLANG_USE_AITER": "1", - "AITER_BF16_FP8_MOE_BOUND": "1", - "SGLANG_USE_ROCM700A": "1", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", - "SGLANG_OPT_DPSK_V4_RADIX": "0", - "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", - "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", - "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", - "SGLANG_OPT_USE_TILELANG_INDEXER": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_OPT_USE_FUSED_COMPRESS": "true", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_OPT_FP8_WO_A_GEMM": "false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", + "SGLANG_OPT_USE_TOPK_V2": "false", + "SGLANG_OPT_USE_AITER_INDEXER": "true", + "SGLANG_OPT_USE_TILELANG_INDEXER": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP": "false", + "SGLANG_ROCM_USE_MULTI_STREAM": "false", + "AITER_BF16_FP8_MOE_BOUND": "0", } -# FP4 variant: FP4 mixed-precision experts. +# FP4 variant FP4_ENV_VARS = { "SGLANG_DSV4_FP4_EXPERTS": "true", - "SGLANG_FORCE_TRITON_MOE_FP8": "0", } @@ -88,11 +82,15 @@ class TestDeepseekV4ProFp4(CustomTestCase): "8", "--disable-radix-cache", "--attention-backend", - "compressed", + "dsv4", "--max-running-requests", "256", "--page-size", "256", + "--mem-fraction-static", + "0.90", + "--swa-full-tokens-ratio", + "0.1", "--chunked-prefill-size", "8192", "--disable-shared-experts-fusion", diff --git a/test/registered/amd/test_deepseek_v4_pro_fp8.py b/test/registered/amd/test_deepseek_v4_pro_fp8.py index 22e304bac..6ef73267d 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp8.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp8.py @@ -37,38 +37,32 @@ DEEPSEEK_V4_PRO_FP8_MODEL_PATH = os.environ.get( # Pro is 1.6T; weight load + warmup is much longer than Flash 285B. SERVER_LAUNCH_TIMEOUT = 5400 -# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: tilelang + AITER + ROCm700A). -# Source of truth: python/run_dsv4.sh. +# Common DeepSeek-V4 env vars (AMD ROCm 7.2 path: AITER indexer + triton attn + ROCm700A). COMMON_ENV_VARS = { - "SGLANG_OPT_USE_FUSED_COMPRESS": "true", - "SGLANG_OPT_USE_OLD_COMPRESSOR": "true", - "SGLANG_OPT_USE_TILELANG_SWA_PREPARE": "false", - "SGLANG_OPT_USE_TRITON_SWA_PREPARE": "true", - "SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK": "false", - "SGLANG_OPT_USE_FUSED_HASH_TOPK": "false", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", - "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", - "SGLANG_OPT_USE_AITER_MHC_PRE": "true", - "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", - "SGLANG_OPT_USE_AITER_MHC_POST": "true", - "SGLANG_ENABLE_THINKING": "1", - "SGLANG_USE_AITER": "1", - "AITER_BF16_FP8_MOE_BOUND": "1", - "SGLANG_USE_ROCM700A": "1", - "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", - "SGLANG_OPT_DPSK_V4_RADIX": "0", - "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "false", - "SGLANG_OPT_USE_FUSED_STORE_CACHE": "false", - "SGLANG_TOPK_TRANSFORM_512_TORCH": "1", - "SGLANG_OPT_USE_TILELANG_INDEXER": "true", - "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_DEFAULT_THINKING": "1", "SGLANG_DSV4_REASONING_EFFORT": "max", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "false", + "SGLANG_USE_AITER": "1", + "SGLANG_USE_ROCM700A": "1", + "SGLANG_OPT_USE_FUSED_COMPRESS": "true", + "SGLANG_OPT_USE_FUSED_COMPRESS_TRITON": "true", + "SGLANG_HACK_FLASHMLA_BACKEND": "triton", + "SGLANG_OPT_FP8_WO_A_GEMM": "false", + "SGLANG_OPT_USE_JIT_INDEXER_METADATA": "false", + "SGLANG_OPT_USE_TOPK_V2": "false", + "SGLANG_OPT_USE_AITER_INDEXER": "true", + "SGLANG_OPT_USE_TILELANG_INDEXER": "false", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "false", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "false", + "SGLANG_FP8_PAGED_MQA_LOGITS_TORCH": "1", + "SGLANG_OPT_USE_MULTI_STREAM_OVERLAP": "false", + "SGLANG_ROCM_USE_MULTI_STREAM": "false", + "AITER_BF16_FP8_MOE_BOUND": "0", } -# FP8 variant: dense-FP8 experts via the Triton MoE FP8 path. +# FP8 variant FP8_ENV_VARS = { "SGLANG_DSV4_FP4_EXPERTS": "false", - "SGLANG_FORCE_TRITON_MOE_FP8": "1", } @@ -88,11 +82,15 @@ class TestDeepseekV4ProFp8(CustomTestCase): "8", "--disable-radix-cache", "--attention-backend", - "compressed", + "dsv4", "--max-running-requests", "256", "--page-size", "256", + "--mem-fraction-static", + "0.90", + "--swa-full-tokens-ratio", + "0.1", "--chunked-prefill-size", "8192", "--disable-shared-experts-fusion",