[CI] Fix rerun-ut workflow: add DeepEP install, RDMA env, Blackwell detection (#19803)
This commit is contained in:
@@ -17,10 +17,16 @@ on:
|
|||||||
required: false
|
required: false
|
||||||
type: string
|
type: string
|
||||||
default: ""
|
default: ""
|
||||||
|
use_deepep:
|
||||||
|
description: "Use ci_install_deepep.sh instead of ci_install_dependency.sh"
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
default: "false"
|
||||||
|
|
||||||
env:
|
env:
|
||||||
SGLANG_IS_IN_CI: true
|
SGLANG_IS_IN_CI: true
|
||||||
SGLANG_CUDA_COREDUMP: "1"
|
SGLANG_CUDA_COREDUMP: "1"
|
||||||
|
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
actions: write
|
actions: write
|
||||||
@@ -32,7 +38,8 @@ jobs:
|
|||||||
timeout-minutes: 120
|
timeout-minutes: 120
|
||||||
env:
|
env:
|
||||||
RUNNER_LABELS: ${{ inputs.runner_label }}
|
RUNNER_LABELS: ${{ inputs.runner_label }}
|
||||||
IS_BLACKWELL: ${{ inputs.runner_label == '1-gpu-5090' && '1' || '' }}
|
IS_BLACKWELL: ${{ (inputs.runner_label == '1-gpu-5090' || contains(inputs.runner_label, 'b200')) && '1' || '' }}
|
||||||
|
SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_label == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }}
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -45,7 +52,11 @@ jobs:
|
|||||||
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
|
if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
|
||||||
source /etc/profile.d/sglang-ci.sh
|
source /etc/profile.d/sglang-ci.sh
|
||||||
fi
|
fi
|
||||||
|
if [[ "${{ inputs.use_deepep }}" == "true" ]]; then
|
||||||
|
bash scripts/ci/cuda/ci_install_deepep.sh
|
||||||
|
else
|
||||||
bash scripts/ci/cuda/ci_install_dependency.sh
|
bash scripts/ci/cuda/ci_install_dependency.sh
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Run test
|
- name: Run test
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
|
|||||||
@@ -424,6 +424,12 @@ CUDA_SUITE_TO_RUNNER = {
|
|||||||
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
"stage-c-test-deepep-8-gpu-h200": "8-gpu-h200",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
DEEPEP_SUITES = {
|
||||||
|
"stage-c-test-8-gpu-h20",
|
||||||
|
"stage-c-test-deepep-4-gpu",
|
||||||
|
"stage-c-test-deepep-8-gpu-h200",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def resolve_test_file(file_part):
|
def resolve_test_file(file_part):
|
||||||
"""
|
"""
|
||||||
@@ -465,7 +471,7 @@ def detect_cuda_suite(file_path_from_test):
|
|||||||
"""
|
"""
|
||||||
Read a test file and extract the suite from register_cuda_ci(suite="...").
|
Read a test file and extract the suite from register_cuda_ci(suite="...").
|
||||||
|
|
||||||
Returns (suite_name, runner_label, error_message).
|
Returns (suite_name, runner_label, use_deepep, error_message).
|
||||||
"""
|
"""
|
||||||
full_path = f"test/{file_path_from_test}"
|
full_path = f"test/{file_path_from_test}"
|
||||||
with open(full_path, "r") as f:
|
with open(full_path, "r") as f:
|
||||||
@@ -478,6 +484,7 @@ def detect_cuda_suite(file_path_from_test):
|
|||||||
return (
|
return (
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
|
False,
|
||||||
(
|
(
|
||||||
f"No `register_cuda_ci()` found in `{full_path}`.\n\n"
|
f"No `register_cuda_ci()` found in `{full_path}`.\n\n"
|
||||||
f"This file may not be a registered CUDA CI test."
|
f"This file may not be a registered CUDA CI test."
|
||||||
@@ -491,12 +498,14 @@ def detect_cuda_suite(file_path_from_test):
|
|||||||
return (
|
return (
|
||||||
suite,
|
suite,
|
||||||
None,
|
None,
|
||||||
|
False,
|
||||||
(
|
(
|
||||||
f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n"
|
f"Unknown CUDA suite `{suite}` in `{full_path}`.\n\n"
|
||||||
f"Known suites: {known}"
|
f"Known suites: {known}"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return suite, runner, None
|
use_deepep = suite in DEEPEP_SUITES
|
||||||
|
return suite, runner, use_deepep, None
|
||||||
|
|
||||||
|
|
||||||
def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
||||||
@@ -541,7 +550,7 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
# Detect suite and runner
|
# Detect suite and runner
|
||||||
suite, runner_label, err = detect_cuda_suite(resolved_path)
|
suite, runner_label, use_deepep, err = detect_cuda_suite(resolved_path)
|
||||||
if err:
|
if err:
|
||||||
comment.create_reaction("confused")
|
comment.create_reaction("confused")
|
||||||
pr.create_issue_comment(f"❌ {err}")
|
pr.create_issue_comment(f"❌ {err}")
|
||||||
@@ -554,7 +563,7 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
|||||||
|
|
||||||
print(
|
print(
|
||||||
f"Resolved: file={resolved_path}, selector={test_selector}, "
|
f"Resolved: file={resolved_path}, selector={test_selector}, "
|
||||||
f"suite={suite}, runner={runner_label}, command='{test_command}'"
|
f"suite={suite}, runner={runner_label}, deepep={use_deepep}, command='{test_command}'"
|
||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -583,12 +592,14 @@ def handle_rerun_ut(gh_repo, pr, comment, user_perms, test_spec, token):
|
|||||||
"test_command": test_command,
|
"test_command": test_command,
|
||||||
"runner_label": runner_label,
|
"runner_label": runner_label,
|
||||||
"pr_head_sha": pr_head_sha,
|
"pr_head_sha": pr_head_sha,
|
||||||
|
"use_deepep": str(use_deepep).lower(),
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
ref = pr.head.ref
|
ref = pr.head.ref
|
||||||
inputs = {
|
inputs = {
|
||||||
"test_command": test_command,
|
"test_command": test_command,
|
||||||
"runner_label": runner_label,
|
"runner_label": runner_label,
|
||||||
|
"use_deepep": str(use_deepep).lower(),
|
||||||
}
|
}
|
||||||
|
|
||||||
dispatch_time = time.time()
|
dispatch_time = time.time()
|
||||||
|
|||||||
Reference in New Issue
Block a user